diff --git a/.github/scripts/check-google-java-format.sh b/.github/scripts/check-google-java-format.sh new file mode 100755 index 0000000000..8e12296a74 --- /dev/null +++ b/.github/scripts/check-google-java-format.sh @@ -0,0 +1,45 @@ +#!/bin/bash +set -u + +script_name="" + +case "$(uname -sr)" in + + Darwin*) + script_name="google-java-format_darwin-arm64" + ;; + + Linux*) + script_name="google-java-format_linux-x86-64" + ;; + *) + echo 'Unsupported OS' + exit 1 + ;; +esac + +JAVA_FILES=$(find . -name "*.java" -type f) + +invalid_files=0 + +echo "Following files are formatted incorrectly:"; +# TODO: remove '--skip-reflowing-long-strings' once https://github.com/google/google-java-format/issues/566 is fixed +for FILE in $JAVA_FILES; do + if [[ "$*" == *--fix* ]]; then + ./$script_name --skip-reflowing-long-strings --replace "$FILE" > /dev/null + else + ./$script_name --set-exit-if-changed --skip-reflowing-long-strings "$FILE" > /dev/null + fi + if [ $? -ne 0 ]; then + echo "$FILE" + ((invalid_files++)) + fi +done + +if [ "$invalid_files" -ne 0 ]; then + echo "Found $invalid_files incorrectly formatted files (listed above), run google-java-format to fix them."; + exit 1 +else + echo "All files are formatted correctly." +fi + diff --git a/.github/scripts/download_reports.sh b/.github/scripts/download_reports.sh deleted file mode 100644 index 79e93f818c..0000000000 --- a/.github/scripts/download_reports.sh +++ /dev/null @@ -1,41 +0,0 @@ -TMPDIR="test" -mkdir $TMPDIR - -REPO="allegro/hermes" -BUILDS_FILE="builds.json" -PAST_BUILDS="$TMPDIR/$BUILDS_FILE" - -UNIT_TEST_ARTIFACT="check-test-report" -E2E_REPORT_ARTIFACT="integrationTest-test-report" - -gh run list --repo $REPO --branch master --workflow ci --json "status,databaseId" --limit 20 >> $PAST_BUILDS - -cd $TMPDIR - -jq -c '.[]' "$BUILDS_FILE" | while read i; do - STATUS=$(echo $i | jq '.status') - if [[ "$STATUS" == 'completed' ]]; then - continue - fi - RUN_ID=$(echo $i | jq '.databaseId') - - echo "downloading results for run: $RUN_ID" - RUN_DIR=$RUN_ID - - mkdir $RUN_DIR - echo "creating dir $RUN_DIR" - cd $RUN_DIR - - mkdir $UNIT_TEST_ARTIFACT - cd $UNIT_TEST_ARTIFACT - gh run download --repo $REPO -n $UNIT_TEST_ARTIFACT $RUN_ID - echo "Downloaded unit test report" - cd .. - - mkdir $E2E_REPORT_ARTIFACT - cd $E2E_REPORT_ARTIFACT - gh run download --repo $REPO -n $E2E_REPORT_ARTIFACT $RUN_ID - echo "Downloaded integrationTest report" - - cd ../.. -done diff --git a/.github/scripts/reporter.py b/.github/scripts/reporter.py deleted file mode 100755 index 5e124b1471..0000000000 --- a/.github/scripts/reporter.py +++ /dev/null @@ -1,118 +0,0 @@ -import dataclasses -import os -import sys -from typing import List, Set, Dict -import xml.etree.ElementTree as ET - - -@dataclasses.dataclass -class TestResults: - failed: Set[str] - passed: Set[str] - run_id: str - test_cnt: int = 0 - skipped_cnt: int = 0 - failed_cnt: int = 0 - error_cnt: int = 0 - - -def get_test_files(dir: str) -> List[str]: - files = [ - os.path.join(dp, f) - for dp, dn, filenames in os.walk(dir) - for f in filenames - if os.path.splitext(f)[1] == '.xml' - and os.path.splitext(f)[0].startswith("TEST-") - ] - return files - - -def parse_test_file(file: str, run_id: str) -> TestResults: - root = ET.parse(file).getroot() - result = TestResults( - skipped_cnt=int(root.get("skipped")), - test_cnt=int(root.get("tests")), - failed_cnt=int(root.get("failures")), - error_cnt=int(root.get("errors")), - failed=set(), - passed=set(), - run_id=run_id - ) - name = root.get("name") - name = '.'.join(name.split('.')[4:]) # remove common prefix - - for testcase in root.findall("testcase"): - testname = testcase.get("name") - test_failed = testcase.findall("failure") - if test_failed: - result.failed.add(f"{name}#{testname}") - else: - result.passed.add(f"{name}#{testname}") - return result - - -def aggregate_results(test_dir: str, run_id: str) -> TestResults: - test_files = get_test_files(test_dir) - results = [] - for test_file in test_files: - result = parse_test_file(test_file, run_id) - results.append(result) - - agg = TestResults(set(), set(), run_id) - - for result in results: - agg.test_cnt += result.test_cnt - agg.skipped_cnt += result.skipped_cnt - agg.error_cnt += result.error_cnt - agg.failed_cnt += result.failed_cnt - for fail in result.failed: - agg.failed.add(fail) - for pas in result.passed: - agg.passed.add(pas) - return agg - - -def report(type: str, runs: List[TestResults]) -> str: - failed = {} - markdown = "" - for run in runs: - for fail in run.failed: - if fail in failed: - failed[fail]["count"] += 1 - failed[fail]["runs"].append(run.run_id) - else: - failed[fail] = {"count": 1, "runs": [run.run_id]} - markdown += f"## {type} \n" - markdown += "| Test name | Fail count | Failed in runs |\n" - markdown += "|--|--|--|\n" - for k, v in sorted(failed.items(), key=lambda item: -item[1]["count"]): - markdown += f"| {k} | {v['count']} | {v['runs']} |\n" - markdown += "\n" - return markdown - - -if __name__ == '__main__': - root = sys.argv[1] - print(f"Parsing tests results from directory: {root}") - run_dirs = [f.path for f in os.scandir(root) if f.is_dir()] - print(f"Found {len(run_dirs)} run dirs") - unittest_runs = [] - e2e_runs = [] - - for run in run_dirs: - run_id = os.path.basename(os.path.normpath(run)) - unit_test_dir = os.path.join(run, "check-test-report") - unit_test_results = aggregate_results(unit_test_dir, run_id) - unittest_runs.append(unit_test_results) - - e2e_test_dir = os.path.join(run, "integrationTest-test-report") - e2e_test_results = aggregate_results(e2e_test_dir, run_id) - e2e_runs.append(e2e_test_results) - - step_summary = "# Failing tests report\n" - step_summary += report("Unit tests", unittest_runs) - step_summary += report("Integration tests", e2e_runs) - - result_file = os.environ["GITHUB_STEP_SUMMARY"] - with open(result_file, 'w') as f: - f.write(step_summary) diff --git a/.github/workflows/checkstyle.yml b/.github/workflows/checkstyle.yml deleted file mode 100644 index 42f4ad6dc6..0000000000 --- a/.github/workflows/checkstyle.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Checkstyle - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - checkstyle: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: reviewdog/action-setup@v1 - with: - reviewdog_version: latest - - name: Set up JDK 17 - uses: actions/setup-java@v4 - with: - java-version: 17 - distribution: 'temurin' - - name: Setup Gradle - uses: gradle/gradle-build-action@v2 - - name: Run check style - # ignore lengthy console setup tasks - run: ./gradlew --continue clean checkstyleMain checkstyleTest checkstyleIntegrationTest checkstyleSlowIntegrationTest checkstyleJmh -PmaxCheckstyleWarnings=0 -x attachHermesConsole -x prepareIndexTemplate - - name: Run reviewdog - if: ${{ success() || failure() }} - env: - REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - for f in $(find . -regex '.*/build/reports/checkstyle/.*\.xml'); do - module_name=$(echo "$f" | cut -d "/" -f2) - reviewdog -f=checkstyle -level=warning -filter-mode=nofilter -reporter=github-check -name="checkstyle-$module_name" < $f - done diff --git a/.github/workflows/google-java-format.yml b/.github/workflows/google-java-format.yml new file mode 100644 index 0000000000..2ec1c5bf8b --- /dev/null +++ b/.github/workflows/google-java-format.yml @@ -0,0 +1,26 @@ +name: Google java format + +on: + workflow_dispatch: + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.head_ref }} + + - name: Download and run google java format + run: | + ls -la + curl -sSLO "https://github.com/google/google-java-format/releases/download/v$VERSION/google-java-format_linux-x86-64" + chmod a+x google-java-format_linux-x86-64 + ./.github/scripts/check-google-java-format.sh + shell: bash + env: + VERSION: 1.23.0 + diff --git a/.github/workflows/test_report.yml b/.github/workflows/test_report.yml deleted file mode 100644 index 20d78c3a41..0000000000 --- a/.github/workflows/test_report.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: "Test report" -on: - push: - branches: [ master ] - -jobs: - validation: - name: "Test report" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Grant execute permission for report downloader - run: chmod +x ./.github/scripts/download_reports.sh - - name: Download past reports - run: ./.github/scripts/download_reports.sh - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - name: Aggregate reports - run: python ./.github/scripts/reporter.py "test" diff --git a/.gitignore b/.gitignore index d085ce8969..475005bac9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ build .gradle classes -.idea *.iml *.ipr *.iws @@ -39,3 +38,38 @@ scripts/lib/ scripts/pip-selfcheck.json .DS_Store + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +.idea/artifacts +.idea/compiler.xml +.idea/jarRepositories.xml +.idea/modules.xml +.idea/*.iml +.idea/modules + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries diff --git a/.idea/externalDependencies.xml b/.idea/externalDependencies.xml new file mode 100644 index 0000000000..679f74ca17 --- /dev/null +++ b/.idea/externalDependencies.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/google-java-format.xml b/.idea/google-java-format.xml new file mode 100644 index 0000000000..8b57f4527a --- /dev/null +++ b/.idea/google-java-format.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000000..4336a05b15 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,58 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000000..35eb1ddfbb --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/README.md b/README.md index 9b0545b9c9..eadb83bfd6 100644 --- a/README.md +++ b/README.md @@ -20,3 +20,40 @@ If you have any question or idea regarding the project, please feel free to reac ## License **hermes** is published under [Apache License 2.0](http://www.apache.org/licenses/LICENSE-2.0). + +## Development + +### Code formatting +For code formatting we use [google-java-format](https://github.com/google/google-java-format/tree/master). +Following steps are required for optimal dev experience in IJ: + +1. Download [google-java-format plugin](https://plugins.jetbrains.com/plugin/8527-google-java-format) +2. [Set custom VM options required for IJ plugin](https://github.com/google/google-java-format/tree/master?tab=readme-ov-file#intellij-jre-config) +3. Go to `Settings > google-java-format` and click `Enable google java-format` (should be checked by default) +4. Go to `Settings > Tools > Actions on Save` and enable `Reformat code` and `Optimize imports` for Java files + +Each save should automatically trigger reformat. + +If you want to debug the CLI check on macOS: + +```shell +wget https://github.com/google/google-java-format/releases/download/v1.23.0/google-java-format_darwin-arm64 +chmod a+x google-java-format_darwin-arm64 +chmod a+x .github/scripts/check-google-java-format.sh +./.github/scripts/check-google-java-format.sh +``` + +or if you are on Linux: + +```shell +wget https://github.com/google/google-java-format/releases/download/v1.23.0/google-java-format_linux-x86-64 +chmod a+x google-java-format_linux-x86-64 +chmod a+x .github/scripts/check-google-java-format.sh +./.github/scripts/check-google-java-format.sh +``` + +You can also run the following command to fix formatting for the whole project: + +```shell +./.github/scripts/check-google-java-format.sh --fix +``` diff --git a/build.gradle b/build.gradle index b1b59aa29f..02f86f4e6d 100644 --- a/build.gradle +++ b/build.gradle @@ -39,7 +39,6 @@ nexusPublishing { allprojects { apply plugin: 'java' apply plugin: 'groovy' - apply plugin: 'checkstyle' group = 'pl.allegro.tech.hermes' version = scmVersion.version @@ -206,18 +205,6 @@ subprojects { events 'passed', 'skipped', 'failed' } } - - tasks.withType(Checkstyle) { - reports { - xml.required = true - html.required = false - } - } - - checkstyle { - toolVersion '10.3.4' - maxWarnings getIntProperty('maxCheckstyleWarnings', Integer.MAX_VALUE) - } } def getIntProperty(String name, int defaultValue) { diff --git a/config/checkstyle/checkstyle.xml b/config/checkstyle/checkstyle.xml deleted file mode 100644 index 06c50058df..0000000000 --- a/config/checkstyle/checkstyle.xml +++ /dev/null @@ -1,439 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/config/checkstyle/suppressions.xml b/config/checkstyle/suppressions.xml deleted file mode 100644 index 0375f028d1..0000000000 --- a/config/checkstyle/suppressions.xml +++ /dev/null @@ -1,56 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Anonymizable.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Anonymizable.java index ca360e371d..272b8ccccb 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Anonymizable.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Anonymizable.java @@ -1,5 +1,5 @@ package pl.allegro.tech.hermes.api; public interface Anonymizable { - Anonymizable anonymize(); + Anonymizable anonymize(); } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/AvroMediaType.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/AvroMediaType.java index 165befe5a7..6d1faa8844 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/AvroMediaType.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/AvroMediaType.java @@ -2,7 +2,7 @@ public class AvroMediaType { - public static final String AVRO_BINARY = "avro/binary"; + public static final String AVRO_BINARY = "avro/binary"; - public static final String AVRO_JSON = "avro/json"; + public static final String AVRO_JSON = "avro/json"; } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/BatchSubscriptionPolicy.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/BatchSubscriptionPolicy.java index 1c06f0ee03..79b26f3bb5 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/BatchSubscriptionPolicy.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/BatchSubscriptionPolicy.java @@ -3,202 +3,208 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.google.common.base.MoreObjects; import jakarta.validation.constraints.Min; -import pl.allegro.tech.hermes.api.helpers.Patch; - import java.util.Map; import java.util.Objects; +import pl.allegro.tech.hermes.api.helpers.Patch; public class BatchSubscriptionPolicy { - private static final int DEFAULT_MESSAGE_TTL = 60; - private static final int DEFAULT_MESSAGE_BACKOFF = 500; - private static final int DEFAULT_REQUEST_TIMEOUT = 30 * 1000; - private static final int DEFAULT_BATCH_SIZE = 100; - private static final int DEFAULT_BATCH_TIME = 30 * 1000; - private static final int DEFAULT_BATCH_VOLUME = 64 * 1000; - - @Min(0) - private int messageTtl; - - private boolean retryClientErrors; - - @Min(0) - private int messageBackoff; - - @Min(1) - private int requestTimeout; - - @Min(1) - private int batchSize; - - @Min(1) - private int batchTime; - - @Min(1) - private int batchVolume; - - private BatchSubscriptionPolicy() {} - - public BatchSubscriptionPolicy(int messageTtl, - boolean retryClientErrors, - int messageBackoff, - int requestTimeout, - int batchSize, - int batchTime, - int batchVolume) { - this.messageTtl = messageTtl; - this.retryClientErrors = retryClientErrors; - this.messageBackoff = messageBackoff; - this.requestTimeout = requestTimeout; - this.batchSize = batchSize; - this.batchTime = batchTime; - this.batchVolume = batchVolume; + private static final int DEFAULT_MESSAGE_TTL = 60; + private static final int DEFAULT_MESSAGE_BACKOFF = 500; + private static final int DEFAULT_REQUEST_TIMEOUT = 30 * 1000; + private static final int DEFAULT_BATCH_SIZE = 100; + private static final int DEFAULT_BATCH_TIME = 30 * 1000; + private static final int DEFAULT_BATCH_VOLUME = 64 * 1000; + + @Min(0) + private int messageTtl; + + private boolean retryClientErrors; + + @Min(0) + private int messageBackoff; + + @Min(1) + private int requestTimeout; + + @Min(1) + private int batchSize; + + @Min(1) + private int batchTime; + + @Min(1) + private int batchVolume; + + private BatchSubscriptionPolicy() {} + + public BatchSubscriptionPolicy( + int messageTtl, + boolean retryClientErrors, + int messageBackoff, + int requestTimeout, + int batchSize, + int batchTime, + int batchVolume) { + this.messageTtl = messageTtl; + this.retryClientErrors = retryClientErrors; + this.messageBackoff = messageBackoff; + this.requestTimeout = requestTimeout; + this.batchSize = batchSize; + this.batchTime = batchTime; + this.batchVolume = batchVolume; + } + + @JsonCreator + public static BatchSubscriptionPolicy create(Map properties) { + return new BatchSubscriptionPolicy( + (Integer) properties.getOrDefault("messageTtl", DEFAULT_MESSAGE_TTL), + (Boolean) properties.getOrDefault("retryClientErrors", false), + (Integer) properties.getOrDefault("messageBackoff", DEFAULT_MESSAGE_BACKOFF), + (Integer) properties.getOrDefault("requestTimeout", DEFAULT_REQUEST_TIMEOUT), + (Integer) properties.getOrDefault("batchSize", DEFAULT_BATCH_SIZE), + (Integer) properties.getOrDefault("batchTime", DEFAULT_BATCH_TIME), + (Integer) properties.getOrDefault("batchVolume", DEFAULT_BATCH_VOLUME)); + } + + @Override + public int hashCode() { + return Objects.hash( + messageTtl, + retryClientErrors, + messageBackoff, + requestTimeout, + batchSize, + batchTime, + batchVolume); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; } - - @JsonCreator - public static BatchSubscriptionPolicy create(Map properties) { - return new BatchSubscriptionPolicy( - (Integer) properties.getOrDefault("messageTtl", DEFAULT_MESSAGE_TTL), - (Boolean) properties.getOrDefault("retryClientErrors", false), - (Integer) properties.getOrDefault("messageBackoff", DEFAULT_MESSAGE_BACKOFF), - (Integer) properties.getOrDefault("requestTimeout", DEFAULT_REQUEST_TIMEOUT), - (Integer) properties.getOrDefault("batchSize", DEFAULT_BATCH_SIZE), - (Integer) properties.getOrDefault("batchTime", DEFAULT_BATCH_TIME), - (Integer) properties.getOrDefault("batchVolume", DEFAULT_BATCH_VOLUME) - ); + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final BatchSubscriptionPolicy other = (BatchSubscriptionPolicy) obj; + return Objects.equals(this.messageTtl, other.messageTtl) + && Objects.equals(this.retryClientErrors, other.retryClientErrors) + && Objects.equals(this.messageBackoff, other.messageBackoff) + && Objects.equals(this.requestTimeout, other.requestTimeout) + && Objects.equals(this.batchSize, other.batchSize) + && Objects.equals(this.batchTime, other.batchTime) + && Objects.equals(this.batchVolume, other.batchVolume); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("messageTtl", messageTtl) + .add("messageBackoff", messageBackoff) + .add("retryClientErrors", retryClientErrors) + .add("batchSize", batchSize) + .add("batchTime", batchTime) + .add("batchVolume", batchVolume) + .add("requestTimeout", requestTimeout) + .toString(); + } + + public Integer getMessageTtl() { + return messageTtl; + } + + public Integer getMessageBackoff() { + return messageBackoff; + } + + public Boolean isRetryClientErrors() { + return retryClientErrors; + } + + public Integer getBatchSize() { + return batchSize; + } + + public Integer getBatchTime() { + return batchTime; + } + + public Integer getBatchVolume() { + return batchVolume; + } + + public Integer getRequestTimeout() { + return requestTimeout; + } + + public static class Builder { + + private BatchSubscriptionPolicy subscriptionPolicy; + + public static Builder batchSubscriptionPolicy() { + return new Builder(); } - @Override - public int hashCode() { - return Objects.hash(messageTtl, retryClientErrors, messageBackoff, requestTimeout, batchSize, batchTime, batchVolume); + public Builder() { + subscriptionPolicy = new BatchSubscriptionPolicy(); } - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - final BatchSubscriptionPolicy other = (BatchSubscriptionPolicy) obj; - return Objects.equals(this.messageTtl, other.messageTtl) - && Objects.equals(this.retryClientErrors, other.retryClientErrors) - && Objects.equals(this.messageBackoff, other.messageBackoff) - && Objects.equals(this.requestTimeout, other.requestTimeout) - && Objects.equals(this.batchSize, other.batchSize) - && Objects.equals(this.batchTime, other.batchTime) - && Objects.equals(this.batchVolume, other.batchVolume); + public Builder withMessageTtl(int messageTtl) { + subscriptionPolicy.messageTtl = messageTtl; + return this; } - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("messageTtl", messageTtl) - .add("messageBackoff", messageBackoff) - .add("retryClientErrors", retryClientErrors) - .add("batchSize", batchSize) - .add("batchTime", batchTime) - .add("batchVolume", batchVolume) - .add("requestTimeout", requestTimeout) - .toString(); + public Builder withMessageBackoff(int messageBackoff) { + subscriptionPolicy.messageBackoff = messageBackoff; + return this; } - public Integer getMessageTtl() { - return messageTtl; + public Builder withClientErrorRetry(boolean retryClientErrors) { + subscriptionPolicy.retryClientErrors = retryClientErrors; + return this; } - public Integer getMessageBackoff() { - return messageBackoff; + public Builder withBatchSize(int batchSize) { + subscriptionPolicy.batchSize = batchSize; + return this; } - public Boolean isRetryClientErrors() { - return retryClientErrors; + public Builder withBatchTime(int batchTime) { + subscriptionPolicy.batchTime = batchTime; + return this; } - public Integer getBatchSize() { - return batchSize; + public Builder withBatchVolume(int batchVolume) { + subscriptionPolicy.batchVolume = batchVolume; + return this; } - public Integer getBatchTime() { - return batchTime; + public Builder withRequestTimeout(int requestTimeout) { + subscriptionPolicy.requestTimeout = requestTimeout; + return this; } - public Integer getBatchVolume() { - return batchVolume; + public BatchSubscriptionPolicy build() { + return new BatchSubscriptionPolicy( + subscriptionPolicy.messageTtl, + subscriptionPolicy.retryClientErrors, + subscriptionPolicy.messageBackoff, + subscriptionPolicy.requestTimeout, + subscriptionPolicy.batchSize, + subscriptionPolicy.batchTime, + subscriptionPolicy.batchVolume); } - public Integer getRequestTimeout() { - return requestTimeout; + public Builder applyDefaults() { + return this; } - public static class Builder { - - private BatchSubscriptionPolicy subscriptionPolicy; - - public static Builder batchSubscriptionPolicy() { - return new Builder(); - } - - public Builder() { - subscriptionPolicy = new BatchSubscriptionPolicy(); - } - - public Builder withMessageTtl(int messageTtl) { - subscriptionPolicy.messageTtl = messageTtl; - return this; - } - - public Builder withMessageBackoff(int messageBackoff) { - subscriptionPolicy.messageBackoff = messageBackoff; - return this; - } - - public Builder withClientErrorRetry(boolean retryClientErrors) { - subscriptionPolicy.retryClientErrors = retryClientErrors; - return this; - } - - public Builder withBatchSize(int batchSize) { - subscriptionPolicy.batchSize = batchSize; - return this; - } - - public Builder withBatchTime(int batchTime) { - subscriptionPolicy.batchTime = batchTime; - return this; - } - - public Builder withBatchVolume(int batchVolume) { - subscriptionPolicy.batchVolume = batchVolume; - return this; - } - - public Builder withRequestTimeout(int requestTimeout) { - subscriptionPolicy.requestTimeout = requestTimeout; - return this; - } - - public BatchSubscriptionPolicy build() { - return new BatchSubscriptionPolicy( - subscriptionPolicy.messageTtl, - subscriptionPolicy.retryClientErrors, - subscriptionPolicy.messageBackoff, - subscriptionPolicy.requestTimeout, - subscriptionPolicy.batchSize, - subscriptionPolicy.batchTime, - subscriptionPolicy.batchVolume); - } - - public Builder applyDefaults() { - return this; - } - - public Builder applyPatch(PatchData patch) { - if (patch != null) { - subscriptionPolicy = Patch.apply(subscriptionPolicy, patch); - } - return this; - } + public Builder applyPatch(PatchData patch) { + if (patch != null) { + subscriptionPolicy = Patch.apply(subscriptionPolicy, patch); + } + return this; } + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/BlacklistStatus.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/BlacklistStatus.java index 8be8d8d53b..250b9a54f5 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/BlacklistStatus.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/BlacklistStatus.java @@ -2,39 +2,38 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.Objects; public final class BlacklistStatus { - public static final BlacklistStatus BLACKLISTED = new BlacklistStatus(true); - public static final BlacklistStatus NOT_BLACKLISTED = new BlacklistStatus(false); + public static final BlacklistStatus BLACKLISTED = new BlacklistStatus(true); + public static final BlacklistStatus NOT_BLACKLISTED = new BlacklistStatus(false); - private final boolean blacklisted; + private final boolean blacklisted; - @JsonCreator - private BlacklistStatus(@JsonProperty("blacklisted") boolean blacklisted) { - this.blacklisted = blacklisted; - } + @JsonCreator + private BlacklistStatus(@JsonProperty("blacklisted") boolean blacklisted) { + this.blacklisted = blacklisted; + } - public boolean isBlacklisted() { - return blacklisted; - } + public boolean isBlacklisted() { + return blacklisted; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - BlacklistStatus that = (BlacklistStatus) o; - return blacklisted == that.blacklisted; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(blacklisted); + if (o == null || getClass() != o.getClass()) { + return false; } + BlacklistStatus that = (BlacklistStatus) o; + return blacklisted == that.blacklisted; + } + + @Override + public int hashCode() { + return Objects.hash(blacklisted); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Constraints.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Constraints.java index 60d71c6e40..fcf82f100a 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Constraints.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Constraints.java @@ -3,37 +3,36 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.constraints.Min; - import java.util.Objects; public class Constraints { - @Min(1) - private final int consumersNumber; + @Min(1) + private final int consumersNumber; - @JsonCreator - public Constraints(@JsonProperty("consumersNumber") int consumersNumber) { - this.consumersNumber = consumersNumber; - } + @JsonCreator + public Constraints(@JsonProperty("consumersNumber") int consumersNumber) { + this.consumersNumber = consumersNumber; + } - public int getConsumersNumber() { - return consumersNumber; - } + public int getConsumersNumber() { + return consumersNumber; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Constraints that = (Constraints) o; - return consumersNumber == that.consumersNumber; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(consumersNumber); + if (o == null || getClass() != o.getClass()) { + return false; } + Constraints that = (Constraints) o; + return consumersNumber == that.consumersNumber; + } + + @Override + public int hashCode() { + return Objects.hash(consumersNumber); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/ConsumerGroup.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/ConsumerGroup.java index d7acd8e37e..61528237df 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/ConsumerGroup.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/ConsumerGroup.java @@ -2,62 +2,62 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.Objects; import java.util.Set; public class ConsumerGroup { - private final String clusterName; - private final String groupId; - private final String state; - - private final Set members; - - @JsonCreator - public ConsumerGroup(@JsonProperty("clusterName") String clusterName, - @JsonProperty("groupId") String groupId, - @JsonProperty("state") String state, - @JsonProperty("members") Set members) { - this.clusterName = clusterName; - this.groupId = groupId; - this.state = state; - this.members = members; - } - - public String getClusterName() { - return clusterName; - } - - public String getGroupId() { - return groupId; - } - - public String getState() { - return state; - } - - public Set getMembers() { - return members; + private final String clusterName; + private final String groupId; + private final String state; + + private final Set members; + + @JsonCreator + public ConsumerGroup( + @JsonProperty("clusterName") String clusterName, + @JsonProperty("groupId") String groupId, + @JsonProperty("state") String state, + @JsonProperty("members") Set members) { + this.clusterName = clusterName; + this.groupId = groupId; + this.state = state; + this.members = members; + } + + public String getClusterName() { + return clusterName; + } + + public String getGroupId() { + return groupId; + } + + public String getState() { + return state; + } + + public Set getMembers() { + return members; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ConsumerGroup that = (ConsumerGroup) o; - return Objects.equals(clusterName, that.clusterName) - && Objects.equals(groupId, that.groupId) - && Objects.equals(state, that.state) - && Objects.equals(members, that.members); - } - - @Override - public int hashCode() { - return Objects.hash(clusterName, groupId, state, members); + if (o == null || getClass() != o.getClass()) { + return false; } + ConsumerGroup that = (ConsumerGroup) o; + return Objects.equals(clusterName, that.clusterName) + && Objects.equals(groupId, that.groupId) + && Objects.equals(state, that.state) + && Objects.equals(members, that.members); + } + + @Override + public int hashCode() { + return Objects.hash(clusterName, groupId, state, members); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/ConsumerGroupMember.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/ConsumerGroupMember.java index 62348cb9b6..94d0491b9d 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/ConsumerGroupMember.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/ConsumerGroupMember.java @@ -2,61 +2,61 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.Objects; import java.util.Set; public class ConsumerGroupMember { - private final String consumerId; - private final String clientId; - private final String host; - - private final Set partitions; - - @JsonCreator - public ConsumerGroupMember(@JsonProperty("consumerId") String consumerId, - @JsonProperty("clientId") String clientId, - @JsonProperty("host")String host, - @JsonProperty("partitions") Set partitions) { - this.consumerId = consumerId; - this.clientId = clientId; - this.host = host; - this.partitions = partitions; - } - - public String getConsumerId() { - return consumerId; - } - - public String getClientId() { - return clientId; - } - - public String getHost() { - return host; - } - - public Set getPartitions() { - return partitions; + private final String consumerId; + private final String clientId; + private final String host; + + private final Set partitions; + + @JsonCreator + public ConsumerGroupMember( + @JsonProperty("consumerId") String consumerId, + @JsonProperty("clientId") String clientId, + @JsonProperty("host") String host, + @JsonProperty("partitions") Set partitions) { + this.consumerId = consumerId; + this.clientId = clientId; + this.host = host; + this.partitions = partitions; + } + + public String getConsumerId() { + return consumerId; + } + + public String getClientId() { + return clientId; + } + + public String getHost() { + return host; + } + + public Set getPartitions() { + return partitions; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ConsumerGroupMember that = (ConsumerGroupMember) o; - return Objects.equals(consumerId, that.consumerId) - && Objects.equals(clientId, that.clientId) - && Objects.equals(host, that.host) - && Objects.equals(partitions, that.partitions); - } - - @Override - public int hashCode() { - return Objects.hash(consumerId, clientId, host, partitions); + if (o == null || getClass() != o.getClass()) { + return false; } + ConsumerGroupMember that = (ConsumerGroupMember) o; + return Objects.equals(consumerId, that.consumerId) + && Objects.equals(clientId, that.clientId) + && Objects.equals(host, that.host) + && Objects.equals(partitions, that.partitions); + } + + @Override + public int hashCode() { + return Objects.hash(consumerId, clientId, host, partitions); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/ContentType.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/ContentType.java index 383f1cdf47..1980a07a1e 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/ContentType.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/ContentType.java @@ -1,5 +1,6 @@ package pl.allegro.tech.hermes.api; public enum ContentType { - JSON, AVRO + JSON, + AVRO } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/DatacenterReadiness.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/DatacenterReadiness.java index 88f12af28a..279bffc33e 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/DatacenterReadiness.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/DatacenterReadiness.java @@ -2,56 +2,52 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.Objects; public class DatacenterReadiness { - private final String datacenter; - private final ReadinessStatus status; - - @JsonCreator - public DatacenterReadiness(@JsonProperty("datacenter") String datacenter, - @JsonProperty("status") ReadinessStatus status) { - this.datacenter = datacenter; - this.status = status; - } - - public String getDatacenter() { - return datacenter; - } - - public ReadinessStatus getStatus() { - return status; - } - - @Override - public String toString() { - return "DatacenterReadiness{" - + "datacenter='" + datacenter + '\'' - + ", status=" + status - + '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof DatacenterReadiness)) { - return false; - } - DatacenterReadiness that = (DatacenterReadiness) o; - return status == that.status - && Objects.equals(datacenter, that.datacenter); + private final String datacenter; + private final ReadinessStatus status; + + @JsonCreator + public DatacenterReadiness( + @JsonProperty("datacenter") String datacenter, + @JsonProperty("status") ReadinessStatus status) { + this.datacenter = datacenter; + this.status = status; + } + + public String getDatacenter() { + return datacenter; + } + + public ReadinessStatus getStatus() { + return status; + } + + @Override + public String toString() { + return "DatacenterReadiness{" + "datacenter='" + datacenter + '\'' + ", status=" + status + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(datacenter, status); - } - - public enum ReadinessStatus { - READY, - NOT_READY + if (!(o instanceof DatacenterReadiness)) { + return false; } + DatacenterReadiness that = (DatacenterReadiness) o; + return status == that.status && Objects.equals(datacenter, that.datacenter); + } + + @Override + public int hashCode() { + return Objects.hash(datacenter, status); + } + + public enum ReadinessStatus { + READY, + NOT_READY + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/DeliveryType.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/DeliveryType.java index 9edad2adb3..ade2c08947 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/DeliveryType.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/DeliveryType.java @@ -1,5 +1,6 @@ package pl.allegro.tech.hermes.api; public enum DeliveryType { - SERIAL, BATCH + SERIAL, + BATCH } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/EndpointAddress.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/EndpointAddress.java index 140a932de1..0dfdeacff9 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/EndpointAddress.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/EndpointAddress.java @@ -5,145 +5,145 @@ import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.base.Strings; -import pl.allegro.tech.hermes.api.jackson.EndpointAddressDeserializer; -import pl.allegro.tech.hermes.api.jackson.EndpointAddressSerializer; - import java.net.URI; import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; +import pl.allegro.tech.hermes.api.jackson.EndpointAddressDeserializer; +import pl.allegro.tech.hermes.api.jackson.EndpointAddressSerializer; @JsonDeserialize(using = EndpointAddressDeserializer.class) @JsonSerialize(using = EndpointAddressSerializer.class) public class EndpointAddress implements Anonymizable { - private static final String ANONYMIZED_PASSWORD = "*****"; + private static final String ANONYMIZED_PASSWORD = "*****"; - private static final Pattern URL_PATTERN = Pattern.compile("([a-zA-Z0-9]*)://(([a-zA-Z0-9\\.\\~\\-\\_]*):(.*)@)?(.*)"); + private static final Pattern URL_PATTERN = + Pattern.compile("([a-zA-Z0-9]*)://(([a-zA-Z0-9\\.\\~\\-\\_]*):(.*)@)?(.*)"); - private static final int PROTOCOL_GROUP = 1; + private static final int PROTOCOL_GROUP = 1; - private static final int ADDRESS_GROUP = 5; + private static final int ADDRESS_GROUP = 5; - private static final int USER_INFO_GROUP = 2; + private static final int USER_INFO_GROUP = 2; - private static final int USERNAME_GROUP = 3; + private static final int USERNAME_GROUP = 3; - private static final int PASSWORD_GROUP = 4; + private static final int PASSWORD_GROUP = 4; - private final boolean containsCredentials; + private final boolean containsCredentials; - private final String protocol; + private final String protocol; - private final String username; + private final String username; - private final String password; + private final String password; - private final String endpoint; + private final String endpoint; - private final String rawEndpoint; + private final String rawEndpoint; - public EndpointAddress(String endpoint) { - this.rawEndpoint = endpoint; + public EndpointAddress(String endpoint) { + this.rawEndpoint = endpoint; - Matcher matcher = URL_PATTERN.matcher(endpoint); - if (matcher.matches()) { - this.protocol = matcher.group(PROTOCOL_GROUP); - this.containsCredentials = !Strings.isNullOrEmpty(matcher.group(USER_INFO_GROUP)); + Matcher matcher = URL_PATTERN.matcher(endpoint); + if (matcher.matches()) { + this.protocol = matcher.group(PROTOCOL_GROUP); + this.containsCredentials = !Strings.isNullOrEmpty(matcher.group(USER_INFO_GROUP)); - this.username = containsCredentials ? matcher.group(USERNAME_GROUP) : null; - this.password = containsCredentials ? matcher.group(PASSWORD_GROUP) : null; + this.username = containsCredentials ? matcher.group(USERNAME_GROUP) : null; + this.password = containsCredentials ? matcher.group(PASSWORD_GROUP) : null; - this.endpoint = containsCredentials ? protocol + "://" + matcher.group(ADDRESS_GROUP) : endpoint; - } else { - this.protocol = null; - this.containsCredentials = false; - this.username = null; - this.password = null; - this.endpoint = endpoint; - } + this.endpoint = + containsCredentials ? protocol + "://" + matcher.group(ADDRESS_GROUP) : endpoint; + } else { + this.protocol = null; + this.containsCredentials = false; + this.username = null; + this.password = null; + this.endpoint = endpoint; } + } - private EndpointAddress(String protocol, String endpoint, String username) { - this.protocol = protocol; - this.endpoint = endpoint; - this.containsCredentials = true; - this.username = username; - this.password = ANONYMIZED_PASSWORD; + private EndpointAddress(String protocol, String endpoint, String username) { + this.protocol = protocol; + this.endpoint = endpoint; + this.containsCredentials = true; + this.username = username; + this.password = ANONYMIZED_PASSWORD; - this.rawEndpoint = protocol + "://" + username + ":" + password + "@" + endpoint.replace(protocol + "://", ""); - } + this.rawEndpoint = + protocol + "://" + username + ":" + password + "@" + endpoint.replace(protocol + "://", ""); + } - public static EndpointAddress of(String endpoint) { - return new EndpointAddress(endpoint); - } + public static EndpointAddress of(String endpoint) { + return new EndpointAddress(endpoint); + } - public static EndpointAddress of(URI endpoint) { - return new EndpointAddress(endpoint.toString()); - } + public static EndpointAddress of(URI endpoint) { + return new EndpointAddress(endpoint.toString()); + } - public static String extractProtocolFromAddress(String endpoint) { - Preconditions.checkArgument(endpoint.indexOf(':') != -1); + public static String extractProtocolFromAddress(String endpoint) { + Preconditions.checkArgument(endpoint.indexOf(':') != -1); - return endpoint.substring(0, endpoint.indexOf(':')); - } + return endpoint.substring(0, endpoint.indexOf(':')); + } - public String getEndpoint() { - return endpoint; - } + public String getEndpoint() { + return endpoint; + } - public String getRawEndpoint() { - return rawEndpoint; - } + public String getRawEndpoint() { + return rawEndpoint; + } - public URI getUri() { - return URI.create(endpoint); - } + public URI getUri() { + return URI.create(endpoint); + } - public String getProtocol() { - return protocol; - } + public String getProtocol() { + return protocol; + } - @Override - public int hashCode() { - return Objects.hash(rawEndpoint); - } + @Override + public int hashCode() { + return Objects.hash(rawEndpoint); + } - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - final EndpointAddress other = (EndpointAddress) obj; - return Objects.equals(this.rawEndpoint, other.rawEndpoint); + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("endpoint", endpoint) - .toString(); + if (obj == null || getClass() != obj.getClass()) { + return false; } + final EndpointAddress other = (EndpointAddress) obj; + return Objects.equals(this.rawEndpoint, other.rawEndpoint); + } - public boolean containsCredentials() { - return containsCredentials; - } + @Override + public String toString() { + return MoreObjects.toStringHelper(this).add("endpoint", endpoint).toString(); + } - public String getPassword() { - return password; - } + public boolean containsCredentials() { + return containsCredentials; + } - public String getUsername() { - return username; - } + public String getPassword() { + return password; + } - public EndpointAddress anonymize() { - if (containsCredentials) { - return new EndpointAddress(protocol, endpoint, username); - } - return this; - } + public String getUsername() { + return username; + } + + public EndpointAddress anonymize() { + if (containsCredentials) { + return new EndpointAddress(protocol, endpoint, username); + } + return this; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/EndpointAddressResolverMetadata.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/EndpointAddressResolverMetadata.java index 8cd3f20bd8..cf863ee8d9 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/EndpointAddressResolverMetadata.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/EndpointAddressResolverMetadata.java @@ -7,7 +7,6 @@ import com.fasterxml.jackson.databind.ser.std.StdSerializer; import com.google.common.collect.ImmutableMap; import jakarta.validation.constraints.NotNull; - import java.io.IOException; import java.util.Collections; import java.util.HashMap; @@ -15,81 +14,84 @@ import java.util.Objects; import java.util.Optional; -@JsonSerialize(using = EndpointAddressResolverMetadata.EndpointAddressResolverMetadataSerializer.class) +@JsonSerialize( + using = EndpointAddressResolverMetadata.EndpointAddressResolverMetadataSerializer.class) public class EndpointAddressResolverMetadata { - private static final EndpointAddressResolverMetadata EMPTY_INSTANCE = new EndpointAddressResolverMetadata(Collections.emptyMap()); + private static final EndpointAddressResolverMetadata EMPTY_INSTANCE = + new EndpointAddressResolverMetadata(Collections.emptyMap()); - @NotNull - private Map entries; + @NotNull private Map entries; - @JsonCreator - public EndpointAddressResolverMetadata(Map entries) { - this.entries = ImmutableMap.copyOf(entries); - } + @JsonCreator + public EndpointAddressResolverMetadata(Map entries) { + this.entries = ImmutableMap.copyOf(entries); + } - public static EndpointAddressResolverMetadata empty() { - return EMPTY_INSTANCE; - } + public static EndpointAddressResolverMetadata empty() { + return EMPTY_INSTANCE; + } - public static Builder endpointAddressResolverMetadata() { - return new Builder(); - } + public static Builder endpointAddressResolverMetadata() { + return new Builder(); + } - public Optional get(String key) { - return Optional.ofNullable(entries.get(key)); - } + public Optional get(String key) { + return Optional.ofNullable(entries.get(key)); + } - @SuppressWarnings("unchecked") - public T getOrDefault(String key, T defaultValue) { - return (T) entries.getOrDefault(key, defaultValue); - } + @SuppressWarnings("unchecked") + public T getOrDefault(String key, T defaultValue) { + return (T) entries.getOrDefault(key, defaultValue); + } - public Map getEntries() { - return entries; - } + public Map getEntries() { + return entries; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - EndpointAddressResolverMetadata that = (EndpointAddressResolverMetadata) o; - return Objects.equals(entries, that.entries); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(entries); + if (o == null || getClass() != o.getClass()) { + return false; } + EndpointAddressResolverMetadata that = (EndpointAddressResolverMetadata) o; + return Objects.equals(entries, that.entries); + } - public static class EndpointAddressResolverMetadataSerializer extends StdSerializer { + @Override + public int hashCode() { + return Objects.hash(entries); + } - protected EndpointAddressResolverMetadataSerializer() { - super(EndpointAddressResolverMetadata.class); - } + public static class EndpointAddressResolverMetadataSerializer + extends StdSerializer { - @Override - public void serialize(EndpointAddressResolverMetadata metadata, JsonGenerator jgen, SerializerProvider provider) - throws IOException { - jgen.writeObject(metadata.entries); - } + protected EndpointAddressResolverMetadataSerializer() { + super(EndpointAddressResolverMetadata.class); } - public static class Builder { + @Override + public void serialize( + EndpointAddressResolverMetadata metadata, JsonGenerator jgen, SerializerProvider provider) + throws IOException { + jgen.writeObject(metadata.entries); + } + } + + public static class Builder { - private Map entries = new HashMap<>(); + private Map entries = new HashMap<>(); - public Builder withEntry(String key, Object value) { - entries.put(key, value); - return this; - } + public Builder withEntry(String key, Object value) { + entries.put(key, value); + return this; + } - public EndpointAddressResolverMetadata build() { - return new EndpointAddressResolverMetadata(entries); - } + public EndpointAddressResolverMetadata build() { + return new EndpointAddressResolverMetadata(entries); } + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/ErrorCode.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/ErrorCode.java index 953e468f97..a000f436e1 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/ErrorCode.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/ErrorCode.java @@ -1,7 +1,5 @@ package pl.allegro.tech.hermes.api; -import jakarta.ws.rs.core.Response; - import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; import static jakarta.ws.rs.core.Response.Status.FORBIDDEN; import static jakarta.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR; @@ -10,72 +8,74 @@ import static jakarta.ws.rs.core.Response.Status.REQUEST_TIMEOUT; import static jakarta.ws.rs.core.Response.Status.SERVICE_UNAVAILABLE; +import jakarta.ws.rs.core.Response; + public enum ErrorCode { - TIMEOUT(REQUEST_TIMEOUT), - TOPIC_ALREADY_EXISTS(BAD_REQUEST), - TOPIC_NOT_EXISTS(NOT_FOUND), - GROUP_NOT_EXISTS(NOT_FOUND), - GROUP_NAME_IS_INVALID(BAD_REQUEST), - SUBSCRIPTION_NOT_EXISTS(BAD_REQUEST), - SUBSCRIPTION_ALREADY_EXISTS(BAD_REQUEST), - VALIDATION_ERROR(BAD_REQUEST), - INTERNAL_ERROR(INTERNAL_SERVER_ERROR), - FORMAT_ERROR(BAD_REQUEST), - GROUP_NOT_EMPTY(FORBIDDEN), - TOPIC_NOT_EMPTY(FORBIDDEN), - GROUP_ALREADY_EXISTS(BAD_REQUEST), - OPERATION_DISABLED(NOT_ACCEPTABLE), - OTHER(INTERNAL_SERVER_ERROR), - UNAVAILABLE_RATE(BAD_REQUEST), - SINGLE_MESSAGE_READER_EXCEPTION(INTERNAL_SERVER_ERROR), - PARTITIONS_NOT_FOUND_FOR_TOPIC(NOT_FOUND), - OFFSET_NOT_FOUND_EXCEPTION(NOT_FOUND), - OFFSETS_NOT_AVAILABLE_EXCEPTION(INTERNAL_SERVER_ERROR), - UNABLE_TO_MOVE_OFFSETS_EXCEPTION(INTERNAL_SERVER_ERROR), - BROKERS_CLUSTER_NOT_FOUND_EXCEPTION(NOT_FOUND), - BROKERS_CLUSTER_COMMUNICATION_EXCEPTION(INTERNAL_SERVER_ERROR), - SIMPLE_CONSUMER_POOL_EXCEPTION(INTERNAL_SERVER_ERROR), - RETRANSMISSION_EXCEPTION(INTERNAL_SERVER_ERROR), - TOKEN_NOT_PROVIDED(FORBIDDEN), - GROUP_NOT_PROVIDED(FORBIDDEN), - AUTH_ERROR(FORBIDDEN), - SCHEMA_REPOSITORY_INTERNAL_ERROR(INTERNAL_SERVER_ERROR), - SCHEMA_BAD_REQUEST(BAD_REQUEST), - SCHEMA_COULD_NOT_BE_LOADED(INTERNAL_SERVER_ERROR), - SCHEMA_VERSION_DOES_NOT_EXIST(BAD_REQUEST), - SCHEMA_ALREADY_EXISTS(BAD_REQUEST), - AVRO_SCHEMA_INVALID_METADATA(BAD_REQUEST), - SUBSCRIPTION_ENDPOINT_ADDRESS_CHANGE_EXCEPTION(INTERNAL_SERVER_ERROR), - OAUTH_PROVIDER_NOT_EXISTS(NOT_FOUND), - OAUTH_PROVIDER_ALREADY_EXISTS(BAD_REQUEST), - TOPIC_BLACKLISTED(FORBIDDEN), - THROUGHPUT_QUOTA_VIOLATION(429), - TOPIC_NOT_UNBLACKLISTED(BAD_REQUEST), - TOPIC_CONSTRAINTS_ALREADY_EXIST(BAD_REQUEST), - TOPIC_CONSTRAINTS_DO_NOT_EXIST(BAD_REQUEST), - SUBSCRIPTION_CONSTRAINTS_ALREADY_EXIST(BAD_REQUEST), - SUBSCRIPTION_CONSTRAINTS_DO_NOT_EXIST(BAD_REQUEST), - OWNER_SOURCE_NOT_FOUND(NOT_FOUND), - OWNER_SOURCE_DOESNT_SUPPORT_AUTOCOMPLETE(BAD_REQUEST), - OWNER_NOT_FOUND(NOT_FOUND), - PERMISSION_DENIED(FORBIDDEN), - UNKNOWN_MIGRATION(NOT_FOUND), - INVALID_QUERY(BAD_REQUEST), - IMPLEMENTATION_ABSENT(NOT_FOUND), - MOVING_SUBSCRIPTION_OFFSETS_VALIDATION_ERROR(BAD_REQUEST), - SENDING_TO_KAFKA_TIMEOUT(SERVICE_UNAVAILABLE); + TIMEOUT(REQUEST_TIMEOUT), + TOPIC_ALREADY_EXISTS(BAD_REQUEST), + TOPIC_NOT_EXISTS(NOT_FOUND), + GROUP_NOT_EXISTS(NOT_FOUND), + GROUP_NAME_IS_INVALID(BAD_REQUEST), + SUBSCRIPTION_NOT_EXISTS(BAD_REQUEST), + SUBSCRIPTION_ALREADY_EXISTS(BAD_REQUEST), + VALIDATION_ERROR(BAD_REQUEST), + INTERNAL_ERROR(INTERNAL_SERVER_ERROR), + FORMAT_ERROR(BAD_REQUEST), + GROUP_NOT_EMPTY(FORBIDDEN), + TOPIC_NOT_EMPTY(FORBIDDEN), + GROUP_ALREADY_EXISTS(BAD_REQUEST), + OPERATION_DISABLED(NOT_ACCEPTABLE), + OTHER(INTERNAL_SERVER_ERROR), + UNAVAILABLE_RATE(BAD_REQUEST), + SINGLE_MESSAGE_READER_EXCEPTION(INTERNAL_SERVER_ERROR), + PARTITIONS_NOT_FOUND_FOR_TOPIC(NOT_FOUND), + OFFSET_NOT_FOUND_EXCEPTION(NOT_FOUND), + OFFSETS_NOT_AVAILABLE_EXCEPTION(INTERNAL_SERVER_ERROR), + UNABLE_TO_MOVE_OFFSETS_EXCEPTION(INTERNAL_SERVER_ERROR), + BROKERS_CLUSTER_NOT_FOUND_EXCEPTION(NOT_FOUND), + BROKERS_CLUSTER_COMMUNICATION_EXCEPTION(INTERNAL_SERVER_ERROR), + SIMPLE_CONSUMER_POOL_EXCEPTION(INTERNAL_SERVER_ERROR), + RETRANSMISSION_EXCEPTION(INTERNAL_SERVER_ERROR), + TOKEN_NOT_PROVIDED(FORBIDDEN), + GROUP_NOT_PROVIDED(FORBIDDEN), + AUTH_ERROR(FORBIDDEN), + SCHEMA_REPOSITORY_INTERNAL_ERROR(INTERNAL_SERVER_ERROR), + SCHEMA_BAD_REQUEST(BAD_REQUEST), + SCHEMA_COULD_NOT_BE_LOADED(INTERNAL_SERVER_ERROR), + SCHEMA_VERSION_DOES_NOT_EXIST(BAD_REQUEST), + SCHEMA_ALREADY_EXISTS(BAD_REQUEST), + AVRO_SCHEMA_INVALID_METADATA(BAD_REQUEST), + SUBSCRIPTION_ENDPOINT_ADDRESS_CHANGE_EXCEPTION(INTERNAL_SERVER_ERROR), + OAUTH_PROVIDER_NOT_EXISTS(NOT_FOUND), + OAUTH_PROVIDER_ALREADY_EXISTS(BAD_REQUEST), + TOPIC_BLACKLISTED(FORBIDDEN), + THROUGHPUT_QUOTA_VIOLATION(429), + TOPIC_NOT_UNBLACKLISTED(BAD_REQUEST), + TOPIC_CONSTRAINTS_ALREADY_EXIST(BAD_REQUEST), + TOPIC_CONSTRAINTS_DO_NOT_EXIST(BAD_REQUEST), + SUBSCRIPTION_CONSTRAINTS_ALREADY_EXIST(BAD_REQUEST), + SUBSCRIPTION_CONSTRAINTS_DO_NOT_EXIST(BAD_REQUEST), + OWNER_SOURCE_NOT_FOUND(NOT_FOUND), + OWNER_SOURCE_DOESNT_SUPPORT_AUTOCOMPLETE(BAD_REQUEST), + OWNER_NOT_FOUND(NOT_FOUND), + PERMISSION_DENIED(FORBIDDEN), + UNKNOWN_MIGRATION(NOT_FOUND), + INVALID_QUERY(BAD_REQUEST), + IMPLEMENTATION_ABSENT(NOT_FOUND), + MOVING_SUBSCRIPTION_OFFSETS_VALIDATION_ERROR(BAD_REQUEST), + SENDING_TO_KAFKA_TIMEOUT(SERVICE_UNAVAILABLE); - private final int httpCode; + private final int httpCode; - ErrorCode(Response.Status httpCode) { - this.httpCode = httpCode.getStatusCode(); - } + ErrorCode(Response.Status httpCode) { + this.httpCode = httpCode.getStatusCode(); + } - ErrorCode(int httpCode) { - this.httpCode = httpCode; - } + ErrorCode(int httpCode) { + this.httpCode = httpCode; + } - public int getHttpCode() { - return httpCode; - } + public int getHttpCode() { + return httpCode; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/ErrorDescription.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/ErrorDescription.java index 2e542f4ec2..bb4038fcd2 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/ErrorDescription.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/ErrorDescription.java @@ -5,24 +5,25 @@ public class ErrorDescription { - private final String message; - private final ErrorCode code; + private final String message; + private final ErrorCode code; - public static ErrorDescription error(String message, ErrorCode errorCode) { - return new ErrorDescription(message, errorCode); - } + public static ErrorDescription error(String message, ErrorCode errorCode) { + return new ErrorDescription(message, errorCode); + } - @JsonCreator - public ErrorDescription(@JsonProperty("message") String message, @JsonProperty("code") ErrorCode code) { - this.message = message; - this.code = code; - } + @JsonCreator + public ErrorDescription( + @JsonProperty("message") String message, @JsonProperty("code") ErrorCode code) { + this.message = message; + this.code = code; + } - public String getMessage() { - return message; - } + public String getMessage() { + return message; + } - public ErrorCode getCode() { - return code; - } + public ErrorCode getCode() { + return code; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Group.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Group.java index 6d58c2f8b2..e0a05eb857 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Group.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Group.java @@ -3,47 +3,45 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.constraints.NotNull; - import java.util.Objects; public class Group { - @NotNull - private final String groupName; + @NotNull private final String groupName; - @JsonCreator - public Group(@JsonProperty("groupName") String groupName) { - this.groupName = groupName; - } + @JsonCreator + public Group(@JsonProperty("groupName") String groupName) { + this.groupName = groupName; + } - public static Group from(String groupName) { - return new Group(groupName); - } + public static Group from(String groupName) { + return new Group(groupName); + } - public String getGroupName() { - return groupName; - } + public String getGroupName() { + return groupName; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof Group)) { - return false; - } - Group group = (Group) o; - - return Objects.equals(this.getGroupName(), group.getGroupName()); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(groupName); + if (!(o instanceof Group)) { + return false; } + Group group = (Group) o; - @Override - public String toString() { - return "Group(" + groupName + ")"; - } + return Objects.equals(this.getGroupName(), group.getGroupName()); + } + + @Override + public int hashCode() { + return Objects.hash(groupName); + } + + @Override + public String toString() { + return "Group(" + groupName + ")"; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Header.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Header.java index af96c6371e..9b11b464fa 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Header.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Header.java @@ -3,46 +3,42 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.constraints.NotNull; - import java.util.Objects; public class Header { - @NotNull - private String name; + @NotNull private String name; - @NotNull - private String value; + @NotNull private String value; - @JsonCreator - public Header(@JsonProperty("name") String name, @JsonProperty("value") String value) { - this.name = name; - this.value = value; - } + @JsonCreator + public Header(@JsonProperty("name") String name, @JsonProperty("value") String value) { + this.name = name; + this.value = value; + } - public String getName() { - return name; - } + public String getName() { + return name; + } - public String getValue() { - return value; - } + public String getValue() { + return value; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Header header = (Header) o; - return Objects.equals(name, header.name) - && Objects.equals(value, header.value); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(name, value); + if (o == null || getClass() != o.getClass()) { + return false; } + Header header = (Header) o; + return Objects.equals(name, header.name) && Objects.equals(value, header.value); + } + + @Override + public int hashCode() { + return Objects.hash(name, value); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/InconsistentGroup.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/InconsistentGroup.java index 2cf02419ed..8eaec6eabf 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/InconsistentGroup.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/InconsistentGroup.java @@ -2,32 +2,32 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.List; public class InconsistentGroup { - private final String name; - private final List inconsistentMetadata; - private final List inconsistentTopics; + private final String name; + private final List inconsistentMetadata; + private final List inconsistentTopics; - @JsonCreator - public InconsistentGroup(@JsonProperty("name") String name, - @JsonProperty("inconsistentMetadata") List inconsistentMetadata, - @JsonProperty("inconsistentTopics") List inconsistentTopics) { - this.name = name; - this.inconsistentMetadata = inconsistentMetadata; - this.inconsistentTopics = inconsistentTopics; - } + @JsonCreator + public InconsistentGroup( + @JsonProperty("name") String name, + @JsonProperty("inconsistentMetadata") List inconsistentMetadata, + @JsonProperty("inconsistentTopics") List inconsistentTopics) { + this.name = name; + this.inconsistentMetadata = inconsistentMetadata; + this.inconsistentTopics = inconsistentTopics; + } - public String getName() { - return name; - } + public String getName() { + return name; + } - public List getInconsistentMetadata() { - return inconsistentMetadata; - } + public List getInconsistentMetadata() { + return inconsistentMetadata; + } - public List getInconsistentTopics() { - return inconsistentTopics; - } + public List getInconsistentTopics() { + return inconsistentTopics; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/InconsistentMetadata.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/InconsistentMetadata.java index 27e6e9bc4d..e94ec573d0 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/InconsistentMetadata.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/InconsistentMetadata.java @@ -4,21 +4,21 @@ import com.fasterxml.jackson.annotation.JsonProperty; public class InconsistentMetadata { - private final String datacenter; - private final String content; + private final String datacenter; + private final String content; - @JsonCreator - public InconsistentMetadata(@JsonProperty("datacenter") String datacenter, - @JsonProperty("content") String content) { - this.datacenter = datacenter; - this.content = content; - } + @JsonCreator + public InconsistentMetadata( + @JsonProperty("datacenter") String datacenter, @JsonProperty("content") String content) { + this.datacenter = datacenter; + this.content = content; + } - public String getDatacenter() { - return datacenter; - } + public String getDatacenter() { + return datacenter; + } - public String getContent() { - return content; - } + public String getContent() { + return content; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/InconsistentSubscription.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/InconsistentSubscription.java index 7418459f55..27e239078e 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/InconsistentSubscription.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/InconsistentSubscription.java @@ -2,25 +2,25 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.List; public class InconsistentSubscription { - private final String name; - private final List inconsistentMetadata; + private final String name; + private final List inconsistentMetadata; - @JsonCreator - public InconsistentSubscription(@JsonProperty("name") String name, - @JsonProperty("inconsistentMetadata") List inconsistentMetadata) { - this.name = name; - this.inconsistentMetadata = inconsistentMetadata; - } + @JsonCreator + public InconsistentSubscription( + @JsonProperty("name") String name, + @JsonProperty("inconsistentMetadata") List inconsistentMetadata) { + this.name = name; + this.inconsistentMetadata = inconsistentMetadata; + } - public String getName() { - return name; - } + public String getName() { + return name; + } - public List getInconsistentMetadata() { - return inconsistentMetadata; - } + public List getInconsistentMetadata() { + return inconsistentMetadata; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/InconsistentTopic.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/InconsistentTopic.java index 3b4c609efc..549f30554e 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/InconsistentTopic.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/InconsistentTopic.java @@ -2,32 +2,33 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.List; public class InconsistentTopic { - private final String name; - private final List inconsistentMetadata; - private final List inconsistentSubscriptions; + private final String name; + private final List inconsistentMetadata; + private final List inconsistentSubscriptions; - @JsonCreator - public InconsistentTopic(@JsonProperty("name") String name, - @JsonProperty("inconsistentMetadata") List inconsistentMetadata, - @JsonProperty("inconsistentSubscriptions") List inconsistentSubscriptions) { - this.name = name; - this.inconsistentMetadata = inconsistentMetadata; - this.inconsistentSubscriptions = inconsistentSubscriptions; - } + @JsonCreator + public InconsistentTopic( + @JsonProperty("name") String name, + @JsonProperty("inconsistentMetadata") List inconsistentMetadata, + @JsonProperty("inconsistentSubscriptions") + List inconsistentSubscriptions) { + this.name = name; + this.inconsistentMetadata = inconsistentMetadata; + this.inconsistentSubscriptions = inconsistentSubscriptions; + } - public String getName() { - return name; - } + public String getName() { + return name; + } - public List getInconsistentMetadata() { - return inconsistentMetadata; - } + public List getInconsistentMetadata() { + return inconsistentMetadata; + } - public List getInconsistentSubscriptions() { - return inconsistentSubscriptions; - } + public List getInconsistentSubscriptions() { + return inconsistentSubscriptions; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MessageFilterSpecification.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MessageFilterSpecification.java index 018d8d7c11..bc0293ebbb 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MessageFilterSpecification.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MessageFilterSpecification.java @@ -2,69 +2,67 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonValue; - import java.util.Map; import java.util.Objects; public class MessageFilterSpecification { - private final String type; - private final Map spec; + private final String type; + private final Map spec; - @JsonCreator - public MessageFilterSpecification(Map spec) { - this.spec = spec; - this.type = getStringValue("type"); - } + @JsonCreator + public MessageFilterSpecification(Map spec) { + this.spec = spec; + this.type = getStringValue("type"); + } - public String getType() { - return type; - } + public String getType() { + return type; + } - public String getPath() { - return getStringValue("path"); - } + public String getPath() { + return getStringValue("path"); + } - public String getMatcher() { - return getStringValue("matcher"); - } + public String getMatcher() { + return getStringValue("matcher"); + } - public String getHeader() { - return getStringValue("header"); - } + public String getHeader() { + return getStringValue("header"); + } - public String getMatchingStrategy() { - return getStringValue("matchingStrategy"); - } + public String getMatchingStrategy() { + return getStringValue("matchingStrategy"); + } - @SuppressWarnings("unchecked") - public T getFieldValue(String key) { - return (T) spec.get(key); - } + @SuppressWarnings("unchecked") + public T getFieldValue(String key) { + return (T) spec.get(key); + } - public String getStringValue(String key) { - return getFieldValue(key); - } + public String getStringValue(String key) { + return getFieldValue(key); + } - @JsonValue - public Object getJsonValue() { - return spec; - } + @JsonValue + public Object getJsonValue() { + return spec; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - MessageFilterSpecification that = (MessageFilterSpecification) o; - return Objects.equals(type, that.type) - && Objects.equals(spec, that.spec); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(type, spec); + if (o == null || getClass() != o.getClass()) { + return false; } + MessageFilterSpecification that = (MessageFilterSpecification) o; + return Objects.equals(type, that.type) && Objects.equals(spec, that.spec); + } + + @Override + public int hashCode() { + return Objects.hash(type, spec); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MessageFiltersVerificationInput.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MessageFiltersVerificationInput.java index eecb9e6dda..2d81c2d371 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MessageFiltersVerificationInput.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MessageFiltersVerificationInput.java @@ -3,27 +3,26 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.constraints.NotNull; - import java.util.List; public class MessageFiltersVerificationInput { - private final List filters; + private final List filters; - @NotNull - private final byte[] message; + @NotNull private final byte[] message; - @JsonCreator - public MessageFiltersVerificationInput(@JsonProperty("filters") List filters, - @JsonProperty("message") byte[] message) { - this.filters = filters; - this.message = message; - } + @JsonCreator + public MessageFiltersVerificationInput( + @JsonProperty("filters") List filters, + @JsonProperty("message") byte[] message) { + this.filters = filters; + this.message = message; + } - public List getFilters() { - return filters; - } + public List getFilters() { + return filters; + } - public byte[] getMessage() { - return message; - } + public byte[] getMessage() { + return message; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MessageFiltersVerificationResult.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MessageFiltersVerificationResult.java index 17ea37cb0f..39f0bfb01f 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MessageFiltersVerificationResult.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MessageFiltersVerificationResult.java @@ -4,27 +4,28 @@ import com.fasterxml.jackson.annotation.JsonProperty; public class MessageFiltersVerificationResult { - private final VerificationStatus status; - private final String errorMessage; + private final VerificationStatus status; + private final String errorMessage; - public enum VerificationStatus { - NOT_MATCHED, - MATCHED, - ERROR - } + public enum VerificationStatus { + NOT_MATCHED, + MATCHED, + ERROR + } - @JsonCreator - public MessageFiltersVerificationResult(@JsonProperty("status") VerificationStatus status, - @JsonProperty("errorMessage") String errorMessage) { - this.status = status; - this.errorMessage = errorMessage; - } + @JsonCreator + public MessageFiltersVerificationResult( + @JsonProperty("status") VerificationStatus status, + @JsonProperty("errorMessage") String errorMessage) { + this.status = status; + this.errorMessage = errorMessage; + } - public VerificationStatus getStatus() { - return status; - } + public VerificationStatus getStatus() { + return status; + } - public String getErrorMessage() { - return errorMessage; - } + public String getErrorMessage() { + return errorMessage; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MessageTextPreview.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MessageTextPreview.java index fe475434bd..c756ddb4e0 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MessageTextPreview.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MessageTextPreview.java @@ -5,21 +5,22 @@ public class MessageTextPreview { - private final String content; + private final String content; - private final boolean truncated; + private final boolean truncated; - @JsonCreator - public MessageTextPreview(@JsonProperty("content") String content, @JsonProperty("truncated") boolean truncated) { - this.content = content; - this.truncated = truncated; - } + @JsonCreator + public MessageTextPreview( + @JsonProperty("content") String content, @JsonProperty("truncated") boolean truncated) { + this.content = content; + this.truncated = truncated; + } - public String getContent() { - return content; - } + public String getContent() { + return content; + } - public boolean isTruncated() { - return truncated; - } + public boolean isTruncated() { + return truncated; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MessageTrace.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MessageTrace.java index d84f209e26..4250dc00bc 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MessageTrace.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MessageTrace.java @@ -1,4 +1,3 @@ package pl.allegro.tech.hermes.api; -public interface MessageTrace { -} +public interface MessageTrace {} diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MetricDecimalValue.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MetricDecimalValue.java index c412b1c3a4..3fa2e56fbe 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MetricDecimalValue.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MetricDecimalValue.java @@ -1,81 +1,76 @@ package pl.allegro.tech.hermes.api; +import static java.lang.Double.parseDouble; + import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonValue; - import java.util.Objects; -import static java.lang.Double.parseDouble; - public class MetricDecimalValue { - private static final String UNAVAILABLE_STRING = "unavailable"; - private static final MetricDecimalValue UNAVAILABLE = new MetricDecimalValue(false, "-1.0"); - private static final MetricDecimalValue DEFAULT_VALUE = new MetricDecimalValue(true, "0.0"); - - private final boolean available; - private final String value; - - private MetricDecimalValue(boolean available, String value) { - this.available = available; - this.value = value; + private static final String UNAVAILABLE_STRING = "unavailable"; + private static final MetricDecimalValue UNAVAILABLE = new MetricDecimalValue(false, "-1.0"); + private static final MetricDecimalValue DEFAULT_VALUE = new MetricDecimalValue(true, "0.0"); + + private final boolean available; + private final String value; + + private MetricDecimalValue(boolean available, String value) { + this.available = available; + this.value = value; + } + + public static MetricDecimalValue unavailable() { + return UNAVAILABLE; + } + + public static MetricDecimalValue defaultValue() { + return DEFAULT_VALUE; + } + + public static MetricDecimalValue of(String value) { + return new MetricDecimalValue(true, value); + } + + @JsonCreator + public static MetricDecimalValue deserialize(String value) { + if (UNAVAILABLE_STRING.equals(value)) { + return unavailable(); } - - public static MetricDecimalValue unavailable() { - return UNAVAILABLE; - } - - public static MetricDecimalValue defaultValue() { - return DEFAULT_VALUE; + return of(value); + } + + @JsonValue + public String asString() { + return available ? value : UNAVAILABLE_STRING; + } + + public double toDouble() { + return parseDouble(value); + } + + public boolean isAvailable() { + return available; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - public static MetricDecimalValue of(String value) { - return new MetricDecimalValue(true, value); - } - - @JsonCreator - public static MetricDecimalValue deserialize(String value) { - if (UNAVAILABLE_STRING.equals(value)) { - return unavailable(); - } - return of(value); - } - - @JsonValue - public String asString() { - return available ? value : UNAVAILABLE_STRING; - } - - public double toDouble() { - return parseDouble(value); - } - - public boolean isAvailable() { - return available; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - MetricDecimalValue that = (MetricDecimalValue) o; - return available == that.available - && Objects.equals(value, that.value); - } - - @Override - public int hashCode() { - return Objects.hash(available, value); - } - - @Override - public String toString() { - return "MetricDecimalValue{" - + "available=" + available - + ", value='" + value + '\'' - + '}'; + if (o == null || getClass() != o.getClass()) { + return false; } + MetricDecimalValue that = (MetricDecimalValue) o; + return available == that.available && Objects.equals(value, that.value); + } + + @Override + public int hashCode() { + return Objects.hash(available, value); + } + + @Override + public String toString() { + return "MetricDecimalValue{" + "available=" + available + ", value='" + value + '\'' + '}'; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MetricLongValue.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MetricLongValue.java index 77753fb51a..648154b301 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MetricLongValue.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MetricLongValue.java @@ -2,65 +2,63 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonValue; - import java.util.Objects; public class MetricLongValue { - private static final String UNAVAILABLE_STRING = "unavailable"; - private static final MetricLongValue UNAVAILABLE = new MetricLongValue(false, -1); + private static final String UNAVAILABLE_STRING = "unavailable"; + private static final MetricLongValue UNAVAILABLE = new MetricLongValue(false, -1); - private final boolean available; - private final long value; + private final boolean available; + private final long value; - private MetricLongValue(boolean available, long value) { - this.available = available; - this.value = value; - } + private MetricLongValue(boolean available, long value) { + this.available = available; + this.value = value; + } - public static MetricLongValue unavailable() { - return UNAVAILABLE; - } + public static MetricLongValue unavailable() { + return UNAVAILABLE; + } - public static MetricLongValue of(long value) { - return new MetricLongValue(true, value); - } + public static MetricLongValue of(long value) { + return new MetricLongValue(true, value); + } - @JsonCreator - public static MetricLongValue deserialize(String value) { - if (UNAVAILABLE_STRING.equals(value)) { - return unavailable(); - } - return of(Long.valueOf(value)); + @JsonCreator + public static MetricLongValue deserialize(String value) { + if (UNAVAILABLE_STRING.equals(value)) { + return unavailable(); } + return of(Long.valueOf(value)); + } - @JsonValue - public String asString() { - return available ? String.valueOf(value) : UNAVAILABLE_STRING; - } + @JsonValue + public String asString() { + return available ? String.valueOf(value) : UNAVAILABLE_STRING; + } - public long toLong() { - return value; - } + public long toLong() { + return value; + } - public boolean isAvailable() { - return available; - } + public boolean isAvailable() { + return available; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - MetricLongValue that = (MetricLongValue) o; - return available == that.available - && value == that.value; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(available, value); + if (o == null || getClass() != o.getClass()) { + return false; } + MetricLongValue that = (MetricLongValue) o; + return available == that.available && value == that.value; + } + + @Override + public int hashCode() { + return Objects.hash(available, value); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MonitoringDetails.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MonitoringDetails.java index d35e99cc26..502a646c2d 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MonitoringDetails.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/MonitoringDetails.java @@ -3,65 +3,58 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.constraints.NotNull; - import java.util.Objects; public final class MonitoringDetails { - public static final MonitoringDetails EMPTY = new MonitoringDetails(Severity.NON_IMPORTANT, ""); + public static final MonitoringDetails EMPTY = new MonitoringDetails(Severity.NON_IMPORTANT, ""); - @NotNull - private final Severity severity; + @NotNull private final Severity severity; - @NotNull - private final String reaction; + @NotNull private final String reaction; - @JsonCreator - public MonitoringDetails(@JsonProperty("severity") Severity severity, - @JsonProperty("reaction") String reaction) { - this.severity = severity; - this.reaction = reaction; - } + @JsonCreator + public MonitoringDetails( + @JsonProperty("severity") Severity severity, @JsonProperty("reaction") String reaction) { + this.severity = severity; + this.reaction = reaction; + } - public Severity getSeverity() { - return severity; - } + public Severity getSeverity() { + return severity; + } - public String getReaction() { - return reaction; - } + public String getReaction() { + return reaction; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof MonitoringDetails)) { - return false; - } - MonitoringDetails that = (MonitoringDetails) o; - return severity == that.severity - && Objects.equals(reaction, that.reaction); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(severity, reaction); - } - - @Override - public String toString() { - return "MonitoringDetails{" - + "severity=" + severity - + ", reaction='" + reaction + '\'' - + '}'; - } - - public enum Severity { - @JsonProperty("CRITICAL") - CRITICAL, - @JsonProperty("IMPORTANT") - IMPORTANT, - @JsonProperty("NON_IMPORTANT") - NON_IMPORTANT + if (!(o instanceof MonitoringDetails)) { + return false; } + MonitoringDetails that = (MonitoringDetails) o; + return severity == that.severity && Objects.equals(reaction, that.reaction); + } + + @Override + public int hashCode() { + return Objects.hash(severity, reaction); + } + + @Override + public String toString() { + return "MonitoringDetails{" + "severity=" + severity + ", reaction='" + reaction + '\'' + '}'; + } + + public enum Severity { + @JsonProperty("CRITICAL") + CRITICAL, + @JsonProperty("IMPORTANT") + IMPORTANT, + @JsonProperty("NON_IMPORTANT") + NON_IMPORTANT + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/OAuthProvider.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/OAuthProvider.java index 2654d1990d..a9accea130 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/OAuthProvider.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/OAuthProvider.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.api; +import static pl.allegro.tech.hermes.api.constraints.Names.ALLOWED_NAME_REGEX; + import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.constraints.Max; @@ -7,119 +9,121 @@ import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; import jakarta.validation.constraints.Pattern; - import java.util.Objects; -import static pl.allegro.tech.hermes.api.constraints.Names.ALLOWED_NAME_REGEX; - public class OAuthProvider implements Anonymizable { - private static final String ANONYMIZED_CLIENT_SECRET = "******"; - private static final int DEFAULT_SOCKET_TIMEOUT = 0; - - @NotEmpty - @Pattern(regexp = ALLOWED_NAME_REGEX) - private final String name; - - @NotEmpty - private final String tokenEndpoint; - - @NotEmpty - private final String clientId; - - @NotEmpty - private final String clientSecret; - - @Min(0) - @Max(3600_000) - @NotNull - private final Integer tokenRequestInitialDelay; - - @Min(0) - @Max(3600_000) - @NotNull - private final Integer tokenRequestMaxDelay; - - @Min(100) - @Max(60_000) - @NotNull - private Integer requestTimeout; - - @Min(0) - @Max(60_000) - @NotNull - private Integer socketTimeout; - - @JsonCreator - public OAuthProvider(@JsonProperty("name") String name, - @JsonProperty("tokenEndpoint") String tokenEndpoint, - @JsonProperty("clientId") String clientId, - @JsonProperty("clientSecret") String clientSecret, - @JsonProperty("tokenRequestInitialDelay") Integer tokenRequestInitialDelay, - @JsonProperty("tokenRequestMaxDelay") Integer tokenRequestMaxDelay, - @JsonProperty("requestTimeout") Integer requestTimeout, - @JsonProperty("socketTimeout") Integer socketTimeout) { - this.name = name; - this.tokenEndpoint = tokenEndpoint; - this.clientId = clientId; - this.clientSecret = clientSecret; - this.tokenRequestInitialDelay = tokenRequestInitialDelay; - this.tokenRequestMaxDelay = tokenRequestMaxDelay; - this.requestTimeout = requestTimeout; - this.socketTimeout = socketTimeout != null ? socketTimeout : DEFAULT_SOCKET_TIMEOUT; - } - - public String getName() { - return name; - } - - public String getTokenEndpoint() { - return tokenEndpoint; - } - - public String getClientId() { - return clientId; - } - - public String getClientSecret() { - return clientSecret; + private static final String ANONYMIZED_CLIENT_SECRET = "******"; + private static final int DEFAULT_SOCKET_TIMEOUT = 0; + + @NotEmpty + @Pattern(regexp = ALLOWED_NAME_REGEX) + private final String name; + + @NotEmpty private final String tokenEndpoint; + + @NotEmpty private final String clientId; + + @NotEmpty private final String clientSecret; + + @Min(0) + @Max(3600_000) + @NotNull + private final Integer tokenRequestInitialDelay; + + @Min(0) + @Max(3600_000) + @NotNull + private final Integer tokenRequestMaxDelay; + + @Min(100) + @Max(60_000) + @NotNull + private Integer requestTimeout; + + @Min(0) + @Max(60_000) + @NotNull + private Integer socketTimeout; + + @JsonCreator + public OAuthProvider( + @JsonProperty("name") String name, + @JsonProperty("tokenEndpoint") String tokenEndpoint, + @JsonProperty("clientId") String clientId, + @JsonProperty("clientSecret") String clientSecret, + @JsonProperty("tokenRequestInitialDelay") Integer tokenRequestInitialDelay, + @JsonProperty("tokenRequestMaxDelay") Integer tokenRequestMaxDelay, + @JsonProperty("requestTimeout") Integer requestTimeout, + @JsonProperty("socketTimeout") Integer socketTimeout) { + this.name = name; + this.tokenEndpoint = tokenEndpoint; + this.clientId = clientId; + this.clientSecret = clientSecret; + this.tokenRequestInitialDelay = tokenRequestInitialDelay; + this.tokenRequestMaxDelay = tokenRequestMaxDelay; + this.requestTimeout = requestTimeout; + this.socketTimeout = socketTimeout != null ? socketTimeout : DEFAULT_SOCKET_TIMEOUT; + } + + public String getName() { + return name; + } + + public String getTokenEndpoint() { + return tokenEndpoint; + } + + public String getClientId() { + return clientId; + } + + public String getClientSecret() { + return clientSecret; + } + + public Integer getTokenRequestInitialDelay() { + return tokenRequestInitialDelay; + } + + public Integer getTokenRequestMaxDelay() { + return tokenRequestMaxDelay; + } + + public Integer getRequestTimeout() { + return requestTimeout; + } + + public Integer getSocketTimeout() { + return socketTimeout; + } + + public OAuthProvider anonymize() { + return new OAuthProvider( + name, + tokenEndpoint, + clientId, + ANONYMIZED_CLIENT_SECRET, + tokenRequestInitialDelay, + tokenRequestMaxDelay, + requestTimeout, + socketTimeout); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - public Integer getTokenRequestInitialDelay() { - return tokenRequestInitialDelay; - } - - public Integer getTokenRequestMaxDelay() { - return tokenRequestMaxDelay; - } - - public Integer getRequestTimeout() { - return requestTimeout; - } - - public Integer getSocketTimeout() { - return socketTimeout; - } - - public OAuthProvider anonymize() { - return new OAuthProvider(name, tokenEndpoint, clientId, ANONYMIZED_CLIENT_SECRET, - tokenRequestInitialDelay, tokenRequestMaxDelay, requestTimeout, socketTimeout); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - OAuthProvider that = (OAuthProvider) o; - return Objects.equals(name, that.name); - } - - @Override - public int hashCode() { - return Objects.hash(name); + if (o == null || getClass() != o.getClass()) { + return false; } + OAuthProvider that = (OAuthProvider) o; + return Objects.equals(name, that.name); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/OfflineRetentionTime.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/OfflineRetentionTime.java index 850b2fa389..9aad1854a6 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/OfflineRetentionTime.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/OfflineRetentionTime.java @@ -2,52 +2,51 @@ import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.constraints.Min; - import java.util.Objects; public class OfflineRetentionTime { - @Min(1) - private final Integer duration; + @Min(1) + private final Integer duration; - private final boolean infinite; + private final boolean infinite; - public OfflineRetentionTime(@JsonProperty("duration") Integer duration, @JsonProperty("infinite") boolean infinite) { - this.infinite = infinite; - this.duration = infinite ? null : duration; - } + public OfflineRetentionTime( + @JsonProperty("duration") Integer duration, @JsonProperty("infinite") boolean infinite) { + this.infinite = infinite; + this.duration = infinite ? null : duration; + } - public static OfflineRetentionTime of(int duration) { - return new OfflineRetentionTime(duration, false); - } + public static OfflineRetentionTime of(int duration) { + return new OfflineRetentionTime(duration, false); + } - public static OfflineRetentionTime infinite() { - return new OfflineRetentionTime(null, false); - } + public static OfflineRetentionTime infinite() { + return new OfflineRetentionTime(null, false); + } - public Integer getDuration() { - return duration; - } + public Integer getDuration() { + return duration; + } - public boolean isInfinite() { - return infinite; - } + public boolean isInfinite() { + return infinite; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof OfflineRetentionTime)) { - return false; - } - OfflineRetentionTime that = (OfflineRetentionTime) o; - return infinite == that.infinite - && Objects.equals(duration, that.duration); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(duration, infinite); + if (!(o instanceof OfflineRetentionTime)) { + return false; } + OfflineRetentionTime that = (OfflineRetentionTime) o; + return infinite == that.infinite && Objects.equals(duration, that.duration); + } + + @Override + public int hashCode() { + return Objects.hash(duration, infinite); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/OfflineRetransmissionRequest.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/OfflineRetransmissionRequest.java index a63b80f073..dfa21e8569 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/OfflineRetransmissionRequest.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/OfflineRetransmissionRequest.java @@ -5,98 +5,102 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import pl.allegro.tech.hermes.api.constraints.OneSourceRetransmission; -import pl.allegro.tech.hermes.api.jackson.InstantIsoSerializer; - import java.time.Instant; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.util.List; import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import pl.allegro.tech.hermes.api.constraints.OneSourceRetransmission; +import pl.allegro.tech.hermes.api.jackson.InstantIsoSerializer; @OneSourceRetransmission public class OfflineRetransmissionRequest { - private static final List formatters = List.of( - DateTimeFormatter.ISO_INSTANT, - DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'").withZone(ZoneId.of("UTC")), - DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm'Z'").withZone(ZoneId.of("UTC")) - ); - private static final Logger logger = LoggerFactory.getLogger(OfflineRetransmissionRequest.class); + private static final List formatters = + List.of( + DateTimeFormatter.ISO_INSTANT, + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'").withZone(ZoneId.of("UTC")), + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm'Z'").withZone(ZoneId.of("UTC"))); + private static final Logger logger = LoggerFactory.getLogger(OfflineRetransmissionRequest.class); - private final String sourceViewPath; - private final String sourceTopic; - @NotEmpty - private final String targetTopic; - @NotNull - private Instant startTimestamp; - @NotNull - private Instant endTimestamp; + private final String sourceViewPath; + private final String sourceTopic; + @NotEmpty private final String targetTopic; + @NotNull private Instant startTimestamp; + @NotNull private Instant endTimestamp; - @JsonCreator - public OfflineRetransmissionRequest( - @JsonProperty("sourceViewPath") String sourceViewPath, - @JsonProperty("sourceTopic") String sourceTopic, - @JsonProperty("targetTopic") String targetTopic, - @JsonProperty("startTimestamp") String startTimestamp, - @JsonProperty("endTimestamp") String endTimestamp) { - this.sourceViewPath = sourceViewPath; - this.sourceTopic = sourceTopic; - this.targetTopic = targetTopic; - this.startTimestamp = initializeTimestamp(startTimestamp); - this.endTimestamp = initializeTimestamp(endTimestamp); - } + @JsonCreator + public OfflineRetransmissionRequest( + @JsonProperty("sourceViewPath") String sourceViewPath, + @JsonProperty("sourceTopic") String sourceTopic, + @JsonProperty("targetTopic") String targetTopic, + @JsonProperty("startTimestamp") String startTimestamp, + @JsonProperty("endTimestamp") String endTimestamp) { + this.sourceViewPath = sourceViewPath; + this.sourceTopic = sourceTopic; + this.targetTopic = targetTopic; + this.startTimestamp = initializeTimestamp(startTimestamp); + this.endTimestamp = initializeTimestamp(endTimestamp); + } - private Instant initializeTimestamp(String timestamp) { - if (timestamp == null) { - return null; - } - - for (DateTimeFormatter formatter : formatters) { - try { - return formatter.parse(timestamp, Instant::from); - } catch (DateTimeParseException e) { - // ignore - } - } - - logger.warn("Provided date [{}] has an invalid format", timestamp); - return null; + private Instant initializeTimestamp(String timestamp) { + if (timestamp == null) { + return null; } - public Optional getSourceViewPath() { - return Optional.ofNullable(sourceViewPath); + for (DateTimeFormatter formatter : formatters) { + try { + return formatter.parse(timestamp, Instant::from); + } catch (DateTimeParseException e) { + // ignore + } } - public Optional getSourceTopic() { - return Optional.ofNullable(sourceTopic); - } + logger.warn("Provided date [{}] has an invalid format", timestamp); + return null; + } - public String getTargetTopic() { - return targetTopic; - } + public Optional getSourceViewPath() { + return Optional.ofNullable(sourceViewPath); + } - @JsonSerialize(using = InstantIsoSerializer.class) - public Instant getStartTimestamp() { - return startTimestamp; - } + public Optional getSourceTopic() { + return Optional.ofNullable(sourceTopic); + } - @JsonSerialize(using = InstantIsoSerializer.class) - public Instant getEndTimestamp() { - return endTimestamp; - } + public String getTargetTopic() { + return targetTopic; + } - @Override - public String toString() { - return "OfflineRetransmissionRequest{" - + "sourceTopic='" + sourceTopic + '\'' - + ", sourceViewPath='" + sourceViewPath + '\'' - + ", targetTopic='" + targetTopic + '\'' - + ", startTimestamp=" + startTimestamp - + ", endTimestamp=" + endTimestamp - + '}'; - } + @JsonSerialize(using = InstantIsoSerializer.class) + public Instant getStartTimestamp() { + return startTimestamp; + } + + @JsonSerialize(using = InstantIsoSerializer.class) + public Instant getEndTimestamp() { + return endTimestamp; + } + + @Override + public String toString() { + return "OfflineRetransmissionRequest{" + + "sourceTopic='" + + sourceTopic + + '\'' + + ", sourceViewPath='" + + sourceViewPath + + '\'' + + ", targetTopic='" + + targetTopic + + '\'' + + ", startTimestamp=" + + startTimestamp + + ", endTimestamp=" + + endTimestamp + + '}'; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/OfflineRetransmissionTask.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/OfflineRetransmissionTask.java index c3fb94f05c..a94a6d54c9 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/OfflineRetransmissionTask.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/OfflineRetransmissionTask.java @@ -4,81 +4,80 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import pl.allegro.tech.hermes.api.jackson.InstantIsoSerializer; - import java.time.Instant; import java.util.Optional; +import pl.allegro.tech.hermes.api.jackson.InstantIsoSerializer; public class OfflineRetransmissionTask { - private final String taskId; - private final OfflineRetransmissionRequest request; - private final Instant createdAt; + private final String taskId; + private final OfflineRetransmissionRequest request; + private final Instant createdAt; - @JsonCreator - public OfflineRetransmissionTask( - @JsonProperty("taskId") String taskId, - @JsonProperty("sourceViewPath") String sourceViewPath, - @JsonProperty("sourceTopic") String sourceTopic, - @JsonProperty("targetTopic") String targetTopic, - @JsonProperty("startTimestamp") Instant startTimestamp, - @JsonProperty("endTimestamp") Instant endTimestamp, - @JsonProperty("createdAt") Instant createdAt) { - this(taskId, new OfflineRetransmissionRequest( - sourceViewPath, - sourceTopic, - targetTopic, - startTimestamp.toString(), - endTimestamp.toString()), - createdAt); - } + @JsonCreator + public OfflineRetransmissionTask( + @JsonProperty("taskId") String taskId, + @JsonProperty("sourceViewPath") String sourceViewPath, + @JsonProperty("sourceTopic") String sourceTopic, + @JsonProperty("targetTopic") String targetTopic, + @JsonProperty("startTimestamp") Instant startTimestamp, + @JsonProperty("endTimestamp") Instant endTimestamp, + @JsonProperty("createdAt") Instant createdAt) { + this( + taskId, + new OfflineRetransmissionRequest( + sourceViewPath, + sourceTopic, + targetTopic, + startTimestamp.toString(), + endTimestamp.toString()), + createdAt); + } - public OfflineRetransmissionTask(String taskId, OfflineRetransmissionRequest request, Instant createdAt) { - this.taskId = taskId; - this.request = request; - this.createdAt = createdAt; - } + public OfflineRetransmissionTask( + String taskId, OfflineRetransmissionRequest request, Instant createdAt) { + this.taskId = taskId; + this.request = request; + this.createdAt = createdAt; + } - public String getTaskId() { - return taskId; - } + public String getTaskId() { + return taskId; + } - public Optional getSourceTopic() { - return request.getSourceTopic(); - } + public Optional getSourceTopic() { + return request.getSourceTopic(); + } - public Optional getSourceViewPath() { - return request.getSourceViewPath(); - } + public Optional getSourceViewPath() { + return request.getSourceViewPath(); + } - public String getTargetTopic() { - return request.getTargetTopic(); - } + public String getTargetTopic() { + return request.getTargetTopic(); + } - @JsonSerialize(using = InstantIsoSerializer.class) - public Instant getStartTimestamp() { - return request.getStartTimestamp(); - } + @JsonSerialize(using = InstantIsoSerializer.class) + public Instant getStartTimestamp() { + return request.getStartTimestamp(); + } - @JsonSerialize(using = InstantIsoSerializer.class) - public Instant getEndTimestamp() { - return request.getEndTimestamp(); - } + @JsonSerialize(using = InstantIsoSerializer.class) + public Instant getEndTimestamp() { + return request.getEndTimestamp(); + } - @JsonSerialize(using = InstantIsoSerializer.class) - public Instant getCreatedAt() { - return createdAt; - } + @JsonSerialize(using = InstantIsoSerializer.class) + public Instant getCreatedAt() { + return createdAt; + } - @JsonIgnore - public OfflineRetransmissionRequest getRequest() { - return request; - } + @JsonIgnore + public OfflineRetransmissionRequest getRequest() { + return request; + } - @Override - public String toString() { - return "OfflineRetransmissionTask{" - + "taskId='" + taskId + '\'' - + ", request=" + request - + '}'; - } + @Override + public String toString() { + return "OfflineRetransmissionTask{" + "taskId='" + taskId + '\'' + ", request=" + request + '}'; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/OffsetRetransmissionDate.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/OffsetRetransmissionDate.java index acd1270616..63e8d202b8 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/OffsetRetransmissionDate.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/OffsetRetransmissionDate.java @@ -3,21 +3,20 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import jakarta.validation.constraints.NotNull; -import pl.allegro.tech.hermes.api.jackson.OffsetDateTimeSerializer; - import java.time.OffsetDateTime; +import pl.allegro.tech.hermes.api.jackson.OffsetDateTimeSerializer; public class OffsetRetransmissionDate { - @NotNull - private final OffsetDateTime retransmissionDate; + @NotNull private final OffsetDateTime retransmissionDate; - public OffsetRetransmissionDate(@JsonProperty("retransmissionDate") OffsetDateTime retransmissionDate) { - this.retransmissionDate = retransmissionDate; - } + public OffsetRetransmissionDate( + @JsonProperty("retransmissionDate") OffsetDateTime retransmissionDate) { + this.retransmissionDate = retransmissionDate; + } - @JsonSerialize(using = OffsetDateTimeSerializer.class) - public OffsetDateTime getRetransmissionDate() { - return retransmissionDate; - } + @JsonSerialize(using = OffsetDateTimeSerializer.class) + public OffsetDateTime getRetransmissionDate() { + return retransmissionDate; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Owner.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Owner.java index e3f960303f..29f0ba1f25 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Owner.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Owner.java @@ -4,65 +4,61 @@ public class Owner { - private final String id; - private final String name; - private final String url; + private final String id; + private final String name; + private final String url; - public Owner() { - this.id = null; - this.name = null; - this.url = null; - } + public Owner() { + this.id = null; + this.name = null; + this.url = null; + } - public Owner(String id, String name) { - this.id = id; - this.name = name; - this.url = null; - } + public Owner(String id, String name) { + this.id = id; + this.name = name; + this.url = null; + } - public Owner(String id, String name, String url) { - this.id = id; - this.name = name; - this.url = url; - } + public Owner(String id, String name, String url) { + this.id = id; + this.name = name; + this.url = url; + } - public String getId() { - return id; - } + public String getId() { + return id; + } - public String getName() { - return name; - } + public String getName() { + return name; + } - public String getUrl() { - return url; - } + public String getUrl() { + return url; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Owner owner = (Owner) o; - return Objects.equals(id, owner.id) - && Objects.equals(name, owner.name) - && Objects.equals(url, owner.url); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(id, name, url); + if (o == null || getClass() != o.getClass()) { + return false; } + Owner owner = (Owner) o; + return Objects.equals(id, owner.id) + && Objects.equals(name, owner.name) + && Objects.equals(url, owner.url); + } - @Override - public String toString() { - return "Owner{" - + "id='" + id + '\'' - + ", name='" + name + '\'' - + ", url='" + url + '\'' - + '}'; - } + @Override + public int hashCode() { + return Objects.hash(id, name, url); + } + + @Override + public String toString() { + return "Owner{" + "id='" + id + '\'' + ", name='" + name + '\'' + ", url='" + url + '\'' + '}'; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/OwnerId.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/OwnerId.java index b8838b9279..50afd1fbfc 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/OwnerId.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/OwnerId.java @@ -3,55 +3,47 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.constraints.NotNull; - import java.util.Objects; public final class OwnerId { - @NotNull - private final String source; + @NotNull private final String source; - @NotNull - private final String id; + @NotNull private final String id; - @JsonCreator - public OwnerId(@JsonProperty("source") String source, - @JsonProperty("id") String id) { - this.source = source; - this.id = id; - } + @JsonCreator + public OwnerId(@JsonProperty("source") String source, @JsonProperty("id") String id) { + this.source = source; + this.id = id; + } - public String getSource() { - return source; - } + public String getSource() { + return source; + } - public String getId() { - return id; - } + public String getId() { + return id; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - OwnerId that = (OwnerId) o; - return Objects.equals(source, that.source) - && Objects.equals(id, that.id); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(source, id); - } - - @Override - public String toString() { - return "OwnerId{" - + "source='" + source + '\'' - + ", id='" + id + '\'' - + '}'; + if (o == null || getClass() != o.getClass()) { + return false; } + OwnerId that = (OwnerId) o; + return Objects.equals(source, that.source) && Objects.equals(id, that.id); + } + + @Override + public int hashCode() { + return Objects.hash(source, id); + } + + @Override + public String toString() { + return "OwnerId{" + "source='" + source + '\'' + ", id='" + id + '\'' + '}'; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/PatchData.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/PatchData.java index 5cb408bba5..d4a1aaaed0 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/PatchData.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/PatchData.java @@ -2,49 +2,48 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import pl.allegro.tech.hermes.api.jackson.PatchDataDeserializer; -import pl.allegro.tech.hermes.api.jackson.PatchDataSerializer; - import java.util.HashMap; import java.util.Map; +import pl.allegro.tech.hermes.api.jackson.PatchDataDeserializer; +import pl.allegro.tech.hermes.api.jackson.PatchDataSerializer; @JsonDeserialize(using = PatchDataDeserializer.class) @JsonSerialize(using = PatchDataSerializer.class) public class PatchData { - private final Map patch; + private final Map patch; - public PatchData(Map patch) { - this.patch = patch; - } + public PatchData(Map patch) { + this.patch = patch; + } - public static PatchData from(Map patch) { - return new PatchData(patch); - } + public static PatchData from(Map patch) { + return new PatchData(patch); + } - public static Builder patchData() { - return new Builder(); - } + public static Builder patchData() { + return new Builder(); + } - public Map getPatch() { - return patch; - } + public Map getPatch() { + return patch; + } - public boolean valueChanged(String field, Object originalValue) { - return patch.containsKey(field) && !patch.get(field).equals(originalValue); - } + public boolean valueChanged(String field, Object originalValue) { + return patch.containsKey(field) && !patch.get(field).equals(originalValue); + } - public static class Builder { + public static class Builder { - private final Map map = new HashMap<>(); + private final Map map = new HashMap<>(); - public PatchData build() { - return PatchData.from(map); - } + public PatchData build() { + return PatchData.from(map); + } - public Builder set(String field, Object value) { - this.map.put(field, value); - return this; - } + public Builder set(String field, Object value) { + this.map.put(field, value); + return this; } + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/PersistentSubscriptionMetrics.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/PersistentSubscriptionMetrics.java index b3cf53ee4f..0ad876225e 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/PersistentSubscriptionMetrics.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/PersistentSubscriptionMetrics.java @@ -5,40 +5,41 @@ public class PersistentSubscriptionMetrics { - private long delivered; - private long discarded; - private long volume; - - @JsonCreator - public PersistentSubscriptionMetrics(@JsonProperty("delivered") long delivered, - @JsonProperty("discarded") long discarded, - @JsonProperty("volume") long volume) { - this.delivered = delivered; - this.discarded = discarded; - this.volume = volume; - } - - public long getDelivered() { - return delivered; - } - - public void setDelivered(long delivered) { - this.delivered = delivered; - } - - public long getDiscarded() { - return discarded; - } - - public void setDiscarded(long discarded) { - this.discarded = discarded; - } - - public long getVolume() { - return volume; - } - - public void setVolume(long volume) { - this.volume = volume; - } + private long delivered; + private long discarded; + private long volume; + + @JsonCreator + public PersistentSubscriptionMetrics( + @JsonProperty("delivered") long delivered, + @JsonProperty("discarded") long discarded, + @JsonProperty("volume") long volume) { + this.delivered = delivered; + this.discarded = discarded; + this.volume = volume; + } + + public long getDelivered() { + return delivered; + } + + public void setDelivered(long delivered) { + this.delivered = delivered; + } + + public long getDiscarded() { + return discarded; + } + + public void setDiscarded(long discarded) { + this.discarded = discarded; + } + + public long getVolume() { + return volume; + } + + public void setVolume(long volume) { + this.volume = volume; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/PublishedMessageTrace.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/PublishedMessageTrace.java index 471a7dd407..bda7e00508 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/PublishedMessageTrace.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/PublishedMessageTrace.java @@ -4,100 +4,100 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.Objects; @JsonIgnoreProperties(ignoreUnknown = true) public class PublishedMessageTrace implements MessageTrace { - private final String messageId; - private final long timestamp; - private final PublishedMessageTraceStatus status; - private final TopicName topicName; - private final String reason; - private final String message; - private final String cluster; - private final String extraRequestHeaders; - private final String storageDatacenter; - - @JsonCreator - public PublishedMessageTrace(@JsonProperty("messageId") String messageId, - @JsonProperty("timestamp") Long timestamp, - @JsonProperty("topicName") String topicName, - @JsonProperty("status") PublishedMessageTraceStatus status, - @JsonProperty("reason") String reason, - @JsonProperty("message") String message, - @JsonProperty("cluster") String cluster, - @JsonProperty("extraRequestHeaders") String extraRequestHeaders, - @JsonProperty("storageDc") String storageDatacenter) { - this.messageId = messageId; - this.timestamp = timestamp; - this.status = status; - this.topicName = TopicName.fromQualifiedName(topicName); - this.reason = reason; - this.message = message; - this.cluster = cluster; - this.extraRequestHeaders = extraRequestHeaders; - this.storageDatacenter = storageDatacenter; - } - - public String getMessageId() { - return messageId; - } - - public Long getTimestamp() { - return timestamp; - } - - public String getReason() { - return reason; - } - - @JsonProperty("topicName") - public String getQualifiedTopicName() { - return TopicName.toQualifiedName(topicName); - } - - @JsonIgnore - public TopicName getTopicName() { - return topicName; - } - - public String getMessage() { - return message; - } - - public PublishedMessageTraceStatus getStatus() { - return status; - } - - public String getCluster() { - return cluster; - } - - public String getExtraRequestHeaders() { - return extraRequestHeaders; - } - - @JsonProperty("storageDc") - public String getStorageDatacenter() { - return storageDatacenter; + private final String messageId; + private final long timestamp; + private final PublishedMessageTraceStatus status; + private final TopicName topicName; + private final String reason; + private final String message; + private final String cluster; + private final String extraRequestHeaders; + private final String storageDatacenter; + + @JsonCreator + public PublishedMessageTrace( + @JsonProperty("messageId") String messageId, + @JsonProperty("timestamp") Long timestamp, + @JsonProperty("topicName") String topicName, + @JsonProperty("status") PublishedMessageTraceStatus status, + @JsonProperty("reason") String reason, + @JsonProperty("message") String message, + @JsonProperty("cluster") String cluster, + @JsonProperty("extraRequestHeaders") String extraRequestHeaders, + @JsonProperty("storageDc") String storageDatacenter) { + this.messageId = messageId; + this.timestamp = timestamp; + this.status = status; + this.topicName = TopicName.fromQualifiedName(topicName); + this.reason = reason; + this.message = message; + this.cluster = cluster; + this.extraRequestHeaders = extraRequestHeaders; + this.storageDatacenter = storageDatacenter; + } + + public String getMessageId() { + return messageId; + } + + public Long getTimestamp() { + return timestamp; + } + + public String getReason() { + return reason; + } + + @JsonProperty("topicName") + public String getQualifiedTopicName() { + return TopicName.toQualifiedName(topicName); + } + + @JsonIgnore + public TopicName getTopicName() { + return topicName; + } + + public String getMessage() { + return message; + } + + public PublishedMessageTraceStatus getStatus() { + return status; + } + + public String getCluster() { + return cluster; + } + + public String getExtraRequestHeaders() { + return extraRequestHeaders; + } + + @JsonProperty("storageDc") + public String getStorageDatacenter() { + return storageDatacenter; + } + + @Override + public int hashCode() { + return Objects.hash(messageId); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(messageId); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - final PublishedMessageTrace other = (PublishedMessageTrace) obj; - return Objects.equals(this.messageId, other.messageId); + if (obj == null || getClass() != obj.getClass()) { + return false; } + final PublishedMessageTrace other = (PublishedMessageTrace) obj; + return Objects.equals(this.messageId, other.messageId); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/PublishedMessageTraceStatus.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/PublishedMessageTraceStatus.java index ea3bab63be..d386e91c13 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/PublishedMessageTraceStatus.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/PublishedMessageTraceStatus.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.api; public enum PublishedMessageTraceStatus { - INFLIGHT, SUCCESS, ERROR + INFLIGHT, + SUCCESS, + ERROR } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/PublishingAuth.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/PublishingAuth.java index 3b09192b47..1ef9ee9a2b 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/PublishingAuth.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/PublishingAuth.java @@ -2,63 +2,63 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.ArrayList; import java.util.List; import java.util.Objects; public class PublishingAuth { - private final List publishers; - private final boolean enabled; - private final boolean unauthenticatedAccessEnabled; + private final List publishers; + private final boolean enabled; + private final boolean unauthenticatedAccessEnabled; - @JsonCreator - public PublishingAuth(@JsonProperty("publishers") List publishers, - @JsonProperty("enabled") boolean enabled, - @JsonProperty("unauthenticatedAccessEnabled") boolean unauthenticatedAccessEnabled) { + @JsonCreator + public PublishingAuth( + @JsonProperty("publishers") List publishers, + @JsonProperty("enabled") boolean enabled, + @JsonProperty("unauthenticatedAccessEnabled") boolean unauthenticatedAccessEnabled) { - this.publishers = publishers; - this.enabled = enabled; - this.unauthenticatedAccessEnabled = unauthenticatedAccessEnabled; - } + this.publishers = publishers; + this.enabled = enabled; + this.unauthenticatedAccessEnabled = unauthenticatedAccessEnabled; + } - public boolean isEnabled() { - return enabled; - } + public boolean isEnabled() { + return enabled; + } - public boolean isUnauthenticatedAccessEnabled() { - return unauthenticatedAccessEnabled; - } + public boolean isUnauthenticatedAccessEnabled() { + return unauthenticatedAccessEnabled; + } - public boolean hasPermission(String publisher) { - return publishers.contains(publisher); - } + public boolean hasPermission(String publisher) { + return publishers.contains(publisher); + } - public List getPublishers() { - return publishers; - } + public List getPublishers() { + return publishers; + } - public static PublishingAuth disabled() { - return new PublishingAuth(new ArrayList<>(), false, true); - } + public static PublishingAuth disabled() { + return new PublishingAuth(new ArrayList<>(), false, true); + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - PublishingAuth that = (PublishingAuth) o; - return enabled == that.enabled - && unauthenticatedAccessEnabled == that.unauthenticatedAccessEnabled - && Objects.equals(publishers, that.publishers); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(publishers, enabled, unauthenticatedAccessEnabled); + if (o == null || getClass() != o.getClass()) { + return false; } + PublishingAuth that = (PublishingAuth) o; + return enabled == that.enabled + && unauthenticatedAccessEnabled == that.unauthenticatedAccessEnabled + && Objects.equals(publishers, that.publishers); + } + + @Override + public int hashCode() { + return Objects.hash(publishers, enabled, unauthenticatedAccessEnabled); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/PublishingChaosPolicy.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/PublishingChaosPolicy.java index ebe9620fc8..1dfb0e3d4a 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/PublishingChaosPolicy.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/PublishingChaosPolicy.java @@ -2,42 +2,45 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.Collections; import java.util.Map; -public record PublishingChaosPolicy(ChaosMode mode, ChaosPolicy globalPolicy, Map datacenterPolicies) { +public record PublishingChaosPolicy( + ChaosMode mode, ChaosPolicy globalPolicy, Map datacenterPolicies) { - @JsonCreator - public PublishingChaosPolicy(@JsonProperty("mode") ChaosMode mode, - @JsonProperty("globalPolicy") ChaosPolicy globalPolicy, - @JsonProperty("datacenterPolicies") Map datacenterPolicies) { - this.mode = mode; - this.globalPolicy = globalPolicy; - this.datacenterPolicies = datacenterPolicies == null ? Map.of() : datacenterPolicies; - } + @JsonCreator + public PublishingChaosPolicy( + @JsonProperty("mode") ChaosMode mode, + @JsonProperty("globalPolicy") ChaosPolicy globalPolicy, + @JsonProperty("datacenterPolicies") Map datacenterPolicies) { + this.mode = mode; + this.globalPolicy = globalPolicy; + this.datacenterPolicies = datacenterPolicies == null ? Map.of() : datacenterPolicies; + } - public static PublishingChaosPolicy disabled() { - return new PublishingChaosPolicy(ChaosMode.DISABLED, null, Collections.emptyMap()); - } + public static PublishingChaosPolicy disabled() { + return new PublishingChaosPolicy(ChaosMode.DISABLED, null, Collections.emptyMap()); + } - public record ChaosPolicy(int probability, int delayFrom, int delayTo, boolean completeWithError) { + public record ChaosPolicy( + int probability, int delayFrom, int delayTo, boolean completeWithError) { - @JsonCreator - public ChaosPolicy(@JsonProperty("probability") int probability, - @JsonProperty("delayFrom") int delayFrom, - @JsonProperty("delayTo") int delayTo, - @JsonProperty("completeWithError") boolean completeWithError) { - this.probability = probability; - this.delayFrom = delayFrom; - this.delayTo = delayTo; - this.completeWithError = completeWithError; - } + @JsonCreator + public ChaosPolicy( + @JsonProperty("probability") int probability, + @JsonProperty("delayFrom") int delayFrom, + @JsonProperty("delayTo") int delayTo, + @JsonProperty("completeWithError") boolean completeWithError) { + this.probability = probability; + this.delayFrom = delayFrom; + this.delayTo = delayTo; + this.completeWithError = completeWithError; } + } - public enum ChaosMode { - DISABLED, - GLOBAL, - DATACENTER - } + public enum ChaosMode { + DISABLED, + GLOBAL, + DATACENTER + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Query.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Query.java index a004e97f46..28a53016a7 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Query.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Query.java @@ -5,15 +5,15 @@ public interface Query { - Stream filter(Stream input); + Stream filter(Stream input); - default Stream filter(Collection input) { - return filter(input.stream()); - } + default Stream filter(Collection input) { + return filter(input.stream()); + } - Stream filterNames(Stream input); + Stream filterNames(Stream input); - default Stream filterNames(Collection input) { - return filterNames(input.stream()); - } + default Stream filterNames(Collection input) { + return filterNames(input.stream()); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/RawSchema.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/RawSchema.java index 132566500a..9554f2d8d6 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/RawSchema.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/RawSchema.java @@ -1,47 +1,47 @@ package pl.allegro.tech.hermes.api; -import java.util.Objects; - import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Strings.emptyToNull; +import java.util.Objects; + public final class RawSchema { - private final String value; + private final String value; - private RawSchema(String value) { - this.value = checkNotNull(emptyToNull(value)); - } + private RawSchema(String value) { + this.value = checkNotNull(emptyToNull(value)); + } - public static RawSchema valueOf(String schema) { - return new RawSchema(schema); - } + public static RawSchema valueOf(String schema) { + return new RawSchema(schema); + } - public String value() { - return value; - } + public String value() { + return value; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } - RawSchema that = (RawSchema) o; + RawSchema that = (RawSchema) o; - return value.equals(that.value); - } + return value.equals(that.value); + } - @Override - public int hashCode() { - return Objects.hashCode(value); - } + @Override + public int hashCode() { + return Objects.hashCode(value); + } - @Override - public String toString() { - return "RawSource(" + value + ")"; - } + @Override + public String toString() { + return "RawSource(" + value + ")"; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/RawSchemaWithMetadata.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/RawSchemaWithMetadata.java index bb6b833172..4e9094b00e 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/RawSchemaWithMetadata.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/RawSchemaWithMetadata.java @@ -1,66 +1,67 @@ package pl.allegro.tech.hermes.api; import com.google.common.base.MoreObjects; - import java.util.Objects; public final class RawSchemaWithMetadata { - private final RawSchema schema; - private final int id; - private final int version; + private final RawSchema schema; + private final int id; + private final int version; - private RawSchemaWithMetadata(RawSchema schema, int id, int version) { - this.schema = schema; - this.id = id; - this.version = version; - } + private RawSchemaWithMetadata(RawSchema schema, int id, int version) { + this.schema = schema; + this.id = id; + this.version = version; + } - public static RawSchemaWithMetadata of(String schema, int id, int version) { - return new RawSchemaWithMetadata(RawSchema.valueOf(schema), id, version); - } + public static RawSchemaWithMetadata of(String schema, int id, int version) { + return new RawSchemaWithMetadata(RawSchema.valueOf(schema), id, version); + } - public RawSchema getSchema() { - return schema; - } + public RawSchema getSchema() { + return schema; + } - public String getSchemaString() { - return schema.value(); - } + public String getSchemaString() { + return schema.value(); + } - public int getId() { - return id; - } + public int getId() { + return id; + } - public int getVersion() { - return version; - } + public int getVersion() { + return version; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } - RawSchemaWithMetadata that = (RawSchemaWithMetadata) o; + RawSchemaWithMetadata that = (RawSchemaWithMetadata) o; - return schema.equals(that.schema) && Objects.equals(id, that.id) && Objects.equals(version, that.version); - } + return schema.equals(that.schema) + && Objects.equals(id, that.id) + && Objects.equals(version, that.version); + } - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("schema", schema) - .add("id", id) - .add("version", version) - .toString(); - } + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("schema", schema) + .add("id", id) + .add("version", version) + .toString(); + } - @Override - public int hashCode() { - return Objects.hash(schema, id, version); - } + @Override + public int hashCode() { + return Objects.hash(schema, id, version); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Readiness.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Readiness.java index 4223df0e1c..d730b467e2 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Readiness.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Readiness.java @@ -6,16 +6,15 @@ import jakarta.validation.constraints.NotNull; public class Readiness { - @NotNull - private final boolean isReady; + @NotNull private final boolean isReady; - @JsonCreator - public Readiness(@JsonProperty("isReady") boolean isReady) { - this.isReady = isReady; - } + @JsonCreator + public Readiness(@JsonProperty("isReady") boolean isReady) { + this.isReady = isReady; + } - @JsonGetter("isReady") - public boolean isReady() { - return isReady; - } + @JsonGetter("isReady") + public boolean isReady() { + return isReady; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/RetentionTime.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/RetentionTime.java index 762465ce9b..8d2c8668dc 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/RetentionTime.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/RetentionTime.java @@ -3,60 +3,61 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.constraints.Min; - import java.util.EnumSet; import java.util.Objects; import java.util.Set; import java.util.concurrent.TimeUnit; public class RetentionTime { - private static final TimeUnit DEFAULT_UNIT = TimeUnit.DAYS; + private static final TimeUnit DEFAULT_UNIT = TimeUnit.DAYS; - public static RetentionTime MAX = new RetentionTime(7, TimeUnit.DAYS); - public static Set allowedUnits = EnumSet.of(TimeUnit.SECONDS, TimeUnit.MINUTES, TimeUnit.HOURS, TimeUnit.DAYS); + public static RetentionTime MAX = new RetentionTime(7, TimeUnit.DAYS); + public static Set allowedUnits = + EnumSet.of(TimeUnit.SECONDS, TimeUnit.MINUTES, TimeUnit.HOURS, TimeUnit.DAYS); + @Min(0) + private final int duration; - @Min(0) - private final int duration; + private final TimeUnit retentionUnit; - private final TimeUnit retentionUnit; + public RetentionTime( + @JsonProperty("duration") int duration, @JsonProperty("retentionUnit") TimeUnit unit) { + this.duration = duration; + this.retentionUnit = unit == null ? DEFAULT_UNIT : unit; + } - public RetentionTime(@JsonProperty("duration") int duration, @JsonProperty("retentionUnit") TimeUnit unit) { - this.duration = duration; - this.retentionUnit = unit == null ? DEFAULT_UNIT : unit; - } + public static RetentionTime of(int duration, TimeUnit unit) { + return new RetentionTime(duration, unit); + } - public static RetentionTime of(int duration, TimeUnit unit) { - return new RetentionTime(duration, unit); - } + public int getDuration() { + return duration; + } - public int getDuration() { - return duration; - } + @JsonIgnore + public long getDurationInMillis() { + return retentionUnit.toMillis(duration); + } - @JsonIgnore - public long getDurationInMillis() { - return retentionUnit.toMillis(duration); - } + public TimeUnit getRetentionUnit() { + return retentionUnit; + } - public TimeUnit getRetentionUnit() { - return retentionUnit; - } + @Override + public int hashCode() { + return Objects.hash(duration); + } - @Override - public int hashCode() { - return Objects.hash(duration); + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - final RetentionTime other = (RetentionTime) obj; - return Objects.equals(this.duration, other.duration) && Objects.equals(this.retentionUnit, other.retentionUnit); + if (obj == null || getClass() != obj.getClass()) { + return false; } + final RetentionTime other = (RetentionTime) obj; + return Objects.equals(this.duration, other.duration) + && Objects.equals(this.retentionUnit, other.retentionUnit); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SentMessageTrace.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SentMessageTrace.java index eb03b91350..494660d899 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SentMessageTrace.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SentMessageTrace.java @@ -4,191 +4,202 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.Objects; @JsonIgnoreProperties(ignoreUnknown = true) public class SentMessageTrace implements MessageTrace { + private final String messageId; + private final String batchId; + private final String subscription; + private final long timestamp; + private final SentMessageTraceStatus status; + private final Integer partition; + private final Long offset; + private final TopicName topicName; + private final String reason; + private final String message; + private final String cluster; + + @JsonCreator + public SentMessageTrace( + @JsonProperty("messageId") String messageId, + @JsonProperty("batchId") String batchId, + @JsonProperty("timestamp") Long timestamp, + @JsonProperty("subscription") String subscription, + @JsonProperty("topicName") String topicName, + @JsonProperty("status") SentMessageTraceStatus status, + @JsonProperty("reason") String reason, + @JsonProperty("message") String message, + @JsonProperty("partition") Integer partition, + @JsonProperty("offset") Long offset, + @JsonProperty("cluster") String cluster) { + this.messageId = messageId; + this.batchId = batchId; + this.timestamp = timestamp; + this.subscription = subscription; + this.status = status; + this.partition = partition; + this.offset = offset; + this.topicName = TopicName.fromQualifiedName(topicName); + this.reason = reason; + this.message = message; + this.cluster = cluster; + } + + public String getMessageId() { + return messageId; + } + + public String getBatchId() { + return batchId; + } + + public Long getTimestamp() { + return timestamp; + } + + public String getSubscription() { + return subscription; + } + + public String getReason() { + return reason; + } + + public Integer getPartition() { + return partition; + } + + public Long getOffset() { + return offset; + } + + @JsonProperty("topicName") + public String getQualifiedTopicName() { + return TopicName.toQualifiedName(topicName); + } + + @JsonIgnore + public TopicName getTopicName() { + return topicName; + } + + public String getMessage() { + return message; + } + + public SentMessageTraceStatus getStatus() { + return status; + } + + public String getCluster() { + return cluster; + } + + @Override + public int hashCode() { + return Objects.hash(messageId); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final SentMessageTrace other = (SentMessageTrace) obj; + return Objects.equals(this.messageId, other.messageId) + && Objects.equals(this.subscription, other.subscription) + && Objects.equals(this.timestamp, other.timestamp) + && Objects.equals(this.status, other.status); + } + + public static class Builder { + private final String messageId; private final String batchId; - private final String subscription; - private final long timestamp; private final SentMessageTraceStatus status; - private final Integer partition; - private final Long offset; - private final TopicName topicName; - private final String reason; - private final String message; - private final String cluster; - - @JsonCreator - public SentMessageTrace(@JsonProperty("messageId") String messageId, - @JsonProperty("batchId") String batchId, - @JsonProperty("timestamp") Long timestamp, - @JsonProperty("subscription") String subscription, - @JsonProperty("topicName") String topicName, - @JsonProperty("status") SentMessageTraceStatus status, - @JsonProperty("reason") String reason, - @JsonProperty("message") String message, - @JsonProperty("partition") Integer partition, - @JsonProperty("offset") Long offset, - @JsonProperty("cluster") String cluster) { - this.messageId = messageId; - this.batchId = batchId; - this.timestamp = timestamp; - this.subscription = subscription; - this.status = status; - this.partition = partition; - this.offset = offset; - this.topicName = TopicName.fromQualifiedName(topicName); - this.reason = reason; - this.message = message; - this.cluster = cluster; - } - - public String getMessageId() { - return messageId; - } - - public String getBatchId() { - return batchId; - } - public Long getTimestamp() { - return timestamp; - } + private String subscription; + private long timestamp; + private Integer partition; + private Long offset; + private String topicName; + private String reason; + private String message; + private String cluster; - public String getSubscription() { - return subscription; + private Builder(String messageId, String batchId, SentMessageTraceStatus status) { + this.messageId = messageId; + this.batchId = batchId; + this.status = status; } - public String getReason() { - return reason; + public Builder withSubscription(String subscription) { + this.subscription = subscription; + return this; } - public Integer getPartition() { - return partition; + public Builder withTimestamp(long timestamp) { + this.timestamp = timestamp; + return this; } - public Long getOffset() { - return offset; + public Builder withPartition(Integer partition) { + this.partition = partition; + return this; } - @JsonProperty("topicName") - public String getQualifiedTopicName() { - return TopicName.toQualifiedName(topicName); + public Builder withOffset(Long offset) { + this.offset = offset; + return this; } - @JsonIgnore - public TopicName getTopicName() { - return topicName; + public Builder withTopicName(String topicName) { + this.topicName = topicName; + return this; } - public String getMessage() { - return message; + public Builder withReason(String reason) { + this.reason = reason; + return this; } - public SentMessageTraceStatus getStatus() { - return status; + public Builder withMessage(String message) { + this.message = message; + return this; } - public String getCluster() { - return cluster; + public Builder withCluster(String cluster) { + this.cluster = cluster; + return this; } - @Override - public int hashCode() { - return Objects.hash(messageId); + public static Builder sentMessageTrace( + String messageId, String batchId, SentMessageTraceStatus status) { + return new Builder(messageId, batchId, status); } - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - final SentMessageTrace other = (SentMessageTrace) obj; - return Objects.equals(this.messageId, other.messageId) - && Objects.equals(this.subscription, other.subscription) - && Objects.equals(this.timestamp, other.timestamp) - && Objects.equals(this.status, other.status); + public static Builder undeliveredMessage() { + return new Builder(null, null, SentMessageTraceStatus.DISCARDED); } - public static class Builder { - - private final String messageId; - private final String batchId; - private final SentMessageTraceStatus status; - - private String subscription; - private long timestamp; - private Integer partition; - private Long offset; - private String topicName; - private String reason; - private String message; - private String cluster; - - private Builder(String messageId, String batchId, SentMessageTraceStatus status) { - this.messageId = messageId; - this.batchId = batchId; - this.status = status; - } - - public Builder withSubscription(String subscription) { - this.subscription = subscription; - return this; - } - - public Builder withTimestamp(long timestamp) { - this.timestamp = timestamp; - return this; - } - - public Builder withPartition(Integer partition) { - this.partition = partition; - return this; - } - - public Builder withOffset(Long offset) { - this.offset = offset; - return this; - } - - public Builder withTopicName(String topicName) { - this.topicName = topicName; - return this; - } - - public Builder withReason(String reason) { - this.reason = reason; - return this; - } - - public Builder withMessage(String message) { - this.message = message; - return this; - } - - public Builder withCluster(String cluster) { - this.cluster = cluster; - return this; - } - - public static Builder sentMessageTrace(String messageId, String batchId, SentMessageTraceStatus status) { - return new Builder(messageId, batchId, status); - } - - public static Builder undeliveredMessage() { - return new Builder(null, null, SentMessageTraceStatus.DISCARDED); - } - - public SentMessageTrace build() { - return new SentMessageTrace(messageId, batchId, timestamp, subscription, topicName, status, - reason, message, partition, offset, cluster); - } + public SentMessageTrace build() { + return new SentMessageTrace( + messageId, + batchId, + timestamp, + subscription, + topicName, + status, + reason, + message, + partition, + offset, + cluster); } + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SentMessageTraceStatus.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SentMessageTraceStatus.java index f194a7cb63..5597d71be9 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SentMessageTraceStatus.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SentMessageTraceStatus.java @@ -1,5 +1,9 @@ package pl.allegro.tech.hermes.api; public enum SentMessageTraceStatus { - INFLIGHT, SUCCESS, FAILED, DISCARDED, FILTERED + INFLIGHT, + SUCCESS, + FAILED, + DISCARDED, + FILTERED } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Stats.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Stats.java index fbcf73a1c3..8dea4566a5 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Stats.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Stats.java @@ -2,52 +2,48 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.Objects; public class Stats { - private final TopicStats topicStats; - private final SubscriptionStats subscriptionStats; - - @JsonCreator - public Stats( - @JsonProperty("topicStats") - TopicStats topicStats, - @JsonProperty("subscriptionStats") SubscriptionStats subscriptionStats) { - this.topicStats = topicStats; - this.subscriptionStats = subscriptionStats; - } - - public TopicStats getTopicStats() { - return topicStats; - } - - public SubscriptionStats getSubscriptionStats() { - return subscriptionStats; + private final TopicStats topicStats; + private final SubscriptionStats subscriptionStats; + + @JsonCreator + public Stats( + @JsonProperty("topicStats") TopicStats topicStats, + @JsonProperty("subscriptionStats") SubscriptionStats subscriptionStats) { + this.topicStats = topicStats; + this.subscriptionStats = subscriptionStats; + } + + public TopicStats getTopicStats() { + return topicStats; + } + + public SubscriptionStats getSubscriptionStats() { + return subscriptionStats; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Stats stats = (Stats) o; - return Objects.equals(topicStats, stats.topicStats) && Objects.equals(subscriptionStats, stats.subscriptionStats); - } - - @Override - public int hashCode() { - return Objects.hash(topicStats, subscriptionStats); - } - - @Override - public String toString() { - return "Stats{" - + "topicStats=" + topicStats - + ", subscriptionStats=" + subscriptionStats - + '}'; + if (o == null || getClass() != o.getClass()) { + return false; } + Stats stats = (Stats) o; + return Objects.equals(topicStats, stats.topicStats) + && Objects.equals(subscriptionStats, stats.subscriptionStats); + } + + @Override + public int hashCode() { + return Objects.hash(topicStats, subscriptionStats); + } + + @Override + public String toString() { + return "Stats{" + "topicStats=" + topicStats + ", subscriptionStats=" + subscriptionStats + '}'; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Subscription.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Subscription.java index 14849437df..359206d487 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Subscription.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Subscription.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.api; +import static pl.allegro.tech.hermes.api.constraints.Names.ALLOWED_NAME_REGEX; + import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -8,8 +10,6 @@ import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; import jakarta.validation.constraints.Pattern; -import pl.allegro.tech.hermes.api.constraints.ValidContentType; - import java.time.Instant; import java.util.ArrayList; import java.util.Collections; @@ -17,463 +17,523 @@ import java.util.List; import java.util.Map; import java.util.Objects; - -import static pl.allegro.tech.hermes.api.constraints.Names.ALLOWED_NAME_REGEX; +import pl.allegro.tech.hermes.api.constraints.ValidContentType; @ValidContentType(message = "AVRO content type is not supported in BATCH delivery mode") -@JsonIgnoreProperties(value = {"createdAt", "modifiedAt"}, allowGetters = true) +@JsonIgnoreProperties( + value = {"createdAt", "modifiedAt"}, + allowGetters = true) public class Subscription implements Anonymizable { - @NotNull - private final MonitoringDetails monitoringDetails; - private final SubscriptionName subscriptionName; - @Valid - @NotNull - private TopicName topicName; - @NotEmpty - @Pattern(regexp = ALLOWED_NAME_REGEX) - private String name; - private State state = State.PENDING; - @NotNull - @Valid - private EndpointAddress endpoint; - @NotNull - private ContentType contentType = ContentType.JSON; - @NotNull - private String description; - @Valid - private SubscriptionPolicy serialSubscriptionPolicy; - @Valid - private BatchSubscriptionPolicy batchSubscriptionPolicy; - /** - * Use trackingMode field instead. - */ - @Deprecated - private boolean trackingEnabled = false; - private TrackingMode trackingMode = TrackingMode.TRACKING_OFF; - private boolean http2Enabled = false; - private boolean profilingEnabled = false; - private long profilingThresholdMs = 0; - @Valid - @NotNull - private OwnerId owner; - @NotNull - private DeliveryType deliveryType = DeliveryType.SERIAL; - @NotNull - private SubscriptionMode mode = SubscriptionMode.ANYCAST; - private List filters = new ArrayList<>(); - - private List
headers; - - private EndpointAddressResolverMetadata endpointAddressResolverMetadata; - - @Valid - private SubscriptionOAuthPolicy oAuthPolicy; - - private boolean subscriptionIdentityHeadersEnabled; - - private boolean autoDeleteWithTopicEnabled; - - private Instant createdAt; - - private Instant modifiedAt; - - private Subscription(TopicName topicName, - String name, - EndpointAddress endpoint, - State state, - String description, - Object subscriptionPolicy, - boolean trackingEnabled, - TrackingMode trackingMode, - OwnerId owner, - MonitoringDetails monitoringDetails, - ContentType contentType, - DeliveryType deliveryType, - List filters, - SubscriptionMode mode, - List
headers, - EndpointAddressResolverMetadata endpointAddressResolverMetadata, - SubscriptionOAuthPolicy oAuthPolicy, - boolean http2Enabled, - boolean profilingEnabled, - long profilingThresholdMs, - boolean subscriptionIdentityHeadersEnabled, - boolean autoDeleteWithTopicEnabled) { - this.topicName = topicName; - this.name = name; - this.endpoint = endpoint; - this.state = state != null ? state : State.PENDING; - this.description = description; - this.trackingEnabled = trackingEnabled; - this.trackingMode = trackingMode; - this.owner = owner; - this.monitoringDetails = monitoringDetails == null ? MonitoringDetails.EMPTY : monitoringDetails; - this.contentType = contentType == null ? ContentType.JSON : contentType; - this.deliveryType = deliveryType; - this.batchSubscriptionPolicy = this.deliveryType == DeliveryType.BATCH ? (BatchSubscriptionPolicy) subscriptionPolicy : null; - this.serialSubscriptionPolicy = this.deliveryType == DeliveryType.SERIAL ? (SubscriptionPolicy) subscriptionPolicy : null; - this.filters = filters; - this.mode = mode; - this.http2Enabled = http2Enabled; - this.profilingEnabled = profilingEnabled; - this.profilingThresholdMs = profilingThresholdMs; - this.subscriptionName = new SubscriptionName(name, topicName); - this.headers = headers; - this.endpointAddressResolverMetadata = endpointAddressResolverMetadata; - this.oAuthPolicy = oAuthPolicy; - this.subscriptionIdentityHeadersEnabled = subscriptionIdentityHeadersEnabled; - this.autoDeleteWithTopicEnabled = autoDeleteWithTopicEnabled; - } - - public static Subscription createSerialSubscription(TopicName topicName, - String name, - EndpointAddress endpoint, - State state, - String description, - SubscriptionPolicy subscriptionPolicy, - boolean trackingEnabled, - TrackingMode trackingMode, - OwnerId owner, - MonitoringDetails monitoringDetails, - ContentType contentType, - List filters, - SubscriptionMode mode, - List
headers, - EndpointAddressResolverMetadata endpointAddressResolverMetadata, - SubscriptionOAuthPolicy oAuthPolicy, - boolean http2Enabled, - boolean profilingEnabled, - long profilingThresholdMs, - boolean subscriptionIdentityHeadersEnabled, - boolean autoDeleteWithTopicEnabled) { - return new Subscription(topicName, name, endpoint, state, description, subscriptionPolicy, trackingEnabled, trackingMode, - owner, monitoringDetails, contentType, DeliveryType.SERIAL, filters, mode, headers, - endpointAddressResolverMetadata, oAuthPolicy, http2Enabled, profilingEnabled, profilingThresholdMs, subscriptionIdentityHeadersEnabled, autoDeleteWithTopicEnabled); - } - - public static Subscription createBatchSubscription(TopicName topicName, - String name, - EndpointAddress endpoint, - State state, - String description, - BatchSubscriptionPolicy subscriptionPolicy, - boolean trackingEnabled, - TrackingMode trackingMode, - OwnerId owner, - MonitoringDetails monitoringDetails, - ContentType contentType, - List filters, - List
headers, - EndpointAddressResolverMetadata endpointAddressResolverMetadata, - SubscriptionOAuthPolicy oAuthPolicy, - boolean http2Enabled, - boolean subscriptionIdentityHeadersEnabled, - boolean autoDeleteWithTopicEnabled) { - return new Subscription(topicName, name, endpoint, state, description, subscriptionPolicy, trackingEnabled, trackingMode, - owner, monitoringDetails, contentType, DeliveryType.BATCH, filters, SubscriptionMode.ANYCAST, headers, - endpointAddressResolverMetadata, oAuthPolicy, http2Enabled, false, 0, subscriptionIdentityHeadersEnabled, autoDeleteWithTopicEnabled); - } - - @JsonCreator - public static Subscription create( - @JsonProperty("topicName") String topicName, - @JsonProperty("name") String name, - @JsonProperty("endpoint") EndpointAddress endpoint, - @JsonProperty("state") State state, - @JsonProperty("description") String description, - @JsonProperty("subscriptionPolicy") Map subscriptionPolicy, - @JsonProperty("trackingEnabled") boolean trackingEnabled, - @JsonProperty("trackingMode") String trackingMode, - @JsonProperty("owner") OwnerId owner, - @JsonProperty("monitoringDetails") MonitoringDetails monitoringDetails, - @JsonProperty("contentType") ContentType contentType, - @JsonProperty("deliveryType") DeliveryType deliveryType, - @JsonProperty("filters") List filters, - @JsonProperty("mode") SubscriptionMode mode, - @JsonProperty("headers") List
headers, - @JsonProperty("endpointAddressResolverMetadata") EndpointAddressResolverMetadata endpointAddressResolverMetadata, - @JsonProperty("oAuthPolicy") SubscriptionOAuthPolicy oAuthPolicy, - @JsonProperty("http2Enabled") boolean http2Enabled, - @JsonProperty("profilingEnabled") boolean profilingEnabled, - @JsonProperty("profilingThresholdMs") long profilingThresholdMs, - @JsonProperty("subscriptionIdentityHeadersEnabled") boolean subscriptionIdentityHeadersEnabled, - @JsonProperty("autoDeleteWithTopicEnabled") boolean autoDeleteWithTopicEnabled) { - - DeliveryType validDeliveryType = deliveryType == null ? DeliveryType.SERIAL : deliveryType; - SubscriptionMode subscriptionMode = mode == null ? SubscriptionMode.ANYCAST : mode; - Map validSubscriptionPolicy = subscriptionPolicy == null ? new HashMap<>() : subscriptionPolicy; - - TrackingMode validTrackingMode = TrackingMode.fromString(trackingMode) - .orElse(trackingEnabled ? TrackingMode.TRACK_ALL : TrackingMode.TRACKING_OFF); - boolean validTrackingEnabled = validTrackingMode != TrackingMode.TRACKING_OFF; - - return new Subscription( - TopicName.fromQualifiedName(topicName), - name, - endpoint, - state, - description, - validDeliveryType == DeliveryType.SERIAL - ? SubscriptionPolicy.create(validSubscriptionPolicy) - : BatchSubscriptionPolicy.create(validSubscriptionPolicy), - validTrackingEnabled, - validTrackingMode, - owner, - monitoringDetails, - contentType, - validDeliveryType, - filters == null ? Collections.emptyList() : filters, - subscriptionMode, - headers == null ? Collections.emptyList() : headers, - endpointAddressResolverMetadata == null ? EndpointAddressResolverMetadata.empty() : endpointAddressResolverMetadata, - oAuthPolicy, - http2Enabled, - profilingEnabled, - profilingThresholdMs, - subscriptionIdentityHeadersEnabled, - autoDeleteWithTopicEnabled - ); - } - - @Override - public int hashCode() { - return Objects.hash(endpoint, topicName, name, description, serialSubscriptionPolicy, batchSubscriptionPolicy, - trackingEnabled, trackingMode, owner, monitoringDetails, contentType, filters, mode, headers, - endpointAddressResolverMetadata, oAuthPolicy, http2Enabled, subscriptionIdentityHeadersEnabled); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - final Subscription other = (Subscription) obj; - - return Objects.equals(this.endpoint, other.endpoint) - && Objects.equals(this.topicName, other.topicName) - && Objects.equals(this.name, other.name) - && Objects.equals(this.description, other.description) - && Objects.equals(this.serialSubscriptionPolicy, other.serialSubscriptionPolicy) - && Objects.equals(this.batchSubscriptionPolicy, other.batchSubscriptionPolicy) - && Objects.equals(this.trackingEnabled, other.trackingEnabled) - && Objects.equals(this.trackingMode, other.trackingMode) - && Objects.equals(this.owner, other.owner) - && Objects.equals(this.monitoringDetails, other.monitoringDetails) - && Objects.equals(this.contentType, other.contentType) - && Objects.equals(this.filters, other.filters) - && Objects.equals(this.mode, other.mode) - && Objects.equals(this.headers, other.headers) - && Objects.equals(this.endpointAddressResolverMetadata, other.endpointAddressResolverMetadata) - && Objects.equals(this.http2Enabled, other.http2Enabled) - && Objects.equals(this.profilingEnabled, other.profilingEnabled) - && Objects.equals(this.profilingThresholdMs, other.profilingThresholdMs) - && Objects.equals(this.oAuthPolicy, other.oAuthPolicy) - && Objects.equals(this.subscriptionIdentityHeadersEnabled, other.subscriptionIdentityHeadersEnabled) - && Objects.equals(this.autoDeleteWithTopicEnabled, other.autoDeleteWithTopicEnabled); - } - - @JsonIgnore - public SubscriptionName getQualifiedName() { - return subscriptionName; - } - - public EndpointAddress getEndpoint() { - return endpoint; - } - - @JsonProperty("topicName") - public String getQualifiedTopicName() { - return TopicName.toQualifiedName(topicName); - } - - @JsonIgnore - public TopicName getTopicName() { - return topicName; - } - - public String getName() { - return name; - } - - public String getDescription() { - return description; - } - - public State getState() { - return state; - } - - public void setState(State state) { - this.state = state; - } - - @JsonProperty("subscriptionPolicy") - public Object getSubscriptionPolicy() { - return isBatchSubscription() ? batchSubscriptionPolicy : serialSubscriptionPolicy; - } - - public boolean isTrackingEnabled() { - return trackingMode != TrackingMode.TRACKING_OFF; - } - - @JsonProperty("trackingMode") - public String getTrackingModeString() { - return trackingMode.getValue(); - } - - @JsonIgnore - public TrackingMode getTrackingMode() { - return trackingMode; - } - - public OwnerId getOwner() { - return owner; - } - - public MonitoringDetails getMonitoringDetails() { - return monitoringDetails; - } - - public ContentType getContentType() { - return contentType; - } - - public DeliveryType getDeliveryType() { - return deliveryType; - } - - public List getFilters() { - return Collections.unmodifiableList(filters); - } - - public List
getHeaders() { - return Collections.unmodifiableList(headers); - } - - public EndpointAddressResolverMetadata getEndpointAddressResolverMetadata() { - return endpointAddressResolverMetadata; - } - - @JsonIgnore - public boolean isBatchSubscription() { - return this.deliveryType == DeliveryType.BATCH; - } - - @JsonIgnore - public BatchSubscriptionPolicy getBatchSubscriptionPolicy() { - return batchSubscriptionPolicy; - } - - @JsonIgnore - public SubscriptionPolicy getSerialSubscriptionPolicy() { - return serialSubscriptionPolicy; - } - - public void setSerialSubscriptionPolicy(SubscriptionPolicy serialSubscriptionPolicy) { - this.serialSubscriptionPolicy = serialSubscriptionPolicy; - } - - @JsonIgnore - public boolean isActive() { - return state == State.ACTIVE || state == State.PENDING; - } - - public SubscriptionMode getMode() { - return mode; - } - - @JsonProperty("oAuthPolicy") - public SubscriptionOAuthPolicy getOAuthPolicy() { - return oAuthPolicy; - } - - @JsonIgnore - public boolean hasOAuthPolicy() { - return oAuthPolicy != null; - } - - @JsonIgnore - public boolean isSeverityNotImportant() { - return getMonitoringDetails().getSeverity() == MonitoringDetails.Severity.NON_IMPORTANT; - } - - public boolean isHttp2Enabled() { - return http2Enabled; - } - - public boolean isProfilingEnabled() { - return profilingEnabled; - } - - public long getProfilingThresholdMs() { - return profilingThresholdMs; - } - - public boolean isSubscriptionIdentityHeadersEnabled() { - return subscriptionIdentityHeadersEnabled; - } - - public Instant getCreatedAt() { - return createdAt; - } - - public void setCreatedAt(Long createdAt) { - this.createdAt = Instant.ofEpochMilli(createdAt); - } - - public Instant getModifiedAt() { - return modifiedAt; - } - - public void setModifiedAt(Long modifiedAt) { - this.modifiedAt = Instant.ofEpochMilli(modifiedAt); - } - - public boolean isAutoDeleteWithTopicEnabled() { - return autoDeleteWithTopicEnabled; - } - - @Override - public Subscription anonymize() { - if (getEndpoint() != null && getEndpoint().containsCredentials() || hasOAuthPolicy()) { - return new Subscription( - topicName, - name, - endpoint.anonymize(), - state, - description, - deliveryType == DeliveryType.BATCH ? batchSubscriptionPolicy : serialSubscriptionPolicy, - trackingEnabled, - trackingMode, - owner, - monitoringDetails, - contentType, - deliveryType, - filters, - mode, - headers, - endpointAddressResolverMetadata, - oAuthPolicy != null ? oAuthPolicy.anonymize() : null, - http2Enabled, - profilingEnabled, - profilingThresholdMs, - subscriptionIdentityHeadersEnabled, - autoDeleteWithTopicEnabled - ); - } - return this; - } - - @Override - public String toString() { - return "Subscription(" + getQualifiedName() + ")"; - } - - public enum State { - PENDING, ACTIVE, SUSPENDED - } + @NotNull private final MonitoringDetails monitoringDetails; + private final SubscriptionName subscriptionName; + @Valid @NotNull private TopicName topicName; + + @NotEmpty + @Pattern(regexp = ALLOWED_NAME_REGEX) + private String name; + + private State state = State.PENDING; + @NotNull @Valid private EndpointAddress endpoint; + @NotNull private ContentType contentType = ContentType.JSON; + @NotNull private String description; + @Valid private SubscriptionPolicy serialSubscriptionPolicy; + @Valid private BatchSubscriptionPolicy batchSubscriptionPolicy; + + /** Use trackingMode field instead. */ + @Deprecated private boolean trackingEnabled = false; + + private TrackingMode trackingMode = TrackingMode.TRACKING_OFF; + private boolean http2Enabled = false; + private boolean profilingEnabled = false; + private long profilingThresholdMs = 0; + @Valid @NotNull private OwnerId owner; + @NotNull private DeliveryType deliveryType = DeliveryType.SERIAL; + @NotNull private SubscriptionMode mode = SubscriptionMode.ANYCAST; + private List filters = new ArrayList<>(); + + private List
headers; + + private EndpointAddressResolverMetadata endpointAddressResolverMetadata; + + @Valid private SubscriptionOAuthPolicy oAuthPolicy; + + private boolean subscriptionIdentityHeadersEnabled; + + private boolean autoDeleteWithTopicEnabled; + + private Instant createdAt; + + private Instant modifiedAt; + + private Subscription( + TopicName topicName, + String name, + EndpointAddress endpoint, + State state, + String description, + Object subscriptionPolicy, + boolean trackingEnabled, + TrackingMode trackingMode, + OwnerId owner, + MonitoringDetails monitoringDetails, + ContentType contentType, + DeliveryType deliveryType, + List filters, + SubscriptionMode mode, + List
headers, + EndpointAddressResolverMetadata endpointAddressResolverMetadata, + SubscriptionOAuthPolicy oAuthPolicy, + boolean http2Enabled, + boolean profilingEnabled, + long profilingThresholdMs, + boolean subscriptionIdentityHeadersEnabled, + boolean autoDeleteWithTopicEnabled) { + this.topicName = topicName; + this.name = name; + this.endpoint = endpoint; + this.state = state != null ? state : State.PENDING; + this.description = description; + this.trackingEnabled = trackingEnabled; + this.trackingMode = trackingMode; + this.owner = owner; + this.monitoringDetails = + monitoringDetails == null ? MonitoringDetails.EMPTY : monitoringDetails; + this.contentType = contentType == null ? ContentType.JSON : contentType; + this.deliveryType = deliveryType; + this.batchSubscriptionPolicy = + this.deliveryType == DeliveryType.BATCH + ? (BatchSubscriptionPolicy) subscriptionPolicy + : null; + this.serialSubscriptionPolicy = + this.deliveryType == DeliveryType.SERIAL ? (SubscriptionPolicy) subscriptionPolicy : null; + this.filters = filters; + this.mode = mode; + this.http2Enabled = http2Enabled; + this.profilingEnabled = profilingEnabled; + this.profilingThresholdMs = profilingThresholdMs; + this.subscriptionName = new SubscriptionName(name, topicName); + this.headers = headers; + this.endpointAddressResolverMetadata = endpointAddressResolverMetadata; + this.oAuthPolicy = oAuthPolicy; + this.subscriptionIdentityHeadersEnabled = subscriptionIdentityHeadersEnabled; + this.autoDeleteWithTopicEnabled = autoDeleteWithTopicEnabled; + } + + public static Subscription createSerialSubscription( + TopicName topicName, + String name, + EndpointAddress endpoint, + State state, + String description, + SubscriptionPolicy subscriptionPolicy, + boolean trackingEnabled, + TrackingMode trackingMode, + OwnerId owner, + MonitoringDetails monitoringDetails, + ContentType contentType, + List filters, + SubscriptionMode mode, + List
headers, + EndpointAddressResolverMetadata endpointAddressResolverMetadata, + SubscriptionOAuthPolicy oAuthPolicy, + boolean http2Enabled, + boolean profilingEnabled, + long profilingThresholdMs, + boolean subscriptionIdentityHeadersEnabled, + boolean autoDeleteWithTopicEnabled) { + return new Subscription( + topicName, + name, + endpoint, + state, + description, + subscriptionPolicy, + trackingEnabled, + trackingMode, + owner, + monitoringDetails, + contentType, + DeliveryType.SERIAL, + filters, + mode, + headers, + endpointAddressResolverMetadata, + oAuthPolicy, + http2Enabled, + profilingEnabled, + profilingThresholdMs, + subscriptionIdentityHeadersEnabled, + autoDeleteWithTopicEnabled); + } + + public static Subscription createBatchSubscription( + TopicName topicName, + String name, + EndpointAddress endpoint, + State state, + String description, + BatchSubscriptionPolicy subscriptionPolicy, + boolean trackingEnabled, + TrackingMode trackingMode, + OwnerId owner, + MonitoringDetails monitoringDetails, + ContentType contentType, + List filters, + List
headers, + EndpointAddressResolverMetadata endpointAddressResolverMetadata, + SubscriptionOAuthPolicy oAuthPolicy, + boolean http2Enabled, + boolean subscriptionIdentityHeadersEnabled, + boolean autoDeleteWithTopicEnabled) { + return new Subscription( + topicName, + name, + endpoint, + state, + description, + subscriptionPolicy, + trackingEnabled, + trackingMode, + owner, + monitoringDetails, + contentType, + DeliveryType.BATCH, + filters, + SubscriptionMode.ANYCAST, + headers, + endpointAddressResolverMetadata, + oAuthPolicy, + http2Enabled, + false, + 0, + subscriptionIdentityHeadersEnabled, + autoDeleteWithTopicEnabled); + } + + @JsonCreator + public static Subscription create( + @JsonProperty("topicName") String topicName, + @JsonProperty("name") String name, + @JsonProperty("endpoint") EndpointAddress endpoint, + @JsonProperty("state") State state, + @JsonProperty("description") String description, + @JsonProperty("subscriptionPolicy") Map subscriptionPolicy, + @JsonProperty("trackingEnabled") boolean trackingEnabled, + @JsonProperty("trackingMode") String trackingMode, + @JsonProperty("owner") OwnerId owner, + @JsonProperty("monitoringDetails") MonitoringDetails monitoringDetails, + @JsonProperty("contentType") ContentType contentType, + @JsonProperty("deliveryType") DeliveryType deliveryType, + @JsonProperty("filters") List filters, + @JsonProperty("mode") SubscriptionMode mode, + @JsonProperty("headers") List
headers, + @JsonProperty("endpointAddressResolverMetadata") + EndpointAddressResolverMetadata endpointAddressResolverMetadata, + @JsonProperty("oAuthPolicy") SubscriptionOAuthPolicy oAuthPolicy, + @JsonProperty("http2Enabled") boolean http2Enabled, + @JsonProperty("profilingEnabled") boolean profilingEnabled, + @JsonProperty("profilingThresholdMs") long profilingThresholdMs, + @JsonProperty("subscriptionIdentityHeadersEnabled") + boolean subscriptionIdentityHeadersEnabled, + @JsonProperty("autoDeleteWithTopicEnabled") boolean autoDeleteWithTopicEnabled) { + + DeliveryType validDeliveryType = deliveryType == null ? DeliveryType.SERIAL : deliveryType; + SubscriptionMode subscriptionMode = mode == null ? SubscriptionMode.ANYCAST : mode; + Map validSubscriptionPolicy = + subscriptionPolicy == null ? new HashMap<>() : subscriptionPolicy; + + TrackingMode validTrackingMode = + TrackingMode.fromString(trackingMode) + .orElse(trackingEnabled ? TrackingMode.TRACK_ALL : TrackingMode.TRACKING_OFF); + boolean validTrackingEnabled = validTrackingMode != TrackingMode.TRACKING_OFF; + + return new Subscription( + TopicName.fromQualifiedName(topicName), + name, + endpoint, + state, + description, + validDeliveryType == DeliveryType.SERIAL + ? SubscriptionPolicy.create(validSubscriptionPolicy) + : BatchSubscriptionPolicy.create(validSubscriptionPolicy), + validTrackingEnabled, + validTrackingMode, + owner, + monitoringDetails, + contentType, + validDeliveryType, + filters == null ? Collections.emptyList() : filters, + subscriptionMode, + headers == null ? Collections.emptyList() : headers, + endpointAddressResolverMetadata == null + ? EndpointAddressResolverMetadata.empty() + : endpointAddressResolverMetadata, + oAuthPolicy, + http2Enabled, + profilingEnabled, + profilingThresholdMs, + subscriptionIdentityHeadersEnabled, + autoDeleteWithTopicEnabled); + } + + @Override + public int hashCode() { + return Objects.hash( + endpoint, + topicName, + name, + description, + serialSubscriptionPolicy, + batchSubscriptionPolicy, + trackingEnabled, + trackingMode, + owner, + monitoringDetails, + contentType, + filters, + mode, + headers, + endpointAddressResolverMetadata, + oAuthPolicy, + http2Enabled, + subscriptionIdentityHeadersEnabled); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final Subscription other = (Subscription) obj; + + return Objects.equals(this.endpoint, other.endpoint) + && Objects.equals(this.topicName, other.topicName) + && Objects.equals(this.name, other.name) + && Objects.equals(this.description, other.description) + && Objects.equals(this.serialSubscriptionPolicy, other.serialSubscriptionPolicy) + && Objects.equals(this.batchSubscriptionPolicy, other.batchSubscriptionPolicy) + && Objects.equals(this.trackingEnabled, other.trackingEnabled) + && Objects.equals(this.trackingMode, other.trackingMode) + && Objects.equals(this.owner, other.owner) + && Objects.equals(this.monitoringDetails, other.monitoringDetails) + && Objects.equals(this.contentType, other.contentType) + && Objects.equals(this.filters, other.filters) + && Objects.equals(this.mode, other.mode) + && Objects.equals(this.headers, other.headers) + && Objects.equals( + this.endpointAddressResolverMetadata, other.endpointAddressResolverMetadata) + && Objects.equals(this.http2Enabled, other.http2Enabled) + && Objects.equals(this.profilingEnabled, other.profilingEnabled) + && Objects.equals(this.profilingThresholdMs, other.profilingThresholdMs) + && Objects.equals(this.oAuthPolicy, other.oAuthPolicy) + && Objects.equals( + this.subscriptionIdentityHeadersEnabled, other.subscriptionIdentityHeadersEnabled) + && Objects.equals(this.autoDeleteWithTopicEnabled, other.autoDeleteWithTopicEnabled); + } + + @JsonIgnore + public SubscriptionName getQualifiedName() { + return subscriptionName; + } + + public EndpointAddress getEndpoint() { + return endpoint; + } + + @JsonProperty("topicName") + public String getQualifiedTopicName() { + return TopicName.toQualifiedName(topicName); + } + + @JsonIgnore + public TopicName getTopicName() { + return topicName; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public State getState() { + return state; + } + + public void setState(State state) { + this.state = state; + } + + @JsonProperty("subscriptionPolicy") + public Object getSubscriptionPolicy() { + return isBatchSubscription() ? batchSubscriptionPolicy : serialSubscriptionPolicy; + } + + public boolean isTrackingEnabled() { + return trackingMode != TrackingMode.TRACKING_OFF; + } + + @JsonProperty("trackingMode") + public String getTrackingModeString() { + return trackingMode.getValue(); + } + + @JsonIgnore + public TrackingMode getTrackingMode() { + return trackingMode; + } + + public OwnerId getOwner() { + return owner; + } + + public MonitoringDetails getMonitoringDetails() { + return monitoringDetails; + } + + public ContentType getContentType() { + return contentType; + } + + public DeliveryType getDeliveryType() { + return deliveryType; + } + + public List getFilters() { + return Collections.unmodifiableList(filters); + } + + public List
getHeaders() { + return Collections.unmodifiableList(headers); + } + + public EndpointAddressResolverMetadata getEndpointAddressResolverMetadata() { + return endpointAddressResolverMetadata; + } + + @JsonIgnore + public boolean isBatchSubscription() { + return this.deliveryType == DeliveryType.BATCH; + } + + @JsonIgnore + public BatchSubscriptionPolicy getBatchSubscriptionPolicy() { + return batchSubscriptionPolicy; + } + + @JsonIgnore + public SubscriptionPolicy getSerialSubscriptionPolicy() { + return serialSubscriptionPolicy; + } + + public void setSerialSubscriptionPolicy(SubscriptionPolicy serialSubscriptionPolicy) { + this.serialSubscriptionPolicy = serialSubscriptionPolicy; + } + + @JsonIgnore + public boolean isActive() { + return state == State.ACTIVE || state == State.PENDING; + } + + public SubscriptionMode getMode() { + return mode; + } + + @JsonProperty("oAuthPolicy") + public SubscriptionOAuthPolicy getOAuthPolicy() { + return oAuthPolicy; + } + + @JsonIgnore + public boolean hasOAuthPolicy() { + return oAuthPolicy != null; + } + + @JsonIgnore + public boolean isSeverityNotImportant() { + return getMonitoringDetails().getSeverity() == MonitoringDetails.Severity.NON_IMPORTANT; + } + + public boolean isHttp2Enabled() { + return http2Enabled; + } + + public boolean isProfilingEnabled() { + return profilingEnabled; + } + + public long getProfilingThresholdMs() { + return profilingThresholdMs; + } + + public boolean isSubscriptionIdentityHeadersEnabled() { + return subscriptionIdentityHeadersEnabled; + } + + public Instant getCreatedAt() { + return createdAt; + } + + public void setCreatedAt(Long createdAt) { + this.createdAt = Instant.ofEpochMilli(createdAt); + } + + public Instant getModifiedAt() { + return modifiedAt; + } + + public void setModifiedAt(Long modifiedAt) { + this.modifiedAt = Instant.ofEpochMilli(modifiedAt); + } + + public boolean isAutoDeleteWithTopicEnabled() { + return autoDeleteWithTopicEnabled; + } + + @Override + public Subscription anonymize() { + if (getEndpoint() != null && getEndpoint().containsCredentials() || hasOAuthPolicy()) { + return new Subscription( + topicName, + name, + endpoint.anonymize(), + state, + description, + deliveryType == DeliveryType.BATCH ? batchSubscriptionPolicy : serialSubscriptionPolicy, + trackingEnabled, + trackingMode, + owner, + monitoringDetails, + contentType, + deliveryType, + filters, + mode, + headers, + endpointAddressResolverMetadata, + oAuthPolicy != null ? oAuthPolicy.anonymize() : null, + http2Enabled, + profilingEnabled, + profilingThresholdMs, + subscriptionIdentityHeadersEnabled, + autoDeleteWithTopicEnabled); + } + return this; + } + + @Override + public String toString() { + return "Subscription(" + getQualifiedName() + ")"; + } + + public enum State { + PENDING, + ACTIVE, + SUSPENDED + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionConstraints.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionConstraints.java index 3ba492ec74..09254c5c8c 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionConstraints.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionConstraints.java @@ -6,22 +6,22 @@ public class SubscriptionConstraints { - private final SubscriptionName subscriptionName; - @Valid - private final Constraints constraints; + private final SubscriptionName subscriptionName; + @Valid private final Constraints constraints; - @JsonCreator - public SubscriptionConstraints(@JsonProperty("subscriptionName") String subscriptionName, - @JsonProperty("constraints") Constraints constraints) { - this.subscriptionName = SubscriptionName.fromString(subscriptionName); - this.constraints = constraints; - } + @JsonCreator + public SubscriptionConstraints( + @JsonProperty("subscriptionName") String subscriptionName, + @JsonProperty("constraints") Constraints constraints) { + this.subscriptionName = SubscriptionName.fromString(subscriptionName); + this.constraints = constraints; + } - public SubscriptionName getSubscriptionName() { - return subscriptionName; - } + public SubscriptionName getSubscriptionName() { + return subscriptionName; + } - public Constraints getConstraints() { - return constraints; - } + public Constraints getConstraints() { + return constraints; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionHealth.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionHealth.java index 033d87ba0c..fca6976c83 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionHealth.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionHealth.java @@ -1,73 +1,73 @@ package pl.allegro.tech.hermes.api; +import static com.google.common.base.Preconditions.checkNotNull; +import static java.util.Collections.emptySet; + import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableSet; - import java.util.Objects; import java.util.Set; -import static com.google.common.base.Preconditions.checkNotNull; -import static java.util.Collections.emptySet; - public final class SubscriptionHealth { - public static final SubscriptionHealth HEALTHY = new SubscriptionHealth(Status.HEALTHY, emptySet()); - public static final SubscriptionHealth NO_DATA = new SubscriptionHealth(Status.NO_DATA, emptySet()); + public static final SubscriptionHealth HEALTHY = + new SubscriptionHealth(Status.HEALTHY, emptySet()); + public static final SubscriptionHealth NO_DATA = + new SubscriptionHealth(Status.NO_DATA, emptySet()); - private final Status status; - private final ImmutableSet problems; + private final Status status; + private final ImmutableSet problems; - @JsonCreator - private SubscriptionHealth(@JsonProperty("status") Status status, - @JsonProperty("problems") Set problems) { - this.status = status; - this.problems = ImmutableSet.copyOf(problems); - } + @JsonCreator + private SubscriptionHealth( + @JsonProperty("status") Status status, + @JsonProperty("problems") Set problems) { + this.status = status; + this.problems = ImmutableSet.copyOf(problems); + } - public Status getStatus() { - return status; - } + public Status getStatus() { + return status; + } - public Set getProblems() { - return problems; - } + public Set getProblems() { + return problems; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof SubscriptionHealth)) { - return false; - } - SubscriptionHealth that = (SubscriptionHealth) o; - return status == that.status - && Objects.equals(problems, that.problems); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(status, problems); + if (!(o instanceof SubscriptionHealth)) { + return false; } + SubscriptionHealth that = (SubscriptionHealth) o; + return status == that.status && Objects.equals(problems, that.problems); + } - @Override - public String toString() { - return "SubscriptionHealth{" - + "status=" + status - + ", problems=" + problems - + '}'; - } + @Override + public int hashCode() { + return Objects.hash(status, problems); + } - public static SubscriptionHealth of(Set problems) { - checkNotNull(problems, "Set of health problems cannot be null"); - if (problems.isEmpty()) { - return HEALTHY; - } else { - return new SubscriptionHealth(Status.UNHEALTHY, problems); - } - } + @Override + public String toString() { + return "SubscriptionHealth{" + "status=" + status + ", problems=" + problems + '}'; + } - public enum Status { - HEALTHY, UNHEALTHY, NO_DATA + public static SubscriptionHealth of(Set problems) { + checkNotNull(problems, "Set of health problems cannot be null"); + if (problems.isEmpty()) { + return HEALTHY; + } else { + return new SubscriptionHealth(Status.UNHEALTHY, problems); } + } + + public enum Status { + HEALTHY, + UNHEALTHY, + NO_DATA + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionHealthProblem.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionHealthProblem.java index 2740a909f5..2f928427af 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionHealthProblem.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionHealthProblem.java @@ -1,10 +1,5 @@ package pl.allegro.tech.hermes.api; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.Objects; - import static java.lang.String.format; import static pl.allegro.tech.hermes.api.SubscriptionHealthProblem.ProblemCode.LAGGING; import static pl.allegro.tech.hermes.api.SubscriptionHealthProblem.ProblemCode.MALFUNCTIONING; @@ -12,93 +7,101 @@ import static pl.allegro.tech.hermes.api.SubscriptionHealthProblem.ProblemCode.TIMING_OUT; import static pl.allegro.tech.hermes.api.SubscriptionHealthProblem.ProblemCode.UNREACHABLE; -public class SubscriptionHealthProblem { - - public enum ProblemCode { - LAGGING, UNREACHABLE, TIMING_OUT, MALFUNCTIONING, RECEIVING_MALFORMED_MESSAGES - } - - private final ProblemCode code; - private final String description; - - @JsonCreator - private SubscriptionHealthProblem(@JsonProperty("code") ProblemCode code, - @JsonProperty("description") String description) { - this.code = code; - this.description = description; - } - - public static SubscriptionHealthProblem lagging(long subscriptionLag, String subscriptionName) { - return new SubscriptionHealthProblem( - LAGGING, - format("Lag is growing on subscription %s, current value is %d messages", subscriptionName, subscriptionLag) - ); - } - - public static SubscriptionHealthProblem malfunctioning(double code5xxErrorsRate, String subscriptionName) { - return new SubscriptionHealthProblem( - MALFUNCTIONING, - format("Consuming service returns a lot of 5xx codes for subscription %s, currently %.0f 5xx/s", - subscriptionName, - code5xxErrorsRate) - ); - } - - public static SubscriptionHealthProblem receivingMalformedMessages(double code4xxErrorsRate, String subscriptionName) { - return new SubscriptionHealthProblem( - RECEIVING_MALFORMED_MESSAGES, - format("Consuming service returns a lot of 4xx codes for subscription %s, currently %.0f 4xx/s", - subscriptionName, - code4xxErrorsRate) - ); - } - - public static SubscriptionHealthProblem timingOut(double timeoutsRate, String subscriptionName) { - return new SubscriptionHealthProblem( - TIMING_OUT, - format("Consuming service times out a lot for subscription %s, currently %.0f timeouts/s", - subscriptionName, - timeoutsRate) - ); - } - - public static SubscriptionHealthProblem unreachable(double otherErrorsRate, String subscriptionName) { - return new SubscriptionHealthProblem( - UNREACHABLE, - format("Unable to connect to consuming service instances for subscription %s, current rate is %.0f failures/s", - subscriptionName, - otherErrorsRate) - ); - } - - public ProblemCode getCode() { - return code; - } - - public String getDescription() { - return description; - } +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Objects; - @Override - public String toString() { - return code.name(); - } +public class SubscriptionHealthProblem { - @Override - public int hashCode() { - return Objects.hash(code, description); + public enum ProblemCode { + LAGGING, + UNREACHABLE, + TIMING_OUT, + MALFUNCTIONING, + RECEIVING_MALFORMED_MESSAGES + } + + private final ProblemCode code; + private final String description; + + @JsonCreator + private SubscriptionHealthProblem( + @JsonProperty("code") ProblemCode code, @JsonProperty("description") String description) { + this.code = code; + this.description = description; + } + + public static SubscriptionHealthProblem lagging(long subscriptionLag, String subscriptionName) { + return new SubscriptionHealthProblem( + LAGGING, + format( + "Lag is growing on subscription %s, current value is %d messages", + subscriptionName, subscriptionLag)); + } + + public static SubscriptionHealthProblem malfunctioning( + double code5xxErrorsRate, String subscriptionName) { + return new SubscriptionHealthProblem( + MALFUNCTIONING, + format( + "Consuming service returns a lot of 5xx codes for subscription %s, currently %.0f 5xx/s", + subscriptionName, code5xxErrorsRate)); + } + + public static SubscriptionHealthProblem receivingMalformedMessages( + double code4xxErrorsRate, String subscriptionName) { + return new SubscriptionHealthProblem( + RECEIVING_MALFORMED_MESSAGES, + format( + "Consuming service returns a lot of 4xx codes for subscription %s, currently %.0f 4xx/s", + subscriptionName, code4xxErrorsRate)); + } + + public static SubscriptionHealthProblem timingOut(double timeoutsRate, String subscriptionName) { + return new SubscriptionHealthProblem( + TIMING_OUT, + format( + "Consuming service times out a lot for subscription %s, currently %.0f timeouts/s", + subscriptionName, timeoutsRate)); + } + + public static SubscriptionHealthProblem unreachable( + double otherErrorsRate, String subscriptionName) { + return new SubscriptionHealthProblem( + UNREACHABLE, + format( + "Unable to connect to consuming service instances for subscription %s, current rate is %.0f failures/s", + subscriptionName, otherErrorsRate)); + } + + public ProblemCode getCode() { + return code; + } + + public String getDescription() { + return description; + } + + @Override + public String toString() { + return code.name(); + } + + @Override + public int hashCode() { + return Objects.hash(code, description); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - SubscriptionHealthProblem other = (SubscriptionHealthProblem) obj; - return Objects.equals(this.code, other.code) - && Objects.equals(this.description, other.description); + if (obj == null || getClass() != obj.getClass()) { + return false; } + SubscriptionHealthProblem other = (SubscriptionHealthProblem) obj; + return Objects.equals(this.code, other.code) + && Objects.equals(this.description, other.description); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionMetrics.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionMetrics.java index 44a6e4cb05..d621922e86 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionMetrics.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionMetrics.java @@ -4,192 +4,192 @@ import com.fasterxml.jackson.annotation.JsonProperty; public class SubscriptionMetrics { - private long delivered; - private long discarded; - private long volume; - private MetricDecimalValue timeouts; - private MetricDecimalValue otherErrors; - private MetricDecimalValue codes2xx; - private MetricDecimalValue codes4xx; - private MetricDecimalValue codes5xx; - private MetricDecimalValue retries; - private MetricLongValue lag; - private Subscription.State state; - private MetricDecimalValue rate; - private MetricDecimalValue throughput; - private MetricDecimalValue batchRate; + private long delivered; + private long discarded; + private long volume; + private MetricDecimalValue timeouts; + private MetricDecimalValue otherErrors; + private MetricDecimalValue codes2xx; + private MetricDecimalValue codes4xx; + private MetricDecimalValue codes5xx; + private MetricDecimalValue retries; + private MetricLongValue lag; + private Subscription.State state; + private MetricDecimalValue rate; + private MetricDecimalValue throughput; + private MetricDecimalValue batchRate; + + private SubscriptionMetrics() {} + + @JsonCreator + public SubscriptionMetrics( + @JsonProperty("delivered") long delivered, + @JsonProperty("discarded") long discarded, + @JsonProperty("volume") long volume, + @JsonProperty("timeouts") MetricDecimalValue timeouts, + @JsonProperty("otherErrors") MetricDecimalValue otherErrors, + @JsonProperty("codes2xx") MetricDecimalValue codes2xx, + @JsonProperty("codes4xx") MetricDecimalValue codes4xx, + @JsonProperty("codes5xx") MetricDecimalValue codes5xx, + @JsonProperty("retries") MetricDecimalValue retries, + @JsonProperty("Subscription") Subscription.State state, + @JsonProperty("rate") MetricDecimalValue rate, + @JsonProperty("throughput") MetricDecimalValue throughput, + @JsonProperty("batchRate") MetricDecimalValue batchRate) { + this.delivered = delivered; + this.discarded = discarded; + this.volume = volume; + this.timeouts = timeouts; + this.otherErrors = otherErrors; + this.codes2xx = codes2xx; + this.codes4xx = codes4xx; + this.codes5xx = codes5xx; + this.retries = retries; + this.state = state; + this.rate = rate; + this.throughput = throughput; + this.batchRate = batchRate; + } + + public long getDelivered() { + return delivered; + } + + public long getDiscarded() { + return discarded; + } - private SubscriptionMetrics() { - } + public MetricDecimalValue getTimeouts() { + return timeouts; + } - @JsonCreator - public SubscriptionMetrics(@JsonProperty("delivered") long delivered, - @JsonProperty("discarded") long discarded, - @JsonProperty("volume") long volume, - @JsonProperty("timeouts") MetricDecimalValue timeouts, - @JsonProperty("otherErrors") MetricDecimalValue otherErrors, - @JsonProperty("codes2xx") MetricDecimalValue codes2xx, - @JsonProperty("codes4xx") MetricDecimalValue codes4xx, - @JsonProperty("codes5xx") MetricDecimalValue codes5xx, - @JsonProperty("retries") MetricDecimalValue retries, - @JsonProperty("Subscription") Subscription.State state, - @JsonProperty("rate") MetricDecimalValue rate, - @JsonProperty("throughput") MetricDecimalValue throughput, - @JsonProperty("batchRate") MetricDecimalValue batchRate) { - this.delivered = delivered; - this.discarded = discarded; - this.volume = volume; - this.timeouts = timeouts; - this.otherErrors = otherErrors; - this.codes2xx = codes2xx; - this.codes4xx = codes4xx; - this.codes5xx = codes5xx; - this.retries = retries; - this.state = state; - this.rate = rate; - this.throughput = throughput; - this.batchRate = batchRate; - } + public MetricLongValue getLag() { + return lag; + } - public long getDelivered() { - return delivered; - } + public MetricDecimalValue getRate() { + return rate; + } - public long getDiscarded() { - return discarded; - } + public MetricDecimalValue getOtherErrors() { + return otherErrors; + } - public MetricDecimalValue getTimeouts() { - return timeouts; - } + public MetricDecimalValue getCodes2xx() { + return codes2xx; + } - public MetricLongValue getLag() { - return lag; - } + public MetricDecimalValue getCodes4xx() { + return codes4xx; + } - public MetricDecimalValue getRate() { - return rate; - } - - public MetricDecimalValue getOtherErrors() { - return otherErrors; - } + public MetricDecimalValue getCodes5xx() { + return codes5xx; + } - public MetricDecimalValue getCodes2xx() { - return codes2xx; - } + public MetricDecimalValue getRetries() { + return retries; + } - public MetricDecimalValue getCodes4xx() { - return codes4xx; - } + public Subscription.State getState() { + return state; + } - public MetricDecimalValue getCodes5xx() { - return codes5xx; - } + public MetricDecimalValue getThroughput() { + return throughput; + } - public MetricDecimalValue getRetries() { - return retries; - } + public MetricDecimalValue getBatchRate() { + return batchRate; + } - public Subscription.State getState() { - return state; - } + public long getVolume() { + return volume; + } - public MetricDecimalValue getThroughput() { - return throughput; - } + public static class Builder { + private SubscriptionMetrics subscriptionMetrics; - public MetricDecimalValue getBatchRate() { - return batchRate; + public Builder() { + subscriptionMetrics = new SubscriptionMetrics(); } - public long getVolume() { - return volume; + public Builder withDelivered(long delivered) { + subscriptionMetrics.delivered = delivered; + return this; } - public static class Builder { - private SubscriptionMetrics subscriptionMetrics; - - public Builder() { - subscriptionMetrics = new SubscriptionMetrics(); - } - - public Builder withDelivered(long delivered) { - subscriptionMetrics.delivered = delivered; - return this; - } - - public Builder withDiscarded(long discarded) { - subscriptionMetrics.discarded = discarded; - return this; - } + public Builder withDiscarded(long discarded) { + subscriptionMetrics.discarded = discarded; + return this; + } - public Builder withVolume(long volume) { - subscriptionMetrics.volume = volume; - return this; - } + public Builder withVolume(long volume) { + subscriptionMetrics.volume = volume; + return this; + } - public Builder withOtherErrors(MetricDecimalValue otherErrors) { - subscriptionMetrics.otherErrors = otherErrors; - return this; - } + public Builder withOtherErrors(MetricDecimalValue otherErrors) { + subscriptionMetrics.otherErrors = otherErrors; + return this; + } - public Builder withTimeouts(MetricDecimalValue timeouts) { - subscriptionMetrics.timeouts = timeouts; - return this; - } + public Builder withTimeouts(MetricDecimalValue timeouts) { + subscriptionMetrics.timeouts = timeouts; + return this; + } - public Builder withCodes2xx(MetricDecimalValue count) { - subscriptionMetrics.codes2xx = count; - return this; - } + public Builder withCodes2xx(MetricDecimalValue count) { + subscriptionMetrics.codes2xx = count; + return this; + } - public Builder withCodes4xx(MetricDecimalValue count) { - subscriptionMetrics.codes4xx = count; - return this; - } + public Builder withCodes4xx(MetricDecimalValue count) { + subscriptionMetrics.codes4xx = count; + return this; + } - public Builder withCodes5xx(MetricDecimalValue count) { - subscriptionMetrics.codes5xx = count; - return this; - } + public Builder withCodes5xx(MetricDecimalValue count) { + subscriptionMetrics.codes5xx = count; + return this; + } - public Builder withRetries(MetricDecimalValue retries) { - subscriptionMetrics.retries = retries; - return this; - } + public Builder withRetries(MetricDecimalValue retries) { + subscriptionMetrics.retries = retries; + return this; + } - public Builder withRate(MetricDecimalValue rate) { - subscriptionMetrics.rate = rate; - return this; - } + public Builder withRate(MetricDecimalValue rate) { + subscriptionMetrics.rate = rate; + return this; + } - public Builder withState(Subscription.State state) { - subscriptionMetrics.state = state; - return this; - } + public Builder withState(Subscription.State state) { + subscriptionMetrics.state = state; + return this; + } - public Builder withLag(MetricLongValue lag) { - subscriptionMetrics.lag = lag; - return this; - } + public Builder withLag(MetricLongValue lag) { + subscriptionMetrics.lag = lag; + return this; + } - public Builder withThroughput(MetricDecimalValue throughput) { - subscriptionMetrics.throughput = throughput; - return this; - } + public Builder withThroughput(MetricDecimalValue throughput) { + subscriptionMetrics.throughput = throughput; + return this; + } - public Builder withBatchRate(MetricDecimalValue batchRate) { - subscriptionMetrics.batchRate = batchRate; - return this; - } + public Builder withBatchRate(MetricDecimalValue batchRate) { + subscriptionMetrics.batchRate = batchRate; + return this; + } - public static Builder subscriptionMetrics() { - return new Builder(); - } + public static Builder subscriptionMetrics() { + return new Builder(); + } - public SubscriptionMetrics build() { - return subscriptionMetrics; - } + public SubscriptionMetrics build() { + return subscriptionMetrics; } + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionMode.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionMode.java index 4d9332d508..743d51ee95 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionMode.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionMode.java @@ -1,5 +1,6 @@ package pl.allegro.tech.hermes.api; public enum SubscriptionMode { - ANYCAST, BROADCAST + ANYCAST, + BROADCAST } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionName.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionName.java index 5ff2d69ef3..58a5685bce 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionName.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionName.java @@ -1,63 +1,65 @@ package pl.allegro.tech.hermes.api; +import static com.google.common.base.Preconditions.checkArgument; + import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.Objects; -import static com.google.common.base.Preconditions.checkArgument; - public class SubscriptionName { - private final String name; - private final TopicName topicName; + private final String name; + private final TopicName topicName; - @JsonCreator - public SubscriptionName(@JsonProperty("name") String name, @JsonProperty("topicName") TopicName topicName) { - this.name = name; - this.topicName = topicName; - } + @JsonCreator + public SubscriptionName( + @JsonProperty("name") String name, @JsonProperty("topicName") TopicName topicName) { + this.name = name; + this.topicName = topicName; + } - @JsonIgnore - public String getQualifiedName() { - return toString(); - } + @JsonIgnore + public String getQualifiedName() { + return toString(); + } - public String getName() { - return name; - } + public String getName() { + return name; + } - public TopicName getTopicName() { - return topicName; - } + public TopicName getTopicName() { + return topicName; + } - public static SubscriptionName fromString(String string) { - String[] tokens = string.split("\\$"); - checkArgument(tokens.length > 1, "Incorrect string format. Expected 'topic$subscription'. Found:'%s'", string); - return new SubscriptionName(tokens[1], TopicName.fromQualifiedName(tokens[0])); - } + public static SubscriptionName fromString(String string) { + String[] tokens = string.split("\\$"); + checkArgument( + tokens.length > 1, + "Incorrect string format. Expected 'topic$subscription'. Found:'%s'", + string); + return new SubscriptionName(tokens[1], TopicName.fromQualifiedName(tokens[0])); + } - @Override - public String toString() { - return topicName.qualifiedName() + "$" + name; - } + @Override + public String toString() { + return topicName.qualifiedName() + "$" + name; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - SubscriptionName that = (SubscriptionName) o; - return Objects.equals(name, that.name) - && Objects.equals(topicName, that.topicName); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(name, topicName); + if (o == null || getClass() != o.getClass()) { + return false; } + SubscriptionName that = (SubscriptionName) o; + return Objects.equals(name, that.name) && Objects.equals(topicName, that.topicName); + } + + @Override + public int hashCode() { + return Objects.hash(name, topicName); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionNameWithMetrics.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionNameWithMetrics.java index e06da8b8ab..a03d8f282a 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionNameWithMetrics.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionNameWithMetrics.java @@ -2,110 +2,124 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.Objects; public class SubscriptionNameWithMetrics { - private final String topicName; - private final String name; - private final long delivered; - private final long discarded; - private final long volume; - private final MetricDecimalValue timeouts; - private final MetricLongValue lag; - private final MetricDecimalValue rate; - private final MetricDecimalValue throughput; - - @JsonCreator - public SubscriptionNameWithMetrics( - @JsonProperty("topicName") String topicName, - @JsonProperty("name") String name, - @JsonProperty("delivered") long delivered, - @JsonProperty("discarded") long discarded, - @JsonProperty("volume") long volume, - @JsonProperty("timeouts") MetricDecimalValue timeouts, - @JsonProperty("lag") MetricLongValue lag, - @JsonProperty("rate") MetricDecimalValue rate, - @JsonProperty("throughput") MetricDecimalValue throughput - ) { - this.topicName = topicName; - this.name = name; - this.delivered = delivered; - this.discarded = discarded; - this.volume = volume; - this.timeouts = timeouts; - this.lag = lag; - this.rate = rate; - this.throughput = throughput; - } - - public static SubscriptionNameWithMetrics from(SubscriptionMetrics metrics, String name, String topicQualifiedName) { - return new SubscriptionNameWithMetrics(topicQualifiedName, name, metrics.getDelivered(), - metrics.getDiscarded(), metrics.getVolume(), metrics.getTimeouts(), metrics.getLag(), - metrics.getRate(), metrics.getThroughput()); - } - - public String getTopicName() { - return topicName; - } - - public String getName() { - return name; - } - - public long getDelivered() { - return delivered; + private final String topicName; + private final String name; + private final long delivered; + private final long discarded; + private final long volume; + private final MetricDecimalValue timeouts; + private final MetricLongValue lag; + private final MetricDecimalValue rate; + private final MetricDecimalValue throughput; + + @JsonCreator + public SubscriptionNameWithMetrics( + @JsonProperty("topicName") String topicName, + @JsonProperty("name") String name, + @JsonProperty("delivered") long delivered, + @JsonProperty("discarded") long discarded, + @JsonProperty("volume") long volume, + @JsonProperty("timeouts") MetricDecimalValue timeouts, + @JsonProperty("lag") MetricLongValue lag, + @JsonProperty("rate") MetricDecimalValue rate, + @JsonProperty("throughput") MetricDecimalValue throughput) { + this.topicName = topicName; + this.name = name; + this.delivered = delivered; + this.discarded = discarded; + this.volume = volume; + this.timeouts = timeouts; + this.lag = lag; + this.rate = rate; + this.throughput = throughput; + } + + public static SubscriptionNameWithMetrics from( + SubscriptionMetrics metrics, String name, String topicQualifiedName) { + return new SubscriptionNameWithMetrics( + topicQualifiedName, + name, + metrics.getDelivered(), + metrics.getDiscarded(), + metrics.getVolume(), + metrics.getTimeouts(), + metrics.getLag(), + metrics.getRate(), + metrics.getThroughput()); + } + + public String getTopicName() { + return topicName; + } + + public String getName() { + return name; + } + + public long getDelivered() { + return delivered; + } + + public long getDiscarded() { + return discarded; + } + + public MetricDecimalValue getTimeouts() { + return timeouts; + } + + public MetricLongValue getLag() { + return lag; + } + + public MetricDecimalValue getRate() { + return rate; + } + + public MetricDecimalValue getThroughput() { + return throughput; + } + + public long getVolume() { + return volume; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - public long getDiscarded() { - return discarded; - } - - public MetricDecimalValue getTimeouts() { - return timeouts; + if (o == null || getClass() != o.getClass()) { + return false; } - public MetricLongValue getLag() { - return lag; - } - - public MetricDecimalValue getRate() { - return rate; - } - - public MetricDecimalValue getThroughput() { - return throughput; - } - - public long getVolume() { - return volume; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - SubscriptionNameWithMetrics that = (SubscriptionNameWithMetrics) o; - - return Objects.equals(this.topicName, that.topicName) - && Objects.equals(this.name, that.name) - && Objects.equals(this.delivered, that.delivered) - && Objects.equals(this.discarded, that.discarded) - && Objects.equals(this.timeouts, that.timeouts) - && Objects.equals(this.lag, that.lag) - && Objects.equals(this.rate, that.rate) - && Objects.equals(this.throughput, that.throughput) - && Objects.equals(this.volume, that.volume); - } - - @Override - public int hashCode() { - return Objects.hash(this.topicName, this.name, this.delivered, this.discarded, - this.timeouts, this.lag, this.rate, this.throughput, this.volume); - } + SubscriptionNameWithMetrics that = (SubscriptionNameWithMetrics) o; + + return Objects.equals(this.topicName, that.topicName) + && Objects.equals(this.name, that.name) + && Objects.equals(this.delivered, that.delivered) + && Objects.equals(this.discarded, that.discarded) + && Objects.equals(this.timeouts, that.timeouts) + && Objects.equals(this.lag, that.lag) + && Objects.equals(this.rate, that.rate) + && Objects.equals(this.throughput, that.throughput) + && Objects.equals(this.volume, that.volume); + } + + @Override + public int hashCode() { + return Objects.hash( + this.topicName, + this.name, + this.delivered, + this.discarded, + this.timeouts, + this.lag, + this.rate, + this.throughput, + this.volume); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionOAuthPolicy.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionOAuthPolicy.java index 769f0d0ae0..a62945e9ae 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionOAuthPolicy.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionOAuthPolicy.java @@ -4,136 +4,135 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonValue; import jakarta.validation.constraints.NotNull; - import java.util.Objects; public class SubscriptionOAuthPolicy { - private static final String ANONYMIZED_PASSWORD = "******"; - @NotNull - private final GrantType grantType; - @NotNull - private final String providerName; - private final String scope; - private final String username; - private final String password; - - @JsonCreator - public SubscriptionOAuthPolicy( - @JsonProperty("grantType") GrantType grantType, - @JsonProperty("providerName") String providerName, - @JsonProperty("scope") String scope, - @JsonProperty("username") String username, - @JsonProperty("password") String password) { - - this.grantType = grantType; - this.providerName = providerName; - this.scope = scope; - this.username = username; - this.password = password; + private static final String ANONYMIZED_PASSWORD = "******"; + @NotNull private final GrantType grantType; + @NotNull private final String providerName; + private final String scope; + private final String username; + private final String password; + + @JsonCreator + public SubscriptionOAuthPolicy( + @JsonProperty("grantType") GrantType grantType, + @JsonProperty("providerName") String providerName, + @JsonProperty("scope") String scope, + @JsonProperty("username") String username, + @JsonProperty("password") String password) { + + this.grantType = grantType; + this.providerName = providerName; + this.scope = scope; + this.username = username; + this.password = password; + } + + public static Builder passwordGrantOAuthPolicy(String providerName) { + return new Builder(providerName, GrantType.USERNAME_PASSWORD); + } + + public static Builder clientCredentialsGrantOAuthPolicy(String providerName) { + return new Builder(providerName, GrantType.CLIENT_CREDENTIALS); + } + + public GrantType getGrantType() { + return grantType; + } + + public String getProviderName() { + return providerName; + } + + public String getScope() { + return scope; + } + + public String getUsername() { + return username; + } + + public String getPassword() { + return password; + } + + public SubscriptionOAuthPolicy anonymize() { + if (GrantType.USERNAME_PASSWORD.equals(grantType)) { + return new SubscriptionOAuthPolicy( + grantType, providerName, scope, username, ANONYMIZED_PASSWORD); } + return this; + } - public static Builder passwordGrantOAuthPolicy(String providerName) { - return new Builder(providerName, GrantType.USERNAME_PASSWORD); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - public static Builder clientCredentialsGrantOAuthPolicy(String providerName) { - return new Builder(providerName, GrantType.CLIENT_CREDENTIALS); + if (o == null || getClass() != o.getClass()) { + return false; } - - public GrantType getGrantType() { - return grantType; - } - - public String getProviderName() { - return providerName; + SubscriptionOAuthPolicy that = (SubscriptionOAuthPolicy) o; + return grantType == that.grantType + && Objects.equals(providerName, that.providerName) + && Objects.equals(scope, that.scope) + && Objects.equals(username, that.username) + && Objects.equals(password, that.password); + } + + @Override + public int hashCode() { + return Objects.hash(grantType, providerName, scope, username, password); + } + + public enum GrantType { + CLIENT_CREDENTIALS("clientCredentials"), + USERNAME_PASSWORD("password"); + + private final String name; + + GrantType(String name) { + this.name = name; } - public String getScope() { - return scope; + @JsonValue + public String getName() { + return name; } + } - public String getUsername() { - return username; - } + public static class Builder { - public String getPassword() { - return password; - } + private final String providerName; + private final GrantType grantType; + private String scope; + private String username; + private String password; - public SubscriptionOAuthPolicy anonymize() { - if (GrantType.USERNAME_PASSWORD.equals(grantType)) { - return new SubscriptionOAuthPolicy(grantType, providerName, scope, username, ANONYMIZED_PASSWORD); - } - return this; + public Builder(String providerName, GrantType grantType) { + this.providerName = providerName; + this.grantType = grantType; } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - SubscriptionOAuthPolicy that = (SubscriptionOAuthPolicy) o; - return grantType == that.grantType - && Objects.equals(providerName, that.providerName) - && Objects.equals(scope, that.scope) - && Objects.equals(username, that.username) - && Objects.equals(password, that.password); + public Builder withScope(String scope) { + this.scope = scope; + return this; } - @Override - public int hashCode() { - return Objects.hash(grantType, providerName, scope, username, password); + public Builder withUsername(String username) { + this.username = username; + return this; } - public enum GrantType { - CLIENT_CREDENTIALS("clientCredentials"), USERNAME_PASSWORD("password"); - - private final String name; - - GrantType(String name) { - this.name = name; - } - - @JsonValue - public String getName() { - return name; - } + public Builder withPassword(String password) { + this.password = password; + return this; } - public static class Builder { - - private final String providerName; - private final GrantType grantType; - private String scope; - private String username; - private String password; - - public Builder(String providerName, GrantType grantType) { - this.providerName = providerName; - this.grantType = grantType; - } - - public Builder withScope(String scope) { - this.scope = scope; - return this; - } - - public Builder withUsername(String username) { - this.username = username; - return this; - } - - public Builder withPassword(String password) { - this.password = password; - return this; - } - - public SubscriptionOAuthPolicy build() { - return new SubscriptionOAuthPolicy(grantType, providerName, scope, username, password); - } + public SubscriptionOAuthPolicy build() { + return new SubscriptionOAuthPolicy(grantType, providerName, scope, username, password); } + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionPolicy.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionPolicy.java index 275013ece1..2a2bc95e34 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionPolicy.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionPolicy.java @@ -5,269 +5,276 @@ import jakarta.annotation.Nullable; import jakarta.validation.constraints.Max; import jakarta.validation.constraints.Min; -import pl.allegro.tech.hermes.api.helpers.Patch; - import java.util.Map; import java.util.Objects; +import pl.allegro.tech.hermes.api.helpers.Patch; public class SubscriptionPolicy { - private static final int DEFAULT_RATE = 400; - private static final int DEFAULT_MESSAGE_TTL = 3600; - private static final int DEFAULT_MESSAGE_BACKOFF = 100; - private static final int DEFAULT_REQUEST_TIMEOUT = 1000; - private static final int DEFAULT_SOCKET_TIMEOUT = 0; - private static final int DEFAULT_SENDING_DELAY = 0; - private static final double DEFAULT_BACKOFF_MULTIPLIER = 1; - private static final int DEFAULT_BACKOFF_MAX_INTERVAL = 600; - - @Min(1) - private int rate = DEFAULT_RATE; - - @Min(0) - @Max(7200) - private int messageTtl = DEFAULT_MESSAGE_TTL; - - @Min(0) - private int messageBackoff = DEFAULT_MESSAGE_BACKOFF; - - @Min(100) - @Max(300_000) - private int requestTimeout = DEFAULT_REQUEST_TIMEOUT; - - @Min(0) - @Max(300_000) - private int socketTimeout = DEFAULT_SOCKET_TIMEOUT; - - @Min(1) - private Integer inflightSize; - - @Min(0) - @Max(5000) - private int sendingDelay = DEFAULT_SENDING_DELAY; - - @Min(1) - @Max(10) - private double backoffMultiplier = DEFAULT_BACKOFF_MULTIPLIER; - - @Min(1) - @Max(600) - private int backoffMaxIntervalInSec = DEFAULT_BACKOFF_MAX_INTERVAL; - - private boolean retryClientErrors = false; - - private SubscriptionPolicy() { - } - - public SubscriptionPolicy(int rate, - int messageTtl, - int requestTimeout, - int socketTimeout, - boolean retryClientErrors, - int messageBackoff, - Integer inflightSize, - int sendingDelay, - double backoffMultiplier, - int backoffMaxIntervalInSec) { - this.rate = rate; - this.messageTtl = messageTtl; - this.requestTimeout = requestTimeout; - this.socketTimeout = socketTimeout; - this.retryClientErrors = retryClientErrors; - this.messageBackoff = messageBackoff; - this.inflightSize = inflightSize; - this.sendingDelay = sendingDelay; - this.backoffMultiplier = backoffMultiplier; - this.backoffMaxIntervalInSec = backoffMaxIntervalInSec; + private static final int DEFAULT_RATE = 400; + private static final int DEFAULT_MESSAGE_TTL = 3600; + private static final int DEFAULT_MESSAGE_BACKOFF = 100; + private static final int DEFAULT_REQUEST_TIMEOUT = 1000; + private static final int DEFAULT_SOCKET_TIMEOUT = 0; + private static final int DEFAULT_SENDING_DELAY = 0; + private static final double DEFAULT_BACKOFF_MULTIPLIER = 1; + private static final int DEFAULT_BACKOFF_MAX_INTERVAL = 600; + + @Min(1) + private int rate = DEFAULT_RATE; + + @Min(0) + @Max(7200) + private int messageTtl = DEFAULT_MESSAGE_TTL; + + @Min(0) + private int messageBackoff = DEFAULT_MESSAGE_BACKOFF; + + @Min(100) + @Max(300_000) + private int requestTimeout = DEFAULT_REQUEST_TIMEOUT; + + @Min(0) + @Max(300_000) + private int socketTimeout = DEFAULT_SOCKET_TIMEOUT; + + @Min(1) + private Integer inflightSize; + + @Min(0) + @Max(5000) + private int sendingDelay = DEFAULT_SENDING_DELAY; + + @Min(1) + @Max(10) + private double backoffMultiplier = DEFAULT_BACKOFF_MULTIPLIER; + + @Min(1) + @Max(600) + private int backoffMaxIntervalInSec = DEFAULT_BACKOFF_MAX_INTERVAL; + + private boolean retryClientErrors = false; + + private SubscriptionPolicy() {} + + public SubscriptionPolicy( + int rate, + int messageTtl, + int requestTimeout, + int socketTimeout, + boolean retryClientErrors, + int messageBackoff, + Integer inflightSize, + int sendingDelay, + double backoffMultiplier, + int backoffMaxIntervalInSec) { + this.rate = rate; + this.messageTtl = messageTtl; + this.requestTimeout = requestTimeout; + this.socketTimeout = socketTimeout; + this.retryClientErrors = retryClientErrors; + this.messageBackoff = messageBackoff; + this.inflightSize = inflightSize; + this.sendingDelay = sendingDelay; + this.backoffMultiplier = backoffMultiplier; + this.backoffMaxIntervalInSec = backoffMaxIntervalInSec; + } + + @JsonCreator + public static SubscriptionPolicy create(Map properties) { + return new SubscriptionPolicy( + (Integer) properties.getOrDefault("rate", DEFAULT_RATE), + (Integer) properties.getOrDefault("messageTtl", DEFAULT_MESSAGE_TTL), + (Integer) properties.getOrDefault("requestTimeout", DEFAULT_REQUEST_TIMEOUT), + (Integer) properties.getOrDefault("socketTimeout", DEFAULT_SOCKET_TIMEOUT), + (Boolean) properties.getOrDefault("retryClientErrors", false), + (Integer) properties.getOrDefault("messageBackoff", DEFAULT_MESSAGE_BACKOFF), + (Integer) properties.getOrDefault("inflightSize", null), + (Integer) properties.getOrDefault("sendingDelay", DEFAULT_SENDING_DELAY), + ((Number) properties.getOrDefault("backoffMultiplier", DEFAULT_BACKOFF_MULTIPLIER)) + .doubleValue(), + (Integer) properties.getOrDefault("backoffMaxIntervalInSec", DEFAULT_BACKOFF_MAX_INTERVAL)); + } + + @Override + public int hashCode() { + return Objects.hash( + rate, + messageTtl, + messageBackoff, + retryClientErrors, + requestTimeout, + socketTimeout, + inflightSize, + sendingDelay, + backoffMultiplier, + backoffMaxIntervalInSec); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; } - - @JsonCreator - public static SubscriptionPolicy create(Map properties) { - return new SubscriptionPolicy( - (Integer) properties.getOrDefault("rate", DEFAULT_RATE), - (Integer) properties.getOrDefault("messageTtl", DEFAULT_MESSAGE_TTL), - (Integer) properties.getOrDefault("requestTimeout", DEFAULT_REQUEST_TIMEOUT), - (Integer) properties.getOrDefault("socketTimeout", DEFAULT_SOCKET_TIMEOUT), - (Boolean) properties.getOrDefault("retryClientErrors", false), - (Integer) properties.getOrDefault("messageBackoff", DEFAULT_MESSAGE_BACKOFF), - (Integer) properties.getOrDefault("inflightSize", null), - (Integer) properties.getOrDefault("sendingDelay", DEFAULT_SENDING_DELAY), - ((Number) properties.getOrDefault("backoffMultiplier", DEFAULT_BACKOFF_MULTIPLIER)).doubleValue(), - (Integer) properties.getOrDefault("backoffMaxIntervalInSec", DEFAULT_BACKOFF_MAX_INTERVAL) - ); + if (obj == null || getClass() != obj.getClass()) { + return false; } - - @Override - public int hashCode() { - return Objects.hash(rate, messageTtl, messageBackoff, retryClientErrors, - requestTimeout, socketTimeout, inflightSize, sendingDelay, backoffMultiplier, - backoffMaxIntervalInSec); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - final SubscriptionPolicy other = (SubscriptionPolicy) obj; - return Objects.equals(this.rate, other.rate) - && Objects.equals(this.messageTtl, other.messageTtl) - && Objects.equals(this.messageBackoff, other.messageBackoff) - && Objects.equals(this.retryClientErrors, other.retryClientErrors) - && Objects.equals(this.requestTimeout, other.requestTimeout) - && Objects.equals(this.socketTimeout, other.socketTimeout) - && Objects.equals(this.inflightSize, other.inflightSize) - && Objects.equals(this.sendingDelay, other.sendingDelay) - && Objects.equals(this.backoffMultiplier, other.backoffMultiplier) - && Objects.equals(this.backoffMaxIntervalInSec, other.backoffMaxIntervalInSec); + final SubscriptionPolicy other = (SubscriptionPolicy) obj; + return Objects.equals(this.rate, other.rate) + && Objects.equals(this.messageTtl, other.messageTtl) + && Objects.equals(this.messageBackoff, other.messageBackoff) + && Objects.equals(this.retryClientErrors, other.retryClientErrors) + && Objects.equals(this.requestTimeout, other.requestTimeout) + && Objects.equals(this.socketTimeout, other.socketTimeout) + && Objects.equals(this.inflightSize, other.inflightSize) + && Objects.equals(this.sendingDelay, other.sendingDelay) + && Objects.equals(this.backoffMultiplier, other.backoffMultiplier) + && Objects.equals(this.backoffMaxIntervalInSec, other.backoffMaxIntervalInSec); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("rate", rate) + .add("messageTtl", messageTtl) + .add("requestTimeout", requestTimeout) + .add("socketTimeout", socketTimeout) + .add("messageBackoff", messageBackoff) + .add("retryClientErrors", retryClientErrors) + .add("inflightSize", inflightSize) + .add("sendingDelay", sendingDelay) + .add("backoffMultiplier", backoffMultiplier) + .add("backoffMaxIntervalInSec", backoffMaxIntervalInSec) + .toString(); + } + + public Integer getRate() { + return rate; + } + + public void setRate(int rate) { + this.rate = rate; + } + + public Integer getMessageTtl() { + return messageTtl; + } + + public Boolean isRetryClientErrors() { + return retryClientErrors; + } + + public Integer getMessageBackoff() { + return messageBackoff; + } + + public Integer getRequestTimeout() { + return requestTimeout; + } + + public Integer getSocketTimeout() { + return socketTimeout; + } + + @Nullable + public Integer getInflightSize() { + return inflightSize; + } + + public Integer getSendingDelay() { + return sendingDelay; + } + + public Double getBackoffMultiplier() { + return backoffMultiplier; + } + + public Integer getBackoffMaxIntervalInSec() { + return backoffMaxIntervalInSec; + } + + public Long getBackoffMaxIntervalMillis() { + return backoffMaxIntervalInSec * 1000L; + } + + public static class Builder { + + private SubscriptionPolicy subscriptionPolicy; + + public Builder() { + subscriptionPolicy = new SubscriptionPolicy(); } - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("rate", rate) - .add("messageTtl", messageTtl) - .add("requestTimeout", requestTimeout) - .add("socketTimeout", socketTimeout) - .add("messageBackoff", messageBackoff) - .add("retryClientErrors", retryClientErrors) - .add("inflightSize", inflightSize) - .add("sendingDelay", sendingDelay) - .add("backoffMultiplier", backoffMultiplier) - .add("backoffMaxIntervalInSec", backoffMaxIntervalInSec) - .toString(); + public static Builder subscriptionPolicy() { + return new Builder(); } - public Integer getRate() { - return rate; + public Builder applyDefaults() { + subscriptionPolicy.rate = DEFAULT_RATE; + subscriptionPolicy.messageTtl = DEFAULT_MESSAGE_TTL; + subscriptionPolicy.backoffMultiplier = DEFAULT_BACKOFF_MULTIPLIER; + return this; } - public void setRate(int rate) { - this.rate = rate; + public Builder withRate(int rate) { + subscriptionPolicy.rate = rate; + return this; } - public Integer getMessageTtl() { - return messageTtl; + public Builder withMessageTtl(int ttl) { + subscriptionPolicy.messageTtl = ttl; + return this; } - public Boolean isRetryClientErrors() { - return retryClientErrors; + public Builder withRequestTimeout(int timeout) { + subscriptionPolicy.requestTimeout = timeout; + return this; } - public Integer getMessageBackoff() { - return messageBackoff; + public Builder withSocketTimeout(int timeout) { + subscriptionPolicy.socketTimeout = timeout; + return this; } - public Integer getRequestTimeout() { - return requestTimeout; + public Builder withInflightSize(Integer inflightSize) { + subscriptionPolicy.inflightSize = inflightSize; + return this; } - public Integer getSocketTimeout() { - return socketTimeout; + public Builder withMessageBackoff(int backoff) { + subscriptionPolicy.messageBackoff = backoff; + return this; } - @Nullable - public Integer getInflightSize() { - return inflightSize; + public Builder withClientErrorRetry() { + subscriptionPolicy.retryClientErrors = true; + return this; } - public Integer getSendingDelay() { - return sendingDelay; + public Builder withSendingDelay(int sendingDelay) { + subscriptionPolicy.sendingDelay = sendingDelay; + return this; } - public Double getBackoffMultiplier() { - return backoffMultiplier; + public Builder withBackoffMultiplier(double backoffMultiplier) { + subscriptionPolicy.backoffMultiplier = backoffMultiplier; + return this; } - public Integer getBackoffMaxIntervalInSec() { - return backoffMaxIntervalInSec; + public Builder withBackoffMaxIntervalInSec(int backoffMaxIntervalInSec) { + subscriptionPolicy.backoffMaxIntervalInSec = backoffMaxIntervalInSec; + return this; } - public Long getBackoffMaxIntervalMillis() { - return backoffMaxIntervalInSec * 1000L; + public SubscriptionPolicy build() { + return subscriptionPolicy; } - public static class Builder { - - private SubscriptionPolicy subscriptionPolicy; - - public Builder() { - subscriptionPolicy = new SubscriptionPolicy(); - } - - public static Builder subscriptionPolicy() { - return new Builder(); - } - - public Builder applyDefaults() { - subscriptionPolicy.rate = DEFAULT_RATE; - subscriptionPolicy.messageTtl = DEFAULT_MESSAGE_TTL; - subscriptionPolicy.backoffMultiplier = DEFAULT_BACKOFF_MULTIPLIER; - return this; - } - - public Builder withRate(int rate) { - subscriptionPolicy.rate = rate; - return this; - } - - public Builder withMessageTtl(int ttl) { - subscriptionPolicy.messageTtl = ttl; - return this; - } - - public Builder withRequestTimeout(int timeout) { - subscriptionPolicy.requestTimeout = timeout; - return this; - } - - public Builder withSocketTimeout(int timeout) { - subscriptionPolicy.socketTimeout = timeout; - return this; - } - - public Builder withInflightSize(Integer inflightSize) { - subscriptionPolicy.inflightSize = inflightSize; - return this; - } - - public Builder withMessageBackoff(int backoff) { - subscriptionPolicy.messageBackoff = backoff; - return this; - } - - public Builder withClientErrorRetry() { - subscriptionPolicy.retryClientErrors = true; - return this; - } - - public Builder withSendingDelay(int sendingDelay) { - subscriptionPolicy.sendingDelay = sendingDelay; - return this; - } - - public Builder withBackoffMultiplier(double backoffMultiplier) { - subscriptionPolicy.backoffMultiplier = backoffMultiplier; - return this; - } - - public Builder withBackoffMaxIntervalInSec(int backoffMaxIntervalInSec) { - subscriptionPolicy.backoffMaxIntervalInSec = backoffMaxIntervalInSec; - return this; - } - - public SubscriptionPolicy build() { - return subscriptionPolicy; - } - - public Builder applyPatch(PatchData update) { - if (update != null) { - subscriptionPolicy = Patch.apply(subscriptionPolicy, update); - } - return this; - } + public Builder applyPatch(PatchData update) { + if (update != null) { + subscriptionPolicy = Patch.apply(subscriptionPolicy, update); + } + return this; } + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionStats.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionStats.java index 743170e0a4..9dabd16d37 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionStats.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/SubscriptionStats.java @@ -1,63 +1,64 @@ package pl.allegro.tech.hermes.api; - import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.Objects; public class SubscriptionStats { - private final long subscriptionCount; - private final long trackingEnabledSubscriptionCount; - private final long avroSubscriptionCount; - - @JsonCreator - public SubscriptionStats( - @JsonProperty("subscriptionCount") long subscriptionCount, - @JsonProperty("trackingEnabledSubscriptionCount") long trackingEnabledSubscriptionCount, - @JsonProperty("avroSubscriptionCount") long avroSubscriptionCount) { - this.subscriptionCount = subscriptionCount; - this.trackingEnabledSubscriptionCount = trackingEnabledSubscriptionCount; - this.avroSubscriptionCount = avroSubscriptionCount; - } + private final long subscriptionCount; + private final long trackingEnabledSubscriptionCount; + private final long avroSubscriptionCount; - public long getSubscriptionCount() { - return subscriptionCount; - } + @JsonCreator + public SubscriptionStats( + @JsonProperty("subscriptionCount") long subscriptionCount, + @JsonProperty("trackingEnabledSubscriptionCount") long trackingEnabledSubscriptionCount, + @JsonProperty("avroSubscriptionCount") long avroSubscriptionCount) { + this.subscriptionCount = subscriptionCount; + this.trackingEnabledSubscriptionCount = trackingEnabledSubscriptionCount; + this.avroSubscriptionCount = avroSubscriptionCount; + } - public long getTrackingEnabledSubscriptionCount() { - return trackingEnabledSubscriptionCount; - } + public long getSubscriptionCount() { + return subscriptionCount; + } - public long getAvroSubscriptionCount() { - return avroSubscriptionCount; - } + public long getTrackingEnabledSubscriptionCount() { + return trackingEnabledSubscriptionCount; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - SubscriptionStats that = (SubscriptionStats) o; - return subscriptionCount == that.subscriptionCount - && trackingEnabledSubscriptionCount == that.trackingEnabledSubscriptionCount - && avroSubscriptionCount == that.avroSubscriptionCount; - } + public long getAvroSubscriptionCount() { + return avroSubscriptionCount; + } - @Override - public int hashCode() { - return Objects.hash(subscriptionCount, trackingEnabledSubscriptionCount, avroSubscriptionCount); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public String toString() { - return "SubscriptionStats{" - + "subscriptionCount=" + subscriptionCount - + ", trackingEnabledSubscriptionCount=" + trackingEnabledSubscriptionCount - + ", avroSubscriptionCount=" + avroSubscriptionCount - + '}'; + if (o == null || getClass() != o.getClass()) { + return false; } + SubscriptionStats that = (SubscriptionStats) o; + return subscriptionCount == that.subscriptionCount + && trackingEnabledSubscriptionCount == that.trackingEnabledSubscriptionCount + && avroSubscriptionCount == that.avroSubscriptionCount; + } + + @Override + public int hashCode() { + return Objects.hash(subscriptionCount, trackingEnabledSubscriptionCount, avroSubscriptionCount); + } + + @Override + public String toString() { + return "SubscriptionStats{" + + "subscriptionCount=" + + subscriptionCount + + ", trackingEnabledSubscriptionCount=" + + trackingEnabledSubscriptionCount + + ", avroSubscriptionCount=" + + avroSubscriptionCount + + '}'; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Topic.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Topic.java index 277857a63f..b6ae679118 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Topic.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/Topic.java @@ -10,274 +10,316 @@ import jakarta.validation.constraints.Max; import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotNull; - import java.time.Instant; import java.util.Collections; import java.util.Objects; import java.util.Set; import java.util.concurrent.TimeUnit; -@JsonIgnoreProperties(value = {"createdAt", "modifiedAt"}, allowGetters = true) +@JsonIgnoreProperties( + value = {"createdAt", "modifiedAt"}, + allowGetters = true) public class Topic { - public static final int MIN_MESSAGE_SIZE = 1024; - public static final int MAX_MESSAGE_SIZE = 2 * 1024 * 1024; - public static final String DEFAULT_SCHEMA_ID_SERIALIZATION_ENABLED_KEY = "defaultSchemaIdAwareSerializationEnabled"; - private static final int DEFAULT_MAX_MESSAGE_SIZE = 50 * 1024; - private final boolean schemaIdAwareSerializationEnabled; - private final TopicDataOfflineStorage offlineStorage; - @Valid - @NotNull - private TopicName name; - @NotNull - private String description; - @Valid - @NotNull - private OwnerId owner; - private boolean jsonToAvroDryRunEnabled = false; - @NotNull - private Ack ack; - public static final String DEFAULT_FALLBACK_TO_REMOTE_DATACENTER_KEY = "defaultFallbackToRemoteDatacenterEnabled"; - private final boolean fallbackToRemoteDatacenterEnabled; - private PublishingChaosPolicy chaos; - @NotNull - private ContentType contentType; - @Min(MIN_MESSAGE_SIZE) - @Max(MAX_MESSAGE_SIZE) - private int maxMessageSize; - @Valid - @NotNull - private RetentionTime retentionTime = RetentionTime.of(1, TimeUnit.DAYS); - private boolean trackingEnabled = false; - private boolean migratedFromJsonType = false; - private boolean subscribingRestricted = false; - - private PublishingAuth publishingAuth; - @Valid - private Set labels; - private Instant createdAt; - private Instant modifiedAt; - - public Topic(TopicName name, String description, OwnerId owner, RetentionTime retentionTime, - boolean migratedFromJsonType, Ack ack, - @JacksonInject(value = DEFAULT_FALLBACK_TO_REMOTE_DATACENTER_KEY, useInput = OptBoolean.TRUE) - Boolean fallbackToRemoteDatacenterEnabled, - PublishingChaosPolicy chaos, boolean trackingEnabled, ContentType contentType, boolean jsonToAvroDryRunEnabled, - @JacksonInject(value = DEFAULT_SCHEMA_ID_SERIALIZATION_ENABLED_KEY, useInput = OptBoolean.TRUE) - Boolean schemaIdAwareSerializationEnabled, - int maxMessageSize, PublishingAuth publishingAuth, boolean subscribingRestricted, - TopicDataOfflineStorage offlineStorage, Set labels, Instant createdAt, Instant modifiedAt) { - this.name = name; - this.description = description; - this.owner = owner; - this.retentionTime = retentionTime; - this.ack = (ack == null ? Ack.LEADER : ack); - this.fallbackToRemoteDatacenterEnabled = fallbackToRemoteDatacenterEnabled; - this.chaos = chaos; - this.trackingEnabled = trackingEnabled; - this.migratedFromJsonType = migratedFromJsonType; - this.contentType = contentType; - this.jsonToAvroDryRunEnabled = jsonToAvroDryRunEnabled; - this.schemaIdAwareSerializationEnabled = schemaIdAwareSerializationEnabled; - this.maxMessageSize = maxMessageSize; - this.publishingAuth = publishingAuth; - this.subscribingRestricted = subscribingRestricted; - this.offlineStorage = offlineStorage; - this.labels = labels; - this.createdAt = createdAt; - this.modifiedAt = modifiedAt; - } - - @JsonCreator - public Topic( - @JsonProperty("name") String qualifiedName, - @JsonProperty("description") String description, - @JsonProperty("owner") OwnerId owner, - @JsonProperty("retentionTime") RetentionTime retentionTime, - @JsonProperty("jsonToAvroDryRun") boolean jsonToAvroDryRunEnabled, - @JsonProperty("ack") Ack ack, - @JacksonInject(value = DEFAULT_FALLBACK_TO_REMOTE_DATACENTER_KEY, useInput = OptBoolean.TRUE) - @JsonProperty("fallbackToRemoteDatacenterEnabled") Boolean fallbackToRemoteDatacenterEnabled, - @JsonProperty("chaos") PublishingChaosPolicy chaos, - @JsonProperty("trackingEnabled") boolean trackingEnabled, - @JsonProperty("migratedFromJsonType") boolean migratedFromJsonType, - @JsonProperty("schemaIdAwareSerializationEnabled") - @JacksonInject(value = DEFAULT_SCHEMA_ID_SERIALIZATION_ENABLED_KEY, useInput = OptBoolean.TRUE) - Boolean schemaIdAwareSerializationEnabled, - @JsonProperty("contentType") ContentType contentType, - @JsonProperty("maxMessageSize") Integer maxMessageSize, - @JsonProperty("auth") PublishingAuth publishingAuth, - @JsonProperty("subscribingRestricted") boolean subscribingRestricted, - @JsonProperty("offlineStorage") TopicDataOfflineStorage offlineStorage, - @JsonProperty("labels") Set labels, - @JsonProperty("createdAt") Instant createdAt, - @JsonProperty("modifiedAt") Instant modifiedAt - ) { - this(TopicName.fromQualifiedName(qualifiedName), description, owner, retentionTime, migratedFromJsonType, ack, - fallbackToRemoteDatacenterEnabled, chaos == null ? PublishingChaosPolicy.disabled() : chaos, - trackingEnabled, contentType, jsonToAvroDryRunEnabled, schemaIdAwareSerializationEnabled, - maxMessageSize == null ? DEFAULT_MAX_MESSAGE_SIZE : maxMessageSize, - publishingAuth == null ? PublishingAuth.disabled() : publishingAuth, subscribingRestricted, - offlineStorage == null ? TopicDataOfflineStorage.defaultOfflineStorage() : offlineStorage, - labels == null ? Collections.emptySet() : labels, - createdAt, modifiedAt - ); - } - - public RetentionTime getRetentionTime() { - return retentionTime; - } - - @Override - public int hashCode() { - return Objects.hash(name, description, owner, retentionTime, migratedFromJsonType, trackingEnabled, ack, - fallbackToRemoteDatacenterEnabled, chaos, contentType, jsonToAvroDryRunEnabled, schemaIdAwareSerializationEnabled, - maxMessageSize, publishingAuth, subscribingRestricted, offlineStorage, labels); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - final Topic other = (Topic) obj; - - return Objects.equals(this.name, other.name) - && Objects.equals(this.description, other.description) - && Objects.equals(this.owner, other.owner) - && Objects.equals(this.retentionTime, other.retentionTime) - && Objects.equals(this.jsonToAvroDryRunEnabled, other.jsonToAvroDryRunEnabled) - && Objects.equals(this.trackingEnabled, other.trackingEnabled) - && Objects.equals(this.migratedFromJsonType, other.migratedFromJsonType) - && Objects.equals(this.schemaIdAwareSerializationEnabled, other.schemaIdAwareSerializationEnabled) - && Objects.equals(this.ack, other.ack) - && Objects.equals(this.fallbackToRemoteDatacenterEnabled, other.fallbackToRemoteDatacenterEnabled) - && Objects.equals(this.chaos, other.chaos) - && Objects.equals(this.contentType, other.contentType) - && Objects.equals(this.maxMessageSize, other.maxMessageSize) - && Objects.equals(this.subscribingRestricted, other.subscribingRestricted) - && Objects.equals(this.publishingAuth, other.publishingAuth) - && Objects.equals(this.offlineStorage, other.offlineStorage) - && Objects.equals(this.labels, other.labels); - } - - @JsonProperty("name") - public String getQualifiedName() { - return TopicName.toQualifiedName(name); - } - - @JsonIgnore - public TopicName getName() { - return name; - } - - public String getDescription() { - return description; - } - - public OwnerId getOwner() { - return owner; - } - - @JsonProperty("jsonToAvroDryRun") - public boolean isJsonToAvroDryRunEnabled() { - return jsonToAvroDryRunEnabled; - } - - public Ack getAck() { - return ack; - } - - public boolean isFallbackToRemoteDatacenterEnabled() { - return fallbackToRemoteDatacenterEnabled; - } - - public PublishingChaosPolicy getChaos() { - return chaos; - } - - public ContentType getContentType() { - return contentType; - } - - public boolean isTrackingEnabled() { - return trackingEnabled; - } - - @JsonProperty("migratedFromJsonType") - public boolean wasMigratedFromJsonType() { - return migratedFromJsonType; - } - - @JsonIgnore - public boolean isReplicationConfirmRequired() { - return getAck() == Ack.ALL; - } - - public boolean isSchemaIdAwareSerializationEnabled() { - return schemaIdAwareSerializationEnabled; - } - - public int getMaxMessageSize() { - return maxMessageSize; - } - - @JsonProperty("auth") - public PublishingAuth getPublishingAuth() { - return publishingAuth; - } - - @JsonIgnore - public boolean isAuthEnabled() { - return publishingAuth.isEnabled(); - } - - @JsonIgnore - public boolean isUnauthenticatedAccessEnabled() { - return publishingAuth.isUnauthenticatedAccessEnabled(); - } - - public boolean hasPermission(String publisher) { - return publishingAuth.hasPermission(publisher); - } - - public boolean isSubscribingRestricted() { - return subscribingRestricted; - } - - public TopicDataOfflineStorage getOfflineStorage() { - return offlineStorage; - } - - public Set getLabels() { - return Collections.unmodifiableSet(labels); - } - - public Instant getCreatedAt() { - return createdAt; - } - - public void setCreatedAt(Long createdAt) { - this.createdAt = Instant.ofEpochMilli(createdAt); - } - - public Instant getModifiedAt() { - return modifiedAt; - } - - public void setModifiedAt(Long modifiedAt) { - this.modifiedAt = Instant.ofEpochMilli(modifiedAt); - } - - @Override - public String toString() { - return "Topic(" + getQualifiedName() + ")"; - } - - public enum Ack { - NONE, LEADER, ALL - } + public static final int MIN_MESSAGE_SIZE = 1024; + public static final int MAX_MESSAGE_SIZE = 2 * 1024 * 1024; + public static final String DEFAULT_SCHEMA_ID_SERIALIZATION_ENABLED_KEY = + "defaultSchemaIdAwareSerializationEnabled"; + private static final int DEFAULT_MAX_MESSAGE_SIZE = 50 * 1024; + private final boolean schemaIdAwareSerializationEnabled; + private final TopicDataOfflineStorage offlineStorage; + @Valid @NotNull private TopicName name; + @NotNull private String description; + @Valid @NotNull private OwnerId owner; + private boolean jsonToAvroDryRunEnabled = false; + @NotNull private Ack ack; + public static final String DEFAULT_FALLBACK_TO_REMOTE_DATACENTER_KEY = + "defaultFallbackToRemoteDatacenterEnabled"; + private final boolean fallbackToRemoteDatacenterEnabled; + private PublishingChaosPolicy chaos; + @NotNull private ContentType contentType; + + @Min(MIN_MESSAGE_SIZE) + @Max(MAX_MESSAGE_SIZE) + private int maxMessageSize; + + @Valid @NotNull private RetentionTime retentionTime = RetentionTime.of(1, TimeUnit.DAYS); + private boolean trackingEnabled = false; + private boolean migratedFromJsonType = false; + private boolean subscribingRestricted = false; + + private PublishingAuth publishingAuth; + @Valid private Set labels; + private Instant createdAt; + private Instant modifiedAt; + + public Topic( + TopicName name, + String description, + OwnerId owner, + RetentionTime retentionTime, + boolean migratedFromJsonType, + Ack ack, + @JacksonInject(value = DEFAULT_FALLBACK_TO_REMOTE_DATACENTER_KEY, useInput = OptBoolean.TRUE) + Boolean fallbackToRemoteDatacenterEnabled, + PublishingChaosPolicy chaos, + boolean trackingEnabled, + ContentType contentType, + boolean jsonToAvroDryRunEnabled, + @JacksonInject( + value = DEFAULT_SCHEMA_ID_SERIALIZATION_ENABLED_KEY, + useInput = OptBoolean.TRUE) + Boolean schemaIdAwareSerializationEnabled, + int maxMessageSize, + PublishingAuth publishingAuth, + boolean subscribingRestricted, + TopicDataOfflineStorage offlineStorage, + Set labels, + Instant createdAt, + Instant modifiedAt) { + this.name = name; + this.description = description; + this.owner = owner; + this.retentionTime = retentionTime; + this.ack = (ack == null ? Ack.LEADER : ack); + this.fallbackToRemoteDatacenterEnabled = fallbackToRemoteDatacenterEnabled; + this.chaos = chaos; + this.trackingEnabled = trackingEnabled; + this.migratedFromJsonType = migratedFromJsonType; + this.contentType = contentType; + this.jsonToAvroDryRunEnabled = jsonToAvroDryRunEnabled; + this.schemaIdAwareSerializationEnabled = schemaIdAwareSerializationEnabled; + this.maxMessageSize = maxMessageSize; + this.publishingAuth = publishingAuth; + this.subscribingRestricted = subscribingRestricted; + this.offlineStorage = offlineStorage; + this.labels = labels; + this.createdAt = createdAt; + this.modifiedAt = modifiedAt; + } + + @JsonCreator + public Topic( + @JsonProperty("name") String qualifiedName, + @JsonProperty("description") String description, + @JsonProperty("owner") OwnerId owner, + @JsonProperty("retentionTime") RetentionTime retentionTime, + @JsonProperty("jsonToAvroDryRun") boolean jsonToAvroDryRunEnabled, + @JsonProperty("ack") Ack ack, + @JacksonInject(value = DEFAULT_FALLBACK_TO_REMOTE_DATACENTER_KEY, useInput = OptBoolean.TRUE) + @JsonProperty("fallbackToRemoteDatacenterEnabled") + Boolean fallbackToRemoteDatacenterEnabled, + @JsonProperty("chaos") PublishingChaosPolicy chaos, + @JsonProperty("trackingEnabled") boolean trackingEnabled, + @JsonProperty("migratedFromJsonType") boolean migratedFromJsonType, + @JsonProperty("schemaIdAwareSerializationEnabled") + @JacksonInject( + value = DEFAULT_SCHEMA_ID_SERIALIZATION_ENABLED_KEY, + useInput = OptBoolean.TRUE) + Boolean schemaIdAwareSerializationEnabled, + @JsonProperty("contentType") ContentType contentType, + @JsonProperty("maxMessageSize") Integer maxMessageSize, + @JsonProperty("auth") PublishingAuth publishingAuth, + @JsonProperty("subscribingRestricted") boolean subscribingRestricted, + @JsonProperty("offlineStorage") TopicDataOfflineStorage offlineStorage, + @JsonProperty("labels") Set labels, + @JsonProperty("createdAt") Instant createdAt, + @JsonProperty("modifiedAt") Instant modifiedAt) { + this( + TopicName.fromQualifiedName(qualifiedName), + description, + owner, + retentionTime, + migratedFromJsonType, + ack, + fallbackToRemoteDatacenterEnabled, + chaos == null ? PublishingChaosPolicy.disabled() : chaos, + trackingEnabled, + contentType, + jsonToAvroDryRunEnabled, + schemaIdAwareSerializationEnabled, + maxMessageSize == null ? DEFAULT_MAX_MESSAGE_SIZE : maxMessageSize, + publishingAuth == null ? PublishingAuth.disabled() : publishingAuth, + subscribingRestricted, + offlineStorage == null ? TopicDataOfflineStorage.defaultOfflineStorage() : offlineStorage, + labels == null ? Collections.emptySet() : labels, + createdAt, + modifiedAt); + } + + public RetentionTime getRetentionTime() { + return retentionTime; + } + + @Override + public int hashCode() { + return Objects.hash( + name, + description, + owner, + retentionTime, + migratedFromJsonType, + trackingEnabled, + ack, + fallbackToRemoteDatacenterEnabled, + chaos, + contentType, + jsonToAvroDryRunEnabled, + schemaIdAwareSerializationEnabled, + maxMessageSize, + publishingAuth, + subscribingRestricted, + offlineStorage, + labels); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final Topic other = (Topic) obj; + + return Objects.equals(this.name, other.name) + && Objects.equals(this.description, other.description) + && Objects.equals(this.owner, other.owner) + && Objects.equals(this.retentionTime, other.retentionTime) + && Objects.equals(this.jsonToAvroDryRunEnabled, other.jsonToAvroDryRunEnabled) + && Objects.equals(this.trackingEnabled, other.trackingEnabled) + && Objects.equals(this.migratedFromJsonType, other.migratedFromJsonType) + && Objects.equals( + this.schemaIdAwareSerializationEnabled, other.schemaIdAwareSerializationEnabled) + && Objects.equals(this.ack, other.ack) + && Objects.equals( + this.fallbackToRemoteDatacenterEnabled, other.fallbackToRemoteDatacenterEnabled) + && Objects.equals(this.chaos, other.chaos) + && Objects.equals(this.contentType, other.contentType) + && Objects.equals(this.maxMessageSize, other.maxMessageSize) + && Objects.equals(this.subscribingRestricted, other.subscribingRestricted) + && Objects.equals(this.publishingAuth, other.publishingAuth) + && Objects.equals(this.offlineStorage, other.offlineStorage) + && Objects.equals(this.labels, other.labels); + } + + @JsonProperty("name") + public String getQualifiedName() { + return TopicName.toQualifiedName(name); + } + + @JsonIgnore + public TopicName getName() { + return name; + } + + public String getDescription() { + return description; + } + + public OwnerId getOwner() { + return owner; + } + + @JsonProperty("jsonToAvroDryRun") + public boolean isJsonToAvroDryRunEnabled() { + return jsonToAvroDryRunEnabled; + } + + public Ack getAck() { + return ack; + } + + public boolean isFallbackToRemoteDatacenterEnabled() { + return fallbackToRemoteDatacenterEnabled; + } + + public PublishingChaosPolicy getChaos() { + return chaos; + } + + public ContentType getContentType() { + return contentType; + } + + public boolean isTrackingEnabled() { + return trackingEnabled; + } + + @JsonProperty("migratedFromJsonType") + public boolean wasMigratedFromJsonType() { + return migratedFromJsonType; + } + + @JsonIgnore + public boolean isReplicationConfirmRequired() { + return getAck() == Ack.ALL; + } + + public boolean isSchemaIdAwareSerializationEnabled() { + return schemaIdAwareSerializationEnabled; + } + + public int getMaxMessageSize() { + return maxMessageSize; + } + + @JsonProperty("auth") + public PublishingAuth getPublishingAuth() { + return publishingAuth; + } + + @JsonIgnore + public boolean isAuthEnabled() { + return publishingAuth.isEnabled(); + } + + @JsonIgnore + public boolean isUnauthenticatedAccessEnabled() { + return publishingAuth.isUnauthenticatedAccessEnabled(); + } + + public boolean hasPermission(String publisher) { + return publishingAuth.hasPermission(publisher); + } + + public boolean isSubscribingRestricted() { + return subscribingRestricted; + } + + public TopicDataOfflineStorage getOfflineStorage() { + return offlineStorage; + } + + public Set getLabels() { + return Collections.unmodifiableSet(labels); + } + + public Instant getCreatedAt() { + return createdAt; + } + + public void setCreatedAt(Long createdAt) { + this.createdAt = Instant.ofEpochMilli(createdAt); + } + + public Instant getModifiedAt() { + return modifiedAt; + } + + public void setModifiedAt(Long modifiedAt) { + this.modifiedAt = Instant.ofEpochMilli(modifiedAt); + } + + @Override + public String toString() { + return "Topic(" + getQualifiedName() + ")"; + } + + public enum Ack { + NONE, + LEADER, + ALL + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicConstraints.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicConstraints.java index e507b9eb35..031fba4a04 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicConstraints.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicConstraints.java @@ -6,22 +6,22 @@ public class TopicConstraints { - private final TopicName topicName; - @Valid - private final Constraints constraints; + private final TopicName topicName; + @Valid private final Constraints constraints; - @JsonCreator - public TopicConstraints(@JsonProperty("topicName") String topicName, - @JsonProperty("constraints") Constraints constraints) { - this.topicName = TopicName.fromQualifiedName(topicName); - this.constraints = constraints; - } + @JsonCreator + public TopicConstraints( + @JsonProperty("topicName") String topicName, + @JsonProperty("constraints") Constraints constraints) { + this.topicName = TopicName.fromQualifiedName(topicName); + this.constraints = constraints; + } - public TopicName getTopicName() { - return topicName; - } + public TopicName getTopicName() { + return topicName; + } - public Constraints getConstraints() { - return constraints; - } + public Constraints getConstraints() { + return constraints; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicDataOfflineStorage.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicDataOfflineStorage.java index 2ac58c2ada..eb99107b8c 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicDataOfflineStorage.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicDataOfflineStorage.java @@ -4,54 +4,52 @@ import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.Valid; import jakarta.validation.constraints.NotNull; - import java.util.Objects; /** - * Topic offline storage metadata - not used in Hermes, but exposed as part of API for other systems to use. + * Topic offline storage metadata - not used in Hermes, but exposed as part of API for other systems + * to use. */ public class TopicDataOfflineStorage { - private final boolean enabled; + private final boolean enabled; - @Valid - @NotNull - private final OfflineRetentionTime retentionTime; + @Valid @NotNull private final OfflineRetentionTime retentionTime; - @JsonCreator - public TopicDataOfflineStorage(@JsonProperty("enabled") boolean enabled, - @JsonProperty("retentionTime") OfflineRetentionTime retentionTime) { - this.enabled = enabled; - this.retentionTime = retentionTime; - } + @JsonCreator + public TopicDataOfflineStorage( + @JsonProperty("enabled") boolean enabled, + @JsonProperty("retentionTime") OfflineRetentionTime retentionTime) { + this.enabled = enabled; + this.retentionTime = retentionTime; + } - public static TopicDataOfflineStorage defaultOfflineStorage() { - return new TopicDataOfflineStorage(false, OfflineRetentionTime.of(0)); - } + public static TopicDataOfflineStorage defaultOfflineStorage() { + return new TopicDataOfflineStorage(false, OfflineRetentionTime.of(0)); + } - public boolean isEnabled() { - return enabled; - } + public boolean isEnabled() { + return enabled; + } - public OfflineRetentionTime getRetentionTime() { - return retentionTime; - } + public OfflineRetentionTime getRetentionTime() { + return retentionTime; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof TopicDataOfflineStorage)) { - return false; - } - TopicDataOfflineStorage that = (TopicDataOfflineStorage) o; - return enabled == that.enabled - && Objects.equals(retentionTime, that.retentionTime); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(enabled, retentionTime); + if (!(o instanceof TopicDataOfflineStorage)) { + return false; } + TopicDataOfflineStorage that = (TopicDataOfflineStorage) o; + return enabled == that.enabled && Objects.equals(retentionTime, that.retentionTime); + } + + @Override + public int hashCode() { + return Objects.hash(enabled, retentionTime); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicLabel.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicLabel.java index 96528ce2e5..e02ef4bc4d 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicLabel.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicLabel.java @@ -7,37 +7,36 @@ public class TopicLabel { - @NotEmpty - private final String value; + @NotEmpty private final String value; - @JsonCreator - public TopicLabel(@JsonProperty("value") String value) { - this.value = value; - } + @JsonCreator + public TopicLabel(@JsonProperty("value") String value) { + this.value = value; + } - public String getValue() { - return value; - } + public String getValue() { + return value; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - TopicLabel that = (TopicLabel) o; - return Objects.equal(value, that.value); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hashCode(value); - } - - @Override - public String toString() { - return "TopicLabel(" + value + ")"; + if (o == null || getClass() != o.getClass()) { + return false; } + TopicLabel that = (TopicLabel) o; + return Objects.equal(value, that.value); + } + + @Override + public int hashCode() { + return Objects.hashCode(value); + } + + @Override + public String toString() { + return "TopicLabel(" + value + ")"; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicMetrics.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicMetrics.java index ddc89bd6cc..564653a28f 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicMetrics.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicMetrics.java @@ -1,116 +1,117 @@ package pl.allegro.tech.hermes.api; -import java.util.Objects; - import static pl.allegro.tech.hermes.api.MetricDecimalValue.of; +import java.util.Objects; + public class TopicMetrics { - private long published; - private long volume; - private MetricDecimalValue rate = of("0.0"); - private MetricDecimalValue deliveryRate = of("0.0"); - private int subscriptions; - private MetricDecimalValue throughput = of("0.0"); - - public long getPublished() { - return published; + private long published; + private long volume; + private MetricDecimalValue rate = of("0.0"); + private MetricDecimalValue deliveryRate = of("0.0"); + private int subscriptions; + private MetricDecimalValue throughput = of("0.0"); + + public long getPublished() { + return published; + } + + public long getVolume() { + return volume; + } + + public MetricDecimalValue getRate() { + return rate; + } + + public MetricDecimalValue getDeliveryRate() { + return deliveryRate; + } + + public int getSubscriptions() { + return subscriptions; + } + + public MetricDecimalValue getThroughput() { + return throughput; + } + + @Override + public int hashCode() { + return Objects.hash(published, rate, deliveryRate, subscriptions, throughput, volume); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; } - - public long getVolume() { - return volume; + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final TopicMetrics other = (TopicMetrics) obj; + return Objects.equals(this.published, other.published) + && Objects.equals(this.rate, other.rate) + && Objects.equals(this.deliveryRate, other.deliveryRate) + && Objects.equals(this.subscriptions, other.subscriptions) + && Objects.equals(this.throughput, other.throughput) + && Objects.equals(this.volume, other.volume); + } + + public static TopicMetrics unavailable() { + return Builder.topicMetrics() + .withRate(MetricDecimalValue.unavailable()) + .withDeliveryRate(MetricDecimalValue.unavailable()) + .withPublished(0) + .withVolume(0) + .withSubscriptions(0) + .withThroughput(MetricDecimalValue.unavailable()) + .build(); + } + + public static class Builder { + private final TopicMetrics topicMetrics; + + public Builder() { + topicMetrics = new TopicMetrics(); } - public MetricDecimalValue getRate() { - return rate; + public Builder withPublished(long published) { + topicMetrics.published = published; + return this; } - public MetricDecimalValue getDeliveryRate() { - return deliveryRate; + public Builder withRate(MetricDecimalValue rate) { + topicMetrics.rate = rate; + return this; } - public int getSubscriptions() { - return subscriptions; + public Builder withDeliveryRate(MetricDecimalValue deliveryRate) { + topicMetrics.deliveryRate = deliveryRate; + return this; } - public MetricDecimalValue getThroughput() { - return throughput; + public Builder withSubscriptions(int subscriptions) { + topicMetrics.subscriptions = subscriptions; + return this; } - @Override - public int hashCode() { - return Objects.hash(published, rate, deliveryRate, subscriptions, throughput, volume); + public Builder withThroughput(MetricDecimalValue throughput) { + topicMetrics.throughput = throughput; + return this; } - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - final TopicMetrics other = (TopicMetrics) obj; - return Objects.equals(this.published, other.published) - && Objects.equals(this.rate, other.rate) - && Objects.equals(this.deliveryRate, other.deliveryRate) - && Objects.equals(this.subscriptions, other.subscriptions) - && Objects.equals(this.throughput, other.throughput) - && Objects.equals(this.volume, other.volume); + public static Builder topicMetrics() { + return new Builder(); } - public static TopicMetrics unavailable() { - return Builder.topicMetrics().withRate(MetricDecimalValue.unavailable()) - .withDeliveryRate(MetricDecimalValue.unavailable()) - .withPublished(0) - .withVolume(0) - .withSubscriptions(0) - .withThroughput(MetricDecimalValue.unavailable()) - .build(); + public TopicMetrics build() { + return topicMetrics; } - public static class Builder { - private final TopicMetrics topicMetrics; - - public Builder() { - topicMetrics = new TopicMetrics(); - } - - public Builder withPublished(long published) { - topicMetrics.published = published; - return this; - } - - public Builder withRate(MetricDecimalValue rate) { - topicMetrics.rate = rate; - return this; - } - - public Builder withDeliveryRate(MetricDecimalValue deliveryRate) { - topicMetrics.deliveryRate = deliveryRate; - return this; - } - - public Builder withSubscriptions(int subscriptions) { - topicMetrics.subscriptions = subscriptions; - return this; - } - - public Builder withThroughput(MetricDecimalValue throughput) { - topicMetrics.throughput = throughput; - return this; - } - - public static Builder topicMetrics() { - return new Builder(); - } - - public TopicMetrics build() { - return topicMetrics; - } - - public Builder withVolume(long volume) { - topicMetrics.volume = volume; - return this; - } + public Builder withVolume(long volume) { + topicMetrics.volume = volume; + return this; } + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicName.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicName.java index 3c9704f211..5e30d9d631 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicName.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicName.java @@ -1,86 +1,84 @@ package pl.allegro.tech.hermes.api; +import static pl.allegro.tech.hermes.api.constraints.Names.ALLOWED_NAME_REGEX; + import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Strings; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.Pattern; - import java.util.Objects; -import static pl.allegro.tech.hermes.api.constraints.Names.ALLOWED_NAME_REGEX; - public class TopicName { - public static final char GROUP_SEPARATOR = '.'; + public static final char GROUP_SEPARATOR = '.'; + + @NotEmpty + @Pattern(regexp = ALLOWED_NAME_REGEX) + private final String groupName; + + @NotEmpty + @Pattern(regexp = ALLOWED_NAME_REGEX) + private final String name; - @NotEmpty - @Pattern(regexp = ALLOWED_NAME_REGEX) - private final String groupName; + @JsonCreator + public TopicName(@JsonProperty("groupName") String groupName, @JsonProperty("name") String name) { + this.groupName = groupName; + this.name = name; + } - @NotEmpty - @Pattern(regexp = ALLOWED_NAME_REGEX) - private final String name; + public static String toQualifiedName(TopicName topicName) { + return topicName != null ? topicName.qualifiedName() : null; + } - @JsonCreator - public TopicName(@JsonProperty("groupName") String groupName, @JsonProperty("name") String name) { - this.groupName = groupName; - this.name = name; + public static TopicName fromQualifiedName(String qualifiedName) { + if (Strings.isNullOrEmpty(qualifiedName)) { + return null; } - public static String toQualifiedName(TopicName topicName) { - return topicName != null ? topicName.qualifiedName() : null; + int index = qualifiedName.lastIndexOf(GROUP_SEPARATOR); + if (index == -1) { + throw new IllegalArgumentException("Invalid qualified name " + qualifiedName); } - public static TopicName fromQualifiedName(String qualifiedName) { - if (Strings.isNullOrEmpty(qualifiedName)) { - return null; - } + String groupName = qualifiedName.substring(0, index); + String topicName = qualifiedName.substring(index + 1, qualifiedName.length()); + return new TopicName(groupName, topicName); + } - int index = qualifiedName.lastIndexOf(GROUP_SEPARATOR); - if (index == -1) { - throw new IllegalArgumentException("Invalid qualified name " + qualifiedName); - } + public String getGroupName() { + return groupName; + } - String groupName = qualifiedName.substring(0, index); - String topicName = qualifiedName.substring(index + 1, qualifiedName.length()); - return new TopicName(groupName, topicName); - } + public String getName() { + return name; + } - public String getGroupName() { - return groupName; - } + public String qualifiedName() { + return groupName + GROUP_SEPARATOR + name; + } - public String getName() { - return name; - } + @Override + public String toString() { + return qualifiedName(); + } - public String qualifiedName() { - return groupName + GROUP_SEPARATOR + name; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public String toString() { - return qualifiedName(); + if (o == null || getClass() != o.getClass()) { + return false; } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } + TopicName that = (TopicName) o; - TopicName that = (TopicName) o; + return Objects.equals(this.name, that.name) && Objects.equals(this.groupName, that.groupName); + } - return Objects.equals(this.name, that.name) - && Objects.equals(this.groupName, that.groupName); - } - - @Override - public int hashCode() { - return Objects.hash(name, groupName); - } + @Override + public int hashCode() { + return Objects.hash(name, groupName); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicNameWithMetrics.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicNameWithMetrics.java index 3d4d2c9368..8f36fb7149 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicNameWithMetrics.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicNameWithMetrics.java @@ -2,102 +2,100 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.Objects; public class TopicNameWithMetrics { - private final long published; - private final MetricDecimalValue rate; - private final MetricDecimalValue deliveryRate; - private final int subscriptions; - private final MetricDecimalValue throughput; - private final long volume; - - private final TopicName topicName; - - @JsonCreator - public TopicNameWithMetrics( - @JsonProperty("published") long published, - @JsonProperty("rate") MetricDecimalValue rate, - @JsonProperty("deliveryRate") MetricDecimalValue deliveryRate, - @JsonProperty("subscriptions") int subscriptions, - @JsonProperty("throughput") MetricDecimalValue throughput, - @JsonProperty("volume") long volume, - @JsonProperty("name") String qualifiedName - ) { - this.published = published; - this.rate = rate; - this.deliveryRate = deliveryRate; - this.subscriptions = subscriptions; - this.throughput = throughput; - this.volume = volume; - this.topicName = TopicName.fromQualifiedName(qualifiedName); - } - - public static TopicNameWithMetrics from(TopicMetrics metrics, String qualifiedName) { - return new TopicNameWithMetrics( - metrics.getPublished(), - metrics.getRate(), - metrics.getDeliveryRate(), - metrics.getSubscriptions(), - metrics.getThroughput(), - metrics.getVolume(), - qualifiedName - ); - } - - public long getPublished() { - return published; + private final long published; + private final MetricDecimalValue rate; + private final MetricDecimalValue deliveryRate; + private final int subscriptions; + private final MetricDecimalValue throughput; + private final long volume; + + private final TopicName topicName; + + @JsonCreator + public TopicNameWithMetrics( + @JsonProperty("published") long published, + @JsonProperty("rate") MetricDecimalValue rate, + @JsonProperty("deliveryRate") MetricDecimalValue deliveryRate, + @JsonProperty("subscriptions") int subscriptions, + @JsonProperty("throughput") MetricDecimalValue throughput, + @JsonProperty("volume") long volume, + @JsonProperty("name") String qualifiedName) { + this.published = published; + this.rate = rate; + this.deliveryRate = deliveryRate; + this.subscriptions = subscriptions; + this.throughput = throughput; + this.volume = volume; + this.topicName = TopicName.fromQualifiedName(qualifiedName); + } + + public static TopicNameWithMetrics from(TopicMetrics metrics, String qualifiedName) { + return new TopicNameWithMetrics( + metrics.getPublished(), + metrics.getRate(), + metrics.getDeliveryRate(), + metrics.getSubscriptions(), + metrics.getThroughput(), + metrics.getVolume(), + qualifiedName); + } + + public long getPublished() { + return published; + } + + public long getVolume() { + return volume; + } + + public MetricDecimalValue getRate() { + return rate; + } + + public MetricDecimalValue getDeliveryRate() { + return deliveryRate; + } + + public int getSubscriptions() { + return subscriptions; + } + + public MetricDecimalValue getThroughput() { + return throughput; + } + + @JsonProperty("name") + public String getName() { + return topicName.qualifiedName(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - public long getVolume() { - return volume; - } - - public MetricDecimalValue getRate() { - return rate; - } - - public MetricDecimalValue getDeliveryRate() { - return deliveryRate; + if (o == null || getClass() != o.getClass()) { + return false; } - public int getSubscriptions() { - return subscriptions; - } - - public MetricDecimalValue getThroughput() { - return throughput; - } - - @JsonProperty("name") - public String getName() { - return topicName.qualifiedName(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - TopicNameWithMetrics that = (TopicNameWithMetrics) o; - - return Objects.equals(this.published, that.published) - && Objects.equals(this.rate, that.rate) - && Objects.equals(this.deliveryRate, that.deliveryRate) - && Objects.equals(this.subscriptions, that.subscriptions) - && Objects.equals(this.throughput, that.throughput) - && Objects.equals(this.volume, that.volume) - && Objects.equals(this.topicName, that.topicName); - } - - @Override - public int hashCode() { - return Objects.hash(published, rate, deliveryRate, subscriptions, throughput, topicName, volume); - } + TopicNameWithMetrics that = (TopicNameWithMetrics) o; + + return Objects.equals(this.published, that.published) + && Objects.equals(this.rate, that.rate) + && Objects.equals(this.deliveryRate, that.deliveryRate) + && Objects.equals(this.subscriptions, that.subscriptions) + && Objects.equals(this.throughput, that.throughput) + && Objects.equals(this.volume, that.volume) + && Objects.equals(this.topicName, that.topicName); + } + + @Override + public int hashCode() { + return Objects.hash( + published, rate, deliveryRate, subscriptions, throughput, topicName, volume); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicPartition.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicPartition.java index 760a935e1b..c664068e87 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicPartition.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicPartition.java @@ -2,87 +2,88 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.Objects; public class TopicPartition { - private final int partition; - private final String topic; - private final long currentOffset; - private final long logEndOffset; - private final long lag; - private final String offsetMetadata; - private final ContentType contentType; + private final int partition; + private final String topic; + private final long currentOffset; + private final long logEndOffset; + private final long lag; + private final String offsetMetadata; + private final ContentType contentType; - @JsonCreator - public TopicPartition(@JsonProperty("partition") int partition, - @JsonProperty("topic") String topic, - @JsonProperty("currentOffset") long currentOffset, - @JsonProperty("logEndOffset") long logEndOffset, - @JsonProperty("offsetMetadata") String offsetMetadata, - @JsonProperty("contentType") ContentType contentType) { - this.partition = partition; - this.topic = topic; - this.currentOffset = currentOffset; - this.logEndOffset = logEndOffset; - this.offsetMetadata = offsetMetadata; - this.contentType = contentType; + @JsonCreator + public TopicPartition( + @JsonProperty("partition") int partition, + @JsonProperty("topic") String topic, + @JsonProperty("currentOffset") long currentOffset, + @JsonProperty("logEndOffset") long logEndOffset, + @JsonProperty("offsetMetadata") String offsetMetadata, + @JsonProperty("contentType") ContentType contentType) { + this.partition = partition; + this.topic = topic; + this.currentOffset = currentOffset; + this.logEndOffset = logEndOffset; + this.offsetMetadata = offsetMetadata; + this.contentType = contentType; - this.lag = calculateLag(currentOffset, logEndOffset); - } + this.lag = calculateLag(currentOffset, logEndOffset); + } - private long calculateLag(long currentOffset, long logEndOffset) { - return logEndOffset - currentOffset; - } + private long calculateLag(long currentOffset, long logEndOffset) { + return logEndOffset - currentOffset; + } - public int getPartition() { - return partition; - } + public int getPartition() { + return partition; + } - public String getTopic() { - return topic; - } + public String getTopic() { + return topic; + } - public long getCurrentOffset() { - return currentOffset; - } + public long getCurrentOffset() { + return currentOffset; + } - public String getOffsetMetadata() { - return offsetMetadata; - } + public String getOffsetMetadata() { + return offsetMetadata; + } - public long getLogEndOffset() { - return logEndOffset; - } + public long getLogEndOffset() { + return logEndOffset; + } - public long getLag() { - return lag; - } + public long getLag() { + return lag; + } - public ContentType getContentType() { - return contentType; - } + public ContentType getContentType() { + return contentType; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - TopicPartition that = (TopicPartition) o; - return partition == that.partition - && currentOffset == that.currentOffset - && logEndOffset == that.logEndOffset - && contentType == that.contentType - && lag == that.lag - && Objects.equals(topic, that.topic) - && Objects.equals(offsetMetadata, that.offsetMetadata); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(partition, topic, currentOffset, logEndOffset, contentType, lag, offsetMetadata); + if (o == null || getClass() != o.getClass()) { + return false; } + TopicPartition that = (TopicPartition) o; + return partition == that.partition + && currentOffset == that.currentOffset + && logEndOffset == that.logEndOffset + && contentType == that.contentType + && lag == that.lag + && Objects.equals(topic, that.topic) + && Objects.equals(offsetMetadata, that.offsetMetadata); + } + + @Override + public int hashCode() { + return Objects.hash( + partition, topic, currentOffset, logEndOffset, contentType, lag, offsetMetadata); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicStats.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicStats.java index 9571b8a214..741c07cb69 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicStats.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicStats.java @@ -2,69 +2,73 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.Objects; public class TopicStats { - private final long topicCount; - private final long ackAllTopicCount; - private final long trackingEnabledTopicCount; - private final long avroTopicCount; + private final long topicCount; + private final long ackAllTopicCount; + private final long trackingEnabledTopicCount; + private final long avroTopicCount; - @JsonCreator - public TopicStats(@JsonProperty("topicCount") long topicCount, - @JsonProperty("ackAllTopicCount") long ackAllTopicCount, - @JsonProperty("trackingEnabledTopicCount") long trackingEnabledTopicCount, - @JsonProperty("avroTopicCount") long avroTopicCount) { - this.topicCount = topicCount; - this.ackAllTopicCount = ackAllTopicCount; - this.trackingEnabledTopicCount = trackingEnabledTopicCount; - this.avroTopicCount = avroTopicCount; - } + @JsonCreator + public TopicStats( + @JsonProperty("topicCount") long topicCount, + @JsonProperty("ackAllTopicCount") long ackAllTopicCount, + @JsonProperty("trackingEnabledTopicCount") long trackingEnabledTopicCount, + @JsonProperty("avroTopicCount") long avroTopicCount) { + this.topicCount = topicCount; + this.ackAllTopicCount = ackAllTopicCount; + this.trackingEnabledTopicCount = trackingEnabledTopicCount; + this.avroTopicCount = avroTopicCount; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - TopicStats that = (TopicStats) o; - return topicCount == that.topicCount - && ackAllTopicCount == that.ackAllTopicCount - && trackingEnabledTopicCount == that.trackingEnabledTopicCount - && avroTopicCount == that.avroTopicCount; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(topicCount, ackAllTopicCount, trackingEnabledTopicCount, avroTopicCount); + if (o == null || getClass() != o.getClass()) { + return false; } + TopicStats that = (TopicStats) o; + return topicCount == that.topicCount + && ackAllTopicCount == that.ackAllTopicCount + && trackingEnabledTopicCount == that.trackingEnabledTopicCount + && avroTopicCount == that.avroTopicCount; + } - @Override - public String toString() { - return "TopicStats{" - + "topicCount=" + topicCount - + ", ackAllTopicCount=" + ackAllTopicCount - + ", trackingEnabledTopicCount=" + trackingEnabledTopicCount - + ", avroTopicCount=" + avroTopicCount - + '}'; - } + @Override + public int hashCode() { + return Objects.hash(topicCount, ackAllTopicCount, trackingEnabledTopicCount, avroTopicCount); + } - public long getTopicCount() { - return topicCount; - } + @Override + public String toString() { + return "TopicStats{" + + "topicCount=" + + topicCount + + ", ackAllTopicCount=" + + ackAllTopicCount + + ", trackingEnabledTopicCount=" + + trackingEnabledTopicCount + + ", avroTopicCount=" + + avroTopicCount + + '}'; + } - public long getAckAllTopicCount() { - return ackAllTopicCount; - } + public long getTopicCount() { + return topicCount; + } - public long getTrackingEnabledTopicCount() { - return trackingEnabledTopicCount; - } + public long getAckAllTopicCount() { + return ackAllTopicCount; + } - public long getAvroTopicCount() { - return avroTopicCount; - } + public long getTrackingEnabledTopicCount() { + return trackingEnabledTopicCount; + } + + public long getAvroTopicCount() { + return avroTopicCount; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicWithSchema.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicWithSchema.java index 2719cc0e5f..7294e4e049 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicWithSchema.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TopicWithSchema.java @@ -5,101 +5,151 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.OptBoolean; - import java.time.Instant; import java.util.Objects; import java.util.Set; public class TopicWithSchema extends Topic { - private final Topic topic; + private final Topic topic; - private String schema; + private String schema; - public TopicWithSchema(Topic topic, String schema) { - this(schema, topic.getQualifiedName(), topic.getDescription(), topic.getOwner(), topic.getRetentionTime(), - topic.isJsonToAvroDryRunEnabled(), topic.getAck(), topic.isFallbackToRemoteDatacenterEnabled(), - topic.getChaos(), topic.isTrackingEnabled(), topic.wasMigratedFromJsonType(), topic.isSchemaIdAwareSerializationEnabled(), - topic.getContentType(), topic.getMaxMessageSize(), topic.getPublishingAuth(), topic.isSubscribingRestricted(), - topic.getOfflineStorage(), topic.getLabels(), topic.getCreatedAt(), topic.getModifiedAt()); - } + public TopicWithSchema(Topic topic, String schema) { + this( + schema, + topic.getQualifiedName(), + topic.getDescription(), + topic.getOwner(), + topic.getRetentionTime(), + topic.isJsonToAvroDryRunEnabled(), + topic.getAck(), + topic.isFallbackToRemoteDatacenterEnabled(), + topic.getChaos(), + topic.isTrackingEnabled(), + topic.wasMigratedFromJsonType(), + topic.isSchemaIdAwareSerializationEnabled(), + topic.getContentType(), + topic.getMaxMessageSize(), + topic.getPublishingAuth(), + topic.isSubscribingRestricted(), + topic.getOfflineStorage(), + topic.getLabels(), + topic.getCreatedAt(), + topic.getModifiedAt()); + } - @JsonCreator - public TopicWithSchema(@JsonProperty("schema") String schema, - @JsonProperty("name") String qualifiedName, - @JsonProperty("description") String description, - @JsonProperty("owner") OwnerId owner, - @JsonProperty("retentionTime") RetentionTime retentionTime, - @JsonProperty("jsonToAvroDryRun") boolean jsonToAvroDryRunEnabled, - @JsonProperty("ack") Ack ack, - @JsonProperty("fallbackToRemoteDatacenterEnabled") - @JacksonInject(value = DEFAULT_FALLBACK_TO_REMOTE_DATACENTER_KEY, useInput = OptBoolean.TRUE) - boolean fallbackToRemoteDatacenterEnabled, - @JsonProperty("chaos") PublishingChaosPolicy chaos, - @JsonProperty("trackingEnabled") boolean trackingEnabled, - @JsonProperty("migratedFromJsonType") boolean migratedFromJsonType, - @JsonProperty("schemaIdAwareSerializationEnabled") - @JacksonInject(value = Topic.DEFAULT_SCHEMA_ID_SERIALIZATION_ENABLED_KEY, useInput = OptBoolean.TRUE) - Boolean schemaIdAwareSerializationEnabled, - @JsonProperty("contentType") ContentType contentType, - @JsonProperty("maxMessageSize") Integer maxMessageSize, - @JsonProperty("auth") PublishingAuth publishingAuth, - @JsonProperty("subscribingRestricted") boolean subscribingRestricted, - @JsonProperty("offlineStorage") TopicDataOfflineStorage offlineStorage, - @JsonProperty("labels") Set labels, - @JsonProperty("createdAt") Instant createdAt, - @JsonProperty("modifiedAt") Instant modifiedAt) { - super(qualifiedName, description, owner, retentionTime, jsonToAvroDryRunEnabled, ack, - fallbackToRemoteDatacenterEnabled, chaos, trackingEnabled, migratedFromJsonType, schemaIdAwareSerializationEnabled, - contentType, maxMessageSize, publishingAuth, subscribingRestricted, offlineStorage, labels, createdAt, - modifiedAt); - this.topic = convertToTopic(); - this.schema = schema; - } + @JsonCreator + public TopicWithSchema( + @JsonProperty("schema") String schema, + @JsonProperty("name") String qualifiedName, + @JsonProperty("description") String description, + @JsonProperty("owner") OwnerId owner, + @JsonProperty("retentionTime") RetentionTime retentionTime, + @JsonProperty("jsonToAvroDryRun") boolean jsonToAvroDryRunEnabled, + @JsonProperty("ack") Ack ack, + @JsonProperty("fallbackToRemoteDatacenterEnabled") + @JacksonInject( + value = DEFAULT_FALLBACK_TO_REMOTE_DATACENTER_KEY, + useInput = OptBoolean.TRUE) + boolean fallbackToRemoteDatacenterEnabled, + @JsonProperty("chaos") PublishingChaosPolicy chaos, + @JsonProperty("trackingEnabled") boolean trackingEnabled, + @JsonProperty("migratedFromJsonType") boolean migratedFromJsonType, + @JsonProperty("schemaIdAwareSerializationEnabled") + @JacksonInject( + value = Topic.DEFAULT_SCHEMA_ID_SERIALIZATION_ENABLED_KEY, + useInput = OptBoolean.TRUE) + Boolean schemaIdAwareSerializationEnabled, + @JsonProperty("contentType") ContentType contentType, + @JsonProperty("maxMessageSize") Integer maxMessageSize, + @JsonProperty("auth") PublishingAuth publishingAuth, + @JsonProperty("subscribingRestricted") boolean subscribingRestricted, + @JsonProperty("offlineStorage") TopicDataOfflineStorage offlineStorage, + @JsonProperty("labels") Set labels, + @JsonProperty("createdAt") Instant createdAt, + @JsonProperty("modifiedAt") Instant modifiedAt) { + super( + qualifiedName, + description, + owner, + retentionTime, + jsonToAvroDryRunEnabled, + ack, + fallbackToRemoteDatacenterEnabled, + chaos, + trackingEnabled, + migratedFromJsonType, + schemaIdAwareSerializationEnabled, + contentType, + maxMessageSize, + publishingAuth, + subscribingRestricted, + offlineStorage, + labels, + createdAt, + modifiedAt); + this.topic = convertToTopic(); + this.schema = schema; + } - public static TopicWithSchema topicWithSchema(Topic topic, String schema) { - return new TopicWithSchema(topic, schema); - } + public static TopicWithSchema topicWithSchema(Topic topic, String schema) { + return new TopicWithSchema(topic, schema); + } - public static TopicWithSchema topicWithSchema(Topic topic) { - return new TopicWithSchema(topic, null); - } + public static TopicWithSchema topicWithSchema(Topic topic) { + return new TopicWithSchema(topic, null); + } - private Topic convertToTopic() { - return new Topic(this.getQualifiedName(), this.getDescription(), this.getOwner(), this.getRetentionTime(), - this.isJsonToAvroDryRunEnabled(), this.getAck(), this.isFallbackToRemoteDatacenterEnabled(), this.getChaos(), - this.isTrackingEnabled(), this.wasMigratedFromJsonType(), this.isSchemaIdAwareSerializationEnabled(), - this.getContentType(), this.getMaxMessageSize(), this.getPublishingAuth(), this.isSubscribingRestricted(), - this.getOfflineStorage(), this.getLabels(), this.getCreatedAt(), this.getModifiedAt()); - } + private Topic convertToTopic() { + return new Topic( + this.getQualifiedName(), + this.getDescription(), + this.getOwner(), + this.getRetentionTime(), + this.isJsonToAvroDryRunEnabled(), + this.getAck(), + this.isFallbackToRemoteDatacenterEnabled(), + this.getChaos(), + this.isTrackingEnabled(), + this.wasMigratedFromJsonType(), + this.isSchemaIdAwareSerializationEnabled(), + this.getContentType(), + this.getMaxMessageSize(), + this.getPublishingAuth(), + this.isSubscribingRestricted(), + this.getOfflineStorage(), + this.getLabels(), + this.getCreatedAt(), + this.getModifiedAt()); + } - public String getSchema() { - return schema; - } + public String getSchema() { + return schema; + } - @JsonIgnore - public Topic getTopic() { - return topic; - } + @JsonIgnore + public Topic getTopic() { + return topic; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - if (!super.equals(o)) { - return false; - } - TopicWithSchema that = (TopicWithSchema) o; - return Objects.equals(topic, that.topic) - && Objects.equals(schema, that.schema); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), topic, schema); + if (o == null || getClass() != o.getClass()) { + return false; } + if (!super.equals(o)) { + return false; + } + TopicWithSchema that = (TopicWithSchema) o; + return Objects.equals(topic, that.topic) && Objects.equals(schema, that.schema); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), topic, schema); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TrackingMode.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TrackingMode.java index d55d2041d3..650dd24540 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TrackingMode.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/TrackingMode.java @@ -3,40 +3,39 @@ import java.util.Optional; public enum TrackingMode { + TRACK_ALL("trackingAll"), + TRACK_DISCARDED_ONLY("discardedOnly"), + TRACKING_OFF("trackingOff"); - TRACK_ALL("trackingAll"), - TRACK_DISCARDED_ONLY("discardedOnly"), - TRACKING_OFF("trackingOff"); + private String value; - private String value; + TrackingMode(String s) { + this.value = s; + } - TrackingMode(String s) { - this.value = s; - } + public static Optional fromString(String trackingMode) { - public static Optional fromString(String trackingMode) { - - if (trackingMode == null) { - return Optional.empty(); - } - - switch (trackingMode) { - case "trackingAll": - return Optional.of(TRACK_ALL); - case "discardedOnly": - return Optional.of(TRACK_DISCARDED_ONLY); - case "trackingOff": - default: - return Optional.of(TRACKING_OFF); - } + if (trackingMode == null) { + return Optional.empty(); } - public String getValue() { - return value; + switch (trackingMode) { + case "trackingAll": + return Optional.of(TRACK_ALL); + case "discardedOnly": + return Optional.of(TRACK_DISCARDED_ONLY); + case "trackingOff": + default: + return Optional.of(TRACKING_OFF); } + } - @Override - public String toString() { - return this.value; - } + public String getValue() { + return value; + } + + @Override + public String toString() { + return this.value; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/UnhealthySubscription.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/UnhealthySubscription.java index ce27767c37..a265585404 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/UnhealthySubscription.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/UnhealthySubscription.java @@ -2,84 +2,90 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.Objects; import java.util.Set; public class UnhealthySubscription { - private final String name; - private final String qualifiedTopicName; - private final MonitoringDetails.Severity severity; - private final Set problems; + private final String name; + private final String qualifiedTopicName; + private final MonitoringDetails.Severity severity; + private final Set problems; - @JsonCreator - public UnhealthySubscription( - @JsonProperty("name") String name, - @JsonProperty("topicName") String qualifiedTopicName, - @JsonProperty("severity") MonitoringDetails.Severity severity, - @JsonProperty("problems") Set problems) { - this.name = name; - this.qualifiedTopicName = qualifiedTopicName; - this.severity = severity; - this.problems = problems; - } + @JsonCreator + public UnhealthySubscription( + @JsonProperty("name") String name, + @JsonProperty("topicName") String qualifiedTopicName, + @JsonProperty("severity") MonitoringDetails.Severity severity, + @JsonProperty("problems") Set problems) { + this.name = name; + this.qualifiedTopicName = qualifiedTopicName; + this.severity = severity; + this.problems = problems; + } - public static UnhealthySubscription from(Subscription subscription, SubscriptionHealth subscriptionHealth) { - return new UnhealthySubscription( - subscription.getName(), - subscription.getQualifiedTopicName(), - subscription.getMonitoringDetails().getSeverity(), - subscriptionHealth.getProblems()); - } + public static UnhealthySubscription from( + Subscription subscription, SubscriptionHealth subscriptionHealth) { + return new UnhealthySubscription( + subscription.getName(), + subscription.getQualifiedTopicName(), + subscription.getMonitoringDetails().getSeverity(), + subscriptionHealth.getProblems()); + } - @JsonProperty("name") - public String getName() { - return name; - } + @JsonProperty("name") + public String getName() { + return name; + } - @JsonProperty("topicName") - public String getQualifiedTopicName() { - return qualifiedTopicName; - } + @JsonProperty("topicName") + public String getQualifiedTopicName() { + return qualifiedTopicName; + } - @JsonProperty("severity") - public MonitoringDetails.Severity getSeverity() { - return severity; - } + @JsonProperty("severity") + public MonitoringDetails.Severity getSeverity() { + return severity; + } - @JsonProperty("problems") - public Set getProblems() { - return problems; - } + @JsonProperty("problems") + public Set getProblems() { + return problems; + } - @Override - public String toString() { - return "UnhealthySubscription{" - + "name='" + name + '\'' - + ", qualifiedTopicName='" + qualifiedTopicName + '\'' - + ", severity=" + severity - + ", problems=" + problems - + '}'; - } + @Override + public String toString() { + return "UnhealthySubscription{" + + "name='" + + name + + '\'' + + ", qualifiedTopicName='" + + qualifiedTopicName + + '\'' + + ", severity=" + + severity + + ", problems=" + + problems + + '}'; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - UnhealthySubscription that = (UnhealthySubscription) o; - return Objects.equals(name, that.name) - && Objects.equals(qualifiedTopicName, that.qualifiedTopicName) - && severity == that.severity - && Objects.equals(problems, that.problems); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(name, qualifiedTopicName, severity, problems); + if (o == null || getClass() != o.getClass()) { + return false; } + UnhealthySubscription that = (UnhealthySubscription) o; + return Objects.equals(name, that.name) + && Objects.equals(qualifiedTopicName, that.qualifiedTopicName) + && severity == that.severity + && Objects.equals(problems, that.problems); + } + + @Override + public int hashCode() { + return Objects.hash(name, qualifiedTopicName, severity, problems); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/constraints/AdminPermitted.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/constraints/AdminPermitted.java index 9fe56ddffc..41e36084ee 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/constraints/AdminPermitted.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/constraints/AdminPermitted.java @@ -1,4 +1,3 @@ package pl.allegro.tech.hermes.api.constraints; -public interface AdminPermitted { -} +public interface AdminPermitted {} diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/constraints/ContentTypeValidator.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/constraints/ContentTypeValidator.java index 1874fbff9c..3f90dfffad 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/constraints/ContentTypeValidator.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/constraints/ContentTypeValidator.java @@ -8,13 +8,12 @@ public class ContentTypeValidator implements ConstraintValidator { - @Override - public void initialize(ValidContentType constraintAnnotation) { + @Override + public void initialize(ValidContentType constraintAnnotation) {} - } - - @Override - public boolean isValid(Subscription subscription, ConstraintValidatorContext context) { - return !(subscription.getDeliveryType() == DeliveryType.BATCH && subscription.getContentType() == ContentType.AVRO); - } + @Override + public boolean isValid(Subscription subscription, ConstraintValidatorContext context) { + return !(subscription.getDeliveryType() == DeliveryType.BATCH + && subscription.getContentType() == ContentType.AVRO); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/constraints/Names.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/constraints/Names.java index 5214757d71..582487d271 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/constraints/Names.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/constraints/Names.java @@ -1,5 +1,5 @@ package pl.allegro.tech.hermes.api.constraints; public final class Names { - public static final String ALLOWED_NAME_REGEX = "[a-zA-Z0-9_.-]+"; + public static final String ALLOWED_NAME_REGEX = "[a-zA-Z0-9_.-]+"; } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/constraints/OneSourceRetransmission.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/constraints/OneSourceRetransmission.java index 9c920a322b..4c7fda0a5d 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/constraints/OneSourceRetransmission.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/constraints/OneSourceRetransmission.java @@ -1,22 +1,22 @@ package pl.allegro.tech.hermes.api.constraints; -import jakarta.validation.Constraint; +import static java.lang.annotation.ElementType.TYPE; +import jakarta.validation.Constraint; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; -import static java.lang.annotation.ElementType.TYPE; - @Documented @Retention(RetentionPolicy.RUNTIME) @Target({TYPE}) @Constraint(validatedBy = OneSourceRetransmissionValidator.class) public @interface OneSourceRetransmission { - String message() default "must contain one defined source of retransmission data - source topic or source view"; + String message() default + "must contain one defined source of retransmission data - source topic or source view"; - Class[] groups() default {}; + Class[] groups() default {}; - Class[] payload() default {}; + Class[] payload() default {}; } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/constraints/OneSourceRetransmissionValidator.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/constraints/OneSourceRetransmissionValidator.java index a311b4b263..f0a89a888c 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/constraints/OneSourceRetransmissionValidator.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/constraints/OneSourceRetransmissionValidator.java @@ -4,21 +4,23 @@ import jakarta.validation.ConstraintValidatorContext; import pl.allegro.tech.hermes.api.OfflineRetransmissionRequest; -public class OneSourceRetransmissionValidator implements ConstraintValidator { +public class OneSourceRetransmissionValidator + implements ConstraintValidator { - public static final String EMPTY_STRING = ""; + public static final String EMPTY_STRING = ""; - @Override - public boolean isValid(OfflineRetransmissionRequest offlineRetransmissionRequest, ConstraintValidatorContext context) { - var sourceViewPath = offlineRetransmissionRequest.getSourceViewPath(); - var sourceTopic = offlineRetransmissionRequest.getSourceTopic(); + @Override + public boolean isValid( + OfflineRetransmissionRequest offlineRetransmissionRequest, + ConstraintValidatorContext context) { + var sourceViewPath = offlineRetransmissionRequest.getSourceViewPath(); + var sourceTopic = offlineRetransmissionRequest.getSourceTopic(); - return (nonBlank(sourceViewPath.orElse(EMPTY_STRING)) && sourceTopic.isEmpty()) - || (nonBlank(sourceTopic.orElse(EMPTY_STRING)) && sourceViewPath.isEmpty()); - } - - private static boolean nonBlank(String value) { - return value != null && !value.isBlank(); - } + return (nonBlank(sourceViewPath.orElse(EMPTY_STRING)) && sourceTopic.isEmpty()) + || (nonBlank(sourceTopic.orElse(EMPTY_STRING)) && sourceViewPath.isEmpty()); + } + private static boolean nonBlank(String value) { + return value != null && !value.isBlank(); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/constraints/ValidContentType.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/constraints/ValidContentType.java index 7c4785c578..30344e8903 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/constraints/ValidContentType.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/constraints/ValidContentType.java @@ -1,22 +1,21 @@ package pl.allegro.tech.hermes.api.constraints; -import jakarta.validation.Constraint; +import static java.lang.annotation.ElementType.TYPE; +import jakarta.validation.Constraint; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; -import static java.lang.annotation.ElementType.TYPE; - @Documented @Retention(RetentionPolicy.RUNTIME) @Target({TYPE}) @Constraint(validatedBy = ContentTypeValidator.class) public @interface ValidContentType { - String message(); + String message(); - Class[] groups() default {}; + Class[] groups() default {}; - Class[] payload() default {}; + Class[] payload() default {}; } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/helpers/Patch.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/helpers/Patch.java index 92b9363cf5..91c57594e8 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/helpers/Patch.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/helpers/Patch.java @@ -1,45 +1,44 @@ package pl.allegro.tech.hermes.api.helpers; +import static com.google.common.base.Preconditions.checkNotNull; + import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import pl.allegro.tech.hermes.api.PatchData; - import java.util.HashMap; import java.util.Map; - -import static com.google.common.base.Preconditions.checkNotNull; +import pl.allegro.tech.hermes.api.PatchData; public class Patch { - private static final ObjectMapper MAPPER = new ObjectMapper() - .setSerializationInclusion(JsonInclude.Include.NON_NULL) - .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) - .disable(SerializationFeature.WRITE_NULL_MAP_VALUES) - .registerModule(new JavaTimeModule()); - - @SuppressWarnings("unchecked") - public static T apply(T object, PatchData patch) { - checkNotNull(object); - checkNotNull(patch); - Map objectMap = MAPPER.convertValue(object, Map.class); - return (T) MAPPER.convertValue(merge(objectMap, patch.getPatch()), object.getClass()); + private static final ObjectMapper MAPPER = + new ObjectMapper() + .setSerializationInclusion(JsonInclude.Include.NON_NULL) + .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) + .disable(SerializationFeature.WRITE_NULL_MAP_VALUES) + .registerModule(new JavaTimeModule()); + + @SuppressWarnings("unchecked") + public static T apply(T object, PatchData patch) { + checkNotNull(object); + checkNotNull(patch); + Map objectMap = MAPPER.convertValue(object, Map.class); + return (T) MAPPER.convertValue(merge(objectMap, patch.getPatch()), object.getClass()); + } + + @SuppressWarnings("unchecked") + private static Map merge(Map left, Map right) { + Map merged = new HashMap<>(left); + for (Map.Entry entry : right.entrySet()) { + if (entry.getValue() instanceof Map && merged.containsKey(entry.getKey())) { + Map nested = (Map) merged.get(entry.getKey()); + nested.putAll(merge(nested, (Map) entry.getValue())); + } else { + merged.put(entry.getKey(), entry.getValue()); + } } - - @SuppressWarnings("unchecked") - private static Map merge(Map left, Map right) { - Map merged = new HashMap<>(left); - for (Map.Entry entry : right.entrySet()) { - if (entry.getValue() instanceof Map && merged.containsKey(entry.getKey())) { - Map nested = (Map) merged.get(entry.getKey()); - nested.putAll(merge(nested, (Map) entry.getValue())); - } else { - merged.put(entry.getKey(), entry.getValue()); - } - } - return merged; - } - + return merged; + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/helpers/Replacer.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/helpers/Replacer.java index 099b0e1466..48365bdb07 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/helpers/Replacer.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/helpers/Replacer.java @@ -6,7 +6,9 @@ public final class Replacer { - public static List replaceInAll(String regex, String replacement, String... strings) { - return Arrays.stream(strings).map(s -> s.replaceAll(regex, replacement)).collect(Collectors.toList()); - } + public static List replaceInAll(String regex, String replacement, String... strings) { + return Arrays.stream(strings) + .map(s -> s.replaceAll(regex, replacement)) + .collect(Collectors.toList()); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/jackson/EndpointAddressDeserializer.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/jackson/EndpointAddressDeserializer.java index 78e919cc2d..946c0ac6af 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/jackson/EndpointAddressDeserializer.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/jackson/EndpointAddressDeserializer.java @@ -5,15 +5,15 @@ import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.JsonNode; -import pl.allegro.tech.hermes.api.EndpointAddress; - import java.io.IOException; +import pl.allegro.tech.hermes.api.EndpointAddress; public class EndpointAddressDeserializer extends JsonDeserializer { - @Override - public EndpointAddress deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException, JsonProcessingException { - JsonNode node = jp.getCodec().readTree(jp); - return new EndpointAddress(node.asText()); - } + @Override + public EndpointAddress deserialize(JsonParser jp, DeserializationContext ctxt) + throws IOException, JsonProcessingException { + JsonNode node = jp.getCodec().readTree(jp); + return new EndpointAddress(node.asText()); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/jackson/EndpointAddressSerializer.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/jackson/EndpointAddressSerializer.java index dda407497e..ce8bf658cb 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/jackson/EndpointAddressSerializer.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/jackson/EndpointAddressSerializer.java @@ -3,13 +3,13 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.SerializerProvider; -import pl.allegro.tech.hermes.api.EndpointAddress; - import java.io.IOException; +import pl.allegro.tech.hermes.api.EndpointAddress; public class EndpointAddressSerializer extends JsonSerializer { - @Override - public void serialize(EndpointAddress value, JsonGenerator jgen, SerializerProvider provider) throws IOException { - jgen.writeString(value.getRawEndpoint()); - } + @Override + public void serialize(EndpointAddress value, JsonGenerator jgen, SerializerProvider provider) + throws IOException { + jgen.writeString(value.getRawEndpoint()); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/jackson/InstantIsoSerializer.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/jackson/InstantIsoSerializer.java index 3b8cd3f2c1..20bf9bcdce 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/jackson/InstantIsoSerializer.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/jackson/InstantIsoSerializer.java @@ -1,18 +1,18 @@ package pl.allegro.tech.hermes.api.jackson; +import static java.time.format.DateTimeFormatter.ISO_INSTANT; + import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.SerializerProvider; - import java.io.IOException; import java.time.Instant; -import static java.time.format.DateTimeFormatter.ISO_INSTANT; - public class InstantIsoSerializer extends JsonSerializer { - @Override - public void serialize(Instant value, JsonGenerator jgen, SerializerProvider provider) throws IOException { - jgen.writeString(ISO_INSTANT.format(value)); - } + @Override + public void serialize(Instant value, JsonGenerator jgen, SerializerProvider provider) + throws IOException { + jgen.writeString(ISO_INSTANT.format(value)); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/jackson/OffsetDateTimeSerializer.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/jackson/OffsetDateTimeSerializer.java index dcabc9b10b..d82a36e324 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/jackson/OffsetDateTimeSerializer.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/jackson/OffsetDateTimeSerializer.java @@ -1,18 +1,18 @@ package pl.allegro.tech.hermes.api.jackson; +import static java.time.format.DateTimeFormatter.ISO_OFFSET_DATE_TIME; + import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.SerializerProvider; - import java.io.IOException; import java.time.OffsetDateTime; -import static java.time.format.DateTimeFormatter.ISO_OFFSET_DATE_TIME; - public class OffsetDateTimeSerializer extends JsonSerializer { - @Override - public void serialize(OffsetDateTime value, JsonGenerator jgen, SerializerProvider provider) throws IOException { - jgen.writeString(ISO_OFFSET_DATE_TIME.format(value)); - } + @Override + public void serialize(OffsetDateTime value, JsonGenerator jgen, SerializerProvider provider) + throws IOException { + jgen.writeString(ISO_OFFSET_DATE_TIME.format(value)); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/jackson/PatchDataDeserializer.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/jackson/PatchDataDeserializer.java index e1775303f9..305b3cd76e 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/jackson/PatchDataDeserializer.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/jackson/PatchDataDeserializer.java @@ -4,16 +4,16 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; -import pl.allegro.tech.hermes.api.PatchData; - import java.io.IOException; import java.util.Map; +import pl.allegro.tech.hermes.api.PatchData; public class PatchDataDeserializer extends JsonDeserializer { - @Override - public PatchData deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { - Map patch = p.getCodec().readValue(p, new TypeReference>() {}); - return new PatchData(patch); - } + @Override + public PatchData deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { + Map patch = + p.getCodec().readValue(p, new TypeReference>() {}); + return new PatchData(patch); + } } diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/jackson/PatchDataSerializer.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/jackson/PatchDataSerializer.java index 38ca0100f3..3580f35d55 100644 --- a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/jackson/PatchDataSerializer.java +++ b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/jackson/PatchDataSerializer.java @@ -4,20 +4,20 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.SerializerProvider; -import pl.allegro.tech.hermes.api.PatchData; - import java.io.IOException; import java.util.Map; +import pl.allegro.tech.hermes.api.PatchData; public class PatchDataSerializer extends JsonSerializer { - @Override - public void serialize(PatchData value, JsonGenerator gen, SerializerProvider serializers) throws IOException, JsonProcessingException { - gen.writeStartObject(); + @Override + public void serialize(PatchData value, JsonGenerator gen, SerializerProvider serializers) + throws IOException, JsonProcessingException { + gen.writeStartObject(); - for (Map.Entry entry : value.getPatch().entrySet()) { - gen.writeObjectField(entry.getKey(), entry.getValue()); - } - - gen.writeEndObject(); + for (Map.Entry entry : value.getPatch().entrySet()) { + gen.writeObjectField(entry.getKey(), entry.getValue()); } + + gen.writeEndObject(); + } } diff --git a/hermes-api/src/test/java/pl/allegro/tech/hermes/api/SubscriptionTest.java b/hermes-api/src/test/java/pl/allegro/tech/hermes/api/SubscriptionTest.java index fc5030e018..a3392ace04 100644 --- a/hermes-api/src/test/java/pl/allegro/tech/hermes/api/SubscriptionTest.java +++ b/hermes-api/src/test/java/pl/allegro/tech/hermes/api/SubscriptionTest.java @@ -1,9 +1,5 @@ package pl.allegro.tech.hermes.api; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.Test; -import pl.allegro.tech.hermes.api.helpers.Patch; - import static org.assertj.core.api.Assertions.assertThat; import static pl.allegro.tech.hermes.api.PatchData.patchData; import static pl.allegro.tech.hermes.api.SubscriptionOAuthPolicy.GrantType.CLIENT_CREDENTIALS; @@ -11,109 +7,119 @@ import static pl.allegro.tech.hermes.api.SubscriptionPolicy.Builder.subscriptionPolicy; import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.Test; +import pl.allegro.tech.hermes.api.helpers.Patch; + public class SubscriptionTest { - private final ObjectMapper mapper = new ObjectMapper(); + private final ObjectMapper mapper = new ObjectMapper(); - @Test - public void shouldDeserializeSubscription() throws Exception { - // given - String json = "{" + @Test + public void shouldDeserializeSubscription() throws Exception { + // given + String json = + "{" + "\"name\": \"test\", " + "\"topicName\": \"g1.t1\", " + "\"endpoint\": \"http://localhost:8888\"" + "}"; - // when - Subscription subscription = mapper.readValue(json, Subscription.class); - - // then - assertThat(subscription.getName()).isEqualTo("test"); - assertThat(subscription.getEndpoint().getEndpoint()).isEqualTo("http://localhost:8888"); - } - - @Test - public void shouldDeserializeSubscriptionWithoutTopicName() throws Exception { - // given - String json = "{\"name\": \"test\", \"endpoint\": \"http://localhost:8888\"}"; - - // when - Subscription subscription = mapper.readValue(json, Subscription.class); - - // then - assertThat(subscription.getName()).isEqualTo("test"); - assertThat(subscription.getEndpoint().getEndpoint()).isEqualTo("http://localhost:8888"); - } - - @Test - public void shouldDeserializeSubscriptionWithoutBackoff() throws Exception { - // given - String json = "{\"name\": \"test\", \"endpoint\": \"http://localhost:8888\", \"subscriptionPolicy\": {\"messageTtl\": 100}}"; - - // when - Subscription subscription = mapper.readValue(json, Subscription.class); - - // then - assertThat(subscription.getSerialSubscriptionPolicy().getMessageBackoff()).isEqualTo(100); - } - - @Test - public void shouldDeserializeSubscriptionWithDefaultTracking() throws Exception { - // given - String json = "{" + // when + Subscription subscription = mapper.readValue(json, Subscription.class); + + // then + assertThat(subscription.getName()).isEqualTo("test"); + assertThat(subscription.getEndpoint().getEndpoint()).isEqualTo("http://localhost:8888"); + } + + @Test + public void shouldDeserializeSubscriptionWithoutTopicName() throws Exception { + // given + String json = "{\"name\": \"test\", \"endpoint\": \"http://localhost:8888\"}"; + + // when + Subscription subscription = mapper.readValue(json, Subscription.class); + + // then + assertThat(subscription.getName()).isEqualTo("test"); + assertThat(subscription.getEndpoint().getEndpoint()).isEqualTo("http://localhost:8888"); + } + + @Test + public void shouldDeserializeSubscriptionWithoutBackoff() throws Exception { + // given + String json = + "{\"name\": \"test\", \"endpoint\": \"http://localhost:8888\", \"subscriptionPolicy\": {\"messageTtl\": 100}}"; + + // when + Subscription subscription = mapper.readValue(json, Subscription.class); + + // then + assertThat(subscription.getSerialSubscriptionPolicy().getMessageBackoff()).isEqualTo(100); + } + + @Test + public void shouldDeserializeSubscriptionWithDefaultTracking() throws Exception { + // given + String json = + "{" + "\"name\": \"test\", " + "\"topicName\": \"g1.t1\", " + "\"endpoint\": \"http://localhost:8888\"" + "}"; - // when - Subscription subscription = mapper.readValue(json, Subscription.class); + // when + Subscription subscription = mapper.readValue(json, Subscription.class); - // then - assertThat(subscription.isTrackingEnabled()).isFalse(); - assertThat(subscription.getTrackingMode()).isEqualTo(TrackingMode.TRACKING_OFF); - } + // then + assertThat(subscription.isTrackingEnabled()).isFalse(); + assertThat(subscription.getTrackingMode()).isEqualTo(TrackingMode.TRACKING_OFF); + } - @Test - public void shouldDeserializeSubscriptionWithTrackAllMode() throws Exception { - // given - String json = "{" + @Test + public void shouldDeserializeSubscriptionWithTrackAllMode() throws Exception { + // given + String json = + "{" + "\"name\": \"test\", " + "\"topicName\": \"g1.t1\", " + "\"endpoint\": \"http://localhost:8888\", " + "\"trackingMode\": \"trackingAll\"" + "}"; - // when - Subscription subscription = mapper.readValue(json, Subscription.class); + // when + Subscription subscription = mapper.readValue(json, Subscription.class); - // then - assertThat(subscription.isTrackingEnabled()).isTrue(); - assertThat(subscription.getTrackingMode()).isEqualTo(TrackingMode.TRACK_ALL); - } + // then + assertThat(subscription.isTrackingEnabled()).isTrue(); + assertThat(subscription.getTrackingMode()).isEqualTo(TrackingMode.TRACK_ALL); + } - @Test - public void shouldDeserializeSubscriptionWithTrackEnabled() throws Exception { - // given - String json = "{" + @Test + public void shouldDeserializeSubscriptionWithTrackEnabled() throws Exception { + // given + String json = + "{" + "\"name\": \"test\", " + "\"topicName\": \"g1.t1\", " + "\"endpoint\": \"http://localhost:8888\", " + "\"trackingEnabled\": \"true\"" + "}"; - // when - Subscription subscription = mapper.readValue(json, Subscription.class); + // when + Subscription subscription = mapper.readValue(json, Subscription.class); - // then - assertThat(subscription.isTrackingEnabled()).isTrue(); - assertThat(subscription.getTrackingMode()).isEqualTo(TrackingMode.TRACK_ALL); - } + // then + assertThat(subscription.isTrackingEnabled()).isTrue(); + assertThat(subscription.getTrackingMode()).isEqualTo(TrackingMode.TRACK_ALL); + } - @Test - public void shouldDeserializeSubscriptionWithTrackEnabledAndTrackMode() throws Exception { - // given - String json = "{" + @Test + public void shouldDeserializeSubscriptionWithTrackEnabledAndTrackMode() throws Exception { + // given + String json = + "{" + "\"name\": \"test\", " + "\"topicName\": \"g1.t1\", " + "\"endpoint\": \"http://localhost:8888\", " @@ -121,70 +127,74 @@ public void shouldDeserializeSubscriptionWithTrackEnabledAndTrackMode() throws E + "\"trackingMode\": \"discardedOnly\"" + "}"; - // when - Subscription subscription = mapper.readValue(json, Subscription.class); - - // then - assertThat(subscription.isTrackingEnabled()).isTrue(); - assertThat(subscription.getTrackingMode()).isEqualTo(TrackingMode.TRACK_DISCARDED_ONLY); - } - - @Test - public void shouldApplyPatchToSubscriptionPolicy() { - //given - PatchData patch = patchData().set("rate", 8).build(); - - //when - SubscriptionPolicy subscription = subscriptionPolicy() - .withRate(1) - .applyPatch(patch).build(); - - //then - assertThat(subscription.getRate()).isEqualTo(8); - } - - @Test - public void shouldAnonymizePassword() { - // given - Subscription subscription = subscription("group.topic", "subscription").withEndpoint("http://user:password@service/path").build(); - - // when & then - assertThat(subscription.anonymize().getEndpoint()).isEqualTo(new EndpointAddress("http://user:*****@service/path")); - } - - @Test - public void shouldApplyPatchChangingSubscriptionOAuthPolicyGrantType() { - // given - Subscription subscription = subscription("group.topic", "subscription") - .withOAuthPolicy(new SubscriptionOAuthPolicy(CLIENT_CREDENTIALS, "myProvider", "repo", null, null)) - .build(); - PatchData oAuthPolicyPatchData = patchData() - .set("grantType", SubscriptionOAuthPolicy.GrantType.USERNAME_PASSWORD.getName()) - .set("username", "user1") - .set("password", "abc123") - .build(); - PatchData patch = patchData() - .set("oAuthPolicy", oAuthPolicyPatchData) - .build(); - - // when - Subscription updated = Patch.apply(subscription, patch); - - // then - SubscriptionOAuthPolicy updatedPolicy = updated.getOAuthPolicy(); - assertThat(updatedPolicy.getGrantType()).isEqualTo(USERNAME_PASSWORD); - assertThat(updatedPolicy.getUsername()).isEqualTo("user1"); - } - - @Test - public void shouldReadIntBackoffMultiplier() throws Exception { - // given - String json = "{\"name\": \"test\", \"endpoint\": \"http://localhost:8888\", \"subscriptionPolicy\": {\"messageBackoff\": 1000, \"backoffMultiplier\": 3}}"; - - // when - Subscription subscription = mapper.readValue(json, Subscription.class); - - // then - assertThat(subscription.getSerialSubscriptionPolicy().getBackoffMultiplier()).isEqualTo(3); - } + // when + Subscription subscription = mapper.readValue(json, Subscription.class); + + // then + assertThat(subscription.isTrackingEnabled()).isTrue(); + assertThat(subscription.getTrackingMode()).isEqualTo(TrackingMode.TRACK_DISCARDED_ONLY); + } + + @Test + public void shouldApplyPatchToSubscriptionPolicy() { + // given + PatchData patch = patchData().set("rate", 8).build(); + + // when + SubscriptionPolicy subscription = subscriptionPolicy().withRate(1).applyPatch(patch).build(); + + // then + assertThat(subscription.getRate()).isEqualTo(8); + } + + @Test + public void shouldAnonymizePassword() { + // given + Subscription subscription = + subscription("group.topic", "subscription") + .withEndpoint("http://user:password@service/path") + .build(); + + // when & then + assertThat(subscription.anonymize().getEndpoint()) + .isEqualTo(new EndpointAddress("http://user:*****@service/path")); + } + + @Test + public void shouldApplyPatchChangingSubscriptionOAuthPolicyGrantType() { + // given + Subscription subscription = + subscription("group.topic", "subscription") + .withOAuthPolicy( + new SubscriptionOAuthPolicy(CLIENT_CREDENTIALS, "myProvider", "repo", null, null)) + .build(); + PatchData oAuthPolicyPatchData = + patchData() + .set("grantType", SubscriptionOAuthPolicy.GrantType.USERNAME_PASSWORD.getName()) + .set("username", "user1") + .set("password", "abc123") + .build(); + PatchData patch = patchData().set("oAuthPolicy", oAuthPolicyPatchData).build(); + + // when + Subscription updated = Patch.apply(subscription, patch); + + // then + SubscriptionOAuthPolicy updatedPolicy = updated.getOAuthPolicy(); + assertThat(updatedPolicy.getGrantType()).isEqualTo(USERNAME_PASSWORD); + assertThat(updatedPolicy.getUsername()).isEqualTo("user1"); + } + + @Test + public void shouldReadIntBackoffMultiplier() throws Exception { + // given + String json = + "{\"name\": \"test\", \"endpoint\": \"http://localhost:8888\", \"subscriptionPolicy\": {\"messageBackoff\": 1000, \"backoffMultiplier\": 3}}"; + + // when + Subscription subscription = mapper.readValue(json, Subscription.class); + + // then + assertThat(subscription.getSerialSubscriptionPolicy().getBackoffMultiplier()).isEqualTo(3); + } } diff --git a/hermes-api/src/test/java/pl/allegro/tech/hermes/api/TopicNameTest.java b/hermes-api/src/test/java/pl/allegro/tech/hermes/api/TopicNameTest.java index fcaaddb968..ddf6ddb833 100644 --- a/hermes-api/src/test/java/pl/allegro/tech/hermes/api/TopicNameTest.java +++ b/hermes-api/src/test/java/pl/allegro/tech/hermes/api/TopicNameTest.java @@ -1,40 +1,39 @@ package pl.allegro.tech.hermes.api; -import org.junit.Test; - import static org.assertj.core.api.Assertions.assertThat; -public class TopicNameTest { +import org.junit.Test; - @Test - public void shouldConvertQualifiedName() { - // given - TopicName topicName = new TopicName("group1", "topic1"); - - // when & then - assertThat(TopicName.fromQualifiedName(topicName.qualifiedName())).isEqualTo(topicName); - } - - @Test - public void shouldHandleDottedGroupName() { - // given - TopicName topicName = TopicName.fromQualifiedName("group.topic"); - - // when & then - assertThat(topicName.getGroupName()).isEqualTo("group"); - assertThat(topicName.getName()).isEqualTo("topic"); - } - - @Test(expected = IllegalArgumentException.class) - public void shouldThrowExceptionWhenInvalidQualifiedNameProvided() { - // when & then - TopicName.fromQualifiedName("invalidQualifiedName"); - } - - @Test - public void shouldBeNullSafe() { - // when & then - assertThat(TopicName.fromQualifiedName(null)).isNull(); - } +public class TopicNameTest { + @Test + public void shouldConvertQualifiedName() { + // given + TopicName topicName = new TopicName("group1", "topic1"); + + // when & then + assertThat(TopicName.fromQualifiedName(topicName.qualifiedName())).isEqualTo(topicName); + } + + @Test + public void shouldHandleDottedGroupName() { + // given + TopicName topicName = TopicName.fromQualifiedName("group.topic"); + + // when & then + assertThat(topicName.getGroupName()).isEqualTo("group"); + assertThat(topicName.getName()).isEqualTo("topic"); + } + + @Test(expected = IllegalArgumentException.class) + public void shouldThrowExceptionWhenInvalidQualifiedNameProvided() { + // when & then + TopicName.fromQualifiedName("invalidQualifiedName"); + } + + @Test + public void shouldBeNullSafe() { + // when & then + assertThat(TopicName.fromQualifiedName(null)).isNull(); + } } diff --git a/hermes-api/src/test/java/pl/allegro/tech/hermes/api/TopicTest.java b/hermes-api/src/test/java/pl/allegro/tech/hermes/api/TopicTest.java index 27185dbeed..b983210192 100644 --- a/hermes-api/src/test/java/pl/allegro/tech/hermes/api/TopicTest.java +++ b/hermes-api/src/test/java/pl/allegro/tech/hermes/api/TopicTest.java @@ -1,109 +1,114 @@ package pl.allegro.tech.hermes.api; +import static org.assertj.core.api.Assertions.assertThat; + import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.InjectableValues; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.Test; -import static org.assertj.core.api.Assertions.assertThat; - public class TopicTest { - private final ObjectMapper objectMapper = createObjectMapper(false); - - @Test - public void shouldDeserializeTopicWithDefaults() throws Exception { - // given - String json = "{\"name\":\"foo.bar\", \"description\": \"description\"}"; - - // when - Topic topic = objectMapper.readValue(json, Topic.class); - - // then - assertThat(topic.getName().getName()).isEqualTo("bar"); - assertThat(topic.getName().getGroupName()).isEqualTo("foo"); - assertThat(topic.getDescription()).isEqualTo("description"); - assertThat(topic.isSchemaIdAwareSerializationEnabled()).isEqualTo(true); - } - - @Test - public void shouldDeserializeTopic() throws Exception { - // given - String json = "{\"name\":\"foo.bar\", \"description\": \"description\", \"schemaIdAwareSerializationEnabled\": \"false\"}"; - - // when - Topic topic = objectMapper.readValue(json, Topic.class); - - // then - assertThat(topic.getName().getName()).isEqualTo("bar"); - assertThat(topic.getName().getGroupName()).isEqualTo("foo"); - assertThat(topic.getDescription()).isEqualTo("description"); - assertThat(topic.isSchemaIdAwareSerializationEnabled()).isEqualTo(false); - } - - @Test - public void shouldSetDefaultAckIfNotPresentInJson() throws Exception { - // given - String json = "{\"name\":\"foo.bar\", \"description\": \"description\"}"; - - // when - Topic topic = objectMapper.readValue(json, Topic.class); - - // then - assertThat(topic.isReplicationConfirmRequired()).isEqualTo(false); - } - - @Test - public void shouldSkippedDeserializedOldSchemaVersionId() throws Exception { - // given - String json = "{\"name\":\"foo.bar\", \"description\": \"description\", \"schemaVersionAwareSerializationEnabled\": false}"; - - // when - Topic topic = objectMapper.readValue(json, Topic.class); - - // then - assertThat(topic.getName().getName()).isEqualTo("bar"); - } - - @Test - public void shouldDeserializeFallbackToRemoteDatacenterWithDefaults() throws Exception { - // given - String json = "{\"name\":\"foo.bar\", \"description\": \"description\"}"; - - // when - Topic topic = objectMapper.readValue(json, Topic.class); - - // then - assertThat(topic.isFallbackToRemoteDatacenterEnabled()).isEqualTo(false); - - // and when - Topic topic2 = createObjectMapper(true).readValue(json, Topic.class); - - // then - assertThat(topic2.isFallbackToRemoteDatacenterEnabled()).isEqualTo(true); - } - - @Test - public void shouldDeserializeFallbackToRemoteDatacenter() throws Exception { - // given - String json = "{\"name\":\"foo.bar\", \"description\": \"description\", \"fallbackToRemoteDatacenterEnabled\": true}"; - - // when - Topic topic = objectMapper.readValue(json, Topic.class); - - // then - assertThat(topic.isFallbackToRemoteDatacenterEnabled()).isEqualTo(true); - } - - private ObjectMapper createObjectMapper(boolean fallbackToRemoteDatacenterEnabled) { - ObjectMapper mapper = new ObjectMapper(); - - final InjectableValues defaultSchemaIdAwareSerializationEnabled = new InjectableValues - .Std().addValue(Topic.DEFAULT_SCHEMA_ID_SERIALIZATION_ENABLED_KEY, true) - .addValue(Topic.DEFAULT_FALLBACK_TO_REMOTE_DATACENTER_KEY, fallbackToRemoteDatacenterEnabled); - - mapper.setInjectableValues(defaultSchemaIdAwareSerializationEnabled); - mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); - return mapper; - } + private final ObjectMapper objectMapper = createObjectMapper(false); + + @Test + public void shouldDeserializeTopicWithDefaults() throws Exception { + // given + String json = "{\"name\":\"foo.bar\", \"description\": \"description\"}"; + + // when + Topic topic = objectMapper.readValue(json, Topic.class); + + // then + assertThat(topic.getName().getName()).isEqualTo("bar"); + assertThat(topic.getName().getGroupName()).isEqualTo("foo"); + assertThat(topic.getDescription()).isEqualTo("description"); + assertThat(topic.isSchemaIdAwareSerializationEnabled()).isEqualTo(true); + } + + @Test + public void shouldDeserializeTopic() throws Exception { + // given + String json = + "{\"name\":\"foo.bar\", \"description\": \"description\", \"schemaIdAwareSerializationEnabled\": \"false\"}"; + + // when + Topic topic = objectMapper.readValue(json, Topic.class); + + // then + assertThat(topic.getName().getName()).isEqualTo("bar"); + assertThat(topic.getName().getGroupName()).isEqualTo("foo"); + assertThat(topic.getDescription()).isEqualTo("description"); + assertThat(topic.isSchemaIdAwareSerializationEnabled()).isEqualTo(false); + } + + @Test + public void shouldSetDefaultAckIfNotPresentInJson() throws Exception { + // given + String json = "{\"name\":\"foo.bar\", \"description\": \"description\"}"; + + // when + Topic topic = objectMapper.readValue(json, Topic.class); + + // then + assertThat(topic.isReplicationConfirmRequired()).isEqualTo(false); + } + + @Test + public void shouldSkippedDeserializedOldSchemaVersionId() throws Exception { + // given + String json = + "{\"name\":\"foo.bar\", \"description\": \"description\", \"schemaVersionAwareSerializationEnabled\": false}"; + + // when + Topic topic = objectMapper.readValue(json, Topic.class); + + // then + assertThat(topic.getName().getName()).isEqualTo("bar"); + } + + @Test + public void shouldDeserializeFallbackToRemoteDatacenterWithDefaults() throws Exception { + // given + String json = "{\"name\":\"foo.bar\", \"description\": \"description\"}"; + + // when + Topic topic = objectMapper.readValue(json, Topic.class); + + // then + assertThat(topic.isFallbackToRemoteDatacenterEnabled()).isEqualTo(false); + + // and when + Topic topic2 = createObjectMapper(true).readValue(json, Topic.class); + + // then + assertThat(topic2.isFallbackToRemoteDatacenterEnabled()).isEqualTo(true); + } + + @Test + public void shouldDeserializeFallbackToRemoteDatacenter() throws Exception { + // given + String json = + "{\"name\":\"foo.bar\", \"description\": \"description\", \"fallbackToRemoteDatacenterEnabled\": true}"; + + // when + Topic topic = objectMapper.readValue(json, Topic.class); + + // then + assertThat(topic.isFallbackToRemoteDatacenterEnabled()).isEqualTo(true); + } + + private ObjectMapper createObjectMapper(boolean fallbackToRemoteDatacenterEnabled) { + ObjectMapper mapper = new ObjectMapper(); + + final InjectableValues defaultSchemaIdAwareSerializationEnabled = + new InjectableValues.Std() + .addValue(Topic.DEFAULT_SCHEMA_ID_SERIALIZATION_ENABLED_KEY, true) + .addValue( + Topic.DEFAULT_FALLBACK_TO_REMOTE_DATACENTER_KEY, fallbackToRemoteDatacenterEnabled); + + mapper.setInjectableValues(defaultSchemaIdAwareSerializationEnabled); + mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + return mapper; + } } diff --git a/hermes-api/src/test/java/pl/allegro/tech/hermes/api/helper/PatchTest.java b/hermes-api/src/test/java/pl/allegro/tech/hermes/api/helper/PatchTest.java index 2e5315ba50..afb0ae8a8b 100644 --- a/hermes-api/src/test/java/pl/allegro/tech/hermes/api/helper/PatchTest.java +++ b/hermes-api/src/test/java/pl/allegro/tech/hermes/api/helper/PatchTest.java @@ -1,5 +1,12 @@ package pl.allegro.tech.hermes.api.helper; +import static org.assertj.core.api.Assertions.assertThat; +import static pl.allegro.tech.hermes.api.PatchData.patchData; +import static pl.allegro.tech.hermes.api.SubscriptionPolicy.Builder.subscriptionPolicy; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topic; + +import java.util.HashMap; import org.junit.Test; import pl.allegro.tech.hermes.api.PatchData; import pl.allegro.tech.hermes.api.Subscription; @@ -7,79 +14,74 @@ import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.api.helpers.Patch; -import java.util.HashMap; - -import static org.assertj.core.api.Assertions.assertThat; -import static pl.allegro.tech.hermes.api.PatchData.patchData; -import static pl.allegro.tech.hermes.api.SubscriptionPolicy.Builder.subscriptionPolicy; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topic; - public class PatchTest { - @Test - public void shouldApplyPatch() { - // given - SubscriptionPolicy policy = SubscriptionPolicy.create(new HashMap<>()); - PatchData patch = patchData().set("rate", 10).set("messageTtl", 30).build(); - - // when - SubscriptionPolicy patched = Patch.apply(policy, patch); - - // when & then - assertThat(patched.getRate()).isEqualTo(10); - assertThat(patched.getMessageTtl()).isEqualTo(30); - } - - @Test(expected = NullPointerException.class) - public void shouldThrowExceptionForNullValues() { - // when - Patch.apply(null, null); - - // then - // exception is thrown - } - - @Test - public void shouldIgnoreUnknownFields() { - //given - SubscriptionPolicy policy = subscriptionPolicy().withRate(10).build(); - PatchData patch = patchData().set("unknown", 10).set("messageTtl", 30).build(); - - // when - SubscriptionPolicy patched = Patch.apply(policy, patch); - - // then - assertThat(patched.getMessageTtl()).isEqualTo(30); - } - - @Test - public void shouldPatchNestedObjects() { - // given - Subscription subscription = subscription("group.topic", "sub").build(); - PatchData patch = patchData().set( - "subscriptionPolicy", patchData().set("rate", 200).set("messageTtl", 8).build().getPatch() - ).build(); - - // when - SubscriptionPolicy result = Patch.apply(subscription, patch).getSerialSubscriptionPolicy(); - - // then - assertThat(result.getMessageTtl()).isEqualTo(8); - assertThat(result.getRate()).isEqualTo(200); - } - - @Test - public void shouldNotResetPrimitiveFields() { - // given - Topic topic = topic("group.topic").withTrackingEnabled(true).build(); - PatchData patch = patchData().set("schemaIdAwareSerializationEnabled", true).build(); - - // when - Topic patched = Patch.apply(topic, patch); - - // then - assertThat(patched.isTrackingEnabled()).isTrue(); - assertThat(patched.isSchemaIdAwareSerializationEnabled()).isTrue(); - } + @Test + public void shouldApplyPatch() { + // given + SubscriptionPolicy policy = SubscriptionPolicy.create(new HashMap<>()); + PatchData patch = patchData().set("rate", 10).set("messageTtl", 30).build(); + + // when + SubscriptionPolicy patched = Patch.apply(policy, patch); + + // when & then + assertThat(patched.getRate()).isEqualTo(10); + assertThat(patched.getMessageTtl()).isEqualTo(30); + } + + @Test(expected = NullPointerException.class) + public void shouldThrowExceptionForNullValues() { + // when + Patch.apply(null, null); + + // then + // exception is thrown + } + + @Test + public void shouldIgnoreUnknownFields() { + // given + SubscriptionPolicy policy = subscriptionPolicy().withRate(10).build(); + PatchData patch = patchData().set("unknown", 10).set("messageTtl", 30).build(); + + // when + SubscriptionPolicy patched = Patch.apply(policy, patch); + + // then + assertThat(patched.getMessageTtl()).isEqualTo(30); + } + + @Test + public void shouldPatchNestedObjects() { + // given + Subscription subscription = subscription("group.topic", "sub").build(); + PatchData patch = + patchData() + .set( + "subscriptionPolicy", + patchData().set("rate", 200).set("messageTtl", 8).build().getPatch()) + .build(); + + // when + SubscriptionPolicy result = Patch.apply(subscription, patch).getSerialSubscriptionPolicy(); + + // then + assertThat(result.getMessageTtl()).isEqualTo(8); + assertThat(result.getRate()).isEqualTo(200); + } + + @Test + public void shouldNotResetPrimitiveFields() { + // given + Topic topic = topic("group.topic").withTrackingEnabled(true).build(); + PatchData patch = patchData().set("schemaIdAwareSerializationEnabled", true).build(); + + // when + Topic patched = Patch.apply(topic, patch); + + // then + assertThat(patched.isTrackingEnabled()).isTrue(); + assertThat(patched.isSchemaIdAwareSerializationEnabled()).isTrue(); + } } diff --git a/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/HermesServerBenchmark.java b/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/HermesServerBenchmark.java index 90bf020797..562e435ec3 100644 --- a/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/HermesServerBenchmark.java +++ b/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/HermesServerBenchmark.java @@ -1,5 +1,6 @@ package pl.allegro.tech.hermes.benchmark; +import java.util.concurrent.TimeUnit; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Mode; @@ -13,40 +14,38 @@ import org.openjdk.jmh.runner.options.TimeValue; import pl.allegro.tech.hermes.benchmark.environment.HermesServerEnvironment; -import java.util.concurrent.TimeUnit; - @State(Scope.Benchmark) public class HermesServerBenchmark { - @Benchmark - @BenchmarkMode(Mode.Throughput) - @OutputTimeUnit(TimeUnit.SECONDS) - public int benchmarkPublishingThroughput(HermesServerEnvironment hermesServerEnvironment) { - return hermesServerEnvironment.publisher().publish(); - } - - @Benchmark - @BenchmarkMode(Mode.SampleTime) - @OutputTimeUnit(TimeUnit.MILLISECONDS) - public int benchmarkPublishingLatency(HermesServerEnvironment hermesServerEnvironment) { - return hermesServerEnvironment.publisher().publish(); - } - - public static void main(String[] args) throws RunnerException { - Options opt = new OptionsBuilder() - .include(".*" + HermesServerBenchmark.class.getSimpleName() + ".*") - .warmupIterations(4) - .measurementIterations(4) - //.addProfiler(JmhFlightRecorderProfiler.class) - //.jvmArgs("-XX:+UnlockCommercialFeatures") - .measurementTime(TimeValue.seconds(60)) - .warmupTime(TimeValue.seconds(40)) - .forks(1) - .threads(2) - .syncIterations(false) - .build(); - - new Runner(opt).run(); - } - + @Benchmark + @BenchmarkMode(Mode.Throughput) + @OutputTimeUnit(TimeUnit.SECONDS) + public int benchmarkPublishingThroughput(HermesServerEnvironment hermesServerEnvironment) { + return hermesServerEnvironment.publisher().publish(); + } + + @Benchmark + @BenchmarkMode(Mode.SampleTime) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public int benchmarkPublishingLatency(HermesServerEnvironment hermesServerEnvironment) { + return hermesServerEnvironment.publisher().publish(); + } + + public static void main(String[] args) throws RunnerException { + Options opt = + new OptionsBuilder() + .include(".*" + HermesServerBenchmark.class.getSimpleName() + ".*") + .warmupIterations(4) + .measurementIterations(4) + // .addProfiler(JmhFlightRecorderProfiler.class) + // .jvmArgs("-XX:+UnlockCommercialFeatures") + .measurementTime(TimeValue.seconds(60)) + .warmupTime(TimeValue.seconds(40)) + .forks(1) + .threads(2) + .syncIterations(false) + .build(); + + new Runner(opt).run(); + } } diff --git a/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/MessageRepositoryBenchmark.java b/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/MessageRepositoryBenchmark.java index a80cd7e429..6c1e0900d0 100644 --- a/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/MessageRepositoryBenchmark.java +++ b/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/MessageRepositoryBenchmark.java @@ -1,5 +1,14 @@ package pl.allegro.tech.hermes.benchmark; +import static java.util.Collections.emptyMap; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topic; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; import net.openhft.chronicle.map.ChronicleMap; import net.openhft.chronicle.map.ChronicleMapBuilder; import org.openjdk.jmh.annotations.Benchmark; @@ -23,16 +32,6 @@ import pl.allegro.tech.hermes.frontend.publishing.message.Message; import pl.allegro.tech.hermes.frontend.publishing.message.MessageIdGenerator; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.TimeUnit; - -import static java.util.Collections.emptyMap; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topic; - @Fork(1) @Warmup(iterations = 5) @Measurement(iterations = 5) @@ -41,93 +40,98 @@ @OutputTimeUnit(TimeUnit.NANOSECONDS) public class MessageRepositoryBenchmark { - @State(Scope.Benchmark) - public static class Repositories { - private static final int ENTRIES = 100; - private static final int AVERAGE_MESSAGE_SIZE = 600; + @State(Scope.Benchmark) + public static class Repositories { + private static final int ENTRIES = 100; + private static final int AVERAGE_MESSAGE_SIZE = 600; + + MessageRepository hermesImplMessageRepository; + MessageRepository baselineMessageRepository; - MessageRepository hermesImplMessageRepository; - MessageRepository baselineMessageRepository; + Message message; + Topic topic; - Message message; - Topic topic; + @Setup + public void setup() throws IOException { + message = generateMessage(); + topic = topic("groupName.topic").build(); - @Setup - public void setup() throws IOException { - message = generateMessage(); - topic = topic("groupName.topic").build(); + hermesImplMessageRepository = + new ChronicleMapMessageRepository(prepareFile(), ENTRIES, AVERAGE_MESSAGE_SIZE); + baselineMessageRepository = + new BaselineChronicleMapMessageRepository(prepareFile(), ENTRIES, AVERAGE_MESSAGE_SIZE); + } - hermesImplMessageRepository = new ChronicleMapMessageRepository(prepareFile(), ENTRIES, AVERAGE_MESSAGE_SIZE); - baselineMessageRepository = new BaselineChronicleMapMessageRepository(prepareFile(), ENTRIES, AVERAGE_MESSAGE_SIZE); - } + private Message generateMessage() { + byte[] messageContent = UUID.randomUUID().toString().getBytes(); + String id = MessageIdGenerator.generate(); + return new JsonMessage( + id, messageContent, System.currentTimeMillis(), "partition-key", emptyMap()); + } - private Message generateMessage() { - byte[] messageContent = UUID.randomUUID().toString().getBytes(); - String id = MessageIdGenerator.generate(); - return new JsonMessage(id, messageContent, System.currentTimeMillis(), "partition-key", emptyMap()); - } + private File prepareFile() throws IOException { - private File prepareFile() throws IOException { + String baseDir = Files.createTempDirectory(null).toFile().getAbsolutePath(); + return new File(baseDir, "messages.dat"); + } + } + + @Benchmark + public void hermesImplSave(Repositories repositories) { + repositories.hermesImplMessageRepository.save(repositories.message, repositories.topic); + } + + @Benchmark + public void baselineSave(Repositories repositories) { + repositories.baselineMessageRepository.save(repositories.message, repositories.topic); + } + + public static class BaselineChronicleMapMessageRepository implements MessageRepository { + private static final boolean SAME_BUILDER_CONFIG = false; + + private final ChronicleMap map; + + public BaselineChronicleMapMessageRepository(File file, int entries, int averageMessageSize) { + try { + map = + ChronicleMapBuilder.of(String.class, ChronicleMapEntryValue.class) + .constantKeySizeBySample(MessageIdGenerator.generate()) + .averageValueSize(averageMessageSize) + .entries(entries) + .sparseFile(true) + .createOrRecoverPersistedTo(file, SAME_BUILDER_CONFIG); + } catch (IOException e) { + throw new ChronicleMapCreationException(e); + } + } - String baseDir = Files.createTempDirectory(null).toFile().getAbsolutePath(); - return new File(baseDir, "messages.dat"); - } + @Override + public void save(Message message, Topic topic) { + ChronicleMapEntryValue entryValue = + new ChronicleMapEntryValue( + message.getData(), + message.getTimestamp(), + topic.getQualifiedName(), + message.getPartitionKey(), + null, + null, + emptyMap()); + map.put(message.getId(), entryValue); } - @Benchmark - public void hermesImplSave(Repositories repositories) { - repositories.hermesImplMessageRepository.save(repositories.message, repositories.topic); + @Override + public void delete(String messageId) { + throw new UnsupportedOperationException(); } - @Benchmark - public void baselineSave(Repositories repositories) { - repositories.baselineMessageRepository.save(repositories.message, repositories.topic); + @Override + public List findAll() { + throw new UnsupportedOperationException(); } - public static class BaselineChronicleMapMessageRepository implements MessageRepository { - private static final boolean SAME_BUILDER_CONFIG = false; - - private final ChronicleMap map; - - public BaselineChronicleMapMessageRepository(File file, int entries, int averageMessageSize) { - try { - map = ChronicleMapBuilder.of(String.class, ChronicleMapEntryValue.class) - .constantKeySizeBySample(MessageIdGenerator.generate()) - .averageValueSize(averageMessageSize) - .entries(entries) - .sparseFile(true) - .createOrRecoverPersistedTo(file, SAME_BUILDER_CONFIG); - } catch (IOException e) { - throw new ChronicleMapCreationException(e); - } - } - - @Override - public void save(Message message, Topic topic) { - ChronicleMapEntryValue entryValue = new ChronicleMapEntryValue( - message.getData(), - message.getTimestamp(), - topic.getQualifiedName(), - message.getPartitionKey(), - null, - null, - emptyMap()); - map.put(message.getId(), entryValue); - } - - @Override - public void delete(String messageId) { - throw new UnsupportedOperationException(); - } - - @Override - public List findAll() { - throw new UnsupportedOperationException(); - } - - @Override - public void close() { - throw new UnsupportedOperationException(); - } + @Override + public void close() { + throw new UnsupportedOperationException(); } + } } diff --git a/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/BenchmarkMessageContentWrapper.java b/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/BenchmarkMessageContentWrapper.java index 39bf8afc61..81801e35c7 100644 --- a/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/BenchmarkMessageContentWrapper.java +++ b/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/BenchmarkMessageContentWrapper.java @@ -1,5 +1,6 @@ package pl.allegro.tech.hermes.benchmark.environment; +import java.util.Map; import org.apache.avro.Schema; import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.common.message.wrapper.AvroMessageContentWrapper; @@ -8,35 +9,44 @@ import pl.allegro.tech.hermes.common.message.wrapper.UnwrappedMessageContent; import pl.allegro.tech.hermes.schema.CompiledSchema; -import java.util.Map; - public class BenchmarkMessageContentWrapper implements MessageContentWrapper { - private final AvroMessageContentWrapper avroMessageContentWrapper; - - public BenchmarkMessageContentWrapper(AvroMessageContentWrapper avroMessageContentWrapper) { - this.avroMessageContentWrapper = avroMessageContentWrapper; - } - - @Override - public UnwrappedMessageContent unwrapAvro(byte[] data, Topic topic, Integer schemaId, Integer schemaVersion) { - throw new UnsupportedOperationException(); - } - - @Override - public UnwrappedMessageContent unwrapJson(byte[] data) { - throw new UnsupportedOperationException(); - } - - @Override - public byte[] wrapAvro(byte[] data, String id, long timestamp, Topic topic, CompiledSchema schema, - Map externalMetadata) { - byte[] wrapped = avroMessageContentWrapper.wrapContent(data, id, timestamp, schema.getSchema(), externalMetadata); - return topic.isSchemaIdAwareSerializationEnabled() ? SchemaAwareSerDe.serialize(schema.getId(), wrapped) : wrapped; - } - - @Override - public byte[] wrapJson(byte[] data, String id, long timestamp, Map externalMetadata) { - throw new UnsupportedOperationException(); - } + private final AvroMessageContentWrapper avroMessageContentWrapper; + + public BenchmarkMessageContentWrapper(AvroMessageContentWrapper avroMessageContentWrapper) { + this.avroMessageContentWrapper = avroMessageContentWrapper; + } + + @Override + public UnwrappedMessageContent unwrapAvro( + byte[] data, Topic topic, Integer schemaId, Integer schemaVersion) { + throw new UnsupportedOperationException(); + } + + @Override + public UnwrappedMessageContent unwrapJson(byte[] data) { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] wrapAvro( + byte[] data, + String id, + long timestamp, + Topic topic, + CompiledSchema schema, + Map externalMetadata) { + byte[] wrapped = + avroMessageContentWrapper.wrapContent( + data, id, timestamp, schema.getSchema(), externalMetadata); + return topic.isSchemaIdAwareSerializationEnabled() + ? SchemaAwareSerDe.serialize(schema.getId(), wrapped) + : wrapped; + } + + @Override + public byte[] wrapJson( + byte[] data, String id, long timestamp, Map externalMetadata) { + throw new UnsupportedOperationException(); + } } diff --git a/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/DisabledReadinessChecker.java b/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/DisabledReadinessChecker.java index 88a8bf689e..433cf53746 100644 --- a/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/DisabledReadinessChecker.java +++ b/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/DisabledReadinessChecker.java @@ -4,24 +4,24 @@ class DisabledReadinessChecker implements ReadinessChecker { - private boolean isReady; + private boolean isReady; - public DisabledReadinessChecker(boolean isReady) { - this.isReady = isReady; - } + public DisabledReadinessChecker(boolean isReady) { + this.isReady = isReady; + } - @Override - public boolean isReady() { - return this.isReady; - } + @Override + public boolean isReady() { + return this.isReady; + } - @Override - public void start() { - this.isReady = true; - } + @Override + public void start() { + this.isReady = true; + } - @Override - public void stop() { - this.isReady = false; - } + @Override + public void stop() { + this.isReady = false; + } } diff --git a/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/HermesPublisher.java b/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/HermesPublisher.java index 65d758873d..41633f4ff6 100644 --- a/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/HermesPublisher.java +++ b/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/HermesPublisher.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.benchmark.environment; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.client.config.CookieSpecs; @@ -15,68 +20,65 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URI; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; - public class HermesPublisher { - private static final int CONNECT_TIMEOUT = 3000; - private static final int SOCKET_TIMEOUT = 3000; - private static final Logger logger = LoggerFactory.getLogger(HermesPublisher.class); + private static final int CONNECT_TIMEOUT = 3000; + private static final int SOCKET_TIMEOUT = 3000; + private static final Logger logger = LoggerFactory.getLogger(HermesPublisher.class); - private final CloseableHttpAsyncClient httpClient; - private final URI targetUrl; - private final HttpEntity body; + private final CloseableHttpAsyncClient httpClient; + private final URI targetUrl; + private final HttpEntity body; - public HermesPublisher(int maxConnectionsPerRoute, String targetUrl, String body) - throws IOReactorException, UnsupportedEncodingException { - this.targetUrl = URI.create(targetUrl); + public HermesPublisher(int maxConnectionsPerRoute, String targetUrl, String body) + throws IOReactorException, UnsupportedEncodingException { + this.targetUrl = URI.create(targetUrl); - RequestConfig requestConfig = RequestConfig.custom() - .setCookieSpec(CookieSpecs.IGNORE_COOKIES) - .setAuthenticationEnabled(false) - .build(); + RequestConfig requestConfig = + RequestConfig.custom() + .setCookieSpec(CookieSpecs.IGNORE_COOKIES) + .setAuthenticationEnabled(false) + .build(); - IOReactorConfig ioReactorConfig = IOReactorConfig.custom() - .setIoThreadCount(Runtime.getRuntime().availableProcessors()) - .setConnectTimeout(CONNECT_TIMEOUT) - .setSoTimeout(SOCKET_TIMEOUT) - .build(); + IOReactorConfig ioReactorConfig = + IOReactorConfig.custom() + .setIoThreadCount(Runtime.getRuntime().availableProcessors()) + .setConnectTimeout(CONNECT_TIMEOUT) + .setSoTimeout(SOCKET_TIMEOUT) + .build(); - PoolingNHttpClientConnectionManager connectionManager = new PoolingNHttpClientConnectionManager( - new DefaultConnectingIOReactor(ioReactorConfig)); - connectionManager.setDefaultMaxPerRoute(maxConnectionsPerRoute); + PoolingNHttpClientConnectionManager connectionManager = + new PoolingNHttpClientConnectionManager(new DefaultConnectingIOReactor(ioReactorConfig)); + connectionManager.setDefaultMaxPerRoute(maxConnectionsPerRoute); - httpClient = HttpAsyncClients.custom() - .setConnectionManager(connectionManager) - .setDefaultRequestConfig(requestConfig) - .build(); + httpClient = + HttpAsyncClients.custom() + .setConnectionManager(connectionManager) + .setDefaultRequestConfig(requestConfig) + .build(); - httpClient.start(); + httpClient.start(); - this.body = new StringEntity(body); - } + this.body = new StringEntity(body); + } - public int publish() { - int response = 0; - HttpPost httpPost = new HttpPost(targetUrl); - httpPost.setEntity(body); - httpPost.setHeader("Content-Type", "application/json"); + public int publish() { + int response = 0; + HttpPost httpPost = new HttpPost(targetUrl); + httpPost.setEntity(body); + httpPost.setHeader("Content-Type", "application/json"); - try { - Future future = httpClient.execute(httpPost, null); - response = future.get().getStatusLine().getStatusCode(); - } catch (RuntimeException | InterruptedException | ExecutionException exception) { - logger.error("Client exception", exception); - } - return response; + try { + Future future = httpClient.execute(httpPost, null); + response = future.get().getStatusLine().getStatusCode(); + } catch (RuntimeException | InterruptedException | ExecutionException exception) { + logger.error("Client exception", exception); } + return response; + } - public void stop() throws IOException { - if (httpClient.isRunning()) { - httpClient.close(); - } + public void stop() throws IOException { + if (httpClient.isRunning()) { + httpClient.close(); } + } } diff --git a/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/HermesServerEnvironment.java b/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/HermesServerEnvironment.java index 2f987a610f..2862a8ddb9 100644 --- a/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/HermesServerEnvironment.java +++ b/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/HermesServerEnvironment.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.benchmark.environment; +import java.io.IOException; +import java.util.Objects; import org.apache.commons.io.IOUtils; import org.openjdk.jmh.annotations.Level; import org.openjdk.jmh.annotations.Scope; @@ -10,55 +12,57 @@ import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.frontend.server.HermesServer; -import java.io.IOException; -import java.util.Objects; - @State(Scope.Benchmark) public class HermesServerEnvironment { - private static final Logger logger = LoggerFactory.getLogger(HermesServerEnvironment.class); - private static final int MAX_CONNECTIONS_PER_ROUTE = 200; - - public static final String BENCHMARK_TOPIC = "bench.topic"; + private static final Logger logger = LoggerFactory.getLogger(HermesServerEnvironment.class); + private static final int MAX_CONNECTIONS_PER_ROUTE = 200; - private HermesPublisher publisher; + public static final String BENCHMARK_TOPIC = "bench.topic"; - private HermesServer hermesServer; + private HermesPublisher publisher; - public static void main(String[] args) throws Exception { - new HermesServerEnvironment().setupEnvironment(); - } + private HermesServer hermesServer; - @Setup(Level.Trial) - public void setupEnvironment() throws Exception { - hermesServer = HermesServerFactory.provideHermesServer(); - hermesServer.start(); - } + public static void main(String[] args) throws Exception { + new HermesServerEnvironment().setupEnvironment(); + } - @Setup(Level.Trial) - public void setupPublisher() throws Exception { + @Setup(Level.Trial) + public void setupEnvironment() throws Exception { + hermesServer = HermesServerFactory.provideHermesServer(); + hermesServer.start(); + } - String messageBody = loadMessageResource("completeMessage"); - publisher = new HermesPublisher(MAX_CONNECTIONS_PER_ROUTE, "http://localhost:8080/topics/" + BENCHMARK_TOPIC, messageBody); - } + @Setup(Level.Trial) + public void setupPublisher() throws Exception { - @TearDown(Level.Trial) - public void shutdownServers() throws Exception { - hermesServer.stop(); - } + String messageBody = loadMessageResource("completeMessage"); + publisher = + new HermesPublisher( + MAX_CONNECTIONS_PER_ROUTE, + "http://localhost:8080/topics/" + BENCHMARK_TOPIC, + messageBody); + } - @TearDown(Level.Trial) - public void shutdownPublisherAndReportMetrics() throws Exception { - publisher.stop(); - } + @TearDown(Level.Trial) + public void shutdownServers() throws Exception { + hermesServer.stop(); + } - public HermesPublisher publisher() { - return publisher; - } + @TearDown(Level.Trial) + public void shutdownPublisherAndReportMetrics() throws Exception { + publisher.stop(); + } - public static String loadMessageResource(String name) throws IOException { - return IOUtils.toString(Objects.requireNonNull(HermesServerEnvironment.class - .getResourceAsStream(String.format("/message/%s.json", name)))); - } + public HermesPublisher publisher() { + return publisher; + } + public static String loadMessageResource(String name) throws IOException { + return IOUtils.toString( + Objects.requireNonNull( + HermesServerEnvironment.class.getResourceAsStream( + String.format("/message/%s.json", name)))); + } } diff --git a/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/HermesServerFactory.java b/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/HermesServerFactory.java index 47c5dbca6f..4c76d4bafe 100644 --- a/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/HermesServerFactory.java +++ b/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/HermesServerFactory.java @@ -1,8 +1,16 @@ package pl.allegro.tech.hermes.benchmark.environment; +import static pl.allegro.tech.hermes.api.ContentType.AVRO; +import static pl.allegro.tech.hermes.benchmark.environment.HermesServerEnvironment.loadMessageResource; +import static pl.allegro.tech.hermes.frontend.publishing.handlers.ThroughputLimiter.QuotaInsight.quotaConfirmed; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topic; + import com.fasterxml.jackson.databind.ObjectMapper; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import io.undertow.server.HttpHandler; +import java.io.IOException; +import java.time.Clock; +import java.util.Collections; import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.common.message.wrapper.AvroMessageContentWrapper; import pl.allegro.tech.hermes.common.metric.MetricsFacade; @@ -33,77 +41,79 @@ import pl.allegro.tech.hermes.schema.SchemaRepository; import pl.allegro.tech.hermes.tracker.frontend.Trackers; -import java.io.IOException; -import java.time.Clock; -import java.util.Collections; - -import static pl.allegro.tech.hermes.api.ContentType.AVRO; -import static pl.allegro.tech.hermes.benchmark.environment.HermesServerEnvironment.loadMessageResource; -import static pl.allegro.tech.hermes.frontend.publishing.handlers.ThroughputLimiter.QuotaInsight.quotaConfirmed; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topic; - class HermesServerFactory { - private static final Topic topic = topic(HermesServerEnvironment.BENCHMARK_TOPIC).withContentType(AVRO).build(); - + private static final Topic topic = + topic(HermesServerEnvironment.BENCHMARK_TOPIC).withContentType(AVRO).build(); - static HermesServer provideHermesServer() throws IOException { - ThroughputLimiter throughputLimiter = (exampleTopic, throughput) -> quotaConfirmed(); - MetricsFacade metricsFacade = new MetricsFacade(new SimpleMeterRegistry()); - TopicsCache topicsCache = new InMemoryTopicsCache(metricsFacade, topic); - BrokerMessageProducer brokerMessageProducer = new InMemoryBrokerMessageProducer(); - RawSchemaClient rawSchemaClient = new InMemorySchemaClient(topic.getName(), loadMessageResource("schema"), 1, 1); - Trackers trackers = new Trackers(Collections.emptyList()); - AvroMessageContentWrapper avroMessageContentWrapper = new AvroMessageContentWrapper(Clock.systemDefaultZone()); - HttpHandler httpHandler = provideHttpHandler(throughputLimiter, topicsCache, - brokerMessageProducer, rawSchemaClient, trackers, avroMessageContentWrapper); - SslProperties sslProperties = new SslProperties(); - HermesServerProperties hermesServerProperties = new HermesServerProperties(); - hermesServerProperties.setGracefulShutdownEnabled(false); + static HermesServer provideHermesServer() throws IOException { + ThroughputLimiter throughputLimiter = (exampleTopic, throughput) -> quotaConfirmed(); + MetricsFacade metricsFacade = new MetricsFacade(new SimpleMeterRegistry()); + TopicsCache topicsCache = new InMemoryTopicsCache(metricsFacade, topic); + BrokerMessageProducer brokerMessageProducer = new InMemoryBrokerMessageProducer(); + RawSchemaClient rawSchemaClient = + new InMemorySchemaClient(topic.getName(), loadMessageResource("schema"), 1, 1); + Trackers trackers = new Trackers(Collections.emptyList()); + AvroMessageContentWrapper avroMessageContentWrapper = + new AvroMessageContentWrapper(Clock.systemDefaultZone()); + HttpHandler httpHandler = + provideHttpHandler( + throughputLimiter, + topicsCache, + brokerMessageProducer, + rawSchemaClient, + trackers, + avroMessageContentWrapper); + SslProperties sslProperties = new SslProperties(); + HermesServerProperties hermesServerProperties = new HermesServerProperties(); + hermesServerProperties.setGracefulShutdownEnabled(false); - return new HermesServer( - sslProperties, - hermesServerProperties, - metricsFacade, - httpHandler, - new HealthCheckService(), - new DisabledReadinessChecker(false), - new NoOpMessagePreviewPersister(), - throughputLimiter, - null, - null); - } + return new HermesServer( + sslProperties, + hermesServerProperties, + metricsFacade, + httpHandler, + new HealthCheckService(), + new DisabledReadinessChecker(false), + new NoOpMessagePreviewPersister(), + throughputLimiter, + null, + null); + } - private static HttpHandler provideHttpHandler(ThroughputLimiter throughputLimiter, - TopicsCache topicsCache, BrokerMessageProducer brokerMessageProducer, - RawSchemaClient rawSchemaClient, Trackers trackers, AvroMessageContentWrapper avroMessageContentWrapper) { - HTTPHeadersProperties httpHeadersProperties = new HTTPHeadersProperties(); - HandlersChainProperties handlersChainProperties = new HandlersChainProperties(); - TrackingHeadersExtractor trackingHeadersExtractor = new DefaultTrackingHeaderExtractor(); - SchemaProperties schemaProperties = new SchemaProperties(); + private static HttpHandler provideHttpHandler( + ThroughputLimiter throughputLimiter, + TopicsCache topicsCache, + BrokerMessageProducer brokerMessageProducer, + RawSchemaClient rawSchemaClient, + Trackers trackers, + AvroMessageContentWrapper avroMessageContentWrapper) { + HTTPHeadersProperties httpHeadersProperties = new HTTPHeadersProperties(); + HandlersChainProperties handlersChainProperties = new HandlersChainProperties(); + TrackingHeadersExtractor trackingHeadersExtractor = new DefaultTrackingHeaderExtractor(); + SchemaProperties schemaProperties = new SchemaProperties(); - return new HandlersChainFactory( - topicsCache, - new MessageErrorProcessor(new ObjectMapper(), trackers, trackingHeadersExtractor), - new MessageEndProcessor(trackers, new BrokerListeners(), trackingHeadersExtractor), - new MessageFactory( - new MessageValidators(Collections.emptyList()), - new MessageContentTypeEnforcer(), - new SchemaRepository( - new DirectSchemaVersionsRepository(rawSchemaClient), - new DirectCompiledSchemaRepository<>(rawSchemaClient, SchemaCompilersFactory.avroSchemaCompiler()) - ), - new DefaultHeadersPropagator(httpHeadersProperties), - new BenchmarkMessageContentWrapper(avroMessageContentWrapper), - Clock.systemDefaultZone(), - schemaProperties.isIdHeaderEnabled() - ), - brokerMessageProducer, - null, - throughputLimiter, - null, - false, - handlersChainProperties - ).provide(); - } + return new HandlersChainFactory( + topicsCache, + new MessageErrorProcessor(new ObjectMapper(), trackers, trackingHeadersExtractor), + new MessageEndProcessor(trackers, new BrokerListeners(), trackingHeadersExtractor), + new MessageFactory( + new MessageValidators(Collections.emptyList()), + new MessageContentTypeEnforcer(), + new SchemaRepository( + new DirectSchemaVersionsRepository(rawSchemaClient), + new DirectCompiledSchemaRepository<>( + rawSchemaClient, SchemaCompilersFactory.avroSchemaCompiler())), + new DefaultHeadersPropagator(httpHeadersProperties), + new BenchmarkMessageContentWrapper(avroMessageContentWrapper), + Clock.systemDefaultZone(), + schemaProperties.isIdHeaderEnabled()), + brokerMessageProducer, + null, + throughputLimiter, + null, + false, + handlersChainProperties) + .provide(); + } } diff --git a/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/InMemoryBrokerMessageProducer.java b/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/InMemoryBrokerMessageProducer.java index 3d06b1c021..994b90777b 100644 --- a/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/InMemoryBrokerMessageProducer.java +++ b/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/InMemoryBrokerMessageProducer.java @@ -7,18 +7,18 @@ public class InMemoryBrokerMessageProducer implements BrokerMessageProducer { - @Override - public void send(Message message, CachedTopic topic, PublishingCallback callback) { - callback.onPublished(message, topic.getTopic()); - } + @Override + public void send(Message message, CachedTopic topic, PublishingCallback callback) { + callback.onPublished(message, topic.getTopic()); + } - @Override - public boolean areAllTopicsAvailable() { - return true; - } + @Override + public boolean areAllTopicsAvailable() { + return true; + } - @Override - public boolean isTopicAvailable(CachedTopic cachedTopic) { - return true; - } + @Override + public boolean isTopicAvailable(CachedTopic cachedTopic) { + return true; + } } diff --git a/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/InMemorySchemaClient.java b/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/InMemorySchemaClient.java index 1c2ec6ec38..f25c6ed395 100644 --- a/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/InMemorySchemaClient.java +++ b/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/InMemorySchemaClient.java @@ -1,6 +1,9 @@ package pl.allegro.tech.hermes.benchmark.environment; import com.google.common.collect.ImmutableList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; import pl.allegro.tech.hermes.api.RawSchema; import pl.allegro.tech.hermes.api.RawSchemaWithMetadata; import pl.allegro.tech.hermes.api.TopicName; @@ -8,54 +11,50 @@ import pl.allegro.tech.hermes.schema.SchemaId; import pl.allegro.tech.hermes.schema.SchemaVersion; -import java.util.List; -import java.util.Objects; -import java.util.Optional; - public class InMemorySchemaClient implements RawSchemaClient { - private final TopicName schemaTopicName; - private final RawSchemaWithMetadata rawSchemaWithMetadata; - - public InMemorySchemaClient(TopicName schemaTopicName, String schemaSource, int id, int version) { - this.schemaTopicName = schemaTopicName; - rawSchemaWithMetadata = RawSchemaWithMetadata.of(schemaSource, id, version); - } - - @Override - public Optional getRawSchemaWithMetadata(TopicName topic, SchemaVersion version) { - return schemaTopicName.equals(topic) && Objects.equals(rawSchemaWithMetadata.getVersion(), version.value()) - ? Optional.of(rawSchemaWithMetadata) : Optional.empty(); - } - - @Override - public Optional getRawSchemaWithMetadata(TopicName topic, SchemaId schemaId) { - return schemaTopicName.equals(topic) && Objects.equals(rawSchemaWithMetadata.getId(), schemaId.value()) - ? Optional.of(rawSchemaWithMetadata) : Optional.empty(); - } - - @Override - public Optional getLatestRawSchemaWithMetadata(TopicName topic) { - return schemaTopicName.equals(topic) ? Optional.of(rawSchemaWithMetadata) : Optional.empty(); - } - - @Override - public List getVersions(TopicName topic) { - return ImmutableList.of(SchemaVersion.valueOf(rawSchemaWithMetadata.getVersion())); - } - - @Override - public void registerSchema(TopicName topic, RawSchema rawSchema) { - - } - - @Override - public void deleteAllSchemaVersions(TopicName topic) { - - } - - @Override - public void validateSchema(TopicName topic, RawSchema rawSchema) { - - } + private final TopicName schemaTopicName; + private final RawSchemaWithMetadata rawSchemaWithMetadata; + + public InMemorySchemaClient(TopicName schemaTopicName, String schemaSource, int id, int version) { + this.schemaTopicName = schemaTopicName; + rawSchemaWithMetadata = RawSchemaWithMetadata.of(schemaSource, id, version); + } + + @Override + public Optional getRawSchemaWithMetadata( + TopicName topic, SchemaVersion version) { + return schemaTopicName.equals(topic) + && Objects.equals(rawSchemaWithMetadata.getVersion(), version.value()) + ? Optional.of(rawSchemaWithMetadata) + : Optional.empty(); + } + + @Override + public Optional getRawSchemaWithMetadata( + TopicName topic, SchemaId schemaId) { + return schemaTopicName.equals(topic) + && Objects.equals(rawSchemaWithMetadata.getId(), schemaId.value()) + ? Optional.of(rawSchemaWithMetadata) + : Optional.empty(); + } + + @Override + public Optional getLatestRawSchemaWithMetadata(TopicName topic) { + return schemaTopicName.equals(topic) ? Optional.of(rawSchemaWithMetadata) : Optional.empty(); + } + + @Override + public List getVersions(TopicName topic) { + return ImmutableList.of(SchemaVersion.valueOf(rawSchemaWithMetadata.getVersion())); + } + + @Override + public void registerSchema(TopicName topic, RawSchema rawSchema) {} + + @Override + public void deleteAllSchemaVersions(TopicName topic) {} + + @Override + public void validateSchema(TopicName topic, RawSchema rawSchema) {} } diff --git a/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/InMemoryTopicsCache.java b/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/InMemoryTopicsCache.java index d7425b67d5..1c054ac63e 100644 --- a/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/InMemoryTopicsCache.java +++ b/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/InMemoryTopicsCache.java @@ -1,6 +1,8 @@ package pl.allegro.tech.hermes.benchmark.environment; import com.codahale.metrics.MetricRegistry; +import java.util.List; +import java.util.Optional; import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.common.kafka.KafkaTopic; import pl.allegro.tech.hermes.common.kafka.KafkaTopicName; @@ -10,46 +12,38 @@ import pl.allegro.tech.hermes.frontend.metric.CachedTopic; import pl.allegro.tech.hermes.frontend.metric.ThroughputRegistry; -import java.util.List; -import java.util.Optional; - class InMemoryTopicsCache implements TopicsCache { - private final MetricsFacade metricsFacade; - private final KafkaTopics kafkaTopics; - private final Topic topic; - private final ThroughputRegistry throughputRegistry; - - - InMemoryTopicsCache(MetricsFacade metricsFacade, Topic topic) { - this.metricsFacade = metricsFacade; - this.topic = topic; - this.kafkaTopics = new KafkaTopics(new KafkaTopic(KafkaTopicName.valueOf(topic.getQualifiedName()), topic.getContentType())); - this.throughputRegistry = new ThroughputRegistry(metricsFacade, new MetricRegistry()); - } - - @Override - public Optional getTopic(String qualifiedTopicName) { - if (qualifiedTopicName.equals(topic.getQualifiedName())) { - return Optional.of( - new CachedTopic( - topic, - metricsFacade, - throughputRegistry, - kafkaTopics - ) - ); - } - return Optional.empty(); - } - - @Override - public List getTopics() { - throw new UnsupportedOperationException(); - } - - @Override - public void start() { - throw new UnsupportedOperationException(); + private final MetricsFacade metricsFacade; + private final KafkaTopics kafkaTopics; + private final Topic topic; + private final ThroughputRegistry throughputRegistry; + + InMemoryTopicsCache(MetricsFacade metricsFacade, Topic topic) { + this.metricsFacade = metricsFacade; + this.topic = topic; + this.kafkaTopics = + new KafkaTopics( + new KafkaTopic( + KafkaTopicName.valueOf(topic.getQualifiedName()), topic.getContentType())); + this.throughputRegistry = new ThroughputRegistry(metricsFacade, new MetricRegistry()); + } + + @Override + public Optional getTopic(String qualifiedTopicName) { + if (qualifiedTopicName.equals(topic.getQualifiedName())) { + return Optional.of(new CachedTopic(topic, metricsFacade, throughputRegistry, kafkaTopics)); } + return Optional.empty(); + } + + @Override + public List getTopics() { + throw new UnsupportedOperationException(); + } + + @Override + public void start() { + throw new UnsupportedOperationException(); + } } diff --git a/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/NoOpMessagePreviewPersister.java b/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/NoOpMessagePreviewPersister.java index d491102a5e..1a47684d23 100644 --- a/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/NoOpMessagePreviewPersister.java +++ b/hermes-benchmark/src/jmh/java/pl/allegro/tech/hermes/benchmark/environment/NoOpMessagePreviewPersister.java @@ -4,11 +4,9 @@ class NoOpMessagePreviewPersister implements MessagePreviewPersister { - @Override - public void start() { - } + @Override + public void start() {} - @Override - public void shutdown() { - } + @Override + public void shutdown() {} } diff --git a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesClient.java b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesClient.java index 76e2b03978..0590e3bf83 100644 --- a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesClient.java +++ b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesClient.java @@ -1,11 +1,6 @@ package pl.allegro.tech.hermes.client; -import net.jodah.failsafe.Failsafe; -import net.jodah.failsafe.RetryPolicy; -import net.jodah.failsafe.event.ExecutionAttemptedEvent; -import net.jodah.failsafe.event.ExecutionCompletedEvent; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static pl.allegro.tech.hermes.client.HermesMessage.hermesMessage; import java.net.URI; import java.time.temporal.ChronoUnit; @@ -21,157 +16,182 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; - -import static pl.allegro.tech.hermes.client.HermesMessage.hermesMessage; +import net.jodah.failsafe.Failsafe; +import net.jodah.failsafe.RetryPolicy; +import net.jodah.failsafe.event.ExecutionAttemptedEvent; +import net.jodah.failsafe.event.ExecutionCompletedEvent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class HermesClient { - private static final Logger logger = LoggerFactory.getLogger(HermesClient.class); - - private final HermesSender sender; - private final String uri; - private final Map defaultHeaders; - private final AtomicInteger currentlySending = new AtomicInteger(0); - private final RetryPolicy retryPolicy; - private final ScheduledExecutorService scheduler; - private volatile boolean shutdown = false; - private final List messageDeliveryListeners = new ArrayList<>(); - - HermesClient(HermesSender sender, - URI uri, - Map defaultHeaders, - int retries, - Predicate retryCondition, - long retrySleepInMillis, - long maxRetrySleepInMillis, - ScheduledExecutorService scheduler) { - this.sender = sender; - this.uri = createUri(uri); - this.defaultHeaders = Collections.unmodifiableMap(new HashMap<>(defaultHeaders)); - this.retryPolicy = createRetryPolicy(retries, retryCondition, retrySleepInMillis, maxRetrySleepInMillis); - this.scheduler = scheduler; - } - - private RetryPolicy createRetryPolicy(int retries, Predicate retryCondition, - long retrySleepInMillis, long maxRetrySleepInMillis) { - RetryPolicy retryPolicy = new RetryPolicy() - .withMaxRetries(retries) - .handleIf((resp, cause) -> retryCondition.test(resp)) - .onRetriesExceeded((e) -> handleMaxRetriesExceeded(e)) - .onRetry((e) -> handleFailedAttempt(e)) - .onFailure((e) -> handleFailure(e)) - .onSuccess((e) -> handleSuccessfulRetry(e)); - - if (retrySleepInMillis > 0) { - retryPolicy.withBackoff(retrySleepInMillis, maxRetrySleepInMillis, ChronoUnit.MILLIS); - } - return retryPolicy; - } - - private String createUri(URI uri) { - String uriString = uri.toString(); - return uriString + (uriString.endsWith("/") ? "" : "/") + "topics/"; - } - - public CompletableFuture publishJSON(String topic, byte[] message) { - return publish(hermesMessage(topic, message).json().build()); - } - - public CompletableFuture publishJSON(String topic, String message) { - return publish(hermesMessage(topic, message).json().build()); - } - - public CompletableFuture publishAvro(String topic, int schemaVersion, byte[] message) { - return publish(hermesMessage(topic, message).avro(schemaVersion).build()); - } - - public CompletableFuture publish(String topic, String message) { - return publish(hermesMessage(topic, message).build()); - } - - public CompletableFuture publish(String topic, String contentType, byte[] message) { - return publish(hermesMessage(topic, message).withContentType(contentType).build()); - } - - public CompletableFuture publish(String topic, String contentType, String message) { - return publish(hermesMessage(topic, message).withContentType(contentType).build()); - } - - public CompletableFuture publish(String topic, String contentType, int schemaVersion, byte[] message) { - return publish(hermesMessage(topic, message).withContentType(contentType).withSchemaVersion(schemaVersion).build()); - } - - public CompletableFuture publish(HermesMessage message) { - if (shutdown) { - return completedWithShutdownException(); - } - HermesMessage.appendDefaults(message, defaultHeaders); - return publishWithRetries(message); - } - - public boolean addMessageDeliveryListener(MessageDeliveryListener listener) { - return messageDeliveryListeners.add(listener); - } - - private CompletableFuture publishWithRetries(HermesMessage message) { - currentlySending.incrementAndGet(); - return Failsafe.with(retryPolicy) - .with(scheduler) - .onComplete((e) -> currentlySending.decrementAndGet()) - .getStageAsync(() -> sendOnce(message)); - } - - private CompletableFuture sendOnce(HermesMessage message) { - long startTime = System.nanoTime(); - - return sender.send(URI.create(uri + message.getTopic()), message) - .exceptionally(e -> HermesResponseBuilder.hermesFailureResponse(e, message)) - .whenComplete((resp, cause) -> { - long latency = System.nanoTime() - startTime; - messageDeliveryListeners.forEach(l -> l.onSend(resp, latency)); - }); - } - - private CompletableFuture completedWithShutdownException() { - CompletableFuture alreadyShutdown = new CompletableFuture<>(); - alreadyShutdown.completeExceptionally(new HermesClientShutdownException()); - return alreadyShutdown; - } - - public CompletableFuture closeAsync(long pollInterval) { - shutdown = true; - return new HermesClientTermination(pollInterval) - .observe(() -> currentlySending.get() == 0) - .whenComplete((response, ex) -> scheduler.shutdown()); - } - - public void close(long pollInterval, long timeout) throws InterruptedException, TimeoutException { - try { - closeAsync(pollInterval).get(timeout, TimeUnit.MILLISECONDS); - } catch (ExecutionException e) { - throw (InterruptedException) e.getCause(); - } - } - - private void handleMaxRetriesExceeded(ExecutionCompletedEvent event) { - if (event.getResult().isSuccess()) { - return; - } - - HermesMessage message = event.getResult().getHermesMessage(); - messageDeliveryListeners.forEach(l -> l.onMaxRetriesExceeded(event.getResult(), event.getAttemptCount())); - logger.error("Failed to send message to topic {} after {} attempts", - message.getTopic(), event.getAttemptCount()); - } - - private void handleFailedAttempt(ExecutionAttemptedEvent event) { - messageDeliveryListeners.forEach(l -> l.onFailedRetry(event.getLastResult(), event.getAttemptCount())); - } - - private void handleFailure(ExecutionCompletedEvent event) { - messageDeliveryListeners.forEach(l -> l.onFailure(event.getResult(), event.getAttemptCount())); - } - - private void handleSuccessfulRetry(ExecutionCompletedEvent event) { - messageDeliveryListeners.forEach(l -> l.onSuccessfulRetry(event.getResult(), event.getAttemptCount())); - } + private static final Logger logger = LoggerFactory.getLogger(HermesClient.class); + + private final HermesSender sender; + private final String uri; + private final Map defaultHeaders; + private final AtomicInteger currentlySending = new AtomicInteger(0); + private final RetryPolicy retryPolicy; + private final ScheduledExecutorService scheduler; + private volatile boolean shutdown = false; + private final List messageDeliveryListeners = new ArrayList<>(); + + HermesClient( + HermesSender sender, + URI uri, + Map defaultHeaders, + int retries, + Predicate retryCondition, + long retrySleepInMillis, + long maxRetrySleepInMillis, + ScheduledExecutorService scheduler) { + this.sender = sender; + this.uri = createUri(uri); + this.defaultHeaders = Collections.unmodifiableMap(new HashMap<>(defaultHeaders)); + this.retryPolicy = + createRetryPolicy(retries, retryCondition, retrySleepInMillis, maxRetrySleepInMillis); + this.scheduler = scheduler; + } + + private RetryPolicy createRetryPolicy( + int retries, + Predicate retryCondition, + long retrySleepInMillis, + long maxRetrySleepInMillis) { + RetryPolicy retryPolicy = + new RetryPolicy() + .withMaxRetries(retries) + .handleIf((resp, cause) -> retryCondition.test(resp)) + .onRetriesExceeded((e) -> handleMaxRetriesExceeded(e)) + .onRetry((e) -> handleFailedAttempt(e)) + .onFailure((e) -> handleFailure(e)) + .onSuccess((e) -> handleSuccessfulRetry(e)); + + if (retrySleepInMillis > 0) { + retryPolicy.withBackoff(retrySleepInMillis, maxRetrySleepInMillis, ChronoUnit.MILLIS); + } + return retryPolicy; + } + + private String createUri(URI uri) { + String uriString = uri.toString(); + return uriString + (uriString.endsWith("/") ? "" : "/") + "topics/"; + } + + public CompletableFuture publishJSON(String topic, byte[] message) { + return publish(hermesMessage(topic, message).json().build()); + } + + public CompletableFuture publishJSON(String topic, String message) { + return publish(hermesMessage(topic, message).json().build()); + } + + public CompletableFuture publishAvro( + String topic, int schemaVersion, byte[] message) { + return publish(hermesMessage(topic, message).avro(schemaVersion).build()); + } + + public CompletableFuture publish(String topic, String message) { + return publish(hermesMessage(topic, message).build()); + } + + public CompletableFuture publish( + String topic, String contentType, byte[] message) { + return publish(hermesMessage(topic, message).withContentType(contentType).build()); + } + + public CompletableFuture publish( + String topic, String contentType, String message) { + return publish(hermesMessage(topic, message).withContentType(contentType).build()); + } + + public CompletableFuture publish( + String topic, String contentType, int schemaVersion, byte[] message) { + return publish( + hermesMessage(topic, message) + .withContentType(contentType) + .withSchemaVersion(schemaVersion) + .build()); + } + + public CompletableFuture publish(HermesMessage message) { + if (shutdown) { + return completedWithShutdownException(); + } + HermesMessage.appendDefaults(message, defaultHeaders); + return publishWithRetries(message); + } + + public boolean addMessageDeliveryListener(MessageDeliveryListener listener) { + return messageDeliveryListeners.add(listener); + } + + private CompletableFuture publishWithRetries(HermesMessage message) { + currentlySending.incrementAndGet(); + return Failsafe.with(retryPolicy) + .with(scheduler) + .onComplete((e) -> currentlySending.decrementAndGet()) + .getStageAsync(() -> sendOnce(message)); + } + + private CompletableFuture sendOnce(HermesMessage message) { + long startTime = System.nanoTime(); + + return sender + .send(URI.create(uri + message.getTopic()), message) + .exceptionally(e -> HermesResponseBuilder.hermesFailureResponse(e, message)) + .whenComplete( + (resp, cause) -> { + long latency = System.nanoTime() - startTime; + messageDeliveryListeners.forEach(l -> l.onSend(resp, latency)); + }); + } + + private CompletableFuture completedWithShutdownException() { + CompletableFuture alreadyShutdown = new CompletableFuture<>(); + alreadyShutdown.completeExceptionally(new HermesClientShutdownException()); + return alreadyShutdown; + } + + public CompletableFuture closeAsync(long pollInterval) { + shutdown = true; + return new HermesClientTermination(pollInterval) + .observe(() -> currentlySending.get() == 0) + .whenComplete((response, ex) -> scheduler.shutdown()); + } + + public void close(long pollInterval, long timeout) throws InterruptedException, TimeoutException { + try { + closeAsync(pollInterval).get(timeout, TimeUnit.MILLISECONDS); + } catch (ExecutionException e) { + throw (InterruptedException) e.getCause(); + } + } + + private void handleMaxRetriesExceeded(ExecutionCompletedEvent event) { + if (event.getResult().isSuccess()) { + return; + } + + HermesMessage message = event.getResult().getHermesMessage(); + messageDeliveryListeners.forEach( + l -> l.onMaxRetriesExceeded(event.getResult(), event.getAttemptCount())); + logger.error( + "Failed to send message to topic {} after {} attempts", + message.getTopic(), + event.getAttemptCount()); + } + + private void handleFailedAttempt(ExecutionAttemptedEvent event) { + messageDeliveryListeners.forEach( + l -> l.onFailedRetry(event.getLastResult(), event.getAttemptCount())); + } + + private void handleFailure(ExecutionCompletedEvent event) { + messageDeliveryListeners.forEach(l -> l.onFailure(event.getResult(), event.getAttemptCount())); + } + + private void handleSuccessfulRetry(ExecutionCompletedEvent event) { + messageDeliveryListeners.forEach( + l -> l.onSuccessfulRetry(event.getResult(), event.getAttemptCount())); + } } diff --git a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesClientBasicRetryCondition.java b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesClientBasicRetryCondition.java index e0cd051370..4e6957b7b2 100644 --- a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesClientBasicRetryCondition.java +++ b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesClientBasicRetryCondition.java @@ -1,20 +1,21 @@ package pl.allegro.tech.hermes.client; -import java.util.function.Predicate; - import static java.net.HttpURLConnection.HTTP_CLIENT_TIMEOUT; +import java.util.function.Predicate; + public class HermesClientBasicRetryCondition implements Predicate { - @Override - public boolean test(HermesResponse response) { - return response == null || (isClientTimeoutOrServerError(response) || isFailedExceptionally(response)); - } + @Override + public boolean test(HermesResponse response) { + return response == null + || (isClientTimeoutOrServerError(response) || isFailedExceptionally(response)); + } - private boolean isClientTimeoutOrServerError(HermesResponse response) { - return response.getHttpStatus() == HTTP_CLIENT_TIMEOUT || response.getHttpStatus() / 100 == 5; - } + private boolean isClientTimeoutOrServerError(HermesResponse response) { + return response.getHttpStatus() == HTTP_CLIENT_TIMEOUT || response.getHttpStatus() / 100 == 5; + } - private boolean isFailedExceptionally(HermesResponse response) { - return response.isFailure() && response.getFailureCause().isPresent(); - } + private boolean isFailedExceptionally(HermesResponse response) { + return response.isFailure() && response.getFailureCause().isPresent(); + } } diff --git a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesClientBuilder.java b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesClientBuilder.java index c318b8e890..6041033150 100644 --- a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesClientBuilder.java +++ b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesClientBuilder.java @@ -1,8 +1,5 @@ package pl.allegro.tech.hermes.client; -import pl.allegro.tech.hermes.client.metrics.MetricsMessageDeliveryListener; -import pl.allegro.tech.hermes.client.metrics.MetricsProvider; - import java.net.URI; import java.util.HashMap; import java.util.Map; @@ -11,82 +8,95 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.function.Predicate; import java.util.function.Supplier; +import pl.allegro.tech.hermes.client.metrics.MetricsMessageDeliveryListener; +import pl.allegro.tech.hermes.client.metrics.MetricsProvider; public class HermesClientBuilder { - private HermesSender sender; - private URI uri = URI.create("http://localhost:8080"); - private final Map defaultHeaders = new HashMap<>(); - private int retries = 3; - private Predicate retryCondition = new HermesClientBasicRetryCondition(); - private long retrySleepInMillis = 100; - private long maxRetrySleepInMillis = 300; - private Supplier schedulerFactory = Executors::newSingleThreadScheduledExecutor; - private Optional metrics = Optional.empty(); - - public HermesClientBuilder(HermesSender sender) { - this.sender = sender; - this.defaultHeaders.put(HermesMessage.CONTENT_TYPE_HEADER, HermesMessage.APPLICATION_JSON); - } - - public static HermesClientBuilder hermesClient(HermesSender sender) { - return new HermesClientBuilder(sender); - } - - public HermesClient build() { - HermesClient hermesClient = new HermesClient(sender, uri, defaultHeaders, retries, retryCondition, - retrySleepInMillis, maxRetrySleepInMillis, schedulerFactory.get()); - - metrics.ifPresent((metricsProvider) -> { - hermesClient.addMessageDeliveryListener(new MetricsMessageDeliveryListener(metricsProvider)); + private HermesSender sender; + private URI uri = URI.create("http://localhost:8080"); + private final Map defaultHeaders = new HashMap<>(); + private int retries = 3; + private Predicate retryCondition = new HermesClientBasicRetryCondition(); + private long retrySleepInMillis = 100; + private long maxRetrySleepInMillis = 300; + private Supplier schedulerFactory = + Executors::newSingleThreadScheduledExecutor; + private Optional metrics = Optional.empty(); + + public HermesClientBuilder(HermesSender sender) { + this.sender = sender; + this.defaultHeaders.put(HermesMessage.CONTENT_TYPE_HEADER, HermesMessage.APPLICATION_JSON); + } + + public static HermesClientBuilder hermesClient(HermesSender sender) { + return new HermesClientBuilder(sender); + } + + public HermesClient build() { + HermesClient hermesClient = + new HermesClient( + sender, + uri, + defaultHeaders, + retries, + retryCondition, + retrySleepInMillis, + maxRetrySleepInMillis, + schedulerFactory.get()); + + metrics.ifPresent( + (metricsProvider) -> { + hermesClient.addMessageDeliveryListener( + new MetricsMessageDeliveryListener(metricsProvider)); }); - return hermesClient; - } - - public HermesClientBuilder withURI(URI uri) { - this.uri = uri; - return this; - } - - public HermesClientBuilder withMetrics(MetricsProvider metrics) { - this.metrics = Optional.of(metrics); - return this; - } - - public HermesClientBuilder withDefaultContentType(String defaultContentType) { - defaultHeaders.put(HermesMessage.CONTENT_TYPE_HEADER, defaultContentType); - return this; - } - - public HermesClientBuilder withDefaultHeaderValue(String header, String value) { - defaultHeaders.put(header, value); - return this; - } - - public HermesClientBuilder withRetries(int retries) { - this.retries = retries; - return this; - } - - public HermesClientBuilder withRetries(int retries, Predicate retryCondition) { - this.retryCondition = retryCondition; - return withRetries(retries); - } - - public HermesClientBuilder withRetrySleep(long retrySleepInMillis) { - this.retrySleepInMillis = retrySleepInMillis; - return this; - } - - public HermesClientBuilder withRetrySleep(long retrySleepInMillis, long maxRetrySleepInMillis) { - this.retrySleepInMillis = retrySleepInMillis; - this.maxRetrySleepInMillis = maxRetrySleepInMillis; - return this; - } - - public HermesClientBuilder withScheduler(ScheduledExecutorService scheduler) { - this.schedulerFactory = () -> scheduler; - return this; - } + return hermesClient; + } + + public HermesClientBuilder withURI(URI uri) { + this.uri = uri; + return this; + } + + public HermesClientBuilder withMetrics(MetricsProvider metrics) { + this.metrics = Optional.of(metrics); + return this; + } + + public HermesClientBuilder withDefaultContentType(String defaultContentType) { + defaultHeaders.put(HermesMessage.CONTENT_TYPE_HEADER, defaultContentType); + return this; + } + + public HermesClientBuilder withDefaultHeaderValue(String header, String value) { + defaultHeaders.put(header, value); + return this; + } + + public HermesClientBuilder withRetries(int retries) { + this.retries = retries; + return this; + } + + public HermesClientBuilder withRetries(int retries, Predicate retryCondition) { + this.retryCondition = retryCondition; + return withRetries(retries); + } + + public HermesClientBuilder withRetrySleep(long retrySleepInMillis) { + this.retrySleepInMillis = retrySleepInMillis; + return this; + } + + public HermesClientBuilder withRetrySleep(long retrySleepInMillis, long maxRetrySleepInMillis) { + this.retrySleepInMillis = retrySleepInMillis; + this.maxRetrySleepInMillis = maxRetrySleepInMillis; + return this; + } + + public HermesClientBuilder withScheduler(ScheduledExecutorService scheduler) { + this.schedulerFactory = () -> scheduler; + return this; + } } diff --git a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesClientShutdownException.java b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesClientShutdownException.java index 1b50292abd..12b4a0a23a 100644 --- a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesClientShutdownException.java +++ b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesClientShutdownException.java @@ -2,11 +2,11 @@ public class HermesClientShutdownException extends RuntimeException { - public HermesClientShutdownException() { - this("Hermes client is already shutdown"); - } + public HermesClientShutdownException() { + this("Hermes client is already shutdown"); + } - public HermesClientShutdownException(String message) { - super(message); - } + public HermesClientShutdownException(String message) { + super(message); + } } diff --git a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesClientTermination.java b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesClientTermination.java index 897e21eb8a..f131f15081 100644 --- a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesClientTermination.java +++ b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesClientTermination.java @@ -7,30 +7,31 @@ class HermesClientTermination { - private final ExecutorService executorService = Executors.newSingleThreadExecutor(); - private final long pollInterval; + private final ExecutorService executorService = Executors.newSingleThreadExecutor(); + private final long pollInterval; - HermesClientTermination(long pollInterval) { - this.pollInterval = pollInterval; - } + HermesClientTermination(long pollInterval) { + this.pollInterval = pollInterval; + } - CompletableFuture observe(BooleanSupplier condition) { - final CompletableFuture result = new CompletableFuture<>(); + CompletableFuture observe(BooleanSupplier condition) { + final CompletableFuture result = new CompletableFuture<>(); - executorService.submit(() -> { - try { - while (!condition.getAsBoolean()) { - Thread.sleep(pollInterval); - } - result.complete(null); - } catch (InterruptedException e) { - result.completeExceptionally(e); - Thread.currentThread().interrupt(); - } finally { - executorService.shutdown(); + executorService.submit( + () -> { + try { + while (!condition.getAsBoolean()) { + Thread.sleep(pollInterval); } + result.complete(null); + } catch (InterruptedException e) { + result.completeExceptionally(e); + Thread.currentThread().interrupt(); + } finally { + executorService.shutdown(); + } }); - return result; - } -} \ No newline at end of file + return result; + } +} diff --git a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesMessage.java b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesMessage.java index 7b7040e007..4bd2fe0fb0 100644 --- a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesMessage.java +++ b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesMessage.java @@ -6,171 +6,163 @@ import java.util.Map; import java.util.function.BiConsumer; -/** - * All information Hermes needs to send a message. - */ +/** All information Hermes needs to send a message. */ public class HermesMessage { - public static final String SCHEMA_VERSION_HEADER = "Schema-Version"; - - static final String APPLICATION_JSON = "application/json;charset=UTF-8"; - static final String CONTENT_TYPE_HEADER = "Content-Type"; - static final String AVRO_BINARY = "avro/binary"; + public static final String SCHEMA_VERSION_HEADER = "Schema-Version"; + + static final String APPLICATION_JSON = "application/json;charset=UTF-8"; + static final String CONTENT_TYPE_HEADER = "Content-Type"; + static final String AVRO_BINARY = "avro/binary"; + + private final String topic; + private final byte[] body; + private final Map headers; + + private HermesMessage(String topic, Map headers, byte[] body) { + this.topic = topic; + this.headers = headers; + this.body = body; + } + + /** Use builder via: HermesMessage#hermesMessage instead. */ + @Deprecated + public HermesMessage(String topic, String contentType, int schemaVersion, byte[] body) { + Map headers = new HashMap<>(); + headers.put(CONTENT_TYPE_HEADER, contentType); + headers.put(SCHEMA_VERSION_HEADER, Integer.toString(schemaVersion)); + + this.topic = topic; + this.headers = headers; + this.body = body; + } + + /** Use builder via: HermesMessage#hermesMessage instead. */ + @Deprecated + public HermesMessage(String topic, String contentType, byte[] body) { + this(topic, contentType, -1, body); + } + + /** + * Message on given topic with given MIME Content Type. + * + *

Use builder via: HermesMessage#hermesMessage instead. + * + * @param topic topic name + * @param contentType MIME content type + * @param body body which will be translated to byte[] using UTF-8 charset + */ + @Deprecated + public HermesMessage(String topic, String contentType, String body) { + this(topic, contentType, body.getBytes(StandardCharsets.UTF_8)); + } + + /** Use builder via: HermesMessage#hermesMessage instead. */ + @Deprecated + public HermesMessage(String topic, String body) { + this(topic, null, body); + } + + public static Builder hermesMessage(String topic, byte[] content) { + return new Builder(topic, content); + } + + public static Builder hermesMessage(String topic, String content) { + return new Builder(topic, content); + } + + /** + * This method modifies the state of HermesMessage in order to avoid additional allocation when + * appending default values. Using same HermesMessage in multiple HermesClient objects with + * different defaults is very unlikely, as is sending the same message in multiple threads (which + * could cause issues with concurrent modification of map). + */ + static HermesMessage appendDefaults(HermesMessage message, Map headers) { + for (Map.Entry entry : headers.entrySet()) { + if (!message.headers.containsKey(entry.getKey())) { + message.headers.put(entry.getKey(), entry.getValue()); + } + } + return message; + } + + public String getTopic() { + return topic; + } + + public byte[] getBody() { + return body; + } + + public String getContentType() { + return headers.get(CONTENT_TYPE_HEADER); + } + + public int getSchemaVersion() { + String schemaVersion = headers.get(SCHEMA_VERSION_HEADER); + return schemaVersion != null ? Integer.parseInt(schemaVersion) : -1; + } + + public boolean schemaVersionDefined() { + return headers.containsKey(SCHEMA_VERSION_HEADER); + } + + public Map getHeaders() { + return Collections.unmodifiableMap(headers); + } + + public void consumeHeaders(BiConsumer consumer) { + headers.forEach(consumer); + } + + @Override + public String toString() { + return new String(getBody(), StandardCharsets.UTF_8); + } + + public static class Builder { private final String topic; private final byte[] body; - private final Map headers; - - private HermesMessage(String topic, Map headers, byte[] body) { - this.topic = topic; - this.headers = headers; - this.body = body; - } - - /** - * Use builder via: HermesMessage#hermesMessage instead. - */ - @Deprecated - public HermesMessage(String topic, String contentType, int schemaVersion, byte[] body) { - Map headers = new HashMap<>(); - headers.put(CONTENT_TYPE_HEADER, contentType); - headers.put(SCHEMA_VERSION_HEADER, Integer.toString(schemaVersion)); - - this.topic = topic; - this.headers = headers; - this.body = body; - } - - /** - * Use builder via: HermesMessage#hermesMessage instead. - */ - @Deprecated - public HermesMessage(String topic, String contentType, byte[] body) { - this(topic, contentType, -1, body); - } - - /** - *

Message on given topic with given MIME Content Type.

- *

Use builder via: HermesMessage#hermesMessage instead.

- * - * @param topic topic name - * @param contentType MIME content type - * @param body body which will be translated to byte[] using UTF-8 charset - */ - @Deprecated - public HermesMessage(String topic, String contentType, String body) { - this(topic, contentType, body.getBytes(StandardCharsets.UTF_8)); - } - - /** - * Use builder via: HermesMessage#hermesMessage instead. - */ - @Deprecated - public HermesMessage(String topic, String body) { - this(topic, null, body); - } - - public static Builder hermesMessage(String topic, byte[] content) { - return new Builder(topic, content); - } - - public static Builder hermesMessage(String topic, String content) { - return new Builder(topic, content); - } - - /** - * This method modifies the state of HermesMessage in order to avoid additional - * allocation when appending default values. Using same HermesMessage in multiple - * HermesClient objects with different defaults is very unlikely, as is sending the same - * message in multiple threads (which could cause issues with concurrent modification of - * map). - */ - static HermesMessage appendDefaults(HermesMessage message, Map headers) { - for (Map.Entry entry : headers.entrySet()) { - if (!message.headers.containsKey(entry.getKey())) { - message.headers.put(entry.getKey(), entry.getValue()); - } - } - return message; - } - - public String getTopic() { - return topic; - } + private final Map headers = new HashMap<>(); - public byte[] getBody() { - return body; + private Builder(String topic, byte[] body) { + this.topic = topic; + this.body = body; } - public String getContentType() { - return headers.get(CONTENT_TYPE_HEADER); + private Builder(String topic, String body) { + this(topic, body.getBytes(StandardCharsets.UTF_8)); } - public int getSchemaVersion() { - String schemaVersion = headers.get(SCHEMA_VERSION_HEADER); - return schemaVersion != null ? Integer.parseInt(schemaVersion) : -1; + public HermesMessage build() { + return new HermesMessage(topic, headers, body); } - public boolean schemaVersionDefined() { - return headers.containsKey(SCHEMA_VERSION_HEADER); + public Builder json() { + this.withContentType(APPLICATION_JSON); + return this; } - public Map getHeaders() { - return Collections.unmodifiableMap(headers); + public Builder avro(int schemaVersion) { + this.withContentType(AVRO_BINARY); + this.withSchemaVersion(schemaVersion); + return this; } - public void consumeHeaders(BiConsumer consumer) { - headers.forEach(consumer); + public Builder withContentType(String contentType) { + this.headers.put(CONTENT_TYPE_HEADER, contentType); + return this; } - @Override - public String toString() { - return new String(getBody(), StandardCharsets.UTF_8); + public Builder withSchemaVersion(int schemaVersion) { + this.headers.put(SCHEMA_VERSION_HEADER, Integer.toString(schemaVersion)); + return this; } - public static class Builder { - - private final String topic; - private final byte[] body; - private final Map headers = new HashMap<>(); - - private Builder(String topic, byte[] body) { - this.topic = topic; - this.body = body; - } - - private Builder(String topic, String body) { - this(topic, body.getBytes(StandardCharsets.UTF_8)); - } - - public HermesMessage build() { - return new HermesMessage(topic, headers, body); - } - - public Builder json() { - this.withContentType(APPLICATION_JSON); - return this; - } - - public Builder avro(int schemaVersion) { - this.withContentType(AVRO_BINARY); - this.withSchemaVersion(schemaVersion); - return this; - } - - public Builder withContentType(String contentType) { - this.headers.put(CONTENT_TYPE_HEADER, contentType); - return this; - } - - public Builder withSchemaVersion(int schemaVersion) { - this.headers.put(SCHEMA_VERSION_HEADER, Integer.toString(schemaVersion)); - return this; - } - - public Builder withHeader(String header, String value) { - this.headers.put(header, value); - return this; - } + public Builder withHeader(String header, String value) { + this.headers.put(header, value); + return this; } + } } diff --git a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesResponse.java b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesResponse.java index 84c6e97c1b..54cdcf3a78 100644 --- a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesResponse.java +++ b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesResponse.java @@ -1,80 +1,78 @@ package pl.allegro.tech.hermes.client; -import java.util.Optional; - import static java.net.HttpURLConnection.HTTP_ACCEPTED; import static java.net.HttpURLConnection.HTTP_CREATED; +import java.util.Optional; + public interface HermesResponse { - String MESSAGE_ID = "Hermes-Message-Id"; - String HTTP_1_1 = "http/1.1"; - - int getHttpStatus(); - - HermesMessage getHermesMessage(); - - @Deprecated - default boolean wasPublished() { - return getHttpStatus() == HTTP_CREATED; - } - - @Deprecated - default boolean wasAccepted() { - return wasPublished() || getHttpStatus() == HTTP_ACCEPTED; - } - - default boolean isSuccess() { - return getHttpStatus() == HTTP_CREATED || getHttpStatus() == HTTP_ACCEPTED; - } - - default boolean isFailure() { - return !isSuccess(); - } - - default Optional getFailureCause() { - return Optional.empty(); - } - - /** - * Retrieves failed HermesMessage. - * - * @deprecated as of Hermes 1.2.4, in favor of {@link #getHermesMessage()} - */ - @Deprecated - default Optional getFailedMessage() { - return Optional.empty(); - } - - default String getBody() { - return ""; - } - - default String getHeader(String header) { - return ""; - } - - default String getMessageId() { - return getHeader(MESSAGE_ID); - } - - default String getProtocol() { - return HTTP_1_1; - } - - default String getDebugLog() { - StringBuilder builder = new StringBuilder("Sending message ") - .append(getMessageId()) - .append(" to Hermes ") - .append(isSuccess() ? "succeeded" : "failed") - .append(", response code: ") - .append(getHttpStatus()) - .append(", body: ") - .append(getBody()); - getFailureCause().ifPresent(ex -> - builder.append(", exception: ") - .append(ex.getMessage()) - ); - return builder.toString(); - } + String MESSAGE_ID = "Hermes-Message-Id"; + String HTTP_1_1 = "http/1.1"; + + int getHttpStatus(); + + HermesMessage getHermesMessage(); + + @Deprecated + default boolean wasPublished() { + return getHttpStatus() == HTTP_CREATED; + } + + @Deprecated + default boolean wasAccepted() { + return wasPublished() || getHttpStatus() == HTTP_ACCEPTED; + } + + default boolean isSuccess() { + return getHttpStatus() == HTTP_CREATED || getHttpStatus() == HTTP_ACCEPTED; + } + + default boolean isFailure() { + return !isSuccess(); + } + + default Optional getFailureCause() { + return Optional.empty(); + } + + /** + * Retrieves failed HermesMessage. + * + * @deprecated as of Hermes 1.2.4, in favor of {@link #getHermesMessage()} + */ + @Deprecated + default Optional getFailedMessage() { + return Optional.empty(); + } + + default String getBody() { + return ""; + } + + default String getHeader(String header) { + return ""; + } + + default String getMessageId() { + return getHeader(MESSAGE_ID); + } + + default String getProtocol() { + return HTTP_1_1; + } + + default String getDebugLog() { + StringBuilder builder = + new StringBuilder("Sending message ") + .append(getMessageId()) + .append(" to Hermes ") + .append(isSuccess() ? "succeeded" : "failed") + .append(", response code: ") + .append(getHttpStatus()) + .append(", body: ") + .append(getBody()); + getFailureCause().ifPresent(ex -> builder.append(", exception: ").append(ex.getMessage())); + return builder.toString(); + } } diff --git a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesResponseBuilder.java b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesResponseBuilder.java index 44a6752606..14b203eb66 100644 --- a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesResponseBuilder.java +++ b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesResponseBuilder.java @@ -5,90 +5,89 @@ public class HermesResponseBuilder { - private int statusCode = -1; - private String body = ""; - private String protocol = "http/1.1"; - private Throwable failureCause; - private Function headerSupplier = (header) -> null; - private HermesMessage hermesMessage; - - public static HermesResponseBuilder hermesResponse(HermesMessage hermesMessage) { - return new HermesResponseBuilder().withHermesMessage(hermesMessage); - } - - public static HermesResponse hermesFailureResponse(Throwable exception, HermesMessage hermesMessage) { - return hermesResponse(hermesMessage) - .withFailureCause(exception) - .build(); - } - - public HermesResponseBuilder withHttpStatus(int statusCode) { - this.statusCode = statusCode; - return this; - } - - public HermesResponseBuilder withBody(String body) { - this.body = body; - return this; - } - - private HermesResponseBuilder withFailureCause(Throwable exception) { - this.failureCause = exception; - return this; - } - - private HermesResponseBuilder withHermesMessage(HermesMessage hermesMessage) { - this.hermesMessage = hermesMessage; - return this; - } - - public HermesResponseBuilder withHeaderSupplier(Function headerSupplier) { - this.headerSupplier = headerSupplier; - return this; - } - - public HermesResponseBuilder withProtocol(String protocol) { - this.protocol = protocol; - return this; - } - - public HermesResponse build() { - return new HermesResponse() { - - @Override - public int getHttpStatus() { - return statusCode; - } - - @Override - public HermesMessage getHermesMessage() { - return hermesMessage; - } - - @Override - public Optional getFailureCause() { - return Optional.ofNullable(failureCause); - } - - @Override - public Optional getFailedMessage() { - return Optional.ofNullable(hermesMessage); - } - - @Override - public String getBody() { - return body; - } - - @Override - public String getHeader(String header) { - return headerSupplier.apply(header); - } - - @Override - public String getProtocol() { - return protocol; - } - }; - } + private int statusCode = -1; + private String body = ""; + private String protocol = "http/1.1"; + private Throwable failureCause; + private Function headerSupplier = (header) -> null; + private HermesMessage hermesMessage; + + public static HermesResponseBuilder hermesResponse(HermesMessage hermesMessage) { + return new HermesResponseBuilder().withHermesMessage(hermesMessage); + } + + public static HermesResponse hermesFailureResponse( + Throwable exception, HermesMessage hermesMessage) { + return hermesResponse(hermesMessage).withFailureCause(exception).build(); + } + + public HermesResponseBuilder withHttpStatus(int statusCode) { + this.statusCode = statusCode; + return this; + } + + public HermesResponseBuilder withBody(String body) { + this.body = body; + return this; + } + + private HermesResponseBuilder withFailureCause(Throwable exception) { + this.failureCause = exception; + return this; + } + + private HermesResponseBuilder withHermesMessage(HermesMessage hermesMessage) { + this.hermesMessage = hermesMessage; + return this; + } + + public HermesResponseBuilder withHeaderSupplier(Function headerSupplier) { + this.headerSupplier = headerSupplier; + return this; + } + + public HermesResponseBuilder withProtocol(String protocol) { + this.protocol = protocol; + return this; + } + + public HermesResponse build() { + return new HermesResponse() { + + @Override + public int getHttpStatus() { + return statusCode; + } + + @Override + public HermesMessage getHermesMessage() { + return hermesMessage; + } + + @Override + public Optional getFailureCause() { + return Optional.ofNullable(failureCause); + } + + @Override + public Optional getFailedMessage() { + return Optional.ofNullable(hermesMessage); + } + + @Override + public String getBody() { + return body; + } + + @Override + public String getHeader(String header) { + return headerSupplier.apply(header); + } + + @Override + public String getProtocol() { + return protocol; + } + }; + } } diff --git a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesSender.java b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesSender.java index a6b096dbc2..fe7fc2710f 100644 --- a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesSender.java +++ b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/HermesSender.java @@ -6,5 +6,5 @@ @FunctionalInterface public interface HermesSender { - CompletableFuture send(URI uri, HermesMessage message); + CompletableFuture send(URI uri, HermesMessage message); } diff --git a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/MessageDeliveryListener.java b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/MessageDeliveryListener.java index bd5a1dcb82..139b706134 100644 --- a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/MessageDeliveryListener.java +++ b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/MessageDeliveryListener.java @@ -2,13 +2,13 @@ public interface MessageDeliveryListener { - void onSend(HermesResponse response, long latency); + void onSend(HermesResponse response, long latency); - void onFailure(HermesResponse message, int attemptCount); + void onFailure(HermesResponse message, int attemptCount); - void onFailedRetry(HermesResponse message, int attemptCount); + void onFailedRetry(HermesResponse message, int attemptCount); - void onSuccessfulRetry(HermesResponse message, int attemptCount); + void onSuccessfulRetry(HermesResponse message, int attemptCount); - void onMaxRetriesExceeded(HermesResponse message, int attemptCount); + void onMaxRetriesExceeded(HermesResponse message, int attemptCount); } diff --git a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/ReactiveHermesClient.java b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/ReactiveHermesClient.java index 0702ecc198..f6ea952c6f 100644 --- a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/ReactiveHermesClient.java +++ b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/ReactiveHermesClient.java @@ -1,10 +1,8 @@ package pl.allegro.tech.hermes.client; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import reactor.core.publisher.Mono; -import reactor.core.scheduler.Scheduler; -import reactor.util.retry.Retry; +import static pl.allegro.tech.hermes.client.HermesMessage.hermesMessage; +import static pl.allegro.tech.hermes.client.HermesResponseBuilder.hermesFailureResponse; +import static reactor.core.Exceptions.isRetryExhausted; import java.net.URI; import java.time.Duration; @@ -16,313 +14,330 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; - -import static pl.allegro.tech.hermes.client.HermesMessage.hermesMessage; -import static pl.allegro.tech.hermes.client.HermesResponseBuilder.hermesFailureResponse; -import static reactor.core.Exceptions.isRetryExhausted; - +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Scheduler; +import reactor.util.retry.Retry; public class ReactiveHermesClient { - private static final Logger logger = LoggerFactory.getLogger(ReactiveHermesClient.class); - private static final String RETRY_CONTEXT_KEY = "hermes-retry-context-key"; - - private final ReactiveHermesSender sender; - private final String uri; - private final Map defaultHeaders; - private final AtomicInteger currentlySending = new AtomicInteger(0); - private final int maxRetries; - private final Predicate retryCondition; - private final Duration retrySleep; - private final Duration maxRetrySleep; - private final Double jitterFactor; - private final Scheduler scheduler; - private volatile boolean shutdown = false; - private final List messageDeliveryListeners; - - ReactiveHermesClient(ReactiveHermesSender sender, - URI uri, - Map defaultHeaders, - int maxRetries, - Predicate retryCondition, - long retrySleepInMillis, - long maxRetrySleepInMillis, - double jitterFactor, - Scheduler scheduler) { - this.sender = sender; - this.uri = createUri(uri); - this.defaultHeaders = Collections.unmodifiableMap(new HashMap<>(defaultHeaders)); - this.maxRetries = maxRetries; - this.retryCondition = retryCondition; - this.retrySleep = Duration.ofMillis(retrySleepInMillis); - this.maxRetrySleep = Duration.ofMillis(maxRetrySleepInMillis); - this.jitterFactor = jitterFactor; - this.scheduler = scheduler; - this.messageDeliveryListeners = new ArrayList<>(); - } - - private String createUri(URI uri) { - String uriString = uri.toString(); - return uriString + (uriString.endsWith("/") ? "" : "/") + "topics/"; - } - - public Mono publishJSON(String topic, byte[] message) { - return publish(hermesMessage(topic, message).json().build()); - } - - public Mono publishJSON(String topic, String message) { - return publish(hermesMessage(topic, message).json().build()); - } - - public Mono publishAvro(String topic, int schemaVersion, byte[] message) { - return publish(hermesMessage(topic, message).avro(schemaVersion).build()); - } - - public Mono publish(String topic, String message) { - return publish(hermesMessage(topic, message).build()); - } - - public Mono publish(String topic, String contentType, byte[] message) { - return publish(hermesMessage(topic, message).withContentType(contentType).build()); - } - - public Mono publish(String topic, String contentType, String message) { - return publish(hermesMessage(topic, message).withContentType(contentType).build()); - } - - public Mono publish(String topic, String contentType, int schemaVersion, byte[] message) { - return publish(hermesMessage(topic, message).withContentType(contentType).withSchemaVersion(schemaVersion).build()); - } - - public Mono publish(HermesMessage message) { - if (shutdown) { - return completedWithShutdownException(); - } - HermesMessage.appendDefaults(message, defaultHeaders); - return publishWithRetries(message); - } + private static final Logger logger = LoggerFactory.getLogger(ReactiveHermesClient.class); + private static final String RETRY_CONTEXT_KEY = "hermes-retry-context-key"; + + private final ReactiveHermesSender sender; + private final String uri; + private final Map defaultHeaders; + private final AtomicInteger currentlySending = new AtomicInteger(0); + private final int maxRetries; + private final Predicate retryCondition; + private final Duration retrySleep; + private final Duration maxRetrySleep; + private final Double jitterFactor; + private final Scheduler scheduler; + private volatile boolean shutdown = false; + private final List messageDeliveryListeners; + + ReactiveHermesClient( + ReactiveHermesSender sender, + URI uri, + Map defaultHeaders, + int maxRetries, + Predicate retryCondition, + long retrySleepInMillis, + long maxRetrySleepInMillis, + double jitterFactor, + Scheduler scheduler) { + this.sender = sender; + this.uri = createUri(uri); + this.defaultHeaders = Collections.unmodifiableMap(new HashMap<>(defaultHeaders)); + this.maxRetries = maxRetries; + this.retryCondition = retryCondition; + this.retrySleep = Duration.ofMillis(retrySleepInMillis); + this.maxRetrySleep = Duration.ofMillis(maxRetrySleepInMillis); + this.jitterFactor = jitterFactor; + this.scheduler = scheduler; + this.messageDeliveryListeners = new ArrayList<>(); + } + + private String createUri(URI uri) { + String uriString = uri.toString(); + return uriString + (uriString.endsWith("/") ? "" : "/") + "topics/"; + } + + public Mono publishJSON(String topic, byte[] message) { + return publish(hermesMessage(topic, message).json().build()); + } + + public Mono publishJSON(String topic, String message) { + return publish(hermesMessage(topic, message).json().build()); + } + + public Mono publishAvro(String topic, int schemaVersion, byte[] message) { + return publish(hermesMessage(topic, message).avro(schemaVersion).build()); + } + + public Mono publish(String topic, String message) { + return publish(hermesMessage(topic, message).build()); + } + + public Mono publish(String topic, String contentType, byte[] message) { + return publish(hermesMessage(topic, message).withContentType(contentType).build()); + } + + public Mono publish(String topic, String contentType, String message) { + return publish(hermesMessage(topic, message).withContentType(contentType).build()); + } + + public Mono publish( + String topic, String contentType, int schemaVersion, byte[] message) { + return publish( + hermesMessage(topic, message) + .withContentType(contentType) + .withSchemaVersion(schemaVersion) + .build()); + } + + public Mono publish(HermesMessage message) { + if (shutdown) { + return completedWithShutdownException(); + } + HermesMessage.appendDefaults(message, defaultHeaders); + return publishWithRetries(message); + } + + public boolean addMessageDeliveryListener(MessageDeliveryListener listener) { + return messageDeliveryListeners.add(listener); + } + + private Mono publishWithRetries(HermesMessage message) { + currentlySending.incrementAndGet(); + Mono sendOnceResult = + sendOnce(message) + .flatMap(this::testRetryCondition) + .onErrorResume(exception -> mapExceptionToFailedAttempt(exception, message)) + .subscribeOn(scheduler); + + return retry(sendOnceResult) + .doOnSuccess( + hr -> { + if (hr.response.isSuccess() || !hr.matchesRetryPolicy) { + handleSuccessfulRetry(hr.response, hr.attempt); + } else { + handleFailure(hr.response, hr.attempt); + } + }) + .map(result -> result.response) + .doFinally(s -> currentlySending.decrementAndGet()); + } + + private Mono retry(Mono sendOnceResult) { + Retry retrySpec = prepareRetrySpec(); + return sendOnceResult + .flatMap(this::unwrapFailedAttemptAsException) + .retryWhen(retrySpec) + .contextWrite(ctx -> ctx.put(RETRY_CONTEXT_KEY, HermesRetryContext.emptyRetryContext())) + .onErrorResume(Exception.class, this::unwrapRetryExhaustedException); + } + + private Mono unwrapRetryExhaustedException(Exception exception) { + if (isRetryExhausted(exception)) { + RetryFailedException rfe = (RetryFailedException) (exception.getCause()); + Failed failedAttempt = rfe.failed; + HermesResponse hermesResponse = failedAttempt.hermesResponse; + handleMaxRetriesExceeded(hermesResponse, failedAttempt.attempt); + Throwable cause = rfe.getCause(); + if (cause instanceof ShouldRetryException) { + ShouldRetryException sre = (ShouldRetryException) cause; + return Mono.just((Attempt) Result.attempt(sre.hermesResponse, failedAttempt.attempt, true)); + } + if (hermesResponse.isFailure()) { + handleFailure(hermesResponse, failedAttempt.attempt); + } + } + return Mono.error(exception); + } + + private Mono unwrapFailedAttemptAsException(Result result) { + if (result instanceof Failed) { + Failed failed = (Failed) result; + return Mono.error(new RetryFailedException(failed)); + } else { + return Mono.just((Attempt) result); + } + } + + private Mono mapExceptionToFailedAttempt( + Throwable throwable, HermesMessage hermesMessage) { + return getNextAttempt() + .map(attempt -> Result.failureByException(throwable, hermesMessage, attempt)); + } + + private Mono testRetryCondition(HermesResponse response) { + return getNextAttempt() + .map( + attempt -> { + if (retryCondition.test(response)) { + return Result.retryableFailure(response, attempt); + } else { + return Result.attempt(response, attempt, false); + } + }); + } + + private Retry prepareRetrySpec() { + if (retrySleep.isZero()) { + return Retry.max(maxRetries).doAfterRetry(this::handleRetryAttempt); + } else { + return Retry.backoff(maxRetries, retrySleep) + .maxBackoff(maxRetrySleep) + .jitter(jitterFactor) + .doAfterRetry(this::handleRetryAttempt); + } + } + + private void handleRetryAttempt(Retry.RetrySignal retrySignal) { + RetryFailedException failedException = (RetryFailedException) retrySignal.failure(); + handleFailedAttempt(failedException.failed.hermesResponse, retrySignal.totalRetries() + 1); + } + + private Mono getNextAttempt() { + return Mono.deferContextual(Mono::just) + .map( + ctx -> + ctx.getOrDefault(RETRY_CONTEXT_KEY, HermesRetryContext.emptyRetryContext()) + .getAndIncrementAttempt()); + } + + private Mono sendOnce(HermesMessage message) { + return Mono.defer( + () -> { + long startTime = System.nanoTime(); + try { + return sender + .sendReactively(URI.create(uri + message.getTopic()), message) + .onErrorResume(e -> Mono.just(hermesFailureResponse(e, message))) + .doOnNext( + resp -> { + long latency = System.nanoTime() - startTime; + messageDeliveryListeners.forEach(l -> l.onSend(resp, latency)); + }); + + } catch (Exception e) { + return Mono.error(e); + } + }); + } + + private Mono completedWithShutdownException() { + return Mono.error(new HermesClientShutdownException()); + } + + public Mono closeAsync(long pollInterval) { + shutdown = true; + CompletableFuture voidCompletableFuture = + new HermesClientTermination(pollInterval) + .observe(() -> currentlySending.get() == 0) + .whenComplete((response, ex) -> scheduler.dispose()); + return Mono.fromFuture(voidCompletableFuture); + } + + public void close(long pollInterval, long timeout) { + closeAsync(pollInterval).block(Duration.ofMillis(timeout)); + } + + private void handleMaxRetriesExceeded(HermesResponse response, int attemptCount) { + messageDeliveryListeners.forEach(l -> l.onMaxRetriesExceeded(response, attemptCount)); + logger.error( + "Failed to send message to topic {} after {} attempts", + response.getHermesMessage().getTopic(), + attemptCount); + } + + private void handleFailedAttempt(HermesResponse response, long attemptCount) { + messageDeliveryListeners.forEach(l -> l.onFailedRetry(response, (int) attemptCount)); + } + + private void handleFailure(HermesResponse response, long attemptCount) { + messageDeliveryListeners.forEach(l -> l.onFailure(response, (int) attemptCount)); + } + + private void handleSuccessfulRetry(HermesResponse response, long attemptCount) { + messageDeliveryListeners.forEach(l -> l.onSuccessfulRetry(response, (int) attemptCount)); + } + + private static class ShouldRetryException extends Exception { + private final HermesResponse hermesResponse; + + public ShouldRetryException(HermesResponse hermesResponse) { + this.hermesResponse = hermesResponse; + } + + public HermesResponse getHermesResponse() { + return hermesResponse; + } + } + + private static class RetryFailedException extends Exception { + private final Failed failed; - public boolean addMessageDeliveryListener(MessageDeliveryListener listener) { - return messageDeliveryListeners.add(listener); + public RetryFailedException(Failed failed) { + super(failed.cause); + this.failed = failed; } + } - private Mono publishWithRetries(HermesMessage message) { - currentlySending.incrementAndGet(); - Mono sendOnceResult = sendOnce(message) - .flatMap(this::testRetryCondition) - .onErrorResume(exception -> mapExceptionToFailedAttempt(exception, message)) - .subscribeOn(scheduler); - - return retry(sendOnceResult) - .doOnSuccess(hr -> { - if (hr.response.isSuccess() || !hr.matchesRetryPolicy) { - handleSuccessfulRetry(hr.response, hr.attempt); - } else { - handleFailure(hr.response, hr.attempt); - } - }) - .map(result -> result.response) - .doFinally(s -> currentlySending.decrementAndGet()); + private interface Result { + static Result attempt(HermesResponse response, int attempt, boolean qualifiedForRetry) { + return new Attempt(response, attempt, qualifiedForRetry); } - private Mono retry(Mono sendOnceResult) { - Retry retrySpec = prepareRetrySpec(); - return sendOnceResult - .flatMap(this::unwrapFailedAttemptAsException) - .retryWhen(retrySpec) - .contextWrite(ctx -> ctx.put(RETRY_CONTEXT_KEY, HermesRetryContext.emptyRetryContext())) - .onErrorResume(Exception.class, this::unwrapRetryExhaustedException); + static Result retryableFailure(HermesResponse response, int attempt) { + return new Failed(response, attempt, new ShouldRetryException(response)); } - private Mono unwrapRetryExhaustedException(Exception exception) { - if (isRetryExhausted(exception)) { - RetryFailedException rfe = (RetryFailedException) (exception.getCause()); - Failed failedAttempt = rfe.failed; - HermesResponse hermesResponse = failedAttempt.hermesResponse; - handleMaxRetriesExceeded(hermesResponse, failedAttempt.attempt); - Throwable cause = rfe.getCause(); - if (cause instanceof ShouldRetryException) { - ShouldRetryException sre = (ShouldRetryException) cause; - return Mono.just((Attempt) Result.attempt(sre.hermesResponse, failedAttempt.attempt, true)); - } - if (hermesResponse.isFailure()) { - handleFailure(hermesResponse, failedAttempt.attempt); - } - } - return Mono.error(exception); + static Result failureByException( + Throwable throwable, HermesMessage hermesMessage, int attempt) { + return new Failed(hermesFailureResponse(throwable, hermesMessage), attempt, throwable); } + } - private Mono unwrapFailedAttemptAsException(Result result) { - if (result instanceof Failed) { - Failed failed = (Failed) result; - return Mono.error(new RetryFailedException(failed)); - } else { - return Mono.just((Attempt) result); - } - } - - private Mono mapExceptionToFailedAttempt(Throwable throwable, HermesMessage hermesMessage) { - return getNextAttempt().map(attempt -> Result.failureByException(throwable, hermesMessage, attempt)); - } - - private Mono testRetryCondition(HermesResponse response) { - return getNextAttempt() - .map(attempt -> { - if (retryCondition.test(response)) { - return Result.retryableFailure(response, attempt); - } else { - return Result.attempt(response, attempt, false); - } - }); - } + private static class Attempt implements Result { + private final HermesResponse response; + private final int attempt; + private final boolean matchesRetryPolicy; - private Retry prepareRetrySpec() { - if (retrySleep.isZero()) { - return Retry.max(maxRetries) - .doAfterRetry(this::handleRetryAttempt); - } else { - return Retry.backoff(maxRetries, retrySleep) - .maxBackoff(maxRetrySleep) - .jitter(jitterFactor) - .doAfterRetry(this::handleRetryAttempt); - } + private Attempt(HermesResponse response, int attempt, boolean matchesRetryPolicy) { + this.response = response; + this.attempt = attempt; + this.matchesRetryPolicy = matchesRetryPolicy; } + } - private void handleRetryAttempt(Retry.RetrySignal retrySignal) { - RetryFailedException failedException = (RetryFailedException) retrySignal.failure(); - handleFailedAttempt(failedException.failed.hermesResponse, retrySignal.totalRetries() + 1); - } + private static class Failed implements Result { + private final HermesResponse hermesResponse; + private final int attempt; + private final Throwable cause; - private Mono getNextAttempt() { - return Mono.deferContextual(Mono::just) - .map(ctx -> ctx.getOrDefault(RETRY_CONTEXT_KEY, HermesRetryContext.emptyRetryContext()) - .getAndIncrementAttempt() - ); + private Failed(HermesResponse hermesResponse, int attempt, Throwable cause) { + this.attempt = attempt; + this.cause = cause; + this.hermesResponse = hermesResponse; } + } - private Mono sendOnce(HermesMessage message) { - return Mono.defer(() -> { - long startTime = System.nanoTime(); - try { - return sender.sendReactively(URI.create(uri + message.getTopic()), message) - .onErrorResume(e -> Mono.just(hermesFailureResponse(e, message))) - .doOnNext(resp -> { - long latency = System.nanoTime() - startTime; - messageDeliveryListeners.forEach(l -> l.onSend(resp, latency)); - }); - - } catch (Exception e) { - return Mono.error(e); - } - } - ); + private static class HermesRetryContext { + static HermesRetryContext emptyRetryContext() { + return new HermesRetryContext(); } - private Mono completedWithShutdownException() { - return Mono.error(new HermesClientShutdownException()); - } + private int attempt; - public Mono closeAsync(long pollInterval) { - shutdown = true; - CompletableFuture voidCompletableFuture = new HermesClientTermination(pollInterval) - .observe(() -> currentlySending.get() == 0) - .whenComplete((response, ex) -> scheduler.dispose()); - return Mono.fromFuture(voidCompletableFuture); + HermesRetryContext() { + attempt = 1; } - public void close(long pollInterval, long timeout) { - closeAsync(pollInterval).block(Duration.ofMillis(timeout)); - } - - private void handleMaxRetriesExceeded(HermesResponse response, int attemptCount) { - messageDeliveryListeners.forEach(l -> l.onMaxRetriesExceeded(response, attemptCount)); - logger.error("Failed to send message to topic {} after {} attempts", - response.getHermesMessage().getTopic(), attemptCount); - } - - private void handleFailedAttempt(HermesResponse response, long attemptCount) { - messageDeliveryListeners.forEach(l -> l.onFailedRetry(response, (int) attemptCount)); - } - - private void handleFailure(HermesResponse response, long attemptCount) { - messageDeliveryListeners.forEach(l -> l.onFailure(response, (int) attemptCount)); - } - - private void handleSuccessfulRetry(HermesResponse response, long attemptCount) { - messageDeliveryListeners.forEach(l -> l.onSuccessfulRetry(response, (int) attemptCount)); - } - - private static class ShouldRetryException extends Exception { - private final HermesResponse hermesResponse; - - public ShouldRetryException(HermesResponse hermesResponse) { - this.hermesResponse = hermesResponse; - } - - public HermesResponse getHermesResponse() { - return hermesResponse; - } - } - - private static class RetryFailedException extends Exception { - private final Failed failed; - - public RetryFailedException(Failed failed) { - super(failed.cause); - this.failed = failed; - } - } - - private interface Result { - static Result attempt(HermesResponse response, int attempt, boolean qualifiedForRetry) { - return new Attempt(response, attempt, qualifiedForRetry); - } - - static Result retryableFailure(HermesResponse response, int attempt) { - return new Failed(response, attempt, new ShouldRetryException(response)); - } - - static Result failureByException(Throwable throwable, HermesMessage hermesMessage, int attempt) { - return new Failed(hermesFailureResponse(throwable, hermesMessage), attempt, throwable); - } - } - - private static class Attempt implements Result { - private final HermesResponse response; - private final int attempt; - private final boolean matchesRetryPolicy; - - private Attempt(HermesResponse response, int attempt, boolean matchesRetryPolicy) { - this.response = response; - this.attempt = attempt; - this.matchesRetryPolicy = matchesRetryPolicy; - } - } - - private static class Failed implements Result { - private final HermesResponse hermesResponse; - private final int attempt; - private final Throwable cause; - - private Failed(HermesResponse hermesResponse, int attempt, Throwable cause) { - this.attempt = attempt; - this.cause = cause; - this.hermesResponse = hermesResponse; - } - } - - private static class HermesRetryContext { - static HermesRetryContext emptyRetryContext() { - return new HermesRetryContext(); - } - - private int attempt; - - HermesRetryContext() { - attempt = 1; - } - - int getAndIncrementAttempt() { - return attempt++; - } + int getAndIncrementAttempt() { + return attempt++; } + } } diff --git a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/ReactiveHermesClientBuilder.java b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/ReactiveHermesClientBuilder.java index 1303acd49a..c25da59ab0 100644 --- a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/ReactiveHermesClientBuilder.java +++ b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/ReactiveHermesClientBuilder.java @@ -1,10 +1,5 @@ package pl.allegro.tech.hermes.client; -import pl.allegro.tech.hermes.client.metrics.MetricsMessageDeliveryListener; -import pl.allegro.tech.hermes.client.metrics.MetricsProvider; -import reactor.core.scheduler.Scheduler; -import reactor.core.scheduler.Schedulers; - import java.net.URI; import java.util.HashMap; import java.util.Map; @@ -12,88 +7,106 @@ import java.util.concurrent.Executors; import java.util.function.Predicate; import java.util.function.Supplier; +import pl.allegro.tech.hermes.client.metrics.MetricsMessageDeliveryListener; +import pl.allegro.tech.hermes.client.metrics.MetricsProvider; +import reactor.core.scheduler.Scheduler; +import reactor.core.scheduler.Schedulers; public class ReactiveHermesClientBuilder { - private final ReactiveHermesSender sender; - private URI uri = URI.create("http://localhost:8080"); - private final Map defaultHeaders = new HashMap<>(); - private int retries = 3; - private Predicate retryCondition = new HermesClientBasicRetryCondition(); - private long retrySleepInMillis = 100; - private long maxRetrySleepInMillis = 300; - private double jitterFactor = 0.5d; - private Supplier schedulerFactory = () -> Schedulers.fromExecutor(Executors.newSingleThreadScheduledExecutor()); - private Optional metrics = Optional.empty(); - - public ReactiveHermesClientBuilder(ReactiveHermesSender sender) { - this.sender = sender; - this.defaultHeaders.put(HermesMessage.CONTENT_TYPE_HEADER, HermesMessage.APPLICATION_JSON); - } - - public static ReactiveHermesClientBuilder hermesClient(ReactiveHermesSender sender) { - return new ReactiveHermesClientBuilder(sender); - } - - public ReactiveHermesClient build() { - ReactiveHermesClient hermesClient = new ReactiveHermesClient(sender, uri, defaultHeaders, retries, retryCondition, - retrySleepInMillis, maxRetrySleepInMillis, jitterFactor, schedulerFactory.get()); - - metrics.ifPresent((metricsProvider) -> { - hermesClient.addMessageDeliveryListener(new MetricsMessageDeliveryListener(metricsProvider)); + private final ReactiveHermesSender sender; + private URI uri = URI.create("http://localhost:8080"); + private final Map defaultHeaders = new HashMap<>(); + private int retries = 3; + private Predicate retryCondition = new HermesClientBasicRetryCondition(); + private long retrySleepInMillis = 100; + private long maxRetrySleepInMillis = 300; + private double jitterFactor = 0.5d; + private Supplier schedulerFactory = + () -> Schedulers.fromExecutor(Executors.newSingleThreadScheduledExecutor()); + private Optional metrics = Optional.empty(); + + public ReactiveHermesClientBuilder(ReactiveHermesSender sender) { + this.sender = sender; + this.defaultHeaders.put(HermesMessage.CONTENT_TYPE_HEADER, HermesMessage.APPLICATION_JSON); + } + + public static ReactiveHermesClientBuilder hermesClient(ReactiveHermesSender sender) { + return new ReactiveHermesClientBuilder(sender); + } + + public ReactiveHermesClient build() { + ReactiveHermesClient hermesClient = + new ReactiveHermesClient( + sender, + uri, + defaultHeaders, + retries, + retryCondition, + retrySleepInMillis, + maxRetrySleepInMillis, + jitterFactor, + schedulerFactory.get()); + + metrics.ifPresent( + (metricsProvider) -> { + hermesClient.addMessageDeliveryListener( + new MetricsMessageDeliveryListener(metricsProvider)); }); - return hermesClient; - } - - public ReactiveHermesClientBuilder withURI(URI uri) { - this.uri = uri; - return this; - } - - public ReactiveHermesClientBuilder withMetrics(MetricsProvider metrics) { - this.metrics = Optional.of(metrics); - return this; - } - - public ReactiveHermesClientBuilder withDefaultContentType(String defaultContentType) { - defaultHeaders.put(HermesMessage.CONTENT_TYPE_HEADER, defaultContentType); - return this; - } - - public ReactiveHermesClientBuilder withDefaultHeaderValue(String header, String value) { - defaultHeaders.put(header, value); - return this; - } - - public ReactiveHermesClientBuilder withRetries(int retries) { - this.retries = retries; - return this; - } - - public ReactiveHermesClientBuilder withRetries(int retries, Predicate retryCondition) { - this.retryCondition = retryCondition; - return withRetries(retries); - } - - public ReactiveHermesClientBuilder withRetrySleep(long retrySleepInMillis) { - this.retrySleepInMillis = retrySleepInMillis; - return this; - } - - public ReactiveHermesClientBuilder withRetrySleep(long retrySleepInMillis, long maxRetrySleepInMillis) { - this.retrySleepInMillis = retrySleepInMillis; - this.maxRetrySleepInMillis = maxRetrySleepInMillis; - return this; - } - - public ReactiveHermesClientBuilder withScheduler(Scheduler scheduler) { - this.schedulerFactory = () -> scheduler; - return this; - } - - public ReactiveHermesClientBuilder withJitter(Double jitterFactor) { - this.jitterFactor = jitterFactor; - return this; - } + return hermesClient; + } + + public ReactiveHermesClientBuilder withURI(URI uri) { + this.uri = uri; + return this; + } + + public ReactiveHermesClientBuilder withMetrics(MetricsProvider metrics) { + this.metrics = Optional.of(metrics); + return this; + } + + public ReactiveHermesClientBuilder withDefaultContentType(String defaultContentType) { + defaultHeaders.put(HermesMessage.CONTENT_TYPE_HEADER, defaultContentType); + return this; + } + + public ReactiveHermesClientBuilder withDefaultHeaderValue(String header, String value) { + defaultHeaders.put(header, value); + return this; + } + + public ReactiveHermesClientBuilder withRetries(int retries) { + this.retries = retries; + return this; + } + + public ReactiveHermesClientBuilder withRetries( + int retries, Predicate retryCondition) { + this.retryCondition = retryCondition; + return withRetries(retries); + } + + public ReactiveHermesClientBuilder withRetrySleep(long retrySleepInMillis) { + this.retrySleepInMillis = retrySleepInMillis; + return this; + } + + public ReactiveHermesClientBuilder withRetrySleep( + long retrySleepInMillis, long maxRetrySleepInMillis) { + this.retrySleepInMillis = retrySleepInMillis; + this.maxRetrySleepInMillis = maxRetrySleepInMillis; + return this; + } + + public ReactiveHermesClientBuilder withScheduler(Scheduler scheduler) { + this.schedulerFactory = () -> scheduler; + return this; + } + + public ReactiveHermesClientBuilder withJitter(Double jitterFactor) { + this.jitterFactor = jitterFactor; + return this; + } } diff --git a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/ReactiveHermesSender.java b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/ReactiveHermesSender.java index b500ed1257..e538458c87 100644 --- a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/ReactiveHermesSender.java +++ b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/ReactiveHermesSender.java @@ -1,11 +1,10 @@ package pl.allegro.tech.hermes.client; -import reactor.core.publisher.Mono; - import java.net.URI; +import reactor.core.publisher.Mono; @FunctionalInterface public interface ReactiveHermesSender { - Mono sendReactively(URI uri, HermesMessage message); + Mono sendReactively(URI uri, HermesMessage message); } diff --git a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/jersey/JerseyHermesSender.java b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/jersey/JerseyHermesSender.java index 3d59d58548..7b21121f57 100644 --- a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/jersey/JerseyHermesSender.java +++ b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/jersey/JerseyHermesSender.java @@ -1,52 +1,53 @@ package pl.allegro.tech.hermes.client.jersey; +import static pl.allegro.tech.hermes.client.HermesResponseBuilder.hermesResponse; + import jakarta.ws.rs.client.Client; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.client.Invocation; import jakarta.ws.rs.client.InvocationCallback; import jakarta.ws.rs.core.Response; +import java.net.URI; +import java.util.concurrent.CompletableFuture; import pl.allegro.tech.hermes.client.HermesMessage; import pl.allegro.tech.hermes.client.HermesResponse; import pl.allegro.tech.hermes.client.HermesSender; -import java.net.URI; -import java.util.concurrent.CompletableFuture; - -import static pl.allegro.tech.hermes.client.HermesResponseBuilder.hermesResponse; - public class JerseyHermesSender implements HermesSender { - private final Client client; - - public JerseyHermesSender(Client client) { - this.client = client; - } - - @Override - public CompletableFuture send(URI uri, HermesMessage message) { - CompletableFuture future = new CompletableFuture<>(); - Invocation.Builder builder = client.target(uri).request(); - message.consumeHeaders(builder::header); - builder.async() - .post(Entity.entity(message.getBody(), message.getContentType()), - new InvocationCallback() { - @Override - public void completed(Response response) { - future.complete(fromJerseyResponse(message, response)); - } - - @Override - public void failed(Throwable exception) { - future.completeExceptionally(exception); - } - }); - return future; - } - - private HermesResponse fromJerseyResponse(HermesMessage message, Response response) { - return hermesResponse(message) - .withHttpStatus(response.getStatus()) - .withBody(response.readEntity(String.class)) - .withHeaderSupplier(response::getHeaderString) - .build(); - } + private final Client client; + + public JerseyHermesSender(Client client) { + this.client = client; + } + + @Override + public CompletableFuture send(URI uri, HermesMessage message) { + CompletableFuture future = new CompletableFuture<>(); + Invocation.Builder builder = client.target(uri).request(); + message.consumeHeaders(builder::header); + builder + .async() + .post( + Entity.entity(message.getBody(), message.getContentType()), + new InvocationCallback() { + @Override + public void completed(Response response) { + future.complete(fromJerseyResponse(message, response)); + } + + @Override + public void failed(Throwable exception) { + future.completeExceptionally(exception); + } + }); + return future; + } + + private HermesResponse fromJerseyResponse(HermesMessage message, Response response) { + return hermesResponse(message) + .withHttpStatus(response.getStatus()) + .withBody(response.readEntity(String.class)) + .withHeaderSupplier(response::getHeaderString) + .build(); + } } diff --git a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/metrics/MetricsMessageDeliveryListener.java b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/metrics/MetricsMessageDeliveryListener.java index 7f72a5ce1f..5fedec10dd 100644 --- a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/metrics/MetricsMessageDeliveryListener.java +++ b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/metrics/MetricsMessageDeliveryListener.java @@ -1,77 +1,76 @@ package pl.allegro.tech.hermes.client.metrics; -import pl.allegro.tech.hermes.client.HermesMessage; -import pl.allegro.tech.hermes.client.HermesResponse; -import pl.allegro.tech.hermes.client.MessageDeliveryListener; +import static java.util.concurrent.TimeUnit.NANOSECONDS; import java.util.HashMap; import java.util.Map; - -import static java.util.concurrent.TimeUnit.NANOSECONDS; +import pl.allegro.tech.hermes.client.HermesMessage; +import pl.allegro.tech.hermes.client.HermesResponse; +import pl.allegro.tech.hermes.client.MessageDeliveryListener; public class MetricsMessageDeliveryListener implements MessageDeliveryListener { - private final MetricsProvider metrics; - - public MetricsMessageDeliveryListener(MetricsProvider metrics) { - this.metrics = metrics; - } - - @Override - public void onSend(HermesResponse response, long latency) { - HermesMessage message = response.getHermesMessage(); - String topic = MetricsUtils.sanitizeTopic(message.getTopic()); - - metrics.timerRecord(topic, "latency", latency, NANOSECONDS); - Map tags = new HashMap<>(); - tags.put("code", String.valueOf(response.getHttpStatus())); - metrics.counterIncrement(topic, "status", tags); - - counterIncrementIf(topic, "publish.failure", response.isFailure()); - } - - @Override - public void onFailure(HermesResponse response, int attemptCount) { - String topic = MetricsUtils.sanitizeTopic(response.getHermesMessage().getTopic()); - metrics.counterIncrement(topic, "failure"); - } - - @Override - public void onFailedRetry(HermesResponse response, int attemptCount) { - String topic = MetricsUtils.sanitizeTopic(response.getHermesMessage().getTopic()); - metrics.counterIncrement(topic, "retries.count"); - metrics.counterIncrement(topic, "failure"); - - metrics.counterIncrement(topic, "publish.retry.failure"); - } - - @Override - public void onSuccessfulRetry(HermesResponse response, int attemptCount) { - String topic = MetricsUtils.sanitizeTopic(response.getHermesMessage().getTopic()); - metrics.counterIncrement(topic, "retries.success"); - metrics.histogramUpdate(topic, "retries.attempts", attemptCount - 1); - - boolean wasRetried = attemptCount > 1; - - metrics.counterIncrement(topic, "publish.attempt"); - counterIncrementIf(topic, "publish.retry.success", response.isSuccess() && wasRetried); - counterIncrementIf(topic, "publish.finally.success", response.isSuccess()); - counterIncrementIf(topic, "publish.retry.failure", response.isFailure() && wasRetried); - counterIncrementIf(topic, "publish.finally.failure", response.isFailure()); - counterIncrementIf(topic, "publish.retry.attempt", wasRetried); - } - - @Override - public void onMaxRetriesExceeded(HermesResponse response, int attemptCount) { - String topic = MetricsUtils.sanitizeTopic(response.getHermesMessage().getTopic()); - metrics.counterIncrement(topic, "retries.exhausted"); - metrics.counterIncrement(topic, "publish.finally.failure"); - metrics.counterIncrement(topic, "publish.attempt"); - metrics.counterIncrement(topic, "publish.retry.attempt"); - } - - private void counterIncrementIf(String topic, String name, boolean condition) { - if (condition) { - metrics.counterIncrement(topic, name); - } + private final MetricsProvider metrics; + + public MetricsMessageDeliveryListener(MetricsProvider metrics) { + this.metrics = metrics; + } + + @Override + public void onSend(HermesResponse response, long latency) { + HermesMessage message = response.getHermesMessage(); + String topic = MetricsUtils.sanitizeTopic(message.getTopic()); + + metrics.timerRecord(topic, "latency", latency, NANOSECONDS); + Map tags = new HashMap<>(); + tags.put("code", String.valueOf(response.getHttpStatus())); + metrics.counterIncrement(topic, "status", tags); + + counterIncrementIf(topic, "publish.failure", response.isFailure()); + } + + @Override + public void onFailure(HermesResponse response, int attemptCount) { + String topic = MetricsUtils.sanitizeTopic(response.getHermesMessage().getTopic()); + metrics.counterIncrement(topic, "failure"); + } + + @Override + public void onFailedRetry(HermesResponse response, int attemptCount) { + String topic = MetricsUtils.sanitizeTopic(response.getHermesMessage().getTopic()); + metrics.counterIncrement(topic, "retries.count"); + metrics.counterIncrement(topic, "failure"); + + metrics.counterIncrement(topic, "publish.retry.failure"); + } + + @Override + public void onSuccessfulRetry(HermesResponse response, int attemptCount) { + String topic = MetricsUtils.sanitizeTopic(response.getHermesMessage().getTopic()); + metrics.counterIncrement(topic, "retries.success"); + metrics.histogramUpdate(topic, "retries.attempts", attemptCount - 1); + + boolean wasRetried = attemptCount > 1; + + metrics.counterIncrement(topic, "publish.attempt"); + counterIncrementIf(topic, "publish.retry.success", response.isSuccess() && wasRetried); + counterIncrementIf(topic, "publish.finally.success", response.isSuccess()); + counterIncrementIf(topic, "publish.retry.failure", response.isFailure() && wasRetried); + counterIncrementIf(topic, "publish.finally.failure", response.isFailure()); + counterIncrementIf(topic, "publish.retry.attempt", wasRetried); + } + + @Override + public void onMaxRetriesExceeded(HermesResponse response, int attemptCount) { + String topic = MetricsUtils.sanitizeTopic(response.getHermesMessage().getTopic()); + metrics.counterIncrement(topic, "retries.exhausted"); + metrics.counterIncrement(topic, "publish.finally.failure"); + metrics.counterIncrement(topic, "publish.attempt"); + metrics.counterIncrement(topic, "publish.retry.attempt"); + } + + private void counterIncrementIf(String topic, String name, boolean condition) { + if (condition) { + metrics.counterIncrement(topic, name); } + } } diff --git a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/metrics/MetricsProvider.java b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/metrics/MetricsProvider.java index 4f077e66d7..52e997ece7 100644 --- a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/metrics/MetricsProvider.java +++ b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/metrics/MetricsProvider.java @@ -5,13 +5,13 @@ public interface MetricsProvider { - String prefix = "hermes-client."; + String prefix = "hermes-client."; - void counterIncrement(String topic, String key); + void counterIncrement(String topic, String key); - void counterIncrement(String topic, String key, Map tags); + void counterIncrement(String topic, String key, Map tags); - void timerRecord(String topic, String key, long duration, TimeUnit unit); + void timerRecord(String topic, String key, long duration, TimeUnit unit); - void histogramUpdate(String topic, String key, int value); + void histogramUpdate(String topic, String key, int value); } diff --git a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/metrics/MetricsUtils.java b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/metrics/MetricsUtils.java index cfb1da88b9..cb2744d01b 100644 --- a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/metrics/MetricsUtils.java +++ b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/metrics/MetricsUtils.java @@ -2,10 +2,10 @@ class MetricsUtils { - static String sanitizeTopic(String topic) { - int lastDot = topic.lastIndexOf("."); - char[] sanitized = topic.replaceAll("\\.", "_").toCharArray(); - sanitized[lastDot] = '.'; - return String.valueOf(sanitized); - } + static String sanitizeTopic(String topic) { + int lastDot = topic.lastIndexOf("."); + char[] sanitized = topic.replaceAll("\\.", "_").toCharArray(); + sanitized[lastDot] = '.'; + return String.valueOf(sanitized); + } } diff --git a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/metrics/MicrometerTaggedMetricsProvider.java b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/metrics/MicrometerTaggedMetricsProvider.java index f25ddee79f..a3dfdc2d78 100644 --- a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/metrics/MicrometerTaggedMetricsProvider.java +++ b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/metrics/MicrometerTaggedMetricsProvider.java @@ -3,7 +3,6 @@ import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Tag; import io.micrometer.core.instrument.Tags; - import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -11,47 +10,59 @@ public class MicrometerTaggedMetricsProvider implements MetricsProvider { - private final MeterRegistry metrics; - - public MicrometerTaggedMetricsProvider(MeterRegistry metrics) { - this.metrics = metrics; - } - - @Override - public void counterIncrement(String topic, String key) { - counterIncrement(topic, key, new HashMap<>()); - } - - @Override - public void counterIncrement(String topic, String key, Map tags) { - tags.put("topic", topic); - metrics.counter(buildCounterName(key), Tags.of(tags.entrySet().stream() - .map(e -> Tag.of(e.getKey(), e.getValue())) - .collect(Collectors.toSet()))) - .increment(); - } - - @Override - public void timerRecord(String topic, String key, long duration, TimeUnit unit) { - Map tags = new HashMap<>(); - tags.put("topic", topic); - metrics.timer(buildCounterName(key), Tags.of(tags.entrySet().stream() - .map(e -> Tag.of(e.getKey(), e.getValue())) - .collect(Collectors.toSet()))) - .record(duration, unit); - } - - @Override - public void histogramUpdate(String topic, String key, int value) { - Map tags = new HashMap<>(); - tags.put("topic", topic); - metrics.summary(buildCounterName(key), Tags.of(tags.entrySet().stream() - .map(e -> Tag.of(e.getKey(), e.getValue())) - .collect(Collectors.toSet()))) - .record(value); - } - - private String buildCounterName(String key) { - return prefix + key; - } + private final MeterRegistry metrics; + + public MicrometerTaggedMetricsProvider(MeterRegistry metrics) { + this.metrics = metrics; + } + + @Override + public void counterIncrement(String topic, String key) { + counterIncrement(topic, key, new HashMap<>()); + } + + @Override + public void counterIncrement(String topic, String key, Map tags) { + tags.put("topic", topic); + metrics + .counter( + buildCounterName(key), + Tags.of( + tags.entrySet().stream() + .map(e -> Tag.of(e.getKey(), e.getValue())) + .collect(Collectors.toSet()))) + .increment(); + } + + @Override + public void timerRecord(String topic, String key, long duration, TimeUnit unit) { + Map tags = new HashMap<>(); + tags.put("topic", topic); + metrics + .timer( + buildCounterName(key), + Tags.of( + tags.entrySet().stream() + .map(e -> Tag.of(e.getKey(), e.getValue())) + .collect(Collectors.toSet()))) + .record(duration, unit); + } + + @Override + public void histogramUpdate(String topic, String key, int value) { + Map tags = new HashMap<>(); + tags.put("topic", topic); + metrics + .summary( + buildCounterName(key), + Tags.of( + tags.entrySet().stream() + .map(e -> Tag.of(e.getKey(), e.getValue())) + .collect(Collectors.toSet()))) + .record(value); + } + + private String buildCounterName(String key) { + return prefix + key; + } } diff --git a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/okhttp/OkHttpHermesSender.java b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/okhttp/OkHttpHermesSender.java index aef4dc7272..c549bcba39 100644 --- a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/okhttp/OkHttpHermesSender.java +++ b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/okhttp/OkHttpHermesSender.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.client.okhttp; +import static pl.allegro.tech.hermes.client.HermesResponseBuilder.hermesResponse; + +import java.io.IOException; +import java.net.URI; +import java.util.concurrent.CompletableFuture; import okhttp3.Call; import okhttp3.Callback; import okhttp3.MediaType; @@ -11,53 +16,48 @@ import pl.allegro.tech.hermes.client.HermesResponse; import pl.allegro.tech.hermes.client.HermesSender; -import java.io.IOException; -import java.net.URI; -import java.util.concurrent.CompletableFuture; - -import static pl.allegro.tech.hermes.client.HermesResponseBuilder.hermesResponse; - public class OkHttpHermesSender implements HermesSender { - private final OkHttpClient client; + private final OkHttpClient client; - public OkHttpHermesSender(OkHttpClient client) { - this.client = client; - } + public OkHttpHermesSender(OkHttpClient client) { + this.client = client; + } - @Override - public CompletableFuture send(URI uri, HermesMessage message) { - CompletableFuture future = new CompletableFuture<>(); + @Override + public CompletableFuture send(URI uri, HermesMessage message) { + CompletableFuture future = new CompletableFuture<>(); - RequestBody body = RequestBody.create(MediaType.parse(message.getContentType()), message.getBody()); - Request.Builder builder = new Request.Builder(); - message.consumeHeaders(builder::addHeader); - Request request = builder - .post(body) - .url(uri.toString()) - .build(); + RequestBody body = + RequestBody.create(MediaType.parse(message.getContentType()), message.getBody()); + Request.Builder builder = new Request.Builder(); + message.consumeHeaders(builder::addHeader); + Request request = builder.post(body).url(uri.toString()).build(); - client.newCall(request).enqueue(new Callback() { - @Override - public void onFailure(Call call, IOException e) { + client + .newCall(request) + .enqueue( + new Callback() { + @Override + public void onFailure(Call call, IOException e) { future.completeExceptionally(e); - } + } - @Override - public void onResponse(Call call, Response response) throws IOException { + @Override + public void onResponse(Call call, Response response) throws IOException { future.complete(fromOkHttpResponse(message, response)); - } - }); - - return future; - } - - HermesResponse fromOkHttpResponse(HermesMessage message, Response response) throws IOException { - return hermesResponse(message) - .withHeaderSupplier(response::header) - .withHttpStatus(response.code()) - .withBody(response.body().string()) - .withProtocol(response.protocol().toString()) - .build(); - } + } + }); + + return future; + } + + HermesResponse fromOkHttpResponse(HermesMessage message, Response response) throws IOException { + return hermesResponse(message) + .withHeaderSupplier(response::header) + .withHttpStatus(response.code()) + .withBody(response.body().string()) + .withProtocol(response.protocol().toString()) + .build(); + } } diff --git a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/webclient/WebClientHermesSender.java b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/webclient/WebClientHermesSender.java index a258aec589..3998f90704 100644 --- a/hermes-client/src/main/java/pl/allegro/tech/hermes/client/webclient/WebClientHermesSender.java +++ b/hermes-client/src/main/java/pl/allegro/tech/hermes/client/webclient/WebClientHermesSender.java @@ -1,5 +1,12 @@ package pl.allegro.tech.hermes.client.webclient; +import static java.util.stream.Collectors.toMap; +import static pl.allegro.tech.hermes.client.HermesResponseBuilder.hermesResponse; + +import java.net.URI; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.CompletableFuture; import org.springframework.web.reactive.function.client.WebClient; import pl.allegro.tech.hermes.client.HermesMessage; import pl.allegro.tech.hermes.client.HermesResponse; @@ -7,53 +14,56 @@ import pl.allegro.tech.hermes.client.ReactiveHermesSender; import reactor.core.publisher.Mono; -import java.net.URI; -import java.util.Map; -import java.util.TreeMap; -import java.util.concurrent.CompletableFuture; +public class WebClientHermesSender implements HermesSender, ReactiveHermesSender { -import static java.util.stream.Collectors.toMap; -import static pl.allegro.tech.hermes.client.HermesResponseBuilder.hermesResponse; + private static final Mono NO_BODY = Mono.just(""); + private final WebClient webClient; -public class WebClientHermesSender implements HermesSender, ReactiveHermesSender { + public WebClientHermesSender(WebClient webClient) { + this.webClient = webClient; + } - private static final Mono NO_BODY = Mono.just(""); - private final WebClient webClient; - - public WebClientHermesSender(WebClient webClient) { - this.webClient = webClient; - } - - @Override - public Mono sendReactively(URI uri, HermesMessage message) { - return webClient.post() - .uri(uri) - .syncBody(message.getBody()) - .headers(httpHeaders -> httpHeaders.setAll(message.getHeaders())) - .exchange() - .flatMap(response -> response - .bodyToMono(String.class) - .switchIfEmpty(NO_BODY) - .map(body -> hermesResponse(message) + @Override + public Mono sendReactively(URI uri, HermesMessage message) { + return webClient + .post() + .uri(uri) + .syncBody(message.getBody()) + .headers(httpHeaders -> httpHeaders.setAll(message.getHeaders())) + .exchange() + .flatMap( + response -> + response + .bodyToMono(String.class) + .switchIfEmpty(NO_BODY) + .map( + body -> + hermesResponse(message) .withBody(body) .withHttpStatus(response.statusCode().value()) - .withHeaderSupplier(header -> - convertToCaseInsensitiveMap(response.headers().asHttpHeaders().toSingleValueMap()).get(header)) + .withHeaderSupplier( + header -> + convertToCaseInsensitiveMap( + response + .headers() + .asHttpHeaders() + .toSingleValueMap()) + .get(header)) .build())); - } - - @Override - public CompletableFuture send(URI uri, HermesMessage message) { - return sendReactively(uri, message).toFuture(); - } - - private TreeMap convertToCaseInsensitiveMap(Map hashMap) { - return hashMap.entrySet().stream() - .collect(toMap( - Map.Entry::getKey, - Map.Entry::getValue, - (oldVal, newVal) -> newVal, - () -> new TreeMap<>(String.CASE_INSENSITIVE_ORDER) - )); - } + } + + @Override + public CompletableFuture send(URI uri, HermesMessage message) { + return sendReactively(uri, message).toFuture(); + } + + private TreeMap convertToCaseInsensitiveMap(Map hashMap) { + return hashMap.entrySet().stream() + .collect( + toMap( + Map.Entry::getKey, + Map.Entry::getValue, + (oldVal, newVal) -> newVal, + () -> new TreeMap<>(String.CASE_INSENSITIVE_ORDER))); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/admin/AdminOperationsCallback.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/admin/AdminOperationsCallback.java index 3a8c08760f..0f418e9c0c 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/admin/AdminOperationsCallback.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/admin/AdminOperationsCallback.java @@ -3,5 +3,5 @@ import pl.allegro.tech.hermes.api.SubscriptionName; public interface AdminOperationsCallback { - void onRetransmissionStarts(SubscriptionName subscription) throws Exception; + void onRetransmissionStarts(SubscriptionName subscription) throws Exception; } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/admin/AdminTool.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/admin/AdminTool.java index 033e58ca3f..7120793822 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/admin/AdminTool.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/admin/AdminTool.java @@ -4,9 +4,9 @@ public interface AdminTool { - void retransmit(SubscriptionName subscriptionName); + void retransmit(SubscriptionName subscriptionName); - enum Operations { - RETRANSMIT - } + enum Operations { + RETRANSMIT + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/admin/AdminToolStartupException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/admin/AdminToolStartupException.java index 4a053eb2b4..d3b257cfc2 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/admin/AdminToolStartupException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/admin/AdminToolStartupException.java @@ -2,8 +2,7 @@ public class AdminToolStartupException extends RuntimeException { - public AdminToolStartupException(Throwable cause) { - super(cause); - } - + public AdminToolStartupException(Throwable cause) { + super(cause); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/admin/zookeeper/ZookeeperAdminCache.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/admin/zookeeper/ZookeeperAdminCache.java index 0ea419fe50..9bab864f80 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/admin/zookeeper/ZookeeperAdminCache.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/admin/zookeeper/ZookeeperAdminCache.java @@ -1,6 +1,11 @@ package pl.allegro.tech.hermes.common.admin.zookeeper; +import static pl.allegro.tech.hermes.common.admin.AdminTool.Operations.RETRANSMIT; + import com.fasterxml.jackson.databind.ObjectMapper; +import java.time.Clock; +import java.util.ArrayList; +import java.util.List; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCache; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; @@ -9,56 +14,52 @@ import pl.allegro.tech.hermes.common.admin.AdminOperationsCallback; import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; -import java.time.Clock; -import java.util.ArrayList; -import java.util.List; - -import static pl.allegro.tech.hermes.common.admin.AdminTool.Operations.RETRANSMIT; - public class ZookeeperAdminCache extends PathChildrenCache implements PathChildrenCacheListener { - private final ObjectMapper objectMapper; - private final List adminCallbacks = new ArrayList<>(); + private final ObjectMapper objectMapper; + private final List adminCallbacks = new ArrayList<>(); - private final long initializationTime; + private final long initializationTime; - public ZookeeperAdminCache(ZookeeperPaths zookeeperPaths, - CuratorFramework client, - ObjectMapper objectMapper, - Clock clock) { - super(client, zookeeperPaths.adminPath(), true); - this.objectMapper = objectMapper; - this.initializationTime = clock.millis(); - getListenable().addListener(this); - } + public ZookeeperAdminCache( + ZookeeperPaths zookeeperPaths, + CuratorFramework client, + ObjectMapper objectMapper, + Clock clock) { + super(client, zookeeperPaths.adminPath(), true); + this.objectMapper = objectMapper; + this.initializationTime = clock.millis(); + getListenable().addListener(this); + } - @Override - public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception { - switch (event.getType()) { - case CHILD_UPDATED: - case CHILD_ADDED: - if (event.getData().getPath().contains(RETRANSMIT.name()) && isYoungerThanThisNode(event)) { - retransmit(event); - } - break; - default: - break; + @Override + public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception { + switch (event.getType()) { + case CHILD_UPDATED: + case CHILD_ADDED: + if (event.getData().getPath().contains(RETRANSMIT.name()) && isYoungerThanThisNode(event)) { + retransmit(event); } + break; + default: + break; } + } - private boolean isYoungerThanThisNode(PathChildrenCacheEvent event) { - return event.getData().getStat().getMtime() > initializationTime; - } + private boolean isYoungerThanThisNode(PathChildrenCacheEvent event) { + return event.getData().getStat().getMtime() > initializationTime; + } - private void retransmit(PathChildrenCacheEvent event) throws Exception { - SubscriptionName subscriptionName = objectMapper.readValue(event.getData().getData(), SubscriptionName.class); + private void retransmit(PathChildrenCacheEvent event) throws Exception { + SubscriptionName subscriptionName = + objectMapper.readValue(event.getData().getData(), SubscriptionName.class); - for (AdminOperationsCallback adminCallback : adminCallbacks) { - adminCallback.onRetransmissionStarts(subscriptionName); - } + for (AdminOperationsCallback adminCallback : adminCallbacks) { + adminCallback.onRetransmissionStarts(subscriptionName); } + } - public void addCallback(AdminOperationsCallback callback) { - adminCallbacks.add(callback); - } + public void addCallback(AdminOperationsCallback callback) { + adminCallbacks.add(callback); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/admin/zookeeper/ZookeeperAdminTool.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/admin/zookeeper/ZookeeperAdminTool.java index b387c5e2f6..aa2aca00a9 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/admin/zookeeper/ZookeeperAdminTool.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/admin/zookeeper/ZookeeperAdminTool.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.common.admin.zookeeper; +import static pl.allegro.tech.hermes.common.admin.AdminTool.Operations.RETRANSMIT; + import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.curator.framework.CuratorFramework; import org.apache.zookeeper.CreateMode; @@ -8,35 +10,35 @@ import pl.allegro.tech.hermes.common.exception.RetransmissionException; import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; -import static pl.allegro.tech.hermes.common.admin.AdminTool.Operations.RETRANSMIT; - public class ZookeeperAdminTool implements AdminTool { - private final ZookeeperPaths zookeeperPaths; - private final CuratorFramework curatorFramework; - private final ObjectMapper objectMapper; - - public ZookeeperAdminTool(ZookeeperPaths zookeeperPaths, CuratorFramework curatorFramework, - ObjectMapper objectMapper) { - this.zookeeperPaths = zookeeperPaths; - this.curatorFramework = curatorFramework; - this.objectMapper = objectMapper; - } - - @Override - public void retransmit(SubscriptionName subscriptionName) { - try { - executeAdminOperation(subscriptionName, RETRANSMIT.name()); - } catch (Exception e) { - throw new RetransmissionException(e); - } + private final ZookeeperPaths zookeeperPaths; + private final CuratorFramework curatorFramework; + private final ObjectMapper objectMapper; + + public ZookeeperAdminTool( + ZookeeperPaths zookeeperPaths, CuratorFramework curatorFramework, ObjectMapper objectMapper) { + this.zookeeperPaths = zookeeperPaths; + this.curatorFramework = curatorFramework; + this.objectMapper = objectMapper; + } + + @Override + public void retransmit(SubscriptionName subscriptionName) { + try { + executeAdminOperation(subscriptionName, RETRANSMIT.name()); + } catch (Exception e) { + throw new RetransmissionException(e); } + } - private void executeAdminOperation(SubscriptionName subscriptionName, String name) throws Exception { - String path = zookeeperPaths.adminOperationPath(name); + private void executeAdminOperation(SubscriptionName subscriptionName, String name) + throws Exception { + String path = zookeeperPaths.adminOperationPath(name); - curatorFramework.create() - .withMode(CreateMode.EPHEMERAL_SEQUENTIAL) - .forPath(path, objectMapper.writeValueAsBytes(subscriptionName)); - } + curatorFramework + .create() + .withMode(CreateMode.EPHEMERAL_SEQUENTIAL) + .forPath(path, objectMapper.writeValueAsBytes(subscriptionName)); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/broker/BrokerDetails.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/broker/BrokerDetails.java index 6a2444f78e..c1d393ee2f 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/broker/BrokerDetails.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/broker/BrokerDetails.java @@ -2,19 +2,19 @@ public class BrokerDetails { - private String host; - private int port; + private String host; + private int port; - public BrokerDetails(String host, int port) { - this.host = host; - this.port = port; - } + public BrokerDetails(String host, int port) { + this.host = host; + this.port = port; + } - public String getHost() { - return host; - } + public String getHost() { + return host; + } - public int getPort() { - return port; - } + public int getPort() { + return port; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/broker/BrokerStorage.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/broker/BrokerStorage.java index fb636dd956..9051919ca4 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/broker/BrokerStorage.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/broker/BrokerStorage.java @@ -1,12 +1,11 @@ package pl.allegro.tech.hermes.common.broker; -import org.apache.kafka.common.TopicPartition; - import java.util.List; +import org.apache.kafka.common.TopicPartition; public interface BrokerStorage { - int readLeaderForPartition(TopicPartition topicAndPartition); + int readLeaderForPartition(TopicPartition topicAndPartition); - List readPartitionsIds(String topicName); + List readPartitionsIds(String topicName); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/broker/KafkaBrokerStorage.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/broker/KafkaBrokerStorage.java index e051d64ec6..e75a225593 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/broker/KafkaBrokerStorage.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/broker/KafkaBrokerStorage.java @@ -1,6 +1,9 @@ package pl.allegro.tech.hermes.common.broker; import com.google.common.collect.Ordering; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.common.KafkaFuture; @@ -9,50 +12,50 @@ import pl.allegro.tech.hermes.common.exception.BrokerNotFoundForPartitionException; import pl.allegro.tech.hermes.common.exception.PartitionsNotFoundForGivenTopicException; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - public class KafkaBrokerStorage implements BrokerStorage { - private final AdminClient kafkaAdminClient; - - public KafkaBrokerStorage(AdminClient kafkaAdminClient) { - this.kafkaAdminClient = kafkaAdminClient; - } - - @Override - public int readLeaderForPartition(TopicPartition topicAndPartition) { - try { - return describeTopic(topicAndPartition.topic()) - .thenApply(description -> description.partitions().get(topicAndPartition.partition()).leader().id()) - .get(); - } catch (Exception exception) { - throw new BrokerNotFoundForPartitionException(topicAndPartition.topic(), topicAndPartition.partition(), exception); - } + private final AdminClient kafkaAdminClient; + + public KafkaBrokerStorage(AdminClient kafkaAdminClient) { + this.kafkaAdminClient = kafkaAdminClient; + } + + @Override + public int readLeaderForPartition(TopicPartition topicAndPartition) { + try { + return describeTopic(topicAndPartition.topic()) + .thenApply( + description -> + description.partitions().get(topicAndPartition.partition()).leader().id()) + .get(); + } catch (Exception exception) { + throw new BrokerNotFoundForPartitionException( + topicAndPartition.topic(), topicAndPartition.partition(), exception); } + } - @Override - public List readPartitionsIds(String topicName) { - try { - List partitions = describeTopic(topicName) - .thenApply(this::resolvePartitionIds) - .get(); - - return Ordering.natural().sortedCopy(partitions); - } catch (Exception exception) { - throw new PartitionsNotFoundForGivenTopicException(topicName, exception); - } - } - - private KafkaFuture describeTopic(String topic) { - return kafkaAdminClient.describeTopics(Collections.singletonList(topic)).all() - .thenApply(topicsMap -> topicsMap.get(topic)); - } + @Override + public List readPartitionsIds(String topicName) { + try { + List partitions = + describeTopic(topicName).thenApply(this::resolvePartitionIds).get(); - private List resolvePartitionIds(TopicDescription topicDescription) { - return topicDescription.partitions().stream() - .map(TopicPartitionInfo::partition) - .collect(Collectors.toList()); + return Ordering.natural().sortedCopy(partitions); + } catch (Exception exception) { + throw new PartitionsNotFoundForGivenTopicException(topicName, exception); } + } + + private KafkaFuture describeTopic(String topic) { + return kafkaAdminClient + .describeTopics(Collections.singletonList(topic)) + .all() + .thenApply(topicsMap -> topicsMap.get(topic)); + } + + private List resolvePartitionIds(TopicDescription topicDescription) { + return topicDescription.partitions().stream() + .map(TopicPartitionInfo::partition) + .collect(Collectors.toList()); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/cache/queue/LinkedHashSetBlockingQueue.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/cache/queue/LinkedHashSetBlockingQueue.java index b061eed052..eaa90f8d51 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/cache/queue/LinkedHashSetBlockingQueue.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/cache/queue/LinkedHashSetBlockingQueue.java @@ -1,23 +1,19 @@ package pl.allegro.tech.hermes.common.cache.queue; /** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ - import java.util.AbstractQueue; import java.util.Collection; import java.util.Iterator; @@ -32,391 +28,357 @@ * A blocking queue implementation backed by a linked hash set for predictable iteration order and * constant time addition, removal and contains operations. * - * Author: Sebastian Schaffert - * Project: apache.marmotta + *

Author: Sebastian Schaffert Project: apache.marmotta */ public class LinkedHashSetBlockingQueue extends AbstractQueue implements BlockingQueue { - private int capacity = Integer.MAX_VALUE; - - /** Current number of elements */ - private final AtomicInteger count = new AtomicInteger(0); - - /** Lock held by take, poll, etc */ - private final ReentrantLock takeLock = new ReentrantLock(); - - /** Wait queue for waiting takes */ - private final Condition notEmpty = takeLock.newCondition(); - - /** Lock held by put, offer, etc */ - private final ReentrantLock putLock = new ReentrantLock(); - - /** Wait queue for waiting puts */ - private final Condition notFull = putLock.newCondition(); - - private final LinkedHashSet delegate; - - public LinkedHashSetBlockingQueue() { - delegate = new LinkedHashSet(); + private int capacity = Integer.MAX_VALUE; + + /** Current number of elements */ + private final AtomicInteger count = new AtomicInteger(0); + + /** Lock held by take, poll, etc */ + private final ReentrantLock takeLock = new ReentrantLock(); + + /** Wait queue for waiting takes */ + private final Condition notEmpty = takeLock.newCondition(); + + /** Lock held by put, offer, etc */ + private final ReentrantLock putLock = new ReentrantLock(); + + /** Wait queue for waiting puts */ + private final Condition notFull = putLock.newCondition(); + + private final LinkedHashSet delegate; + + public LinkedHashSetBlockingQueue() { + delegate = new LinkedHashSet(); + } + + public LinkedHashSetBlockingQueue(int capacity) { + this.delegate = new LinkedHashSet(capacity); + this.capacity = capacity; + } + + @Override + public boolean offer(E e) { + if (e == null) throw new NullPointerException(); + final AtomicInteger count = this.count; + if (count.get() == capacity) return false; + int c = -1; + final ReentrantLock putLock = this.putLock; + putLock.lock(); + try { + if (count.get() < capacity) { + final boolean wasAdded = enqueue(e); + c = wasAdded ? count.getAndIncrement() : count.get(); + if (c + 1 < capacity) notFull.signal(); + } + } finally { + putLock.unlock(); } - - public LinkedHashSetBlockingQueue(int capacity) { - this.delegate = new LinkedHashSet(capacity); - this.capacity = capacity; + if (c == 0) signalNotEmpty(); + return c >= 0; + } + + @Override + public void put(E e) throws InterruptedException { + if (e == null) throw new NullPointerException(); + + int c = -1; + final ReentrantLock putLock = this.putLock; + final AtomicInteger count = this.count; + putLock.lockInterruptibly(); + try { + while (count.get() == capacity) { + notFull.await(); + } + final boolean wasAdded = enqueue(e); + c = wasAdded ? count.getAndIncrement() : count.get(); + if (c + 1 < capacity) notFull.signal(); + } finally { + putLock.unlock(); } - - @Override - public boolean offer(E e) { - if (e == null) throw new NullPointerException(); - final AtomicInteger count = this.count; - if (count.get() == capacity) - return false; - int c = -1; - final ReentrantLock putLock = this.putLock; - putLock.lock(); - try { - if (count.get() < capacity) { - final boolean wasAdded = enqueue(e); - c = wasAdded?count.getAndIncrement():count.get(); - if (c + 1 < capacity) - notFull.signal(); - } - } finally { - putLock.unlock(); - } - if (c == 0) - signalNotEmpty(); - return c >= 0; + if (c == 0) signalNotEmpty(); + } + + @Override + public boolean offer(E e, long timeout, TimeUnit unit) throws InterruptedException { + if (e == null) throw new NullPointerException(); + long nanos = unit.toNanos(timeout); + int c = -1; + final ReentrantLock putLock = this.putLock; + final AtomicInteger count = this.count; + putLock.lockInterruptibly(); + try { + while (count.get() == capacity) { + + if (nanos <= 0) return false; + nanos = notFull.awaitNanos(nanos); + } + final boolean wasAdded = enqueue(e); + c = wasAdded ? count.getAndIncrement() : count.get(); + if (c + 1 < capacity) notFull.signal(); + } finally { + putLock.unlock(); } - - @Override - public void put(E e) throws InterruptedException { - if (e == null) throw new NullPointerException(); - - int c = -1; - final ReentrantLock putLock = this.putLock; - final AtomicInteger count = this.count; - putLock.lockInterruptibly(); - try { - while (count.get() == capacity) { - notFull.await(); - } - final boolean wasAdded = enqueue(e); - c = wasAdded?count.getAndIncrement():count.get(); - if (c + 1 < capacity) - notFull.signal(); - } finally { - putLock.unlock(); - } - if (c == 0) - signalNotEmpty(); + if (c == 0) signalNotEmpty(); + return true; + } + + @Override + public E take() throws InterruptedException { + E x; + int c = -1; + final AtomicInteger count = this.count; + final ReentrantLock takeLock = this.takeLock; + takeLock.lockInterruptibly(); + try { + while (count.get() == 0) { + notEmpty.await(); + } + x = dequeue(); + c = count.getAndDecrement(); + if (c > 1) notEmpty.signal(); + } finally { + takeLock.unlock(); } - - @Override - public boolean offer(E e, long timeout, TimeUnit unit) throws InterruptedException { - if (e == null) throw new NullPointerException(); - long nanos = unit.toNanos(timeout); - int c = -1; - final ReentrantLock putLock = this.putLock; - final AtomicInteger count = this.count; - putLock.lockInterruptibly(); - try { - while (count.get() == capacity) { - - if (nanos <= 0) - return false; - nanos = notFull.awaitNanos(nanos); - } - final boolean wasAdded = enqueue(e); - c = wasAdded?count.getAndIncrement():count.get(); - if (c + 1 < capacity) - notFull.signal(); - } finally { - putLock.unlock(); - } - if (c == 0) - signalNotEmpty(); - return true; + if (c == capacity) signalNotFull(); + return x; + } + + @Override + public E poll(long timeout, TimeUnit unit) throws InterruptedException { + E x = null; + int c = -1; + long nanos = unit.toNanos(timeout); + final AtomicInteger count = this.count; + final ReentrantLock takeLock = this.takeLock; + takeLock.lockInterruptibly(); + try { + while (count.get() == 0) { + if (nanos <= 0) return null; + nanos = notEmpty.awaitNanos(nanos); + } + x = dequeue(); + c = count.getAndDecrement(); + if (c > 1) notEmpty.signal(); + } finally { + takeLock.unlock(); } - - @Override - public E take() throws InterruptedException { - E x; - int c = -1; - final AtomicInteger count = this.count; - final ReentrantLock takeLock = this.takeLock; - takeLock.lockInterruptibly(); - try { - while (count.get() == 0) { - notEmpty.await(); - } - x = dequeue(); - c = count.getAndDecrement(); - if (c > 1) - notEmpty.signal(); - } finally { - takeLock.unlock(); - } - if (c == capacity) - signalNotFull(); - return x; + if (c == capacity) signalNotFull(); + return x; + } + + @Override + public int remainingCapacity() { + return Integer.MAX_VALUE - size(); + } + + @Override + public int drainTo(Collection c) { + return drainTo(c, Integer.MAX_VALUE); + } + + @Override + public int drainTo(Collection c, int maxElements) { + if (c == null) throw new NullPointerException(); + if (c == this) throw new IllegalArgumentException(); + boolean signalNotFull = false; + final ReentrantLock takeLock = this.takeLock; + takeLock.lock(); + try { + int n = Math.min(maxElements, count.get()); + Iterator it = delegate.iterator(); + for (int i = 0; i < n && it.hasNext(); i++) { + E x = it.next(); + c.add(x); + } + count.getAndAdd(-n); + return n; + } finally { + takeLock.unlock(); + if (signalNotFull) signalNotFull(); } - - @Override - public E poll(long timeout, TimeUnit unit) throws InterruptedException { - E x = null; - int c = -1; - long nanos = unit.toNanos(timeout); - final AtomicInteger count = this.count; - final ReentrantLock takeLock = this.takeLock; - takeLock.lockInterruptibly(); - try { - while (count.get() == 0) { - if (nanos <= 0) - return null; - nanos = notEmpty.awaitNanos(nanos); - } - x = dequeue(); - c = count.getAndDecrement(); - if (c > 1) - notEmpty.signal(); - } finally { - takeLock.unlock(); - } - if (c == capacity) - signalNotFull(); - return x; + } + + @Override + public E poll() { + final AtomicInteger count = this.count; + if (count.get() == 0) return null; + E x = null; + int c = -1; + final ReentrantLock takeLock = this.takeLock; + takeLock.lock(); + try { + if (count.get() > 0) { + x = dequeue(); + c = count.getAndDecrement(); + if (c > 1) notEmpty.signal(); + } + } finally { + takeLock.unlock(); } - - @Override - public int remainingCapacity() { - return Integer.MAX_VALUE - size(); + if (c == capacity) signalNotFull(); + return x; + } + + @Override + public E peek() { + if (count.get() == 0) return null; + final ReentrantLock takeLock = this.takeLock; + takeLock.lock(); + try { + Iterator it = delegate.iterator(); + if (it.hasNext()) { + return it.next(); + } else { + return null; + } + } finally { + takeLock.unlock(); } - - @Override - public int drainTo(Collection c) { - return drainTo(c,Integer.MAX_VALUE); + } + + /** + * Creates a node and links it at end of queue. + * + * @param x the item + * @return true if this set did not already contain x + */ + private boolean enqueue(E x) { + synchronized (delegate) { + return delegate.add(x); } - - @Override - public int drainTo(Collection c, int maxElements) { - if (c == null) - throw new NullPointerException(); - if (c == this) - throw new IllegalArgumentException(); - boolean signalNotFull = false; - final ReentrantLock takeLock = this.takeLock; - takeLock.lock(); - try { - int n = Math.min(maxElements, count.get()); - Iterator it = delegate.iterator(); - for(int i=0; i it = delegate.iterator(); + E x = it.next(); + it.remove(); + return x; } - - @Override - public E poll() { - final AtomicInteger count = this.count; - if (count.get() == 0) - return null; - E x = null; - int c = -1; - final ReentrantLock takeLock = this.takeLock; - takeLock.lock(); - try { - if (count.get() > 0) { - x = dequeue(); - c = count.getAndDecrement(); - if (c > 1) - notEmpty.signal(); - } - } finally { - takeLock.unlock(); - } - if (c == capacity) - signalNotFull(); - return x; + } + + /** Lock to prevent both puts and takes. */ + void fullyLock() { + putLock.lock(); + takeLock.lock(); + } + + /** Unlock to allow both puts and takes. */ + void fullyUnlock() { + takeLock.unlock(); + putLock.unlock(); + } + + /** + * Signals a waiting take. Called only from put/offer (which do not otherwise ordinarily lock + * takeLock.) + */ + private void signalNotEmpty() { + final ReentrantLock takeLock = this.takeLock; + takeLock.lock(); + try { + notEmpty.signal(); + } finally { + takeLock.unlock(); } - - - @Override - public E peek() { - if (count.get() == 0) - return null; - final ReentrantLock takeLock = this.takeLock; - takeLock.lock(); - try { - Iterator it = delegate.iterator(); - if(it.hasNext()) { - return it.next(); - } else { - return null; - } - } finally { - takeLock.unlock(); - } - } - - - /** - * Creates a node and links it at end of queue. - * @param x the item - * @return true if this set did not already contain x - */ - private boolean enqueue(E x) { - synchronized (delegate) { - return delegate.add(x); - } - } - - /** - * Removes a node from head of queue. - * @return the node - */ - private E dequeue() { - synchronized (delegate) { - Iterator it = delegate.iterator(); - E x = it.next(); - it.remove(); - return x; - } - } - - - - /** - * Lock to prevent both puts and takes. - */ - void fullyLock() { - putLock.lock(); - takeLock.lock(); - } - - /** - * Unlock to allow both puts and takes. - */ - void fullyUnlock() { - takeLock.unlock(); - putLock.unlock(); - } - - /** - * Signals a waiting take. Called only from put/offer (which do not - * otherwise ordinarily lock takeLock.) - */ - private void signalNotEmpty() { - final ReentrantLock takeLock = this.takeLock; - takeLock.lock(); - try { - notEmpty.signal(); - } finally { - takeLock.unlock(); - } + } + + /** Signals a waiting put. Called only from take/poll. */ + private void signalNotFull() { + final ReentrantLock putLock = this.putLock; + putLock.lock(); + try { + notFull.signal(); + } finally { + putLock.unlock(); } - - /** - * Signals a waiting put. Called only from take/poll. - */ - private void signalNotFull() { - final ReentrantLock putLock = this.putLock; - putLock.lock(); + } + + /** Tells whether both locks are held by current thread. */ + boolean isFullyLocked() { + return (putLock.isHeldByCurrentThread() && takeLock.isHeldByCurrentThread()); + } + + @Override + public Iterator iterator() { + final Iterator it = delegate.iterator(); + return new Iterator() { + @Override + public boolean hasNext() { + fullyLock(); try { - notFull.signal(); + return it.hasNext(); } finally { - putLock.unlock(); + fullyUnlock(); } - } - - /** - * Tells whether both locks are held by current thread. - */ - boolean isFullyLocked() { - return (putLock.isHeldByCurrentThread() && - takeLock.isHeldByCurrentThread()); - } - - @Override - public Iterator iterator() { - final Iterator it = delegate.iterator(); - return new Iterator() { - @Override - public boolean hasNext() { - fullyLock(); - try { - return it.hasNext(); - } finally { - fullyUnlock(); - } - } - - @Override - public E next() { - fullyLock(); - try { - return it.next(); - } finally { - fullyUnlock(); - } - } - - @Override - public void remove() { - fullyLock(); - try { - it.remove(); - - // remove counter - count.getAndDecrement(); - } finally { - fullyUnlock(); - } - } - }; - } - - @Override - public int size() { - return count.get(); - } - - @Override - public boolean remove(Object o) { - if (o == null) return false; + } + @Override + public E next() { fullyLock(); try { - if(delegate.remove(o)) { - if(count.getAndDecrement() == capacity) { - notFull.signal(); - } - return true; - } + return it.next(); } finally { - fullyUnlock(); + fullyUnlock(); } + } - return false; - } - - @Override - public void clear() { + @Override + public void remove() { fullyLock(); try { - delegate.clear(); - count.set(0); + it.remove(); + + // remove counter + count.getAndDecrement(); } finally { - fullyUnlock(); + fullyUnlock(); + } + } + }; + } + + @Override + public int size() { + return count.get(); + } + + @Override + public boolean remove(Object o) { + if (o == null) return false; + + fullyLock(); + try { + if (delegate.remove(o)) { + if (count.getAndDecrement() == capacity) { + notFull.signal(); } + return true; + } + } finally { + fullyUnlock(); } - + return false; + } + + @Override + public void clear() { + fullyLock(); + try { + delegate.clear(); + count.set(0); + } finally { + fullyUnlock(); + } + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/clock/ClockFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/clock/ClockFactory.java index 583717df37..15484df856 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/clock/ClockFactory.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/clock/ClockFactory.java @@ -4,7 +4,7 @@ public class ClockFactory { - public Clock provide() { - return Clock.systemDefaultZone(); - } + public Clock provide() { + return Clock.systemDefaultZone(); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/concurrent/DefaultExecutorServiceFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/concurrent/DefaultExecutorServiceFactory.java index cb490375ce..ff420dda03 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/concurrent/DefaultExecutorServiceFactory.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/concurrent/DefaultExecutorServiceFactory.java @@ -1,16 +1,14 @@ package pl.allegro.tech.hermes.common.concurrent; import com.google.common.util.concurrent.ThreadFactoryBuilder; - import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; public class DefaultExecutorServiceFactory implements ExecutorServiceFactory { - @Override - public ScheduledExecutorService createSingleThreadScheduledExecutor(String nameFormat) { - return Executors.newSingleThreadScheduledExecutor( - new ThreadFactoryBuilder().setNameFormat(nameFormat).build() - ); - } + @Override + public ScheduledExecutorService createSingleThreadScheduledExecutor(String nameFormat) { + return Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder().setNameFormat(nameFormat).build()); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/concurrent/ExecutorServiceFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/concurrent/ExecutorServiceFactory.java index f9c8837ca8..62ffed8525 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/concurrent/ExecutorServiceFactory.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/concurrent/ExecutorServiceFactory.java @@ -4,5 +4,5 @@ public interface ExecutorServiceFactory { - ScheduledExecutorService createSingleThreadScheduledExecutor(String nameFormat); + ScheduledExecutorService createSingleThreadScheduledExecutor(String nameFormat); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/config/KafkaAuthenticationProperties.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/config/KafkaAuthenticationProperties.java index ffe03f74ec..658db7741d 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/config/KafkaAuthenticationProperties.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/config/KafkaAuthenticationProperties.java @@ -2,72 +2,77 @@ public class KafkaAuthenticationProperties { - private boolean enabled = false; - private String mechanism = "PLAIN"; - private String protocol = "SASL_PLAINTEXT"; - private String username = "username"; - private String password = "password"; - private String loginModule = "org.apache.kafka.common.security.plain.PlainLoginModule"; - private String jaasConfig; - - public boolean isEnabled() { - return enabled; - } - - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } - - public String getMechanism() { - return mechanism; - } - - public void setMechanism(String mechanism) { - this.mechanism = mechanism; - } - - public String getProtocol() { - return protocol; - } - - public void setProtocol(String protocol) { - this.protocol = protocol; - } - - public String getUsername() { - return username; - } - - public void setUsername(String username) { - this.username = username; - } - - public String getPassword() { - return password; - } - - public void setPassword(String password) { - this.password = password; - } - - public String getLoginModule() { - return loginModule; - } - - public void setLoginModule(String loginModule) { - this.loginModule = loginModule; - } - - public String getJaasConfig() { - if (jaasConfig != null) { - return jaasConfig; - } - return loginModule + " required\n" - + "username=\"" + username + "\"\n" - + "password=\"" + password + "\";"; - } - - public void setJaasConfig(String jaasConfig) { - this.jaasConfig = jaasConfig; + private boolean enabled = false; + private String mechanism = "PLAIN"; + private String protocol = "SASL_PLAINTEXT"; + private String username = "username"; + private String password = "password"; + private String loginModule = "org.apache.kafka.common.security.plain.PlainLoginModule"; + private String jaasConfig; + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public String getMechanism() { + return mechanism; + } + + public void setMechanism(String mechanism) { + this.mechanism = mechanism; + } + + public String getProtocol() { + return protocol; + } + + public void setProtocol(String protocol) { + this.protocol = protocol; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public String getLoginModule() { + return loginModule; + } + + public void setLoginModule(String loginModule) { + this.loginModule = loginModule; + } + + public String getJaasConfig() { + if (jaasConfig != null) { + return jaasConfig; } + return loginModule + + " required\n" + + "username=\"" + + username + + "\"\n" + + "password=\"" + + password + + "\";"; + } + + public void setJaasConfig(String jaasConfig) { + this.jaasConfig = jaasConfig; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/config/SchemaCacheProperties.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/config/SchemaCacheProperties.java index 84181c41f9..35a7a51cbe 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/config/SchemaCacheProperties.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/config/SchemaCacheProperties.java @@ -1,76 +1,75 @@ package pl.allegro.tech.hermes.common.config; -import pl.allegro.tech.hermes.common.schema.SchemaVersionRepositoryParameters; - import java.time.Duration; +import pl.allegro.tech.hermes.common.schema.SchemaVersionRepositoryParameters; public class SchemaCacheProperties implements SchemaVersionRepositoryParameters { - private Duration refreshAfterWrite = Duration.ofMinutes(10); + private Duration refreshAfterWrite = Duration.ofMinutes(10); - private Duration expireAfterWrite = Duration.ofHours(24); + private Duration expireAfterWrite = Duration.ofHours(24); - private Duration compiledExpireAfterAccess = Duration.ofHours(40); + private Duration compiledExpireAfterAccess = Duration.ofHours(40); - private int reloadThreadPoolSize = 2; + private int reloadThreadPoolSize = 2; - private boolean enabled = true; + private boolean enabled = true; - private int compiledMaximumSize = 2000; + private int compiledMaximumSize = 2000; - @Override - public boolean isCacheEnabled() { - return enabled; - } + @Override + public boolean isCacheEnabled() { + return enabled; + } - @Override - public Duration getRefreshAfterWrite() { - return refreshAfterWrite; - } + @Override + public Duration getRefreshAfterWrite() { + return refreshAfterWrite; + } - public void setRefreshAfterWrite(Duration refreshAfterWrite) { - this.refreshAfterWrite = refreshAfterWrite; - } + public void setRefreshAfterWrite(Duration refreshAfterWrite) { + this.refreshAfterWrite = refreshAfterWrite; + } - @Override - public Duration getExpireAfterWrite() { - return expireAfterWrite; - } + @Override + public Duration getExpireAfterWrite() { + return expireAfterWrite; + } - public void setExpireAfterWrite(Duration expireAfterWrite) { - this.expireAfterWrite = expireAfterWrite; - } + public void setExpireAfterWrite(Duration expireAfterWrite) { + this.expireAfterWrite = expireAfterWrite; + } - public Duration getCompiledExpireAfterAccess() { - return compiledExpireAfterAccess; - } + public Duration getCompiledExpireAfterAccess() { + return compiledExpireAfterAccess; + } - public void setCompiledExpireAfterAccess(Duration compiledExpireAfterAccess) { - this.compiledExpireAfterAccess = compiledExpireAfterAccess; - } + public void setCompiledExpireAfterAccess(Duration compiledExpireAfterAccess) { + this.compiledExpireAfterAccess = compiledExpireAfterAccess; + } - @Override - public int getReloadThreadPoolSize() { - return reloadThreadPoolSize; - } + @Override + public int getReloadThreadPoolSize() { + return reloadThreadPoolSize; + } - public void setReloadThreadPoolSize(int reloadThreadPoolSize) { - this.reloadThreadPoolSize = reloadThreadPoolSize; - } + public void setReloadThreadPoolSize(int reloadThreadPoolSize) { + this.reloadThreadPoolSize = reloadThreadPoolSize; + } - public boolean isEnabled() { - return enabled; - } + public boolean isEnabled() { + return enabled; + } - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } - public int getCompiledMaximumSize() { - return compiledMaximumSize; - } + public int getCompiledMaximumSize() { + return compiledMaximumSize; + } - public void setCompiledMaximumSize(int compiledMaximumSize) { - this.compiledMaximumSize = compiledMaximumSize; - } + public void setCompiledMaximumSize(int compiledMaximumSize) { + this.compiledMaximumSize = compiledMaximumSize; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/config/SchemaRepositoryProperties.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/config/SchemaRepositoryProperties.java index a9746a764f..544c5be382 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/config/SchemaRepositoryProperties.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/config/SchemaRepositoryProperties.java @@ -4,73 +4,73 @@ public class SchemaRepositoryProperties { - private String serverUrl = "http://localhost:8888/"; + private String serverUrl = "http://localhost:8888/"; - private Duration httpReadTimeout = Duration.ofSeconds(2); + private Duration httpReadTimeout = Duration.ofSeconds(2); - private Duration httpConnectTimeout = Duration.ofSeconds(2000); + private Duration httpConnectTimeout = Duration.ofSeconds(2000); - private double onlineCheckPermitsPerSecond = 100.0; + private double onlineCheckPermitsPerSecond = 100.0; - private Duration onlineCheckAcquireWait = Duration.ofMillis(500); + private Duration onlineCheckAcquireWait = Duration.ofMillis(500); - private boolean subjectSuffixEnabled = false; + private boolean subjectSuffixEnabled = false; - private boolean subjectNamespaceEnabled = false; + private boolean subjectNamespaceEnabled = false; - public String getServerUrl() { - return serverUrl; - } + public String getServerUrl() { + return serverUrl; + } - public void setServerUrl(String serverUrl) { - this.serverUrl = serverUrl; - } + public void setServerUrl(String serverUrl) { + this.serverUrl = serverUrl; + } - public Duration getHttpReadTimeout() { - return httpReadTimeout; - } + public Duration getHttpReadTimeout() { + return httpReadTimeout; + } - public void setHttpReadTimeout(Duration httpReadTimeout) { - this.httpReadTimeout = httpReadTimeout; - } + public void setHttpReadTimeout(Duration httpReadTimeout) { + this.httpReadTimeout = httpReadTimeout; + } - public Duration getHttpConnectTimeout() { - return httpConnectTimeout; - } + public Duration getHttpConnectTimeout() { + return httpConnectTimeout; + } - public void setHttpConnectTimeout(Duration httpConnectTimeout) { - this.httpConnectTimeout = httpConnectTimeout; - } + public void setHttpConnectTimeout(Duration httpConnectTimeout) { + this.httpConnectTimeout = httpConnectTimeout; + } - public double getOnlineCheckPermitsPerSecond() { - return onlineCheckPermitsPerSecond; - } + public double getOnlineCheckPermitsPerSecond() { + return onlineCheckPermitsPerSecond; + } - public void setOnlineCheckPermitsPerSecond(double onlineCheckPermitsPerSecond) { - this.onlineCheckPermitsPerSecond = onlineCheckPermitsPerSecond; - } + public void setOnlineCheckPermitsPerSecond(double onlineCheckPermitsPerSecond) { + this.onlineCheckPermitsPerSecond = onlineCheckPermitsPerSecond; + } - public Duration getOnlineCheckAcquireWait() { - return onlineCheckAcquireWait; - } + public Duration getOnlineCheckAcquireWait() { + return onlineCheckAcquireWait; + } - public void setOnlineCheckAcquireWait(Duration onlineCheckAcquireWait) { - this.onlineCheckAcquireWait = onlineCheckAcquireWait; - } + public void setOnlineCheckAcquireWait(Duration onlineCheckAcquireWait) { + this.onlineCheckAcquireWait = onlineCheckAcquireWait; + } - public boolean isSubjectSuffixEnabled() { - return subjectSuffixEnabled; - } + public boolean isSubjectSuffixEnabled() { + return subjectSuffixEnabled; + } - public void setSubjectSuffixEnabled(boolean subjectSuffixEnabled) { - this.subjectSuffixEnabled = subjectSuffixEnabled; - } + public void setSubjectSuffixEnabled(boolean subjectSuffixEnabled) { + this.subjectSuffixEnabled = subjectSuffixEnabled; + } - public boolean isSubjectNamespaceEnabled() { - return subjectNamespaceEnabled; - } + public boolean isSubjectNamespaceEnabled() { + return subjectNamespaceEnabled; + } - public void setSubjectNamespaceEnabled(boolean subjectNamespaceEnabled) { - this.subjectNamespaceEnabled = subjectNamespaceEnabled; - } + public void setSubjectNamespaceEnabled(boolean subjectNamespaceEnabled) { + this.subjectNamespaceEnabled = subjectNamespaceEnabled; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/CuratorClientFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/CuratorClientFactory.java index 351fb66e87..59951d29ac 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/CuratorClientFactory.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/CuratorClientFactory.java @@ -1,6 +1,9 @@ package pl.allegro.tech.hermes.common.di.factories; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ThreadFactory; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.framework.api.ACLProvider; @@ -11,88 +14,88 @@ import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.common.exception.InternalProcessingException; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ThreadFactory; - public class CuratorClientFactory { - public static class ZookeeperAuthorization { - private final String scheme; - private final String user; - private final String password; + public static class ZookeeperAuthorization { + private final String scheme; + private final String user; + private final String password; - public ZookeeperAuthorization(String scheme, String user, String password) { - this.scheme = scheme; - this.user = user; - this.password = password; - } + public ZookeeperAuthorization(String scheme, String user, String password) { + this.scheme = scheme; + this.user = user; + this.password = password; + } - byte[] getAuth() { - return String.join(":", user, password).getBytes(); - } + byte[] getAuth() { + return String.join(":", user, password).getBytes(); } + } - private static final Logger logger = LoggerFactory.getLogger(CuratorClientFactory.class); - private final ZookeeperParameters zookeeperParameters; + private static final Logger logger = LoggerFactory.getLogger(CuratorClientFactory.class); + private final ZookeeperParameters zookeeperParameters; - public CuratorClientFactory(ZookeeperParameters zookeeperParameters) { - this.zookeeperParameters = zookeeperParameters; - } + public CuratorClientFactory(ZookeeperParameters zookeeperParameters) { + this.zookeeperParameters = zookeeperParameters; + } - public CuratorFramework provide(String connectString) { - return provide(connectString, Optional.empty()); - } + public CuratorFramework provide(String connectString) { + return provide(connectString, Optional.empty()); + } - public CuratorFramework provide(String connectString, Optional zookeeperAuthorization) { - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setNameFormat("hermes-curator-%d") - .setUncaughtExceptionHandler((t, e) -> - logger.error("Exception from curator with name {}", t.getName(), e)).build(); - CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder() - .threadFactory(threadFactory) - .connectString(connectString) - .sessionTimeoutMs((int) zookeeperParameters.getSessionTimeout().toMillis()) - .connectionTimeoutMs((int) zookeeperParameters.getConnectionTimeout().toMillis()) - .retryPolicy( - new ExponentialBackoffRetry( - (int) zookeeperParameters.getBaseSleepTime().toMillis(), - zookeeperParameters.getMaxRetries(), - (int) zookeeperParameters.getMaxSleepTime().toMillis() - ) - ); + public CuratorFramework provide( + String connectString, Optional zookeeperAuthorization) { + ThreadFactory threadFactory = + new ThreadFactoryBuilder() + .setNameFormat("hermes-curator-%d") + .setUncaughtExceptionHandler( + (t, e) -> logger.error("Exception from curator with name {}", t.getName(), e)) + .build(); + CuratorFrameworkFactory.Builder builder = + CuratorFrameworkFactory.builder() + .threadFactory(threadFactory) + .connectString(connectString) + .sessionTimeoutMs((int) zookeeperParameters.getSessionTimeout().toMillis()) + .connectionTimeoutMs((int) zookeeperParameters.getConnectionTimeout().toMillis()) + .retryPolicy( + new ExponentialBackoffRetry( + (int) zookeeperParameters.getBaseSleepTime().toMillis(), + zookeeperParameters.getMaxRetries(), + (int) zookeeperParameters.getMaxSleepTime().toMillis())); - zookeeperAuthorization.ifPresent(it -> { - builder.authorization(it.scheme, it.getAuth()); - builder.aclProvider( - new ACLProvider() { - @Override - public List getDefaultAcl() { - return ZooDefs.Ids.CREATOR_ALL_ACL; - } + zookeeperAuthorization.ifPresent( + it -> { + builder.authorization(it.scheme, it.getAuth()); + builder.aclProvider( + new ACLProvider() { + @Override + public List getDefaultAcl() { + return ZooDefs.Ids.CREATOR_ALL_ACL; + } - @Override - public List getAclForPath(String path) { - return ZooDefs.Ids.CREATOR_ALL_ACL; - } - } - ); + @Override + public List getAclForPath(String path) { + return ZooDefs.Ids.CREATOR_ALL_ACL; + } + }); }); - CuratorFramework curatorClient = builder.build(); - startAndWaitForConnection(curatorClient); + CuratorFramework curatorClient = builder.build(); + startAndWaitForConnection(curatorClient); - return curatorClient; - } + return curatorClient; + } - private void startAndWaitForConnection(CuratorFramework curator) { - curator.start(); - try { - curator.blockUntilConnected(); - } catch (InterruptedException interruptedException) { - RuntimeException exception = new InternalProcessingException("Could not start Zookeeper Curator", interruptedException); - logger.error(exception.getMessage(), interruptedException); - throw exception; - } + private void startAndWaitForConnection(CuratorFramework curator) { + curator.start(); + try { + curator.blockUntilConnected(); + } catch (InterruptedException interruptedException) { + RuntimeException exception = + new InternalProcessingException( + "Could not start Zookeeper Curator", interruptedException); + logger.error(exception.getMessage(), interruptedException); + throw exception; } + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/HermesCuratorClientFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/HermesCuratorClientFactory.java index 2817083d2b..6152c55f9c 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/HermesCuratorClientFactory.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/HermesCuratorClientFactory.java @@ -1,32 +1,33 @@ package pl.allegro.tech.hermes.common.di.factories; -import org.apache.curator.framework.CuratorFramework; - import java.util.Optional; +import org.apache.curator.framework.CuratorFramework; public class HermesCuratorClientFactory { - private final ZookeeperParameters zookeeperParameters; - private final CuratorClientFactory curatorClientFactory; + private final ZookeeperParameters zookeeperParameters; + private final CuratorClientFactory curatorClientFactory; - public HermesCuratorClientFactory(ZookeeperParameters zookeeperParameters, CuratorClientFactory curatorClientFactory) { - this.zookeeperParameters = zookeeperParameters; - this.curatorClientFactory = curatorClientFactory; - } - - public CuratorFramework provide() { - String connectString = zookeeperParameters.getConnectionString(); + public HermesCuratorClientFactory( + ZookeeperParameters zookeeperParameters, CuratorClientFactory curatorClientFactory) { + this.zookeeperParameters = zookeeperParameters; + this.curatorClientFactory = curatorClientFactory; + } - Optional authorization = Optional.empty(); + public CuratorFramework provide() { + String connectString = zookeeperParameters.getConnectionString(); - if (zookeeperParameters.isAuthorizationEnabled()) { - authorization = Optional.of(new CuratorClientFactory.ZookeeperAuthorization( - zookeeperParameters.getScheme(), - zookeeperParameters.getUser(), - zookeeperParameters.getPassword()) - ); - } + Optional authorization = Optional.empty(); - return curatorClientFactory.provide(connectString, authorization); + if (zookeeperParameters.isAuthorizationEnabled()) { + authorization = + Optional.of( + new CuratorClientFactory.ZookeeperAuthorization( + zookeeperParameters.getScheme(), + zookeeperParameters.getUser(), + zookeeperParameters.getPassword())); } + + return curatorClientFactory.provide(connectString, authorization); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/MicrometerRegistryParameters.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/MicrometerRegistryParameters.java index 48a74a84ea..0c599b13b2 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/MicrometerRegistryParameters.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/MicrometerRegistryParameters.java @@ -4,9 +4,9 @@ import java.util.List; public interface MicrometerRegistryParameters { - List getPercentiles(); + List getPercentiles(); - boolean zookeeperReporterEnabled(); + boolean zookeeperReporterEnabled(); - Duration zookeeperReportPeriod(); + Duration zookeeperReportPeriod(); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/ModelAwareZookeeperNotifyingCacheFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/ModelAwareZookeeperNotifyingCacheFactory.java index 21326ac177..2bc4d9bd3c 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/ModelAwareZookeeperNotifyingCacheFactory.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/ModelAwareZookeeperNotifyingCacheFactory.java @@ -1,48 +1,58 @@ package pl.allegro.tech.hermes.common.di.factories; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.apache.curator.framework.CuratorFramework; -import pl.allegro.tech.hermes.common.cache.queue.LinkedHashSetBlockingQueue; -import pl.allegro.tech.hermes.common.metric.MetricsFacade; -import pl.allegro.tech.hermes.infrastructure.zookeeper.cache.ModelAwareZookeeperNotifyingCache; - import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import org.apache.curator.framework.CuratorFramework; +import pl.allegro.tech.hermes.common.cache.queue.LinkedHashSetBlockingQueue; +import pl.allegro.tech.hermes.common.metric.MetricsFacade; +import pl.allegro.tech.hermes.infrastructure.zookeeper.cache.ModelAwareZookeeperNotifyingCache; public class ModelAwareZookeeperNotifyingCacheFactory { - private final CuratorFramework curator; - - private final MetricsFacade metricsFacade; - - private final ZookeeperParameters zookeeperParameters; - - public ModelAwareZookeeperNotifyingCacheFactory(CuratorFramework curator, MetricsFacade metricaFacade, ZookeeperParameters zookeeperParameters) { - this.curator = curator; - this.metricsFacade = metricaFacade; - this.zookeeperParameters = zookeeperParameters; - } - - public ModelAwareZookeeperNotifyingCache provide() { - String rootPath = zookeeperParameters.getRoot(); - ExecutorService executor = createExecutor(rootPath, zookeeperParameters.getProcessingThreadPoolSize()); - ModelAwareZookeeperNotifyingCache cache = new ModelAwareZookeeperNotifyingCache( - curator, executor, rootPath - ); - try { - cache.start(); - } catch (Exception e) { - throw new IllegalStateException("Unable to start Zookeeper cache for root path " + rootPath, e); - } - return cache; - } - - private ExecutorService createExecutor(String rootPath, int processingThreadPoolSize) { - ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat(rootPath + "-zk-cache-%d").build(); - ExecutorService executor = new ThreadPoolExecutor(1, processingThreadPoolSize, - Integer.MAX_VALUE, TimeUnit.SECONDS, new LinkedHashSetBlockingQueue<>(), threadFactory); - return metricsFacade.executor().monitor(executor, rootPath + "zk-cache"); + private final CuratorFramework curator; + + private final MetricsFacade metricsFacade; + + private final ZookeeperParameters zookeeperParameters; + + public ModelAwareZookeeperNotifyingCacheFactory( + CuratorFramework curator, + MetricsFacade metricaFacade, + ZookeeperParameters zookeeperParameters) { + this.curator = curator; + this.metricsFacade = metricaFacade; + this.zookeeperParameters = zookeeperParameters; + } + + public ModelAwareZookeeperNotifyingCache provide() { + String rootPath = zookeeperParameters.getRoot(); + ExecutorService executor = + createExecutor(rootPath, zookeeperParameters.getProcessingThreadPoolSize()); + ModelAwareZookeeperNotifyingCache cache = + new ModelAwareZookeeperNotifyingCache(curator, executor, rootPath); + try { + cache.start(); + } catch (Exception e) { + throw new IllegalStateException( + "Unable to start Zookeeper cache for root path " + rootPath, e); } + return cache; + } + + private ExecutorService createExecutor(String rootPath, int processingThreadPoolSize) { + ThreadFactory threadFactory = + new ThreadFactoryBuilder().setNameFormat(rootPath + "-zk-cache-%d").build(); + ExecutorService executor = + new ThreadPoolExecutor( + 1, + processingThreadPoolSize, + Integer.MAX_VALUE, + TimeUnit.SECONDS, + new LinkedHashSetBlockingQueue<>(), + threadFactory); + return metricsFacade.executor().monitor(executor, rootPath + "zk-cache"); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/ObjectMapperFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/ObjectMapperFactory.java index c2b9d89743..8ac12fee47 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/ObjectMapperFactory.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/ObjectMapperFactory.java @@ -10,26 +10,30 @@ public class ObjectMapperFactory { - private final boolean schemaIdSerializationEnabled; - private final boolean fallbackToRemoteDatacenterEnabled; + private final boolean schemaIdSerializationEnabled; + private final boolean fallbackToRemoteDatacenterEnabled; - public ObjectMapperFactory(boolean schemaIdSerializationEnabled, boolean fallbackToRemoteDatacenterEnabled) { - this.schemaIdSerializationEnabled = schemaIdSerializationEnabled; - this.fallbackToRemoteDatacenterEnabled = fallbackToRemoteDatacenterEnabled; - } + public ObjectMapperFactory( + boolean schemaIdSerializationEnabled, boolean fallbackToRemoteDatacenterEnabled) { + this.schemaIdSerializationEnabled = schemaIdSerializationEnabled; + this.fallbackToRemoteDatacenterEnabled = fallbackToRemoteDatacenterEnabled; + } - public ObjectMapper provide() { - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); - objectMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); - objectMapper.disable(SerializationFeature.WRITE_NULL_MAP_VALUES); - objectMapper.registerModule(new JavaTimeModule()); + public ObjectMapper provide() { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + objectMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + objectMapper.disable(SerializationFeature.WRITE_NULL_MAP_VALUES); + objectMapper.registerModule(new JavaTimeModule()); - final InjectableValues defaultSchemaIdAwareSerializationEnabled = new InjectableValues.Std() - .addValue(Topic.DEFAULT_SCHEMA_ID_SERIALIZATION_ENABLED_KEY, schemaIdSerializationEnabled) - .addValue(Topic.DEFAULT_FALLBACK_TO_REMOTE_DATACENTER_KEY, fallbackToRemoteDatacenterEnabled); - objectMapper.setInjectableValues(defaultSchemaIdAwareSerializationEnabled); + final InjectableValues defaultSchemaIdAwareSerializationEnabled = + new InjectableValues.Std() + .addValue( + Topic.DEFAULT_SCHEMA_ID_SERIALIZATION_ENABLED_KEY, schemaIdSerializationEnabled) + .addValue( + Topic.DEFAULT_FALLBACK_TO_REMOTE_DATACENTER_KEY, fallbackToRemoteDatacenterEnabled); + objectMapper.setInjectableValues(defaultSchemaIdAwareSerializationEnabled); - return objectMapper; - } + return objectMapper; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/PrometheusMeterRegistryFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/PrometheusMeterRegistryFactory.java index d2849d9cb1..7d46f2bba2 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/PrometheusMeterRegistryFactory.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/PrometheusMeterRegistryFactory.java @@ -9,61 +9,69 @@ import io.micrometer.core.instrument.distribution.DistributionStatisticConfig; import io.micrometer.prometheus.PrometheusConfig; import io.micrometer.prometheus.PrometheusMeterRegistry; +import java.util.concurrent.TimeUnit; import pl.allegro.tech.hermes.common.metric.counter.CounterStorage; import pl.allegro.tech.hermes.common.metric.counter.zookeeper.ZookeeperCounterReporter; -import java.util.concurrent.TimeUnit; - public class PrometheusMeterRegistryFactory { - private final MicrometerRegistryParameters parameters; - private final PrometheusConfig prometheusConfig; - private final CounterStorage counterStorage; - private final String prefix; + private final MicrometerRegistryParameters parameters; + private final PrometheusConfig prometheusConfig; + private final CounterStorage counterStorage; + private final String prefix; - public PrometheusMeterRegistryFactory(MicrometerRegistryParameters parameters, - PrometheusConfig prometheusConfig, - CounterStorage counterStorage, String prefix) { - this.parameters = parameters; - this.prometheusConfig = prometheusConfig; - this.counterStorage = counterStorage; - this.prefix = prefix + "_"; - } + public PrometheusMeterRegistryFactory( + MicrometerRegistryParameters parameters, + PrometheusConfig prometheusConfig, + CounterStorage counterStorage, + String prefix) { + this.parameters = parameters; + this.prometheusConfig = prometheusConfig; + this.counterStorage = counterStorage; + this.prefix = prefix + "_"; + } - public PrometheusMeterRegistry provide() { - PrometheusMeterRegistry meterRegistry = new PrometheusMeterRegistry(prometheusConfig); - applyFilters(meterRegistry); - if (parameters.zookeeperReporterEnabled()) { - registerZookeeperReporter(meterRegistry); - } - registerJvmMetrics(meterRegistry); - return meterRegistry; + public PrometheusMeterRegistry provide() { + PrometheusMeterRegistry meterRegistry = new PrometheusMeterRegistry(prometheusConfig); + applyFilters(meterRegistry); + if (parameters.zookeeperReporterEnabled()) { + registerZookeeperReporter(meterRegistry); } + registerJvmMetrics(meterRegistry); + return meterRegistry; + } - private void applyFilters(PrometheusMeterRegistry meterRegistry) { - meterRegistry.config().meterFilter(new MeterFilter() { - @Override - public Meter.Id map(Meter.Id id) { + private void applyFilters(PrometheusMeterRegistry meterRegistry) { + meterRegistry + .config() + .meterFilter( + new MeterFilter() { + @Override + public Meter.Id map(Meter.Id id) { return id.withName(prefix + id.getName()); - } + } - @Override - public DistributionStatisticConfig configure(Meter.Id id, DistributionStatisticConfig config) { + @Override + public DistributionStatisticConfig configure( + Meter.Id id, DistributionStatisticConfig config) { return DistributionStatisticConfig.builder() - .percentiles(parameters.getPercentiles() - .stream().mapToDouble(Double::doubleValue).toArray() - ).build().merge(config); - } - }); - } + .percentiles( + parameters.getPercentiles().stream() + .mapToDouble(Double::doubleValue) + .toArray()) + .build() + .merge(config); + } + }); + } - private void registerZookeeperReporter(PrometheusMeterRegistry meterRegistry) { - new ZookeeperCounterReporter(meterRegistry, counterStorage, prefix) - .start(parameters.zookeeperReportPeriod().toSeconds(), TimeUnit.SECONDS); - } + private void registerZookeeperReporter(PrometheusMeterRegistry meterRegistry) { + new ZookeeperCounterReporter(meterRegistry, counterStorage, prefix) + .start(parameters.zookeeperReportPeriod().toSeconds(), TimeUnit.SECONDS); + } - private void registerJvmMetrics(MeterRegistry meterRegistry) { - new JvmMemoryMetrics().bindTo(meterRegistry); - new JvmGcMetrics().bindTo(meterRegistry); - new JvmThreadMetrics().bindTo(meterRegistry); - } + private void registerJvmMetrics(MeterRegistry meterRegistry) { + new JvmMemoryMetrics().bindTo(meterRegistry); + new JvmGcMetrics().bindTo(meterRegistry); + new JvmThreadMetrics().bindTo(meterRegistry); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/ZookeeperParameters.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/ZookeeperParameters.java index 7c8a7b36ee..6123d4926e 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/ZookeeperParameters.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/di/factories/ZookeeperParameters.java @@ -4,27 +4,27 @@ public interface ZookeeperParameters { - String getConnectionString(); + String getConnectionString(); - Duration getBaseSleepTime(); + Duration getBaseSleepTime(); - Duration getMaxSleepTime(); + Duration getMaxSleepTime(); - int getMaxRetries(); + int getMaxRetries(); - Duration getConnectionTimeout(); + Duration getConnectionTimeout(); - Duration getSessionTimeout(); + Duration getSessionTimeout(); - String getRoot(); + String getRoot(); - int getProcessingThreadPoolSize(); + int getProcessingThreadPoolSize(); - boolean isAuthorizationEnabled(); + boolean isAuthorizationEnabled(); - String getScheme(); + String getScheme(); - String getUser(); + String getUser(); - String getPassword(); + String getPassword(); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/BrokerInfoNotAvailableException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/BrokerInfoNotAvailableException.java index 5b1d033b4a..9e99906700 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/BrokerInfoNotAvailableException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/BrokerInfoNotAvailableException.java @@ -2,7 +2,7 @@ public class BrokerInfoNotAvailableException extends InternalProcessingException { - public BrokerInfoNotAvailableException(Integer brokerId, Throwable cause) { - super("Could not find or read info about broker with id " + brokerId, cause); - } + public BrokerInfoNotAvailableException(Integer brokerId, Throwable cause) { + super("Could not find or read info about broker with id " + brokerId, cause); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/BrokerNotFoundForPartitionException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/BrokerNotFoundForPartitionException.java index 8eff999a1d..15604905ee 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/BrokerNotFoundForPartitionException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/BrokerNotFoundForPartitionException.java @@ -3,7 +3,7 @@ @SuppressWarnings("serial") public class BrokerNotFoundForPartitionException extends InternalProcessingException { - public BrokerNotFoundForPartitionException(String topic, int partition, Throwable cause) { - super(String.format("Broker not found for topic %s and partition %d", topic, partition), cause); - } + public BrokerNotFoundForPartitionException(String topic, int partition, Throwable cause) { + super(String.format("Broker not found for topic %s and partition %d", topic, partition), cause); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/EndpointProtocolNotSupportedException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/EndpointProtocolNotSupportedException.java index 5b8c150684..a6d00e08ef 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/EndpointProtocolNotSupportedException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/EndpointProtocolNotSupportedException.java @@ -5,12 +5,15 @@ public class EndpointProtocolNotSupportedException extends HermesException { - public EndpointProtocolNotSupportedException(EndpointAddress endpoint) { - super(String.format("Protocol %s not supported in endpoint %s", endpoint.getProtocol(), endpoint.toString())); - } + public EndpointProtocolNotSupportedException(EndpointAddress endpoint) { + super( + String.format( + "Protocol %s not supported in endpoint %s", + endpoint.getProtocol(), endpoint.toString())); + } - @Override - public ErrorCode getCode() { - return ErrorCode.VALIDATION_ERROR; - } + @Override + public ErrorCode getCode() { + return ErrorCode.VALIDATION_ERROR; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/HermesException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/HermesException.java index 7d21902938..f0153a52bb 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/HermesException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/HermesException.java @@ -2,19 +2,18 @@ import pl.allegro.tech.hermes.api.ErrorCode; - public abstract class HermesException extends RuntimeException { - public HermesException(Throwable t) { - super(t); - } + public HermesException(Throwable t) { + super(t); + } - public HermesException(String message) { - super(message); - } + public HermesException(String message) { + super(message); + } - public HermesException(String message, Throwable cause) { - super(message, cause); - } + public HermesException(String message, Throwable cause) { + super(message, cause); + } - public abstract ErrorCode getCode(); + public abstract ErrorCode getCode(); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/InternalProcessingException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/InternalProcessingException.java index 7983ae724d..5e3ec2d1b9 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/InternalProcessingException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/InternalProcessingException.java @@ -1,15 +1,15 @@ package pl.allegro.tech.hermes.common.exception; public class InternalProcessingException extends RuntimeException { - public InternalProcessingException(Throwable throwable) { - super(throwable); - } + public InternalProcessingException(Throwable throwable) { + super(throwable); + } - public InternalProcessingException(String message) { - super(message); - } + public InternalProcessingException(String message) { + super(message); + } - public InternalProcessingException(String message, Throwable throwable) { - super(message, throwable); - } + public InternalProcessingException(String message, Throwable throwable) { + super(message, throwable); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/PartitionsNotFoundForGivenTopicException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/PartitionsNotFoundForGivenTopicException.java index 7e290ffb5a..b3481d2fdf 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/PartitionsNotFoundForGivenTopicException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/PartitionsNotFoundForGivenTopicException.java @@ -5,12 +5,12 @@ @SuppressWarnings("serial") public class PartitionsNotFoundForGivenTopicException extends HermesException { - public PartitionsNotFoundForGivenTopicException(String topicName, Throwable cause) { - super(String.format("Partitions not found for topic: %s", topicName), cause); - } + public PartitionsNotFoundForGivenTopicException(String topicName, Throwable cause) { + super(String.format("Partitions not found for topic: %s", topicName), cause); + } - @Override - public ErrorCode getCode() { - return ErrorCode.PARTITIONS_NOT_FOUND_FOR_TOPIC; - } + @Override + public ErrorCode getCode() { + return ErrorCode.PARTITIONS_NOT_FOUND_FOR_TOPIC; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/RepositoryNotAvailableException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/RepositoryNotAvailableException.java index dd2a0bbaf0..fef3a94e5b 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/RepositoryNotAvailableException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/RepositoryNotAvailableException.java @@ -2,8 +2,7 @@ public class RepositoryNotAvailableException extends InternalProcessingException { - public RepositoryNotAvailableException(String message) { - super(message); - } - + public RepositoryNotAvailableException(String message) { + super(message); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/RetransmissionException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/RetransmissionException.java index 5d0b43be90..44d13ba790 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/RetransmissionException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/RetransmissionException.java @@ -4,16 +4,16 @@ public class RetransmissionException extends HermesException { - public RetransmissionException(String message) { - super(message); - } + public RetransmissionException(String message) { + super(message); + } - public RetransmissionException(Throwable cause) { - super("Error during retransmitting messages.", cause); - } + public RetransmissionException(Throwable cause) { + super("Error during retransmitting messages.", cause); + } - @Override - public ErrorCode getCode() { - return ErrorCode.RETRANSMISSION_EXCEPTION; - } + @Override + public ErrorCode getCode() { + return ErrorCode.RETRANSMISSION_EXCEPTION; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/SubscriptionEndpointAddressChangeException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/SubscriptionEndpointAddressChangeException.java index a62d93e822..e2c6b7d421 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/SubscriptionEndpointAddressChangeException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/SubscriptionEndpointAddressChangeException.java @@ -3,12 +3,12 @@ import pl.allegro.tech.hermes.api.ErrorCode; public class SubscriptionEndpointAddressChangeException extends HermesException { - public SubscriptionEndpointAddressChangeException(Throwable cause) { - super("Error during change topic endpoint address.", cause); - } + public SubscriptionEndpointAddressChangeException(Throwable cause) { + super("Error during change topic endpoint address.", cause); + } - @Override - public ErrorCode getCode() { - return ErrorCode.SUBSCRIPTION_ENDPOINT_ADDRESS_CHANGE_EXCEPTION; - } + @Override + public ErrorCode getCode() { + return ErrorCode.SUBSCRIPTION_ENDPOINT_ADDRESS_CHANGE_EXCEPTION; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/UnavailableRateException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/UnavailableRateException.java index 1c671f8415..4b5123b8b5 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/UnavailableRateException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/exception/UnavailableRateException.java @@ -6,23 +6,24 @@ @SuppressWarnings("serial") public class UnavailableRateException extends HermesException { - public UnavailableRateException(TopicName topicName, String subscriptionName, Throwable cause) { - super(String.format( - "Rate for %s subscription on %s topic in group %s is unavailable.", - subscriptionName, topicName.getName(), topicName.getGroupName()), cause - ); - } + public UnavailableRateException(TopicName topicName, String subscriptionName, Throwable cause) { + super( + String.format( + "Rate for %s subscription on %s topic in group %s is unavailable.", + subscriptionName, topicName.getName(), topicName.getGroupName()), + cause); + } - public UnavailableRateException(TopicName topicName, Throwable cause) { - super(String.format( - "Rate for %s topic in group %s is unavailable", - topicName.getName(), topicName.getGroupName()), - cause - ); - } + public UnavailableRateException(TopicName topicName, Throwable cause) { + super( + String.format( + "Rate for %s topic in group %s is unavailable", + topicName.getName(), topicName.getGroupName()), + cause); + } - @Override - public ErrorCode getCode() { - return ErrorCode.UNAVAILABLE_RATE; - } + @Override + public ErrorCode getCode() { + return ErrorCode.UNAVAILABLE_RATE; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/http/ExtraRequestHeadersCollector.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/http/ExtraRequestHeadersCollector.java index d334c8b0e2..6f321ebbd8 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/http/ExtraRequestHeadersCollector.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/http/ExtraRequestHeadersCollector.java @@ -9,46 +9,46 @@ import java.util.function.Supplier; import java.util.stream.Collector; -public class ExtraRequestHeadersCollector implements Collector, StringBuilder, String> { - private ExtraRequestHeadersCollector() { - } - - public static ExtraRequestHeadersCollector extraRequestHeadersCollector() { - return new ExtraRequestHeadersCollector(); - } - - @Override - public Supplier supplier() { - return StringBuilder::new; - } - - @Override - public BiConsumer> accumulator() { - return (StringBuilder accumulator, Map.Entry entry) -> { - accumulator.append(entry.getKey()); - accumulator.append(": "); - accumulator.append(entry.getValue()); - accumulator.append('\n'); - }; - } - - @Override - public BinaryOperator combiner() { - return StringBuilder::append; - } - - @Override - public Function finisher() { - return (StringBuilder acc) -> { - if (acc.length() > 0) { - acc.setLength(acc.length() - 1); - } - return acc.toString(); - }; - } - - @Override - public Set characteristics() { - return Collections.emptySet(); - } +public class ExtraRequestHeadersCollector + implements Collector, StringBuilder, String> { + private ExtraRequestHeadersCollector() {} + + public static ExtraRequestHeadersCollector extraRequestHeadersCollector() { + return new ExtraRequestHeadersCollector(); + } + + @Override + public Supplier supplier() { + return StringBuilder::new; + } + + @Override + public BiConsumer> accumulator() { + return (StringBuilder accumulator, Map.Entry entry) -> { + accumulator.append(entry.getKey()); + accumulator.append(": "); + accumulator.append(entry.getValue()); + accumulator.append('\n'); + }; + } + + @Override + public BinaryOperator combiner() { + return StringBuilder::append; + } + + @Override + public Function finisher() { + return (StringBuilder acc) -> { + if (acc.length() > 0) { + acc.setLength(acc.length() - 1); + } + return acc.toString(); + }; + } + + @Override + public Set characteristics() { + return Collections.emptySet(); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/http/MessageMetadataHeaders.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/http/MessageMetadataHeaders.java index d48f8002ce..943502748f 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/http/MessageMetadataHeaders.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/http/MessageMetadataHeaders.java @@ -1,27 +1,26 @@ package pl.allegro.tech.hermes.common.http; public enum MessageMetadataHeaders { + MESSAGE_ID("Hermes-Message-Id"), + BATCH_ID("Hermes-Batch-Id"), + TOPIC_NAME("Hermes-Topic-Name"), + SUBSCRIPTION_NAME("Hermes-Subscription-Name"), + RETRY_COUNT("Hermes-Retry-Count"), + SCHEMA_VERSION("Schema-Version"), + SCHEMA_ID("Schema-Id"), + PARTITION_KEY("Partition-Key"); - MESSAGE_ID("Hermes-Message-Id"), - BATCH_ID("Hermes-Batch-Id"), - TOPIC_NAME("Hermes-Topic-Name"), - SUBSCRIPTION_NAME("Hermes-Subscription-Name"), - RETRY_COUNT("Hermes-Retry-Count"), - SCHEMA_VERSION("Schema-Version"), - SCHEMA_ID("Schema-Id"), - PARTITION_KEY("Partition-Key"); + private final String headerName; - private final String headerName; + MessageMetadataHeaders(String headerName) { + this.headerName = headerName; + } - MessageMetadataHeaders(String headerName) { - this.headerName = headerName; - } + public String getName() { + return this.headerName; + } - public String getName() { - return this.headerName; - } - - public String getCamelCaseName() { - return this.headerName.replace("-", ""); - } + public String getCamelCaseName() { + return this.headerName.replace("-", ""); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/ConsumerGroupId.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/ConsumerGroupId.java index aaff0c6110..b554fc6854 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/ConsumerGroupId.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/ConsumerGroupId.java @@ -1,44 +1,44 @@ package pl.allegro.tech.hermes.common.kafka; -import java.util.Objects; - import static com.google.common.base.Preconditions.checkNotNull; -public class ConsumerGroupId { +import java.util.Objects; - private final String value; +public class ConsumerGroupId { - private ConsumerGroupId(String value) { - this.value = checkNotNull(value); - } + private final String value; - public static ConsumerGroupId valueOf(String value) { - return new ConsumerGroupId(value); - } + private ConsumerGroupId(String value) { + this.value = checkNotNull(value); + } - public String asString() { - return value; - } + public static ConsumerGroupId valueOf(String value) { + return new ConsumerGroupId(value); + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ConsumerGroupId that = (ConsumerGroupId) o; - return Objects.equals(value, that.value); - } + public String asString() { + return value; + } - @Override - public int hashCode() { - return Objects.hash(value); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public String toString() { - return "ConsumerGroupId(" + value + ")"; + if (o == null || getClass() != o.getClass()) { + return false; } + ConsumerGroupId that = (ConsumerGroupId) o; + return Objects.equals(value, that.value); + } + + @Override + public int hashCode() { + return Objects.hash(value); + } + + @Override + public String toString() { + return "ConsumerGroupId(" + value + ")"; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/HTTPHeadersPropagationAsKafkaHeadersProperties.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/HTTPHeadersPropagationAsKafkaHeadersProperties.java index 0c9abaf3b8..a28d0e951a 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/HTTPHeadersPropagationAsKafkaHeadersProperties.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/HTTPHeadersPropagationAsKafkaHeadersProperties.java @@ -1,7 +1,7 @@ package pl.allegro.tech.hermes.common.kafka; public interface HTTPHeadersPropagationAsKafkaHeadersProperties { - boolean isEnabled(); + boolean isEnabled(); - String getPrefix(); + String getPrefix(); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/JsonToAvroMigrationKafkaNamesMapper.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/JsonToAvroMigrationKafkaNamesMapper.java index 92ceaf4834..bfe177419c 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/JsonToAvroMigrationKafkaNamesMapper.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/JsonToAvroMigrationKafkaNamesMapper.java @@ -1,37 +1,45 @@ package pl.allegro.tech.hermes.common.kafka; +import java.util.function.Function; import pl.allegro.tech.hermes.api.ContentType; import pl.allegro.tech.hermes.api.Topic; -import java.util.function.Function; - public class JsonToAvroMigrationKafkaNamesMapper extends NamespaceKafkaNamesMapper { - public JsonToAvroMigrationKafkaNamesMapper(String namespace, String namespaceSeparator) { - super(namespace, namespaceSeparator); - } - - public KafkaTopics toKafkaTopics(Topic topic) { - KafkaTopic primary = mapToKafkaTopic.andThen(appendNamespace).andThen(appendContentTypeSuffix).apply(topic); - - if (topic.wasMigratedFromJsonType()) { - KafkaTopic secondary = mapToJsonKafkaTopic.andThen(appendNamespace).andThen(appendContentTypeSuffix).apply(topic); - return new KafkaTopics(primary, secondary); - } - return new KafkaTopics(primary); + public JsonToAvroMigrationKafkaNamesMapper(String namespace, String namespaceSeparator) { + super(namespace, namespaceSeparator); + } + + public KafkaTopics toKafkaTopics(Topic topic) { + KafkaTopic primary = + mapToKafkaTopic.andThen(appendNamespace).andThen(appendContentTypeSuffix).apply(topic); + + if (topic.wasMigratedFromJsonType()) { + KafkaTopic secondary = + mapToJsonKafkaTopic + .andThen(appendNamespace) + .andThen(appendContentTypeSuffix) + .apply(topic); + return new KafkaTopics(primary, secondary); } + return new KafkaTopics(primary); + } - private final Function mapToJsonKafkaTopic = it -> - new KafkaTopic(KafkaTopicName.valueOf(it.getQualifiedName()), ContentType.JSON); + private final Function mapToJsonKafkaTopic = + it -> new KafkaTopic(KafkaTopicName.valueOf(it.getQualifiedName()), ContentType.JSON); - private final Function appendContentTypeSuffix = kafkaTopic -> { + private final Function appendContentTypeSuffix = + kafkaTopic -> { switch (kafkaTopic.contentType()) { - case JSON: - return kafkaTopic; - case AVRO: - return new KafkaTopic(KafkaTopicName.valueOf(kafkaTopic.name().asString() + "_avro"), kafkaTopic.contentType()); - default: - throw new IllegalStateException(String.format("Unknown content type '%s'", kafkaTopic.contentType())); + case JSON: + return kafkaTopic; + case AVRO: + return new KafkaTopic( + KafkaTopicName.valueOf(kafkaTopic.name().asString() + "_avro"), + kafkaTopic.contentType()); + default: + throw new IllegalStateException( + String.format("Unknown content type '%s'", kafkaTopic.contentType())); } - }; + }; } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaConsumerPool.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaConsumerPool.java index 3014885869..f22fc0b05b 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaConsumerPool.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaConsumerPool.java @@ -1,20 +1,5 @@ package pl.allegro.tech.hermes.common.kafka; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.cache.RemovalListener; -import com.google.common.cache.RemovalNotification; -import com.google.common.util.concurrent.UncheckedExecutionException; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.common.TopicPartition; -import pl.allegro.tech.hermes.common.broker.BrokerStorage; -import pl.allegro.tech.hermes.common.exception.BrokerNotFoundForPartitionException; - -import java.util.Properties; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; - import static org.apache.kafka.clients.CommonClientConfigs.SECURITY_PROTOCOL_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG; @@ -25,87 +10,111 @@ import static org.apache.kafka.common.config.SaslConfigs.SASL_JAAS_CONFIG; import static org.apache.kafka.common.config.SaslConfigs.SASL_MECHANISM; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.cache.RemovalListener; +import com.google.common.cache.RemovalNotification; +import com.google.common.util.concurrent.UncheckedExecutionException; +import java.util.Properties; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.TopicPartition; +import pl.allegro.tech.hermes.common.broker.BrokerStorage; +import pl.allegro.tech.hermes.common.exception.BrokerNotFoundForPartitionException; /** - * This class help us to avoid unnecessarily creating new kafka consumers for the same broker instance, mainly in case of - * retransmission and migrating topic from json to avro. We map topic and its partitions to specific kafka broker which - * without caching would lead to creating exactly the same consumer multiple times in short period of time. - * Additionaly consumers created here are rarely used, so we don't bother about concurrency and thread safety. + * This class help us to avoid unnecessarily creating new kafka consumers for the same broker + * instance, mainly in case of retransmission and migrating topic from json to avro. We map topic + * and its partitions to specific kafka broker which without caching would lead to creating exactly + * the same consumer multiple times in short period of time. Additionaly consumers created here are + * rarely used, so we don't bother about concurrency and thread safety. */ public class KafkaConsumerPool { - private final LoadingCache> kafkaConsumers; - private final BrokerStorage brokerStorage; - - public KafkaConsumerPool(KafkaConsumerPoolConfig poolConfig, BrokerStorage brokerStorage, String configuredBootstrapServers) { - this.brokerStorage = brokerStorage; - this.kafkaConsumers = CacheBuilder.newBuilder() - .expireAfterAccess(poolConfig.getCacheExpirationSeconds(), TimeUnit.SECONDS) - .removalListener(new KafkaConsumerRemoveListener()) - .build(new KafkaConsumerSupplier(poolConfig, configuredBootstrapServers)); + private final LoadingCache> kafkaConsumers; + private final BrokerStorage brokerStorage; + + public KafkaConsumerPool( + KafkaConsumerPoolConfig poolConfig, + BrokerStorage brokerStorage, + String configuredBootstrapServers) { + this.brokerStorage = brokerStorage; + this.kafkaConsumers = + CacheBuilder.newBuilder() + .expireAfterAccess(poolConfig.getCacheExpirationSeconds(), TimeUnit.SECONDS) + .removalListener(new KafkaConsumerRemoveListener()) + .build(new KafkaConsumerSupplier(poolConfig, configuredBootstrapServers)); + } + + public KafkaConsumer get(KafkaTopic topic, int partition) { + return get(topic.name().asString(), partition); + } + + public KafkaConsumer get(String topicName, int partition) { + try { + int leaderId = brokerStorage.readLeaderForPartition(new TopicPartition(topicName, partition)); + return kafkaConsumers.get(leaderId); + + } catch (ExecutionException e) { + String message = + String.format( + "Cannot get KafkaConsumer for topic %s and partition %d", topicName, partition); + throw new KafkaConsumerPoolException(message, e); + } catch (UncheckedExecutionException e) { + if (e.getCause() instanceof BrokerNotFoundForPartitionException) { + throw (BrokerNotFoundForPartitionException) e.getCause(); + } + throw e; } + } - public KafkaConsumer get(KafkaTopic topic, int partition) { - return get(topic.name().asString(), partition); - } + private static class KafkaConsumerSupplier + extends CacheLoader> { + + private final KafkaConsumerPoolConfig poolConfig; + private final String configuredBootstrapServers; - public KafkaConsumer get(String topicName, int partition) { - try { - int leaderId = brokerStorage.readLeaderForPartition(new TopicPartition(topicName, partition)); - return kafkaConsumers.get(leaderId); - - } catch (ExecutionException e) { - String message = String.format("Cannot get KafkaConsumer for topic %s and partition %d", topicName, partition); - throw new KafkaConsumerPoolException(message, e); - } catch (UncheckedExecutionException e) { - if (e.getCause() instanceof BrokerNotFoundForPartitionException) { - throw (BrokerNotFoundForPartitionException) e.getCause(); - } - throw e; - } + KafkaConsumerSupplier(KafkaConsumerPoolConfig poolConfig, String configuredBootstrapServers) { + this.poolConfig = poolConfig; + this.configuredBootstrapServers = configuredBootstrapServers; } - private static class KafkaConsumerSupplier extends CacheLoader> { - - private final KafkaConsumerPoolConfig poolConfig; - private final String configuredBootstrapServers; - - KafkaConsumerSupplier(KafkaConsumerPoolConfig poolConfig, String configuredBootstrapServers) { - this.poolConfig = poolConfig; - this.configuredBootstrapServers = configuredBootstrapServers; - } - - @Override - public KafkaConsumer load(Integer leaderId) throws Exception { - return createKafkaConsumer(); - } - - private KafkaConsumer createKafkaConsumer() { - - Properties props = new Properties(); - props.put(BOOTSTRAP_SERVERS_CONFIG, configuredBootstrapServers); - props.put(GROUP_ID_CONFIG, poolConfig.getIdPrefix() + "_" + poolConfig.getConsumerGroupName()); - props.put(RECEIVE_BUFFER_CONFIG, poolConfig.getBufferSizeBytes()); - props.put(ENABLE_AUTO_COMMIT_CONFIG, false); - props.put(FETCH_MAX_WAIT_MS_CONFIG, poolConfig.getFetchMaxWaitMillis()); - props.put(FETCH_MIN_BYTES_CONFIG, poolConfig.getFetchMinBytes()); - props.put("key.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer"); - props.put("value.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer"); - - if (poolConfig.isSaslEnabled()) { - props.put(SASL_MECHANISM, poolConfig.getSecurityMechanism()); - props.put(SECURITY_PROTOCOL_CONFIG, poolConfig.getSecurityProtocol()); - props.put(SASL_JAAS_CONFIG, poolConfig.getSaslJaasConfig()); - } - return new KafkaConsumer<>(props); - } + @Override + public KafkaConsumer load(Integer leaderId) throws Exception { + return createKafkaConsumer(); } - private static class KafkaConsumerRemoveListener implements RemovalListener> { - @Override - public void onRemoval(RemovalNotification> notification) { - notification.getValue().close(); - } + private KafkaConsumer createKafkaConsumer() { + + Properties props = new Properties(); + props.put(BOOTSTRAP_SERVERS_CONFIG, configuredBootstrapServers); + props.put( + GROUP_ID_CONFIG, poolConfig.getIdPrefix() + "_" + poolConfig.getConsumerGroupName()); + props.put(RECEIVE_BUFFER_CONFIG, poolConfig.getBufferSizeBytes()); + props.put(ENABLE_AUTO_COMMIT_CONFIG, false); + props.put(FETCH_MAX_WAIT_MS_CONFIG, poolConfig.getFetchMaxWaitMillis()); + props.put(FETCH_MIN_BYTES_CONFIG, poolConfig.getFetchMinBytes()); + props.put("key.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer"); + props.put( + "value.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer"); + + if (poolConfig.isSaslEnabled()) { + props.put(SASL_MECHANISM, poolConfig.getSecurityMechanism()); + props.put(SECURITY_PROTOCOL_CONFIG, poolConfig.getSecurityProtocol()); + props.put(SASL_JAAS_CONFIG, poolConfig.getSaslJaasConfig()); + } + return new KafkaConsumer<>(props); } + } + + private static class KafkaConsumerRemoveListener + implements RemovalListener> { + @Override + public void onRemoval( + RemovalNotification> notification) { + notification.getValue().close(); + } + } } - diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaConsumerPoolConfig.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaConsumerPoolConfig.java index 772306fffa..38212a9093 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaConsumerPoolConfig.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaConsumerPoolConfig.java @@ -2,69 +2,77 @@ public class KafkaConsumerPoolConfig { - private final int cacheExpirationSeconds; - private final int bufferSizeBytes; - private final int fetchMaxWaitMillis; - private final int fetchMinBytes; - private final String idPrefix; - private final String consumerGroupName; - private final boolean isSaslEnabled; - private final String securityMechanism; - private final String securityProtocol; - private final String saslJaasConfig; + private final int cacheExpirationSeconds; + private final int bufferSizeBytes; + private final int fetchMaxWaitMillis; + private final int fetchMinBytes; + private final String idPrefix; + private final String consumerGroupName; + private final boolean isSaslEnabled; + private final String securityMechanism; + private final String securityProtocol; + private final String saslJaasConfig; - public KafkaConsumerPoolConfig(int cacheExpirationSeconds, int bufferSize, int fetchMaxWaitMillis, - int fetchMinBytes, String idPrefix, String consumerGroupName, - boolean isSaslEnabled, String securityMechanism, String securityProtocol, String saslJaasConfig) { - this.cacheExpirationSeconds = cacheExpirationSeconds; - this.bufferSizeBytes = bufferSize; - this.fetchMaxWaitMillis = fetchMaxWaitMillis; - this.fetchMinBytes = fetchMinBytes; - this.idPrefix = idPrefix; - this.consumerGroupName = consumerGroupName; - this.isSaslEnabled = isSaslEnabled; - this.securityMechanism = securityMechanism; - this.securityProtocol = securityProtocol; - this.saslJaasConfig = saslJaasConfig; - } + public KafkaConsumerPoolConfig( + int cacheExpirationSeconds, + int bufferSize, + int fetchMaxWaitMillis, + int fetchMinBytes, + String idPrefix, + String consumerGroupName, + boolean isSaslEnabled, + String securityMechanism, + String securityProtocol, + String saslJaasConfig) { + this.cacheExpirationSeconds = cacheExpirationSeconds; + this.bufferSizeBytes = bufferSize; + this.fetchMaxWaitMillis = fetchMaxWaitMillis; + this.fetchMinBytes = fetchMinBytes; + this.idPrefix = idPrefix; + this.consumerGroupName = consumerGroupName; + this.isSaslEnabled = isSaslEnabled; + this.securityMechanism = securityMechanism; + this.securityProtocol = securityProtocol; + this.saslJaasConfig = saslJaasConfig; + } - public int getCacheExpirationSeconds() { - return cacheExpirationSeconds; - } + public int getCacheExpirationSeconds() { + return cacheExpirationSeconds; + } - public int getBufferSizeBytes() { - return bufferSizeBytes; - } + public int getBufferSizeBytes() { + return bufferSizeBytes; + } - public String getIdPrefix() { - return idPrefix; - } + public String getIdPrefix() { + return idPrefix; + } - public String getConsumerGroupName() { - return consumerGroupName; - } + public String getConsumerGroupName() { + return consumerGroupName; + } - public int getFetchMaxWaitMillis() { - return fetchMaxWaitMillis; - } + public int getFetchMaxWaitMillis() { + return fetchMaxWaitMillis; + } - public int getFetchMinBytes() { - return fetchMinBytes; - } + public int getFetchMinBytes() { + return fetchMinBytes; + } - public boolean isSaslEnabled() { - return isSaslEnabled; - } + public boolean isSaslEnabled() { + return isSaslEnabled; + } - public String getSecurityMechanism() { - return securityMechanism; - } + public String getSecurityMechanism() { + return securityMechanism; + } - public String getSecurityProtocol() { - return securityProtocol; - } + public String getSecurityProtocol() { + return securityProtocol; + } - public String getSaslJaasConfig() { - return saslJaasConfig; - } + public String getSaslJaasConfig() { + return saslJaasConfig; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaConsumerPoolException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaConsumerPoolException.java index 150031e460..bc55b8cc6d 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaConsumerPoolException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaConsumerPoolException.java @@ -5,13 +5,12 @@ public class KafkaConsumerPoolException extends HermesException { - public KafkaConsumerPoolException(String message, Throwable t) { - super(message, t); - } - - @Override - public ErrorCode getCode() { - return ErrorCode.SIMPLE_CONSUMER_POOL_EXCEPTION; - } + public KafkaConsumerPoolException(String message, Throwable t) { + super(message, t); + } + @Override + public ErrorCode getCode() { + return ErrorCode.SIMPLE_CONSUMER_POOL_EXCEPTION; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaHeaderNameParameters.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaHeaderNameParameters.java index 63a8274c69..bf6f0bbc57 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaHeaderNameParameters.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaHeaderNameParameters.java @@ -2,10 +2,9 @@ public interface KafkaHeaderNameParameters { - String getSchemaVersion(); + String getSchemaVersion(); - String getSchemaId(); - - String getMessageId(); + String getSchemaId(); + String getMessageId(); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaNamesMapper.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaNamesMapper.java index d926d3f9aa..1613f222ed 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaNamesMapper.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaNamesMapper.java @@ -5,7 +5,7 @@ public interface KafkaNamesMapper { - ConsumerGroupId toConsumerGroupId(SubscriptionName subscription); + ConsumerGroupId toConsumerGroupId(SubscriptionName subscription); - KafkaTopics toKafkaTopics(Topic topic); + KafkaTopics toKafkaTopics(Topic topic); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaParameters.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaParameters.java index a9c0f92e47..113ad8b6d0 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaParameters.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaParameters.java @@ -2,15 +2,15 @@ public interface KafkaParameters { - String getDatacenter(); + String getDatacenter(); - boolean isAuthenticationEnabled(); + boolean isAuthenticationEnabled(); - String getAuthenticationMechanism(); + String getAuthenticationMechanism(); - String getAuthenticationProtocol(); + String getAuthenticationProtocol(); - String getBrokerList(); + String getBrokerList(); - String getJaasConfig(); + String getJaasConfig(); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaTopic.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaTopic.java index e6155a4273..63a786bb66 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaTopic.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaTopic.java @@ -1,53 +1,51 @@ package pl.allegro.tech.hermes.common.kafka; -import com.google.common.base.MoreObjects; -import pl.allegro.tech.hermes.api.ContentType; +import static com.google.common.base.Preconditions.checkNotNull; +import com.google.common.base.MoreObjects; import java.util.Objects; - -import static com.google.common.base.Preconditions.checkNotNull; +import pl.allegro.tech.hermes.api.ContentType; public class KafkaTopic { - private final KafkaTopicName name; - private final ContentType contentType; + private final KafkaTopicName name; + private final ContentType contentType; - public KafkaTopic(KafkaTopicName name, ContentType contentType) { - this.name = checkNotNull(name); - this.contentType = checkNotNull(contentType); - } - - public KafkaTopicName name() { - return name; - } + public KafkaTopic(KafkaTopicName name, ContentType contentType) { + this.name = checkNotNull(name); + this.contentType = checkNotNull(contentType); + } - public ContentType contentType() { - return contentType; - } + public KafkaTopicName name() { + return name; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - KafkaTopic that = (KafkaTopic) o; - return Objects.equals(name, that.name) - && Objects.equals(contentType, that.contentType); - } + public ContentType contentType() { + return contentType; + } - @Override - public int hashCode() { - return Objects.hash(name, contentType); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("name", name) - .add("contentType", contentType) - .toString(); + if (o == null || getClass() != o.getClass()) { + return false; } + KafkaTopic that = (KafkaTopic) o; + return Objects.equals(name, that.name) && Objects.equals(contentType, that.contentType); + } + + @Override + public int hashCode() { + return Objects.hash(name, contentType); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("name", name) + .add("contentType", contentType) + .toString(); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaTopicName.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaTopicName.java index 58f7f438ef..8c2ab7dea6 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaTopicName.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaTopicName.java @@ -1,50 +1,49 @@ package pl.allegro.tech.hermes.common.kafka; +import static com.google.common.base.Preconditions.checkNotNull; + import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonGetter; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.Objects; -import static com.google.common.base.Preconditions.checkNotNull; - public class KafkaTopicName { - private final String value; + private final String value; - private KafkaTopicName(String value) { - this.value = checkNotNull(value); - } + private KafkaTopicName(String value) { + this.value = checkNotNull(value); + } - @JsonCreator - public static KafkaTopicName valueOf(@JsonProperty("value") String value) { - return new KafkaTopicName(value); - } + @JsonCreator + public static KafkaTopicName valueOf(@JsonProperty("value") String value) { + return new KafkaTopicName(value); + } - @JsonGetter(value = "value") - public String asString() { - return value; - } + @JsonGetter(value = "value") + public String asString() { + return value; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - KafkaTopicName that = (KafkaTopicName) o; - return Objects.equals(value, that.value); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(value); - } - - @Override - public String toString() { - return "KafkaTopicName(" + value + ")"; + if (o == null || getClass() != o.getClass()) { + return false; } + KafkaTopicName that = (KafkaTopicName) o; + return Objects.equals(value, that.value); + } + + @Override + public int hashCode() { + return Objects.hash(value); + } + + @Override + public String toString() { + return "KafkaTopicName(" + value + ")"; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaTopics.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaTopics.java index dde42b8944..4093a28bd0 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaTopics.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/KafkaTopics.java @@ -1,45 +1,45 @@ package pl.allegro.tech.hermes.common.kafka; +import static com.google.common.base.Preconditions.checkNotNull; + import java.util.Optional; import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Stream; -import static com.google.common.base.Preconditions.checkNotNull; - public class KafkaTopics { - private final KafkaTopic primary; - private final Optional secondary; + private final KafkaTopic primary; + private final Optional secondary; - public KafkaTopics(KafkaTopic primary) { - this.primary = checkNotNull(primary); - this.secondary = Optional.empty(); - } + public KafkaTopics(KafkaTopic primary) { + this.primary = checkNotNull(primary); + this.secondary = Optional.empty(); + } - public KafkaTopics(KafkaTopic primary, KafkaTopic secondary) { - this.primary = checkNotNull(primary); - this.secondary = Optional.of(secondary); - } + public KafkaTopics(KafkaTopic primary, KafkaTopic secondary) { + this.primary = checkNotNull(primary); + this.secondary = Optional.of(secondary); + } - public KafkaTopic getPrimary() { - return primary; - } + public KafkaTopic getPrimary() { + return primary; + } - public Optional getSecondary() { - return secondary; - } + public Optional getSecondary() { + return secondary; + } - public void forEach(Consumer consumer) { - consumer.accept(primary); - secondary.ifPresent(consumer); - } + public void forEach(Consumer consumer) { + consumer.accept(primary); + secondary.ifPresent(consumer); + } - public boolean allMatch(Function matcher) { - return matcher.apply(primary) && secondary.map(matcher).orElse(true); - } + public boolean allMatch(Function matcher) { + return matcher.apply(primary) && secondary.map(matcher).orElse(true); + } - public Stream stream() { - return secondary.map(secondary -> Stream.of(primary, secondary)).orElse(Stream.of(primary)); - } + public Stream stream() { + return secondary.map(secondary -> Stream.of(primary, secondary)).orElse(Stream.of(primary)); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/NamespaceKafkaNamesMapper.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/NamespaceKafkaNamesMapper.java index d0577e3f46..18a1414f78 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/NamespaceKafkaNamesMapper.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/NamespaceKafkaNamesMapper.java @@ -1,52 +1,54 @@ package pl.allegro.tech.hermes.common.kafka; +import static pl.allegro.tech.hermes.api.helpers.Replacer.replaceInAll; + import com.google.common.base.Joiner; +import java.util.function.Function; import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.api.Topic; -import java.util.function.Function; - -import static pl.allegro.tech.hermes.api.helpers.Replacer.replaceInAll; - public class NamespaceKafkaNamesMapper implements KafkaNamesMapper { - private final String namespace; - private final String namespaceSeparator; + private final String namespace; + private final String namespaceSeparator; - public NamespaceKafkaNamesMapper(String namespace, String namespaceSeparator) { - this.namespace = namespace; - this.namespaceSeparator = namespaceSeparator; - } + public NamespaceKafkaNamesMapper(String namespace, String namespaceSeparator) { + this.namespace = namespace; + this.namespaceSeparator = namespaceSeparator; + } - @Override - public ConsumerGroupId toConsumerGroupId(SubscriptionName subscriptionName) { - return ConsumerGroupId.valueOf( - appendNamespace(subscriptionNameToConsumerId(subscriptionName)) - ); - } + @Override + public ConsumerGroupId toConsumerGroupId(SubscriptionName subscriptionName) { + return ConsumerGroupId.valueOf(appendNamespace(subscriptionNameToConsumerId(subscriptionName))); + } - @Override - public KafkaTopics toKafkaTopics(Topic topic) { - return mapToKafkaTopic.andThen(appendNamespace).andThen(mapToKafkaTopics).apply(topic); - } + @Override + public KafkaTopics toKafkaTopics(Topic topic) { + return mapToKafkaTopic.andThen(appendNamespace).andThen(mapToKafkaTopics).apply(topic); + } - protected Function mapToKafkaTopic = it -> - new KafkaTopic(KafkaTopicName.valueOf(it.getQualifiedName()), it.getContentType()); + protected Function mapToKafkaTopic = + it -> new KafkaTopic(KafkaTopicName.valueOf(it.getQualifiedName()), it.getContentType()); - protected Function appendNamespace = it -> - new KafkaTopic(KafkaTopicName.valueOf(appendNamespace(it.name().asString())), it.contentType()); + protected Function appendNamespace = + it -> + new KafkaTopic( + KafkaTopicName.valueOf(appendNamespace(it.name().asString())), it.contentType()); - protected Function mapToKafkaTopics = KafkaTopics::new; + protected Function mapToKafkaTopics = KafkaTopics::new; - private String subscriptionNameToConsumerId(SubscriptionName subscriptionName) { - return Joiner.on("_").join(replaceInAll("_", "__", + private String subscriptionNameToConsumerId(SubscriptionName subscriptionName) { + return Joiner.on("_") + .join( + replaceInAll( + "_", + "__", subscriptionName.getTopicName().getGroupName(), subscriptionName.getTopicName().getName(), - subscriptionName.getName()) - ); - } + subscriptionName.getName())); + } - private String appendNamespace(String name) { - return namespace.isEmpty() ? name : namespace + namespaceSeparator + name; - } + private String appendNamespace(String name) { + return namespace.isEmpty() ? name : namespace + namespaceSeparator + name; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/offset/PartitionOffset.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/offset/PartitionOffset.java index cc390200ae..459770e508 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/offset/PartitionOffset.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/offset/PartitionOffset.java @@ -2,55 +2,54 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import pl.allegro.tech.hermes.common.kafka.KafkaTopicName; - import java.util.Objects; +import pl.allegro.tech.hermes.common.kafka.KafkaTopicName; public class PartitionOffset { - private final KafkaTopicName topic; - private final int partition; - private final long offset; - - @JsonCreator - public PartitionOffset(@JsonProperty("topic") KafkaTopicName topic, - @JsonProperty("offset") long offset, - @JsonProperty("partition") int partition) { - this.topic = topic; - this.offset = offset; - this.partition = partition; + private final KafkaTopicName topic; + private final int partition; + private final long offset; + + @JsonCreator + public PartitionOffset( + @JsonProperty("topic") KafkaTopicName topic, + @JsonProperty("offset") long offset, + @JsonProperty("partition") int partition) { + this.topic = topic; + this.offset = offset; + this.partition = partition; + } + + public KafkaTopicName getTopic() { + return topic; + } + + public int getPartition() { + return partition; + } + + public long getOffset() { + return offset; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; } - - public KafkaTopicName getTopic() { - return topic; + if (obj == null || getClass() != obj.getClass()) { + return false; } - - public int getPartition() { - return partition; - } - - public long getOffset() { - return offset; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - final PartitionOffset that = (PartitionOffset) obj; - - return Objects.equals(this.topic, that.topic) - && Objects.equals(this.partition, that.partition) - && Objects.equals(this.offset, that.offset); - } - - @Override - public int hashCode() { - return Objects.hash(topic, partition, offset); - } - -} \ No newline at end of file + final PartitionOffset that = (PartitionOffset) obj; + + return Objects.equals(this.topic, that.topic) + && Objects.equals(this.partition, that.partition) + && Objects.equals(this.offset, that.offset); + } + + @Override + public int hashCode() { + return Objects.hash(topic, partition, offset); + } +} diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/offset/PartitionOffsets.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/offset/PartitionOffsets.java index a0f2023064..c0c5b1ef30 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/offset/PartitionOffsets.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/offset/PartitionOffsets.java @@ -6,20 +6,20 @@ public class PartitionOffsets implements Iterable { - private final List offsets = new ArrayList<>(); + private final List offsets = new ArrayList<>(); - public PartitionOffsets add(PartitionOffset offset) { - offsets.add(offset); - return this; - } + public PartitionOffsets add(PartitionOffset offset) { + offsets.add(offset); + return this; + } - public PartitionOffsets addAll(PartitionOffsets offsets) { - offsets.forEach(this::add); - return this; - } + public PartitionOffsets addAll(PartitionOffsets offsets) { + offsets.forEach(this::add); + return this; + } - @Override - public Iterator iterator() { - return offsets.iterator(); - } + @Override + public Iterator iterator() { + return offsets.iterator(); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/offset/SubscriptionOffsetChangeIndicator.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/offset/SubscriptionOffsetChangeIndicator.java index 095b3746a8..863c8a8801 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/offset/SubscriptionOffsetChangeIndicator.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/kafka/offset/SubscriptionOffsetChangeIndicator.java @@ -1,21 +1,32 @@ package pl.allegro.tech.hermes.common.kafka.offset; +import java.util.List; import pl.allegro.tech.hermes.api.TopicName; import pl.allegro.tech.hermes.common.kafka.KafkaTopic; import pl.allegro.tech.hermes.common.kafka.KafkaTopicName; -import java.util.List; - public interface SubscriptionOffsetChangeIndicator { - void setSubscriptionOffset(TopicName topicName, String subscriptionName, String brokersClusterName, PartitionOffset partitionOffset); - - PartitionOffsets getSubscriptionOffsets(TopicName topic, String subscriptionName, String brokersClusterName); - - boolean areOffsetsMoved(TopicName topicName, String subscriptionName, String brokersClusterName, - KafkaTopic kafkaTopic, List partitionIds); - - - void removeOffset(TopicName topicName, String subscriptionName, String brokersClusterName, - KafkaTopicName kafkaTopicName, int partitionId); + void setSubscriptionOffset( + TopicName topicName, + String subscriptionName, + String brokersClusterName, + PartitionOffset partitionOffset); + + PartitionOffsets getSubscriptionOffsets( + TopicName topic, String subscriptionName, String brokersClusterName); + + boolean areOffsetsMoved( + TopicName topicName, + String subscriptionName, + String brokersClusterName, + KafkaTopic kafkaTopic, + List partitionIds); + + void removeOffset( + TopicName topicName, + String subscriptionName, + String brokersClusterName, + KafkaTopicName kafkaTopicName, + int partitionId); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/converter/AvroBinaryDecoders.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/converter/AvroBinaryDecoders.java index 1d092f655b..4c7d397651 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/converter/AvroBinaryDecoders.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/converter/AvroBinaryDecoders.java @@ -1,5 +1,8 @@ package pl.allegro.tech.hermes.common.message.converter; +import java.io.ByteArrayInputStream; +import java.io.Closeable; +import java.io.InputStream; import org.apache.avro.Schema; import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.generic.GenericRecord; @@ -8,39 +11,41 @@ import org.apache.commons.lang3.exception.ExceptionUtils; import tech.allegro.schema.json2avro.converter.AvroConversionException; -import java.io.ByteArrayInputStream; -import java.io.Closeable; -import java.io.InputStream; - public class AvroBinaryDecoders { - private static ThreadLocal threadLocalEmptyInputStream = - ThreadLocal.withInitial(() -> new ByteArrayInputStream(new byte[0])); - - private static ThreadLocal threadLocalBinaryDecoder = - ThreadLocal.withInitial(() -> DecoderFactory.get().binaryDecoder(threadLocalEmptyInputStream.get(), null)); - - static GenericRecord decodeReusingThreadLocalBinaryDecoder(byte[] message, Schema schema) { - try (FlushableBinaryDecoderHolder holder = new FlushableBinaryDecoderHolder()) { - BinaryDecoder binaryDecoder = DecoderFactory.get().binaryDecoder(message, holder.getBinaryDecoder()); - return new GenericDatumReader(schema).read(null, binaryDecoder); - } catch (Exception e) { - String reason = e.getMessage() == null ? ExceptionUtils.getRootCauseMessage(e) : e.getMessage(); - throw new AvroConversionException(String.format("Could not deserialize Avro message with provided schema, reason: %s", reason)); - } + private static ThreadLocal threadLocalEmptyInputStream = + ThreadLocal.withInitial(() -> new ByteArrayInputStream(new byte[0])); + + private static ThreadLocal threadLocalBinaryDecoder = + ThreadLocal.withInitial( + () -> DecoderFactory.get().binaryDecoder(threadLocalEmptyInputStream.get(), null)); + + static GenericRecord decodeReusingThreadLocalBinaryDecoder(byte[] message, Schema schema) { + try (FlushableBinaryDecoderHolder holder = new FlushableBinaryDecoderHolder()) { + BinaryDecoder binaryDecoder = + DecoderFactory.get().binaryDecoder(message, holder.getBinaryDecoder()); + return new GenericDatumReader(schema).read(null, binaryDecoder); + } catch (Exception e) { + String reason = + e.getMessage() == null ? ExceptionUtils.getRootCauseMessage(e) : e.getMessage(); + throw new AvroConversionException( + String.format( + "Could not deserialize Avro message with provided schema, reason: %s", reason)); } + } - static class FlushableBinaryDecoderHolder implements Closeable { + static class FlushableBinaryDecoderHolder implements Closeable { - final BinaryDecoder binaryDecoder = threadLocalBinaryDecoder.get(); + final BinaryDecoder binaryDecoder = threadLocalBinaryDecoder.get(); - BinaryDecoder getBinaryDecoder() { - return binaryDecoder; - } + BinaryDecoder getBinaryDecoder() { + return binaryDecoder; + } - @Override - public void close() { - DecoderFactory.get().binaryDecoder(threadLocalEmptyInputStream.get(), threadLocalBinaryDecoder.get()); - } + @Override + public void close() { + DecoderFactory.get() + .binaryDecoder(threadLocalEmptyInputStream.get(), threadLocalBinaryDecoder.get()); } + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/converter/AvroRecordToBytesConverter.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/converter/AvroRecordToBytesConverter.java index ea4f6fe347..64762fcccc 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/converter/AvroRecordToBytesConverter.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/converter/AvroRecordToBytesConverter.java @@ -1,26 +1,25 @@ package pl.allegro.tech.hermes.common.message.converter; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import org.apache.avro.Schema; import org.apache.avro.generic.GenericDatumWriter; import org.apache.avro.generic.GenericRecord; import org.apache.avro.io.BinaryEncoder; import org.apache.avro.io.EncoderFactory; -import java.io.ByteArrayOutputStream; -import java.io.IOException; - public interface AvroRecordToBytesConverter { - static GenericRecord bytesToRecord(byte[] data, Schema schema) { - return AvroBinaryDecoders.decodeReusingThreadLocalBinaryDecoder(data, schema); - } + static GenericRecord bytesToRecord(byte[] data, Schema schema) { + return AvroBinaryDecoders.decodeReusingThreadLocalBinaryDecoder(data, schema); + } - static byte [] recordToBytes(GenericRecord genericRecord, Schema schema) throws IOException { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - BinaryEncoder binaryEncoder = EncoderFactory.get().binaryEncoder(outputStream, null); - new GenericDatumWriter<>(schema).write(genericRecord, binaryEncoder); - binaryEncoder.flush(); + static byte[] recordToBytes(GenericRecord genericRecord, Schema schema) throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + BinaryEncoder binaryEncoder = EncoderFactory.get().binaryEncoder(outputStream, null); + new GenericDatumWriter<>(schema).write(genericRecord, binaryEncoder); + binaryEncoder.flush(); - return outputStream.toByteArray(); - } + return outputStream.toByteArray(); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/undelivered/LastUndeliveredMessageReader.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/undelivered/LastUndeliveredMessageReader.java index 326060eefc..8cef6b8052 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/undelivered/LastUndeliveredMessageReader.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/undelivered/LastUndeliveredMessageReader.java @@ -1,11 +1,10 @@ package pl.allegro.tech.hermes.common.message.undelivered; +import java.util.Optional; import pl.allegro.tech.hermes.api.SentMessageTrace; import pl.allegro.tech.hermes.api.TopicName; -import java.util.Optional; - public interface LastUndeliveredMessageReader { - Optional last(TopicName topicName, String subscriptionName); + Optional last(TopicName topicName, String subscriptionName); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/undelivered/UndeliveredMessageLog.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/undelivered/UndeliveredMessageLog.java index 6897e76277..c76782e08a 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/undelivered/UndeliveredMessageLog.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/undelivered/UndeliveredMessageLog.java @@ -1,11 +1,10 @@ package pl.allegro.tech.hermes.common.message.undelivered; - import pl.allegro.tech.hermes.api.SentMessageTrace; public interface UndeliveredMessageLog { - void add(SentMessageTrace undeliveredMessage); + void add(SentMessageTrace undeliveredMessage); - void persist(); + void persist(); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/undelivered/UndeliveredMessagePaths.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/undelivered/UndeliveredMessagePaths.java index 54912a787c..b6555312f0 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/undelivered/UndeliveredMessagePaths.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/undelivered/UndeliveredMessagePaths.java @@ -5,15 +5,15 @@ class UndeliveredMessagePaths { - private static final String NODE_NAME = "undelivered"; + private static final String NODE_NAME = "undelivered"; - private final ZookeeperPaths zookeeperPaths; + private final ZookeeperPaths zookeeperPaths; - UndeliveredMessagePaths(ZookeeperPaths zookeeperPaths) { - this.zookeeperPaths = zookeeperPaths; - } + UndeliveredMessagePaths(ZookeeperPaths zookeeperPaths) { + this.zookeeperPaths = zookeeperPaths; + } - String buildPath(TopicName topicName, String subscriptionName) { - return zookeeperPaths.subscriptionPath(topicName, subscriptionName, NODE_NAME); - } + String buildPath(TopicName topicName, String subscriptionName) { + return zookeeperPaths.subscriptionPath(topicName, subscriptionName, NODE_NAME); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/undelivered/ZookeeperLastUndeliveredMessageReader.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/undelivered/ZookeeperLastUndeliveredMessageReader.java index f8c2a37ddd..c65cf2bc75 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/undelivered/ZookeeperLastUndeliveredMessageReader.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/undelivered/ZookeeperLastUndeliveredMessageReader.java @@ -1,48 +1,48 @@ package pl.allegro.tech.hermes.common.message.undelivered; +import static java.lang.String.format; + import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Optional; import org.apache.curator.framework.CuratorFramework; import pl.allegro.tech.hermes.api.SentMessageTrace; import pl.allegro.tech.hermes.api.TopicName; import pl.allegro.tech.hermes.common.exception.InternalProcessingException; import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; -import java.util.Optional; - -import static java.lang.String.format; - public class ZookeeperLastUndeliveredMessageReader implements LastUndeliveredMessageReader { - private final CuratorFramework curator; - private final UndeliveredMessagePaths paths; - private final ObjectMapper mapper; - - public ZookeeperLastUndeliveredMessageReader(CuratorFramework curator, - ZookeeperPaths zookeeperPaths, - ObjectMapper mapper) { - this.curator = curator; - this.paths = new UndeliveredMessagePaths(zookeeperPaths); - this.mapper = mapper; + private final CuratorFramework curator; + private final UndeliveredMessagePaths paths; + private final ObjectMapper mapper; + + public ZookeeperLastUndeliveredMessageReader( + CuratorFramework curator, ZookeeperPaths zookeeperPaths, ObjectMapper mapper) { + this.curator = curator; + this.paths = new UndeliveredMessagePaths(zookeeperPaths); + this.mapper = mapper; + } + + @Override + public Optional last(TopicName topicName, String subscriptionName) { + try { + String path = paths.buildPath(topicName, subscriptionName); + if (exists(path)) { + return Optional.of( + mapper.readValue(curator.getData().forPath(path), SentMessageTrace.class)); + } else { + return Optional.empty(); + } + } catch (Exception e) { + throw new InternalProcessingException( + format( + "Could not read latest undelivered message for topic: %s and subscription: %s .", + topicName.qualifiedName(), subscriptionName), + e); } + } - @Override - public Optional last(TopicName topicName, String subscriptionName) { - try { - String path = paths.buildPath(topicName, subscriptionName); - if (exists(path)) { - return Optional.of(mapper.readValue(curator.getData().forPath(path), SentMessageTrace.class)); - } else { - return Optional.empty(); - } - } catch (Exception e) { - throw new InternalProcessingException( - format("Could not read latest undelivered message for topic: %s and subscription: %s .", - topicName.qualifiedName(), subscriptionName), - e); - } - } - - private boolean exists(String path) throws Exception { - return curator.checkExists().forPath(path) != null; - } + private boolean exists(String path) throws Exception { + return curator.checkExists().forPath(path) != null; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/undelivered/ZookeeperUndeliveredMessageLog.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/undelivered/ZookeeperUndeliveredMessageLog.java index 9baa14c3ea..9f781cd942 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/undelivered/ZookeeperUndeliveredMessageLog.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/undelivered/ZookeeperUndeliveredMessageLog.java @@ -1,6 +1,10 @@ package pl.allegro.tech.hermes.common.message.undelivered; +import static java.lang.String.format; + import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.api.BackgroundPathAndBytesable; import org.slf4j.Logger; @@ -12,66 +16,66 @@ import pl.allegro.tech.hermes.metrics.HermesCounter; import pl.allegro.tech.hermes.metrics.HermesHistogram; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -import static java.lang.String.format; - public class ZookeeperUndeliveredMessageLog implements UndeliveredMessageLog { - private static final Logger LOGGER = LoggerFactory.getLogger(ZookeeperUndeliveredMessageLog.class); + private static final Logger LOGGER = + LoggerFactory.getLogger(ZookeeperUndeliveredMessageLog.class); - private final CuratorFramework curator; - private final UndeliveredMessagePaths paths; - private final ObjectMapper mapper; - private final HermesCounter persistedMessagesMeter; - private final HermesHistogram persistedMessageSizeHistogram; + private final CuratorFramework curator; + private final UndeliveredMessagePaths paths; + private final ObjectMapper mapper; + private final HermesCounter persistedMessagesMeter; + private final HermesHistogram persistedMessageSizeHistogram; - private final ConcurrentMap lastUndeliveredMessages = new ConcurrentHashMap<>(); + private final ConcurrentMap lastUndeliveredMessages = + new ConcurrentHashMap<>(); - public ZookeeperUndeliveredMessageLog(CuratorFramework curator, - ZookeeperPaths zookeeperPaths, - ObjectMapper mapper, - MetricsFacade metricsFacade) { - this.curator = curator; - this.paths = new UndeliveredMessagePaths(zookeeperPaths); - this.mapper = mapper; - persistedMessagesMeter = metricsFacade.undeliveredMessages().undeliveredMessagesCounter(); - persistedMessageSizeHistogram = metricsFacade.undeliveredMessages().undeliveredMessagesSizeHistogram(); - } + public ZookeeperUndeliveredMessageLog( + CuratorFramework curator, + ZookeeperPaths zookeeperPaths, + ObjectMapper mapper, + MetricsFacade metricsFacade) { + this.curator = curator; + this.paths = new UndeliveredMessagePaths(zookeeperPaths); + this.mapper = mapper; + persistedMessagesMeter = metricsFacade.undeliveredMessages().undeliveredMessagesCounter(); + persistedMessageSizeHistogram = + metricsFacade.undeliveredMessages().undeliveredMessagesSizeHistogram(); + } - @Override - public void add(SentMessageTrace message) { - lastUndeliveredMessages.put(new SubscriptionName(message.getSubscription(), message.getTopicName()), message); - } + @Override + public void add(SentMessageTrace message) { + lastUndeliveredMessages.put( + new SubscriptionName(message.getSubscription(), message.getTopicName()), message); + } - @Override - public void persist() { - for (SubscriptionName key : lastUndeliveredMessages.keySet()) { - log(lastUndeliveredMessages.remove(key)); - } + @Override + public void persist() { + for (SubscriptionName key : lastUndeliveredMessages.keySet()) { + log(lastUndeliveredMessages.remove(key)); } + } - private void log(SentMessageTrace messageTrace) { - try { - String undeliveredPath = paths.buildPath(messageTrace.getTopicName(), messageTrace.getSubscription()); - BackgroundPathAndBytesable builder = exists(undeliveredPath) ? curator.setData() : curator.create(); - byte[] bytesToPersist = mapper.writeValueAsBytes(messageTrace); - builder.forPath(undeliveredPath, bytesToPersist); - persistedMessagesMeter.increment(); - persistedMessageSizeHistogram.record(bytesToPersist.length); - } catch (Exception exception) { - LOGGER.warn( - format("Could not log undelivered message for topic: %s and subscription: %s", - messageTrace.getQualifiedTopicName(), - messageTrace.getSubscription() - ), - exception - ); - } + private void log(SentMessageTrace messageTrace) { + try { + String undeliveredPath = + paths.buildPath(messageTrace.getTopicName(), messageTrace.getSubscription()); + BackgroundPathAndBytesable builder = + exists(undeliveredPath) ? curator.setData() : curator.create(); + byte[] bytesToPersist = mapper.writeValueAsBytes(messageTrace); + builder.forPath(undeliveredPath, bytesToPersist); + persistedMessagesMeter.increment(); + persistedMessageSizeHistogram.record(bytesToPersist.length); + } catch (Exception exception) { + LOGGER.warn( + format( + "Could not log undelivered message for topic: %s and subscription: %s", + messageTrace.getQualifiedTopicName(), messageTrace.getSubscription()), + exception); } + } - private boolean exists(String path) throws Exception { - return curator.checkExists().forPath(path) != null; - } + private boolean exists(String path) throws Exception { + return curator.checkExists().forPath(path) != null; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroInvalidMetadataException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroInvalidMetadataException.java index a6dd9d20fe..3e1f7c4d47 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroInvalidMetadataException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroInvalidMetadataException.java @@ -3,7 +3,7 @@ import pl.allegro.tech.hermes.common.exception.InternalProcessingException; public class AvroInvalidMetadataException extends InternalProcessingException { - public AvroInvalidMetadataException(String message, Exception cause) { - super(message, cause); - } + public AvroInvalidMetadataException(String message, Exception cause) { + super(message, cause); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageContentUnwrapper.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageContentUnwrapper.java index 3f6d6ee30e..88549d34ae 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageContentUnwrapper.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageContentUnwrapper.java @@ -5,7 +5,8 @@ interface AvroMessageContentUnwrapper { - AvroMessageContentUnwrapperResult unwrap(byte[] data, Topic topic, @Nullable Integer headerId, @Nullable Integer headerVersion); + AvroMessageContentUnwrapperResult unwrap( + byte[] data, Topic topic, @Nullable Integer headerId, @Nullable Integer headerVersion); - boolean isApplicable(byte[] data, Topic topic, Integer schemaId, Integer schemaVersion); + boolean isApplicable(byte[] data, Topic topic, Integer schemaId, Integer schemaVersion); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageContentUnwrapperResult.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageContentUnwrapperResult.java index 2849a842b6..8e8eeb7791 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageContentUnwrapperResult.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageContentUnwrapperResult.java @@ -2,32 +2,35 @@ class AvroMessageContentUnwrapperResult { - private final UnwrappedMessageContent content; - private final AvroMessageContentUnwrapperResultStatus status; - - private AvroMessageContentUnwrapperResult(UnwrappedMessageContent content, - AvroMessageContentUnwrapperResultStatus status) { - this.content = content; - this.status = status; - } - - public static AvroMessageContentUnwrapperResult success(UnwrappedMessageContent content) { - return new AvroMessageContentUnwrapperResult(content, AvroMessageContentUnwrapperResultStatus.SUCCESS); - } - - public static AvroMessageContentUnwrapperResult failure() { - return new AvroMessageContentUnwrapperResult(null, AvroMessageContentUnwrapperResultStatus.FAILURE); - } - - public UnwrappedMessageContent getContent() { - return content; - } - - public AvroMessageContentUnwrapperResultStatus getStatus() { - return status; - } - - enum AvroMessageContentUnwrapperResultStatus { - SUCCESS, FAILURE - } + private final UnwrappedMessageContent content; + private final AvroMessageContentUnwrapperResultStatus status; + + private AvroMessageContentUnwrapperResult( + UnwrappedMessageContent content, AvroMessageContentUnwrapperResultStatus status) { + this.content = content; + this.status = status; + } + + public static AvroMessageContentUnwrapperResult success(UnwrappedMessageContent content) { + return new AvroMessageContentUnwrapperResult( + content, AvroMessageContentUnwrapperResultStatus.SUCCESS); + } + + public static AvroMessageContentUnwrapperResult failure() { + return new AvroMessageContentUnwrapperResult( + null, AvroMessageContentUnwrapperResultStatus.FAILURE); + } + + public UnwrappedMessageContent getContent() { + return content; + } + + public AvroMessageContentUnwrapperResultStatus getStatus() { + return status; + } + + enum AvroMessageContentUnwrapperResultStatus { + SUCCESS, + FAILURE + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageContentWrapper.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageContentWrapper.java index 9b34fe141e..e0244a4f2c 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageContentWrapper.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageContentWrapper.java @@ -1,10 +1,10 @@ package pl.allegro.tech.hermes.common.message.wrapper; -import org.apache.avro.AvroRuntimeException; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.util.Utf8; -import pl.allegro.tech.hermes.schema.CompiledSchema; +import static pl.allegro.tech.hermes.common.message.converter.AvroRecordToBytesConverter.bytesToRecord; +import static pl.allegro.tech.hermes.common.message.converter.AvroRecordToBytesConverter.recordToBytes; +import static pl.allegro.tech.hermes.common.message.wrapper.AvroMetadataMarker.METADATA_MARKER; +import static pl.allegro.tech.hermes.common.message.wrapper.AvroMetadataMarker.METADATA_MESSAGE_ID_KEY; +import static pl.allegro.tech.hermes.common.message.wrapper.AvroMetadataMarker.METADATA_TIMESTAMP_KEY; import java.time.Clock; import java.util.Collections; @@ -12,95 +12,102 @@ import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; +import org.apache.avro.AvroRuntimeException; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.util.Utf8; +import pl.allegro.tech.hermes.schema.CompiledSchema; -import static pl.allegro.tech.hermes.common.message.converter.AvroRecordToBytesConverter.bytesToRecord; -import static pl.allegro.tech.hermes.common.message.converter.AvroRecordToBytesConverter.recordToBytes; -import static pl.allegro.tech.hermes.common.message.wrapper.AvroMetadataMarker.METADATA_MARKER; -import static pl.allegro.tech.hermes.common.message.wrapper.AvroMetadataMarker.METADATA_MESSAGE_ID_KEY; -import static pl.allegro.tech.hermes.common.message.wrapper.AvroMetadataMarker.METADATA_TIMESTAMP_KEY; - -/** This class deals only with wrapping and unwrapping messages. - * It does not generate any Hermes ID for message in case it is missing. - * Missing Hermes ID will be left as empty. +/** + * This class deals only with wrapping and unwrapping messages. It does not generate any Hermes ID + * for message in case it is missing. Missing Hermes ID will be left as empty. */ public class AvroMessageContentWrapper { - private final Clock clock; + private final Clock clock; - public AvroMessageContentWrapper(Clock clock) { - this.clock = clock; - } + public AvroMessageContentWrapper(Clock clock) { + this.clock = clock; + } - @SuppressWarnings("unchecked") - UnwrappedMessageContent unwrapContent(byte[] data, CompiledSchema schema) { - try { - Map metadata = (Map) bytesToRecord(data, schema.getSchema()).get(METADATA_MARKER); - MessageMetadata messageMetadata = getMetadata(metadata); + @SuppressWarnings("unchecked") + UnwrappedMessageContent unwrapContent(byte[] data, CompiledSchema schema) { + try { + Map metadata = + (Map) bytesToRecord(data, schema.getSchema()).get(METADATA_MARKER); + MessageMetadata messageMetadata = getMetadata(metadata); - return new UnwrappedMessageContent(messageMetadata, data, schema); - } catch (Exception exception) { - throw new UnwrappingException("Could not read avro message", exception); - } + return new UnwrappedMessageContent(messageMetadata, data, schema); + } catch (Exception exception) { + throw new UnwrappingException("Could not read avro message", exception); } - - @SuppressWarnings("unchecked") - private MessageMetadata getMetadata(Map metadata) { - if (metadata == null) { - long timestamp = clock.millis(); - return new MessageMetadata(timestamp, Collections.EMPTY_MAP); - } else { - long timestamp = metadata.containsKey(METADATA_TIMESTAMP_KEY) ? timestampFromMetadata(metadata) : - clock.millis(); - Map extractedMetadata = extractMetadata(metadata); - - return metadata.containsKey(METADATA_MESSAGE_ID_KEY) - ? new MessageMetadata(timestamp, messageIdFromMetadata(metadata), extractedMetadata) - : new MessageMetadata(timestamp, extractedMetadata); - } + } + + @SuppressWarnings("unchecked") + private MessageMetadata getMetadata(Map metadata) { + if (metadata == null) { + long timestamp = clock.millis(); + return new MessageMetadata(timestamp, Collections.EMPTY_MAP); + } else { + long timestamp = + metadata.containsKey(METADATA_TIMESTAMP_KEY) + ? timestampFromMetadata(metadata) + : clock.millis(); + Map extractedMetadata = extractMetadata(metadata); + + return metadata.containsKey(METADATA_MESSAGE_ID_KEY) + ? new MessageMetadata(timestamp, messageIdFromMetadata(metadata), extractedMetadata) + : new MessageMetadata(timestamp, extractedMetadata); } - - public byte[] wrapContent(byte[] message, String id, long timestamp, Schema schema, Map externalMetadata) { - if (schema.getField(METADATA_MARKER) != null) { - GenericRecord genericRecord = bytesToRecord(message, schema); - try { - genericRecord.put(METADATA_MARKER, metadataMap(id, timestamp, externalMetadata)); - return recordToBytes(genericRecord, schema); - } catch (Exception e) { - if (e instanceof AvroRuntimeException && e.getMessage().equals("Not a valid schema field: __metadata")) { - throw new AvroInvalidMetadataException( - "Schema does not contain mandatory __metadata field for Hermes internal metadata. Please fix topic schema.", e); - } - throw new WrappingException("Could not wrap avro message", e); - } - } else { - return message; + } + + public byte[] wrapContent( + byte[] message, + String id, + long timestamp, + Schema schema, + Map externalMetadata) { + if (schema.getField(METADATA_MARKER) != null) { + GenericRecord genericRecord = bytesToRecord(message, schema); + try { + genericRecord.put(METADATA_MARKER, metadataMap(id, timestamp, externalMetadata)); + return recordToBytes(genericRecord, schema); + } catch (Exception e) { + if (e instanceof AvroRuntimeException + && e.getMessage().equals("Not a valid schema field: __metadata")) { + throw new AvroInvalidMetadataException( + "Schema does not contain mandatory __metadata field for Hermes internal metadata. Please fix topic schema.", + e); } + throw new WrappingException("Could not wrap avro message", e); + } + } else { + return message; } - - private Map metadataMap(String id, long timestamp, Map externalMetadata) { - Map metadata = new HashMap<>(); - metadata.put(METADATA_MESSAGE_ID_KEY, new Utf8(id)); - metadata.put(METADATA_TIMESTAMP_KEY, new Utf8(Long.toString(timestamp))); - - externalMetadata.forEach((key, value) -> metadata.put(new Utf8(key), new Utf8(value))); - return metadata; - } - - private long timestampFromMetadata(Map metadata) { - return Long.parseLong(metadata.remove(METADATA_TIMESTAMP_KEY).toString()); - } - - private String messageIdFromMetadata(Map metadata) { - return metadata.remove(METADATA_MESSAGE_ID_KEY).toString(); - } - - private Map extractMetadata(Map metadata) { - return Optional.ofNullable(metadata).orElse(Collections.emptyMap()) - .entrySet() - .stream() - .collect(Collectors.toMap( - entry -> entry.getKey().toString(), - entry -> entry.getValue().toString() - )); - } + } + + private Map metadataMap( + String id, long timestamp, Map externalMetadata) { + Map metadata = new HashMap<>(); + metadata.put(METADATA_MESSAGE_ID_KEY, new Utf8(id)); + metadata.put(METADATA_TIMESTAMP_KEY, new Utf8(Long.toString(timestamp))); + + externalMetadata.forEach((key, value) -> metadata.put(new Utf8(key), new Utf8(value))); + return metadata; + } + + private long timestampFromMetadata(Map metadata) { + return Long.parseLong(metadata.remove(METADATA_TIMESTAMP_KEY).toString()); + } + + private String messageIdFromMetadata(Map metadata) { + return metadata.remove(METADATA_MESSAGE_ID_KEY).toString(); + } + + private Map extractMetadata(Map metadata) { + return Optional.ofNullable(metadata).orElse(Collections.emptyMap()).entrySet().stream() + .collect( + Collectors.toMap( + entry -> entry.getKey().toString(), entry -> entry.getValue().toString())); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageHeaderSchemaIdContentWrapper.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageHeaderSchemaIdContentWrapper.java index 1a87be5b84..5299a5f3b6 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageHeaderSchemaIdContentWrapper.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageHeaderSchemaIdContentWrapper.java @@ -12,47 +12,55 @@ public class AvroMessageHeaderSchemaIdContentWrapper implements AvroMessageContentUnwrapper { - private static final Logger logger = LoggerFactory.getLogger(AvroMessageHeaderSchemaIdContentWrapper.class); + private static final Logger logger = + LoggerFactory.getLogger(AvroMessageHeaderSchemaIdContentWrapper.class); - private final SchemaRepository schemaRepository; - private final AvroMessageContentWrapper avroMessageContentWrapper; + private final SchemaRepository schemaRepository; + private final AvroMessageContentWrapper avroMessageContentWrapper; - private final HermesCounter deserializationWithErrorsUsingHeaderSchemaId; - private final HermesCounter deserializationUsingHeaderSchemaId; - private final boolean schemaIdHeaderEnabled; + private final HermesCounter deserializationWithErrorsUsingHeaderSchemaId; + private final HermesCounter deserializationUsingHeaderSchemaId; + private final boolean schemaIdHeaderEnabled; - public AvroMessageHeaderSchemaIdContentWrapper(SchemaRepository schemaRepository, - AvroMessageContentWrapper avroMessageContentWrapper, - MetricsFacade metrics, - boolean schemaIdHeaderEnabled) { - this.schemaRepository = schemaRepository; - this.avroMessageContentWrapper = avroMessageContentWrapper; + public AvroMessageHeaderSchemaIdContentWrapper( + SchemaRepository schemaRepository, + AvroMessageContentWrapper avroMessageContentWrapper, + MetricsFacade metrics, + boolean schemaIdHeaderEnabled) { + this.schemaRepository = schemaRepository; + this.avroMessageContentWrapper = avroMessageContentWrapper; - this.deserializationWithErrorsUsingHeaderSchemaId = metrics.deserialization().errorsForHeaderSchemaId(); - this.deserializationUsingHeaderSchemaId = metrics.deserialization().usingHeaderSchemaId(); - this.schemaIdHeaderEnabled = schemaIdHeaderEnabled; - } + this.deserializationWithErrorsUsingHeaderSchemaId = + metrics.deserialization().errorsForHeaderSchemaId(); + this.deserializationUsingHeaderSchemaId = metrics.deserialization().usingHeaderSchemaId(); + this.schemaIdHeaderEnabled = schemaIdHeaderEnabled; + } - @Override - public AvroMessageContentUnwrapperResult unwrap(byte[] data, Topic topic, Integer schemaId, Integer schemaVersion) { - try { - deserializationUsingHeaderSchemaId.increment(); - CompiledSchema avroSchema = schemaRepository.getAvroSchema(topic, SchemaId.valueOf(schemaId)); + @Override + public AvroMessageContentUnwrapperResult unwrap( + byte[] data, Topic topic, Integer schemaId, Integer schemaVersion) { + try { + deserializationUsingHeaderSchemaId.increment(); + CompiledSchema avroSchema = + schemaRepository.getAvroSchema(topic, SchemaId.valueOf(schemaId)); - return AvroMessageContentUnwrapperResult.success(avroMessageContentWrapper.unwrapContent(data, avroSchema)); - } catch (Exception ex) { - logger.warn( - "Could not unwrap content for topic [{}] using schema id provided in header [{}] - falling back", - topic.getQualifiedName(), schemaVersion, ex); + return AvroMessageContentUnwrapperResult.success( + avroMessageContentWrapper.unwrapContent(data, avroSchema)); + } catch (Exception ex) { + logger.warn( + "Could not unwrap content for topic [{}] using schema id provided in header [{}] - falling back", + topic.getQualifiedName(), + schemaVersion, + ex); - deserializationWithErrorsUsingHeaderSchemaId.increment(); + deserializationWithErrorsUsingHeaderSchemaId.increment(); - return AvroMessageContentUnwrapperResult.failure(); - } + return AvroMessageContentUnwrapperResult.failure(); } + } - @Override - public boolean isApplicable(byte[] data, Topic topic, Integer schemaId, Integer schemaVersion) { - return schemaIdHeaderEnabled && schemaId != null; - } + @Override + public boolean isApplicable(byte[] data, Topic topic, Integer schemaId, Integer schemaVersion) { + return schemaIdHeaderEnabled && schemaId != null; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageHeaderSchemaVersionContentWrapper.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageHeaderSchemaVersionContentWrapper.java index 6d1d5b803a..4aa622a7b6 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageHeaderSchemaVersionContentWrapper.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageHeaderSchemaVersionContentWrapper.java @@ -12,44 +12,53 @@ public class AvroMessageHeaderSchemaVersionContentWrapper implements AvroMessageContentUnwrapper { - private static final Logger logger = LoggerFactory.getLogger(AvroMessageHeaderSchemaVersionContentWrapper.class); + private static final Logger logger = + LoggerFactory.getLogger(AvroMessageHeaderSchemaVersionContentWrapper.class); - private final SchemaRepository schemaRepository; - private final AvroMessageContentWrapper avroMessageContentWrapper; + private final SchemaRepository schemaRepository; + private final AvroMessageContentWrapper avroMessageContentWrapper; - private final HermesCounter deserializationWithErrorsUsingHeaderSchemaVersion; - private final HermesCounter deserializationUsingHeaderSchemaVersion; + private final HermesCounter deserializationWithErrorsUsingHeaderSchemaVersion; + private final HermesCounter deserializationUsingHeaderSchemaVersion; - public AvroMessageHeaderSchemaVersionContentWrapper(SchemaRepository schemaRepository, - AvroMessageContentWrapper avroMessageContentWrapper, - MetricsFacade metrics) { - this.schemaRepository = schemaRepository; - this.avroMessageContentWrapper = avroMessageContentWrapper; + public AvroMessageHeaderSchemaVersionContentWrapper( + SchemaRepository schemaRepository, + AvroMessageContentWrapper avroMessageContentWrapper, + MetricsFacade metrics) { + this.schemaRepository = schemaRepository; + this.avroMessageContentWrapper = avroMessageContentWrapper; - this.deserializationWithErrorsUsingHeaderSchemaVersion = metrics.deserialization().errorsForHeaderSchemaVersion(); - this.deserializationUsingHeaderSchemaVersion = metrics.deserialization().usingHeaderSchemaVersion(); - } + this.deserializationWithErrorsUsingHeaderSchemaVersion = + metrics.deserialization().errorsForHeaderSchemaVersion(); + this.deserializationUsingHeaderSchemaVersion = + metrics.deserialization().usingHeaderSchemaVersion(); + } - @Override - public AvroMessageContentUnwrapperResult unwrap(byte[] data, Topic topic, Integer schemaId, Integer schemaVersion) { - try { - deserializationUsingHeaderSchemaVersion.increment(); - CompiledSchema avroSchema = schemaRepository.getAvroSchema(topic, SchemaVersion.valueOf(schemaVersion)); + @Override + public AvroMessageContentUnwrapperResult unwrap( + byte[] data, Topic topic, Integer schemaId, Integer schemaVersion) { + try { + deserializationUsingHeaderSchemaVersion.increment(); + CompiledSchema avroSchema = + schemaRepository.getAvroSchema(topic, SchemaVersion.valueOf(schemaVersion)); - return AvroMessageContentUnwrapperResult.success(avroMessageContentWrapper.unwrapContent(data, avroSchema)); - } catch (Exception ex) { - logger.warn( - "Could not unwrap content for topic [{}] using schema version provided in header [{}] - falling back", - topic.getQualifiedName(), schemaVersion, ex); + return AvroMessageContentUnwrapperResult.success( + avroMessageContentWrapper.unwrapContent(data, avroSchema)); + } catch (Exception ex) { + logger.warn( + "Could not unwrap content for topic [{}] using schema version provided in header [{}] - falling back", + topic.getQualifiedName(), + schemaVersion, + ex); - deserializationWithErrorsUsingHeaderSchemaVersion.increment(); + deserializationWithErrorsUsingHeaderSchemaVersion.increment(); - return AvroMessageContentUnwrapperResult.failure(); - } + return AvroMessageContentUnwrapperResult.failure(); } + } - @Override - public boolean isApplicable(byte[] data, Topic topic, Integer schemaId, Integer schemaVersion) { - return schemaVersion != null; - } + @Override + public boolean isApplicable(byte[] data, Topic topic, Integer schemaId, Integer schemaVersion) { + return schemaVersion != null; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageSchemaIdAwareContentWrapper.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageSchemaIdAwareContentWrapper.java index 79e28d67a0..45719da8e6 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageSchemaIdAwareContentWrapper.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageSchemaIdAwareContentWrapper.java @@ -11,58 +11,68 @@ public class AvroMessageSchemaIdAwareContentWrapper implements AvroMessageContentUnwrapper { - private static final Logger logger = LoggerFactory.getLogger(AvroMessageSchemaIdAwareContentWrapper.class); - - private final SchemaRepository schemaRepository; - private final AvroMessageContentWrapper avroMessageContentWrapper; - - private final HermesCounter deserializationUsingSchemaIdAware; - private final HermesCounter deserializationErrorsForSchemaIdAwarePayload; - private final HermesCounter deserializationWithMissingSchemaIdInPayload; - - public AvroMessageSchemaIdAwareContentWrapper(SchemaRepository schemaRepository, - AvroMessageContentWrapper avroMessageContentWrapper, - MetricsFacade metrics) { - this.schemaRepository = schemaRepository; - this.avroMessageContentWrapper = avroMessageContentWrapper; - - this.deserializationErrorsForSchemaIdAwarePayload = metrics.deserialization().errorsForSchemaIdAwarePayload(); - this.deserializationWithMissingSchemaIdInPayload = metrics.deserialization().missingSchemaIdInPayload(); - this.deserializationUsingSchemaIdAware = metrics.deserialization().usingSchemaIdAware(); + private static final Logger logger = + LoggerFactory.getLogger(AvroMessageSchemaIdAwareContentWrapper.class); + + private final SchemaRepository schemaRepository; + private final AvroMessageContentWrapper avroMessageContentWrapper; + + private final HermesCounter deserializationUsingSchemaIdAware; + private final HermesCounter deserializationErrorsForSchemaIdAwarePayload; + private final HermesCounter deserializationWithMissingSchemaIdInPayload; + + public AvroMessageSchemaIdAwareContentWrapper( + SchemaRepository schemaRepository, + AvroMessageContentWrapper avroMessageContentWrapper, + MetricsFacade metrics) { + this.schemaRepository = schemaRepository; + this.avroMessageContentWrapper = avroMessageContentWrapper; + + this.deserializationErrorsForSchemaIdAwarePayload = + metrics.deserialization().errorsForSchemaIdAwarePayload(); + this.deserializationWithMissingSchemaIdInPayload = + metrics.deserialization().missingSchemaIdInPayload(); + this.deserializationUsingSchemaIdAware = metrics.deserialization().usingSchemaIdAware(); + } + + @Override + public AvroMessageContentUnwrapperResult unwrap( + byte[] data, Topic topic, Integer schemaIdFromHeader, Integer schemaVersionFromHeader) { + try { + deserializationUsingSchemaIdAware.increment(); + + SchemaAwarePayload payload = SchemaAwareSerDe.deserialize(data); + CompiledSchema avroSchema = + schemaRepository.getAvroSchema(topic, payload.getSchemaId()); + + return AvroMessageContentUnwrapperResult.success( + avroMessageContentWrapper.unwrapContent(payload.getPayload(), avroSchema)); + } catch (Exception ex) { + logger.warn( + "Could not deserialize schema id aware payload for topic [{}] - falling back", + topic.getQualifiedName(), + ex); + + deserializationErrorsForSchemaIdAwarePayload.increment(); + + return AvroMessageContentUnwrapperResult.failure(); } + } - @Override - public AvroMessageContentUnwrapperResult unwrap(byte[] data, Topic topic, Integer schemaIdFromHeader, Integer schemaVersionFromHeader) { - try { - deserializationUsingSchemaIdAware.increment(); - - SchemaAwarePayload payload = SchemaAwareSerDe.deserialize(data); - CompiledSchema avroSchema = schemaRepository.getAvroSchema(topic, payload.getSchemaId()); - - return AvroMessageContentUnwrapperResult.success(avroMessageContentWrapper.unwrapContent(payload.getPayload(), avroSchema)); - } catch (Exception ex) { - logger.warn("Could not deserialize schema id aware payload for topic [{}] - falling back", topic.getQualifiedName(), ex); + @Override + public boolean isApplicable(byte[] data, Topic topic, Integer schemaId, Integer schemaVersion) { + return isPayloadAwareOfSchemaId(data, topic); + } - deserializationErrorsForSchemaIdAwarePayload.increment(); + private boolean isPayloadAwareOfSchemaId(byte[] data, Topic topic) { + if (topic.isSchemaIdAwareSerializationEnabled()) { + if (SchemaAwareSerDe.startsWithMagicByte(data)) { + return true; + } - return AvroMessageContentUnwrapperResult.failure(); - } + deserializationWithMissingSchemaIdInPayload.increment(); } - @Override - public boolean isApplicable(byte[] data, Topic topic, Integer schemaId, Integer schemaVersion) { - return isPayloadAwareOfSchemaId(data, topic); - } - - private boolean isPayloadAwareOfSchemaId(byte[] data, Topic topic) { - if (topic.isSchemaIdAwareSerializationEnabled()) { - if (SchemaAwareSerDe.startsWithMagicByte(data)) { - return true; - } - - deserializationWithMissingSchemaIdInPayload.increment(); - } - - return false; - } + return false; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageSchemaVersionTruncationContentWrapper.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageSchemaVersionTruncationContentWrapper.java index c4f39e6aa6..b84dcafdb7 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageSchemaVersionTruncationContentWrapper.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageSchemaVersionTruncationContentWrapper.java @@ -10,57 +10,66 @@ import pl.allegro.tech.hermes.schema.SchemaRepository; import pl.allegro.tech.hermes.schema.SchemaVersion; -public class AvroMessageSchemaVersionTruncationContentWrapper implements AvroMessageContentUnwrapper { +public class AvroMessageSchemaVersionTruncationContentWrapper + implements AvroMessageContentUnwrapper { - private static final Logger logger = LoggerFactory.getLogger(AvroMessageSchemaVersionTruncationContentWrapper.class); + private static final Logger logger = + LoggerFactory.getLogger(AvroMessageSchemaVersionTruncationContentWrapper.class); - private final SchemaRepository schemaRepository; - private final AvroMessageContentWrapper avroMessageContentWrapper; - private final boolean magicByteTruncationEnabled; + private final SchemaRepository schemaRepository; + private final AvroMessageContentWrapper avroMessageContentWrapper; + private final boolean magicByteTruncationEnabled; - private final HermesCounter deserializationWithSchemaVersionTruncation; - private final HermesCounter deserializationErrorsWithSchemaVersionTruncation; + private final HermesCounter deserializationWithSchemaVersionTruncation; + private final HermesCounter deserializationErrorsWithSchemaVersionTruncation; - public AvroMessageSchemaVersionTruncationContentWrapper(SchemaRepository schemaRepository, - AvroMessageContentWrapper avroMessageContentWrapper, - MetricsFacade metrics, - boolean schemaVersionTruncationEnabled) { - this.schemaRepository = schemaRepository; - this.avroMessageContentWrapper = avroMessageContentWrapper; - this.magicByteTruncationEnabled = schemaVersionTruncationEnabled; - - this.deserializationErrorsWithSchemaVersionTruncation = metrics.deserialization().errorsForSchemaVersionTruncation(); - this.deserializationWithSchemaVersionTruncation = metrics.deserialization().usingSchemaVersionTruncation(); - } + public AvroMessageSchemaVersionTruncationContentWrapper( + SchemaRepository schemaRepository, + AvroMessageContentWrapper avroMessageContentWrapper, + MetricsFacade metrics, + boolean schemaVersionTruncationEnabled) { + this.schemaRepository = schemaRepository; + this.avroMessageContentWrapper = avroMessageContentWrapper; + this.magicByteTruncationEnabled = schemaVersionTruncationEnabled; + this.deserializationErrorsWithSchemaVersionTruncation = + metrics.deserialization().errorsForSchemaVersionTruncation(); + this.deserializationWithSchemaVersionTruncation = + metrics.deserialization().usingSchemaVersionTruncation(); + } - @Override - public AvroMessageContentUnwrapperResult unwrap(byte[] data, Topic topic, Integer schemaId, Integer schemaVersion) { - try { - deserializationWithSchemaVersionTruncation.increment(); + @Override + public AvroMessageContentUnwrapperResult unwrap( + byte[] data, Topic topic, Integer schemaId, Integer schemaVersion) { + try { + deserializationWithSchemaVersionTruncation.increment(); - byte[] dataWithoutMagicByteAndSchemaId = SchemaAwareSerDe.trimMagicByteAndSchemaVersion(data); - CompiledSchema avroSchema = schemaRepository.getAvroSchema(topic, SchemaVersion.valueOf(schemaVersion)); + byte[] dataWithoutMagicByteAndSchemaId = SchemaAwareSerDe.trimMagicByteAndSchemaVersion(data); + CompiledSchema avroSchema = + schemaRepository.getAvroSchema(topic, SchemaVersion.valueOf(schemaVersion)); - return AvroMessageContentUnwrapperResult.success( - avroMessageContentWrapper.unwrapContent(dataWithoutMagicByteAndSchemaId, avroSchema)); - } catch (Exception e) { - logger.warn( - "Could not unwrap content for topic [{}] using schema id provided in header [{}] - falling back", - topic.getQualifiedName(), schemaVersion, e); + return AvroMessageContentUnwrapperResult.success( + avroMessageContentWrapper.unwrapContent(dataWithoutMagicByteAndSchemaId, avroSchema)); + } catch (Exception e) { + logger.warn( + "Could not unwrap content for topic [{}] using schema id provided in header [{}] - falling back", + topic.getQualifiedName(), + schemaVersion, + e); - deserializationErrorsWithSchemaVersionTruncation.increment(); + deserializationErrorsWithSchemaVersionTruncation.increment(); - return AvroMessageContentUnwrapperResult.failure(); - } + return AvroMessageContentUnwrapperResult.failure(); } + } - @Override - public boolean isApplicable(byte[] data, Topic topic, Integer schemaId, Integer schemaVersion) { - return magicByteTruncationEnabled && containsSchemaVersionInMagicByteAndInHeader(data, schemaVersion); - } + @Override + public boolean isApplicable(byte[] data, Topic topic, Integer schemaId, Integer schemaVersion) { + return magicByteTruncationEnabled + && containsSchemaVersionInMagicByteAndInHeader(data, schemaVersion); + } - private boolean containsSchemaVersionInMagicByteAndInHeader(byte[] data, Integer schemaVersion) { - return SchemaAwareSerDe.startsWithMagicByte(data) && schemaVersion != null; - } + private boolean containsSchemaVersionInMagicByteAndInHeader(byte[] data, Integer schemaVersion) { + return SchemaAwareSerDe.startsWithMagicByte(data) && schemaVersion != null; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMetadataMarker.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMetadataMarker.java index c860040606..c67291753b 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMetadataMarker.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMetadataMarker.java @@ -3,7 +3,7 @@ import org.apache.avro.util.Utf8; public interface AvroMetadataMarker { - String METADATA_MARKER = "__metadata"; - Utf8 METADATA_TIMESTAMP_KEY = new Utf8("timestamp"); - Utf8 METADATA_MESSAGE_ID_KEY = new Utf8("messageId"); + String METADATA_MARKER = "__metadata"; + Utf8 METADATA_TIMESTAMP_KEY = new Utf8("timestamp"); + Utf8 METADATA_MESSAGE_ID_KEY = new Utf8("messageId"); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/CompositeMessageContentWrapper.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/CompositeMessageContentWrapper.java index 53de748f1b..69d424df4a 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/CompositeMessageContentWrapper.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/CompositeMessageContentWrapper.java @@ -1,72 +1,83 @@ package pl.allegro.tech.hermes.common.message.wrapper; +import static java.util.Arrays.asList; +import static pl.allegro.tech.hermes.common.message.wrapper.AvroMessageContentUnwrapperResult.AvroMessageContentUnwrapperResultStatus.SUCCESS; + +import java.util.Collection; +import java.util.Map; import org.apache.avro.Schema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.schema.CompiledSchema; -import java.util.Collection; -import java.util.Map; - -import static java.util.Arrays.asList; -import static pl.allegro.tech.hermes.common.message.wrapper.AvroMessageContentUnwrapperResult.AvroMessageContentUnwrapperResultStatus.SUCCESS; - public class CompositeMessageContentWrapper implements MessageContentWrapper { - private static final Logger logger = LoggerFactory.getLogger(CompositeMessageContentWrapper.class); + private static final Logger logger = + LoggerFactory.getLogger(CompositeMessageContentWrapper.class); - private final JsonMessageContentWrapper jsonMessageContentWrapper; - private final AvroMessageContentWrapper avroMessageContentWrapper; - private final Collection avroMessageContentUnwrappers; + private final JsonMessageContentWrapper jsonMessageContentWrapper; + private final AvroMessageContentWrapper avroMessageContentWrapper; + private final Collection avroMessageContentUnwrappers; - public CompositeMessageContentWrapper(JsonMessageContentWrapper jsonMessageContentWrapper, - AvroMessageContentWrapper avroMessageContentWrapper, - AvroMessageSchemaIdAwareContentWrapper schemaIdAwareContentWrapper, - AvroMessageHeaderSchemaVersionContentWrapper headerSchemaVersionContentWrapper, - AvroMessageHeaderSchemaIdContentWrapper headerSchemaIdContentWrapper, - AvroMessageSchemaVersionTruncationContentWrapper schemaVersionTruncationContentWrapper) { + public CompositeMessageContentWrapper( + JsonMessageContentWrapper jsonMessageContentWrapper, + AvroMessageContentWrapper avroMessageContentWrapper, + AvroMessageSchemaIdAwareContentWrapper schemaIdAwareContentWrapper, + AvroMessageHeaderSchemaVersionContentWrapper headerSchemaVersionContentWrapper, + AvroMessageHeaderSchemaIdContentWrapper headerSchemaIdContentWrapper, + AvroMessageSchemaVersionTruncationContentWrapper schemaVersionTruncationContentWrapper) { - this.jsonMessageContentWrapper = jsonMessageContentWrapper; - this.avroMessageContentWrapper = avroMessageContentWrapper; - this.avroMessageContentUnwrappers = asList( - schemaIdAwareContentWrapper, - schemaVersionTruncationContentWrapper, - headerSchemaVersionContentWrapper, - headerSchemaIdContentWrapper); - } + this.jsonMessageContentWrapper = jsonMessageContentWrapper; + this.avroMessageContentWrapper = avroMessageContentWrapper; + this.avroMessageContentUnwrappers = + asList( + schemaIdAwareContentWrapper, + schemaVersionTruncationContentWrapper, + headerSchemaVersionContentWrapper, + headerSchemaIdContentWrapper); + } - public UnwrappedMessageContent unwrapJson(byte[] data) { - return jsonMessageContentWrapper.unwrapContent(data); - } + public UnwrappedMessageContent unwrapJson(byte[] data) { + return jsonMessageContentWrapper.unwrapContent(data); + } - public UnwrappedMessageContent unwrapAvro(byte[] data, Topic topic, Integer schemaId, Integer schemaVersion) { - for (AvroMessageContentUnwrapper unwrapper : avroMessageContentUnwrappers) { - if (unwrapper.isApplicable(data, topic, schemaId, schemaVersion)) { - AvroMessageContentUnwrapperResult result = unwrapper.unwrap(data, topic, schemaId, schemaVersion); - if (result.getStatus() == SUCCESS) { - return result.getContent(); - } - } + public UnwrappedMessageContent unwrapAvro( + byte[] data, Topic topic, Integer schemaId, Integer schemaVersion) { + for (AvroMessageContentUnwrapper unwrapper : avroMessageContentUnwrappers) { + if (unwrapper.isApplicable(data, topic, schemaId, schemaVersion)) { + AvroMessageContentUnwrapperResult result = + unwrapper.unwrap(data, topic, schemaId, schemaVersion); + if (result.getStatus() == SUCCESS) { + return result.getContent(); } - - logger.error("All attempts to unwrap Avro message for topic {} with schema version {} failed", - topic.getQualifiedName(), - schemaVersion); - throw new SchemaMissingException(topic); + } } - public byte[] wrapAvro(byte[] data, - String id, - long timestamp, - Topic topic, - CompiledSchema schema, - Map externalMetadata) { - byte[] wrapped = avroMessageContentWrapper.wrapContent(data, id, timestamp, schema.getSchema(), externalMetadata); - return topic.isSchemaIdAwareSerializationEnabled() ? SchemaAwareSerDe.serialize(schema.getId(), wrapped) : wrapped; - } + logger.error( + "All attempts to unwrap Avro message for topic {} with schema version {} failed", + topic.getQualifiedName(), + schemaVersion); + throw new SchemaMissingException(topic); + } - public byte[] wrapJson(byte[] data, String id, long timestamp, Map externalMetadata) { - return jsonMessageContentWrapper.wrapContent(data, id, timestamp, externalMetadata); - } + public byte[] wrapAvro( + byte[] data, + String id, + long timestamp, + Topic topic, + CompiledSchema schema, + Map externalMetadata) { + byte[] wrapped = + avroMessageContentWrapper.wrapContent( + data, id, timestamp, schema.getSchema(), externalMetadata); + return topic.isSchemaIdAwareSerializationEnabled() + ? SchemaAwareSerDe.serialize(schema.getId(), wrapped) + : wrapped; + } + + public byte[] wrapJson( + byte[] data, String id, long timestamp, Map externalMetadata) { + return jsonMessageContentWrapper.wrapContent(data, id, timestamp, externalMetadata); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/DeserializationException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/DeserializationException.java index 9cc62105f8..c6a167f83e 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/DeserializationException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/DeserializationException.java @@ -2,7 +2,7 @@ public class DeserializationException extends RuntimeException { - DeserializationException(String message) { - super(message); - } + DeserializationException(String message) { + super(message); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/JsonMessageContentWrapper.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/JsonMessageContentWrapper.java index 3014f3073d..f46cb84ade 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/JsonMessageContentWrapper.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/JsonMessageContentWrapper.java @@ -1,98 +1,103 @@ package pl.allegro.tech.hermes.common.message.wrapper; +import static com.google.common.base.Charsets.UTF_8; +import static com.google.common.primitives.Bytes.indexOf; +import static java.lang.String.format; +import static java.util.Arrays.copyOfRange; + import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.Map; import java.util.UUID; - -import static com.google.common.base.Charsets.UTF_8; -import static com.google.common.primitives.Bytes.indexOf; -import static java.lang.String.format; -import static java.util.Arrays.copyOfRange; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class JsonMessageContentWrapper { - private static final Logger LOGGER = LoggerFactory.getLogger(JsonMessageContentWrapper.class); - - private static final byte[] SEPARATOR = ",".getBytes(UTF_8); - private static final byte[] WRAPPED_MARKER = "\"_w\":true".getBytes(UTF_8); - private static final byte JSON_OPEN = (byte) '{'; - private static final byte JSON_CLOSE = (byte) '}'; - private static final int BRACKET_LENGTH = 1; - private final ObjectMapper mapper; - private final byte[] contentRootField; - private final byte[] metadataRootField; - - public JsonMessageContentWrapper(String contentRootName, String metadataRootName, ObjectMapper mapper) { - this.contentRootField = formatNodeKey(contentRootName); - this.metadataRootField = formatNodeKey(metadataRootName); - this.mapper = mapper; + private static final Logger LOGGER = LoggerFactory.getLogger(JsonMessageContentWrapper.class); + + private static final byte[] SEPARATOR = ",".getBytes(UTF_8); + private static final byte[] WRAPPED_MARKER = "\"_w\":true".getBytes(UTF_8); + private static final byte JSON_OPEN = (byte) '{'; + private static final byte JSON_CLOSE = (byte) '}'; + private static final int BRACKET_LENGTH = 1; + private final ObjectMapper mapper; + private final byte[] contentRootField; + private final byte[] metadataRootField; + + public JsonMessageContentWrapper( + String contentRootName, String metadataRootName, ObjectMapper mapper) { + this.contentRootField = formatNodeKey(contentRootName); + this.metadataRootField = formatNodeKey(metadataRootName); + this.mapper = mapper; + } + + byte[] wrapContent(byte[] json, String id, long timestamp, Map externalMetadata) { + try { + return wrapContent( + mapper.writeValueAsBytes(new MessageMetadata(timestamp, id, externalMetadata)), json); + } catch (IOException e) { + throw new WrappingException("Could not wrap json message", e); } - - byte[] wrapContent(byte[] json, String id, long timestamp, Map externalMetadata) { - try { - return wrapContent(mapper.writeValueAsBytes(new MessageMetadata(timestamp, id, externalMetadata)), json); - } catch (IOException e) { - throw new WrappingException("Could not wrap json message", e); - } + } + + private byte[] wrapContent(byte[] attributes, byte[] message) throws IOException { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + stream.write(JSON_OPEN); + stream.write(WRAPPED_MARKER); + stream.write(SEPARATOR); + stream.write(metadataRootField); + stream.write(attributes); + stream.write(SEPARATOR); + stream.write(contentRootField); + stream.write(message); + stream.write(JSON_CLOSE); + return stream.toByteArray(); + } + + public UnwrappedMessageContent unwrapContent(byte[] json) { + if (isWrapped(json)) { + return unwrapMessageContent(json); + } else { + UUID id = UUID.randomUUID(); + LOGGER.warn("Unwrapped message read by consumer (size={}, id={}).", json.length, id); + return new UnwrappedMessageContent( + new MessageMetadata(1L, id.toString(), ImmutableMap.of()), json); } - - private byte[] wrapContent(byte[] attributes, byte[] message) throws IOException { - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - stream.write(JSON_OPEN); - stream.write(WRAPPED_MARKER); - stream.write(SEPARATOR); - stream.write(metadataRootField); - stream.write(attributes); - stream.write(SEPARATOR); - stream.write(contentRootField); - stream.write(message); - stream.write(JSON_CLOSE); - return stream.toByteArray(); + } + + private byte[] unwrapContent(byte[] json, int rootIndex) { + return copyOfRange(json, rootIndex + contentRootField.length, json.length - BRACKET_LENGTH); + } + + private UnwrappedMessageContent unwrapMessageContent(byte[] json) { + int rootIndex = indexOf(json, contentRootField); + int metadataIndex = indexOf(json, metadataRootField); + try { + return new UnwrappedMessageContent( + unwrapMesssageMetadata(json, metadataIndex, rootIndex), unwrapContent(json, rootIndex)); + } catch (Exception exception) { + throw new UnwrappingException("Could not unwrap json message", exception); } - - public UnwrappedMessageContent unwrapContent(byte[] json) { - if (isWrapped(json)) { - return unwrapMessageContent(json); - } else { - UUID id = UUID.randomUUID(); - LOGGER.warn("Unwrapped message read by consumer (size={}, id={}).", json.length, id); - return new UnwrappedMessageContent(new MessageMetadata(1L, id.toString(), ImmutableMap.of()), json); - } - } - - private byte[] unwrapContent(byte[] json, int rootIndex) { - return copyOfRange(json, rootIndex + contentRootField.length, json.length - BRACKET_LENGTH); - } - - private UnwrappedMessageContent unwrapMessageContent(byte[] json) { - int rootIndex = indexOf(json, contentRootField); - int metadataIndex = indexOf(json, metadataRootField); - try { - return new UnwrappedMessageContent(unwrapMesssageMetadata(json, metadataIndex, rootIndex), unwrapContent(json, rootIndex)); - } catch (Exception exception) { - throw new UnwrappingException("Could not unwrap json message", exception); - } - } - - private MessageMetadata unwrapMesssageMetadata(byte[] json, int metadataIndexStart, int metadataIndexEnd) throws IOException { - return mapper.readValue(unwrapMetadataBytes(json, metadataIndexStart, metadataIndexEnd), MessageMetadata.class); - } - - private byte[] unwrapMetadataBytes(byte[] json, int metadataIndexStart, int metadataIndexEnd) { - return copyOfRange(json, metadataIndexStart + metadataRootField.length, metadataIndexEnd + BRACKET_LENGTH); - } - - private byte[] formatNodeKey(String keyName) { - return format("\"%s\":", keyName).getBytes(UTF_8); - } - - private boolean isWrapped(byte[] json) { - return indexOf(json, WRAPPED_MARKER) > 0; - } - + } + + private MessageMetadata unwrapMesssageMetadata( + byte[] json, int metadataIndexStart, int metadataIndexEnd) throws IOException { + return mapper.readValue( + unwrapMetadataBytes(json, metadataIndexStart, metadataIndexEnd), MessageMetadata.class); + } + + private byte[] unwrapMetadataBytes(byte[] json, int metadataIndexStart, int metadataIndexEnd) { + return copyOfRange( + json, metadataIndexStart + metadataRootField.length, metadataIndexEnd + BRACKET_LENGTH); + } + + private byte[] formatNodeKey(String keyName) { + return format("\"%s\":", keyName).getBytes(UTF_8); + } + + private boolean isWrapped(byte[] json) { + return indexOf(json, WRAPPED_MARKER) > 0; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/MessageContentWrapper.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/MessageContentWrapper.java index ac05f68065..2846279370 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/MessageContentWrapper.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/MessageContentWrapper.java @@ -1,23 +1,24 @@ package pl.allegro.tech.hermes.common.message.wrapper; +import java.util.Map; import org.apache.avro.Schema; import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.schema.CompiledSchema; -import java.util.Map; - public interface MessageContentWrapper { - UnwrappedMessageContent unwrapAvro(byte[] data, Topic topic, Integer schemaId, Integer schemaVersion); + UnwrappedMessageContent unwrapAvro( + byte[] data, Topic topic, Integer schemaId, Integer schemaVersion); - UnwrappedMessageContent unwrapJson(byte[] data); + UnwrappedMessageContent unwrapJson(byte[] data); - byte[] wrapAvro(byte[] data, - String id, - long timestamp, - Topic topic, - CompiledSchema schema, - Map externalMetadata); + byte[] wrapAvro( + byte[] data, + String id, + long timestamp, + Topic topic, + CompiledSchema schema, + Map externalMetadata); - byte[] wrapJson(byte[] data, String id, long timestamp, Map externalMetadata); + byte[] wrapJson(byte[] data, String id, long timestamp, Map externalMetadata); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/MessageMetadata.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/MessageMetadata.java index 493e9d09da..3b98829544 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/MessageMetadata.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/MessageMetadata.java @@ -1,62 +1,62 @@ package pl.allegro.tech.hermes.common.message.wrapper; +import static java.util.Optional.ofNullable; + import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableMap; - import java.util.Map; import java.util.Objects; -import static java.util.Optional.ofNullable; - @JsonIgnoreProperties(ignoreUnknown = true) public class MessageMetadata { - private final long timestamp; - private final String id; - private final Map externalMetadata; - - @JsonCreator - public MessageMetadata(@JsonProperty("timestamp") long timestamp, - @JsonProperty("id") String id, - @JsonProperty("externalMetadata") Map externalMetadata) { - this.id = id; - this.timestamp = timestamp; - this.externalMetadata = ofNullable(externalMetadata).orElseGet(ImmutableMap::of); - } - - public MessageMetadata(long timestamp, Map externalMetadata) { - this(timestamp, "", externalMetadata); - } - - public String getId() { - return id; - } - - public long getTimestamp() { - return timestamp; - } - - public Map getExternalMetadata() { - return ImmutableMap.copyOf(externalMetadata); - } - - @Override - public int hashCode() { - return Objects.hash(id, timestamp); + private final long timestamp; + private final String id; + private final Map externalMetadata; + + @JsonCreator + public MessageMetadata( + @JsonProperty("timestamp") long timestamp, + @JsonProperty("id") String id, + @JsonProperty("externalMetadata") Map externalMetadata) { + this.id = id; + this.timestamp = timestamp; + this.externalMetadata = + ofNullable(externalMetadata).orElseGet(ImmutableMap::of); + } + + public MessageMetadata(long timestamp, Map externalMetadata) { + this(timestamp, "", externalMetadata); + } + + public String getId() { + return id; + } + + public long getTimestamp() { + return timestamp; + } + + public Map getExternalMetadata() { + return ImmutableMap.copyOf(externalMetadata); + } + + @Override + public int hashCode() { + return Objects.hash(id, timestamp); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - final MessageMetadata other = (MessageMetadata) obj; - return Objects.equals(this.id, other.id) - && Objects.equals(this.timestamp, other.timestamp); + if (obj == null || getClass() != obj.getClass()) { + return false; } + final MessageMetadata other = (MessageMetadata) obj; + return Objects.equals(this.id, other.id) && Objects.equals(this.timestamp, other.timestamp); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaAwarePayload.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaAwarePayload.java index 0c8eab2352..f062b066b7 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaAwarePayload.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaAwarePayload.java @@ -3,19 +3,19 @@ import pl.allegro.tech.hermes.schema.SchemaId; public class SchemaAwarePayload { - private final byte[] payload; - private final SchemaId schemaId; + private final byte[] payload; + private final SchemaId schemaId; - public SchemaAwarePayload(byte[] payload, SchemaId schemaId) { - this.payload = payload; - this.schemaId = schemaId; - } + public SchemaAwarePayload(byte[] payload, SchemaId schemaId) { + this.payload = payload; + this.schemaId = schemaId; + } - public byte[] getPayload() { - return payload; - } + public byte[] getPayload() { + return payload; + } - public SchemaId getSchemaId() { - return schemaId; - } + public SchemaId getSchemaId() { + return schemaId; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaAwareSerDe.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaAwareSerDe.java index ad7830ca9a..9c4e307a1e 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaAwareSerDe.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaAwareSerDe.java @@ -1,50 +1,49 @@ package pl.allegro.tech.hermes.common.message.wrapper; -import pl.allegro.tech.hermes.schema.SchemaId; +import static java.lang.String.format; import java.nio.ByteBuffer; - -import static java.lang.String.format; +import pl.allegro.tech.hermes.schema.SchemaId; public class SchemaAwareSerDe { - private static final byte MAGIC_BYTE_VALUE = 0; - private static final byte MAGIC_BYTE_INDEX = 0; - private static final int HEADER_SIZE = 5; - - private SchemaAwareSerDe() { - } - - public static byte[] serialize(SchemaId schemaId, byte[] binaryAvro) { - ByteBuffer buffer = ByteBuffer.allocate(HEADER_SIZE + binaryAvro.length); - buffer.put(MAGIC_BYTE_VALUE); - buffer.putInt(schemaId.value()); - buffer.put(binaryAvro); - return buffer.array(); - } - - public static SchemaAwarePayload deserialize(byte[] payloadWithHeader) { - ByteBuffer buffer = ByteBuffer.wrap(payloadWithHeader); - assertMagicByte(buffer.get()); - int schemaId = buffer.getInt(); - byte[] payload = new byte[payloadWithHeader.length - HEADER_SIZE]; - buffer.get(payload); - return new SchemaAwarePayload(payload, SchemaId.valueOf(schemaId)); - } - - public static boolean startsWithMagicByte(byte[] payload) { - return payload[MAGIC_BYTE_INDEX] == MAGIC_BYTE_VALUE; - } - - private static void assertMagicByte(byte magicByte) { - if (magicByte != MAGIC_BYTE_VALUE) { - throw new DeserializationException(format("Could not deserialize payload, unknown magic byte: \'%s\'.", magicByte)); - } - } - - public static byte[] trimMagicByteAndSchemaVersion(byte[] data) { - int length = data.length - HEADER_SIZE; - byte[] dataWithoutMagicByteAndSchemaVersion = new byte[length]; - System.arraycopy(data, HEADER_SIZE, dataWithoutMagicByteAndSchemaVersion, 0, length); - return dataWithoutMagicByteAndSchemaVersion; + private static final byte MAGIC_BYTE_VALUE = 0; + private static final byte MAGIC_BYTE_INDEX = 0; + private static final int HEADER_SIZE = 5; + + private SchemaAwareSerDe() {} + + public static byte[] serialize(SchemaId schemaId, byte[] binaryAvro) { + ByteBuffer buffer = ByteBuffer.allocate(HEADER_SIZE + binaryAvro.length); + buffer.put(MAGIC_BYTE_VALUE); + buffer.putInt(schemaId.value()); + buffer.put(binaryAvro); + return buffer.array(); + } + + public static SchemaAwarePayload deserialize(byte[] payloadWithHeader) { + ByteBuffer buffer = ByteBuffer.wrap(payloadWithHeader); + assertMagicByte(buffer.get()); + int schemaId = buffer.getInt(); + byte[] payload = new byte[payloadWithHeader.length - HEADER_SIZE]; + buffer.get(payload); + return new SchemaAwarePayload(payload, SchemaId.valueOf(schemaId)); + } + + public static boolean startsWithMagicByte(byte[] payload) { + return payload[MAGIC_BYTE_INDEX] == MAGIC_BYTE_VALUE; + } + + private static void assertMagicByte(byte magicByte) { + if (magicByte != MAGIC_BYTE_VALUE) { + throw new DeserializationException( + format("Could not deserialize payload, unknown magic byte: \'%s\'.", magicByte)); } + } + + public static byte[] trimMagicByteAndSchemaVersion(byte[] data) { + int length = data.length - HEADER_SIZE; + byte[] dataWithoutMagicByteAndSchemaVersion = new byte[length]; + System.arraycopy(data, HEADER_SIZE, dataWithoutMagicByteAndSchemaVersion, 0, length); + return dataWithoutMagicByteAndSchemaVersion; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaMissingException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaMissingException.java index 6df5563fe1..55677e4f09 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaMissingException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaMissingException.java @@ -6,13 +6,12 @@ public class SchemaMissingException extends HermesException { - SchemaMissingException(Topic topic) { - super("Schema for topic " + topic.getQualifiedName() + " was not available"); - } - - @Override - public ErrorCode getCode() { - return ErrorCode.SCHEMA_COULD_NOT_BE_LOADED; - } + SchemaMissingException(Topic topic) { + super("Schema for topic " + topic.getQualifiedName() + " was not available"); + } + @Override + public ErrorCode getCode() { + return ErrorCode.SCHEMA_COULD_NOT_BE_LOADED; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaOnlineChecksWaitingRateLimiter.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaOnlineChecksWaitingRateLimiter.java index e69de29bb2..8b13789179 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaOnlineChecksWaitingRateLimiter.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaOnlineChecksWaitingRateLimiter.java @@ -0,0 +1 @@ + diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/UnsupportedContentTypeException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/UnsupportedContentTypeException.java index 29788ee711..31956447e5 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/UnsupportedContentTypeException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/UnsupportedContentTypeException.java @@ -6,28 +6,24 @@ public class UnsupportedContentTypeException extends InternalProcessingException { - public UnsupportedContentTypeException(Topic topic) { - super(String.format( - "Unsupported content type %s for topic %s", - topic.getContentType(), - topic.getQualifiedName() - )); - } + public UnsupportedContentTypeException(Topic topic) { + super( + String.format( + "Unsupported content type %s for topic %s", + topic.getContentType(), topic.getQualifiedName())); + } - public UnsupportedContentTypeException(Subscription subscription) { - super(String.format( - "Unsupported content type %s for subscription %s", - subscription.getContentType(), - subscription.getQualifiedName() - )); - } - - public UnsupportedContentTypeException(String payloadContentType, Topic topic) { - super(String.format( - "Unsupported payload content type header %s for topic %s", - payloadContentType, - topic.getQualifiedName() - )); - } + public UnsupportedContentTypeException(Subscription subscription) { + super( + String.format( + "Unsupported content type %s for subscription %s", + subscription.getContentType(), subscription.getQualifiedName())); + } + public UnsupportedContentTypeException(String payloadContentType, Topic topic) { + super( + String.format( + "Unsupported payload content type header %s for topic %s", + payloadContentType, topic.getQualifiedName())); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/UnwrappedMessageContent.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/UnwrappedMessageContent.java index 1a5185c785..32ffb3f5b8 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/UnwrappedMessageContent.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/UnwrappedMessageContent.java @@ -1,39 +1,39 @@ package pl.allegro.tech.hermes.common.message.wrapper; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.util.Optional; import org.apache.avro.Schema; import pl.allegro.tech.hermes.schema.CompiledSchema; -import java.util.Optional; - @SuppressFBWarnings("EI_EXPOSE_REP2") public class UnwrappedMessageContent { - private final MessageMetadata messageMetadata; - private final byte[] content; - private final Optional> schema; - - public UnwrappedMessageContent(MessageMetadata messageMetadata, byte[] content) { - this.messageMetadata = messageMetadata; - this.content = content; - this.schema = Optional.empty(); - } - - public UnwrappedMessageContent(MessageMetadata messageMetadata, byte[] content, CompiledSchema schema) { - this.messageMetadata = messageMetadata; - this.content = content; - this.schema = Optional.of(schema); - } - - public byte[] getContent() { - return content; - } - - public MessageMetadata getMessageMetadata() { - return messageMetadata; - } - - public Optional> getSchema() { - return schema; - } + private final MessageMetadata messageMetadata; + private final byte[] content; + private final Optional> schema; + + public UnwrappedMessageContent(MessageMetadata messageMetadata, byte[] content) { + this.messageMetadata = messageMetadata; + this.content = content; + this.schema = Optional.empty(); + } + + public UnwrappedMessageContent( + MessageMetadata messageMetadata, byte[] content, CompiledSchema schema) { + this.messageMetadata = messageMetadata; + this.content = content; + this.schema = Optional.of(schema); + } + + public byte[] getContent() { + return content; + } + + public MessageMetadata getMessageMetadata() { + return messageMetadata; + } + + public Optional> getSchema() { + return schema; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/UnwrappingException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/UnwrappingException.java index 5ec37de19a..ade63a4d4a 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/UnwrappingException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/UnwrappingException.java @@ -3,7 +3,7 @@ import pl.allegro.tech.hermes.common.exception.InternalProcessingException; public class UnwrappingException extends InternalProcessingException { - public UnwrappingException(String message, Exception cause) { - super(message, cause); - } + public UnwrappingException(String message, Exception cause) { + super(message, cause); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/WrappingException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/WrappingException.java index b335f02fb4..60e6e15ddc 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/WrappingException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/message/wrapper/WrappingException.java @@ -3,7 +3,7 @@ import pl.allegro.tech.hermes.common.exception.InternalProcessingException; public class WrappingException extends InternalProcessingException { - public WrappingException(String message, Exception cause) { - super(message, cause); - } + public WrappingException(String message, Exception cause) { + super(message, cause); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/BrokerMetrics.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/BrokerMetrics.java index bbfeac1554..5b9d7b3911 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/BrokerMetrics.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/BrokerMetrics.java @@ -2,24 +2,23 @@ import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Timer; -import pl.allegro.tech.hermes.api.Topic; - import java.time.Duration; +import pl.allegro.tech.hermes.api.Topic; public class BrokerMetrics { - private final MeterRegistry meterRegistry; + private final MeterRegistry meterRegistry; - public BrokerMetrics(MeterRegistry meterRegistry) { - this.meterRegistry = meterRegistry; - } + public BrokerMetrics(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } - public void recordBrokerLatency(String broker, Topic.Ack ack, Duration duration) { - Timer.builder("broker.latency") - .tag("broker", broker) - .tag("ack", ack.name()) - .publishPercentileHistogram() - .maximumExpectedValue(Duration.ofSeconds(5)) - .register(meterRegistry) - .record(duration); - } + public void recordBrokerLatency(String broker, Topic.Ack ack, Duration duration) { + Timer.builder("broker.latency") + .tag("broker", broker) + .tag("ack", ack.name()) + .publishPercentileHistogram() + .maximumExpectedValue(Duration.ofSeconds(5)) + .register(meterRegistry) + .record(duration); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/ConsistencyMetrics.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/ConsistencyMetrics.java index 4324ec0e68..c9c3a06bdb 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/ConsistencyMetrics.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/ConsistencyMetrics.java @@ -1,19 +1,17 @@ package pl.allegro.tech.hermes.common.metric; import io.micrometer.core.instrument.MeterRegistry; - import java.util.function.ToDoubleFunction; - public class ConsistencyMetrics { - private final MeterRegistry meterRegistry; - - ConsistencyMetrics(MeterRegistry meterRegistry) { - this.meterRegistry = meterRegistry; + private final MeterRegistry meterRegistry; - } + ConsistencyMetrics(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } - public void registerStorageConsistencyGauge(T stateObject, ToDoubleFunction valueFunction) { - meterRegistry.gauge("storage.consistency", stateObject, valueFunction); - } + public void registerStorageConsistencyGauge( + T stateObject, ToDoubleFunction valueFunction) { + meterRegistry.gauge("storage.consistency", stateObject, valueFunction); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/ConsumerMetrics.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/ConsumerMetrics.java index c6b3bb2af0..de757afbde 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/ConsumerMetrics.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/ConsumerMetrics.java @@ -1,79 +1,71 @@ package pl.allegro.tech.hermes.common.metric; +import static pl.allegro.tech.hermes.common.metric.SubscriptionTagsFactory.subscriptionTags; + import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Tags; +import java.util.function.ToDoubleFunction; import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.metrics.HermesCounter; import pl.allegro.tech.hermes.metrics.HermesTimer; import pl.allegro.tech.hermes.metrics.counters.HermesCounters; -import java.util.function.ToDoubleFunction; - -import static pl.allegro.tech.hermes.common.metric.SubscriptionTagsFactory.subscriptionTags; - public class ConsumerMetrics { - private final MeterRegistry meterRegistry; - private final GaugeRegistrar gaugeRegistrar; - - public ConsumerMetrics(MeterRegistry meterRegistry) { - this.meterRegistry = meterRegistry; - this.gaugeRegistrar = new GaugeRegistrar(meterRegistry); - } - - public void registerQueueUtilizationGauge(T obj, String queueName, ToDoubleFunction f) { - gaugeRegistrar.registerGauge("queue." + queueName + ".utilization", obj, f); - } - - public HermesCounter queueFailuresCounter(String name) { - return HermesCounters.from( - meterRegistry.counter("queue." + name + ".failures") - ); - } - - public void registerConsumerProcessesThreadsGauge(T obj, ToDoubleFunction f) { - gaugeRegistrar.registerGauge("consumer-processes.threads", obj, f); - } - - public void registerRunningConsumerProcessesGauge(T obj, ToDoubleFunction f) { - meterRegistry.gauge("consumer-processes.running", obj, f); - } - - public void registerDyingConsumerProcessesGauge(T obj, ToDoubleFunction f) { - meterRegistry.gauge("consumer-processes.dying", obj, f); - } - - public void registerBatchBufferTotalBytesGauge(T obj, ToDoubleFunction f) { - gaugeRegistrar.registerGauge("batch-buffer.total-bytes", obj, f); - } - - public void registerBatchBufferAvailableBytesGauge(T obj, ToDoubleFunction f) { - gaugeRegistrar.registerGauge("batch-buffer.available-bytes", obj, f); - } - - public HermesCounter oAuthSubscriptionTokenRequestCounter(Subscription subscription, String providerName) { - return HermesCounters.from( - meterRegistry.counter("oauth.token-requests", Tags.concat( - subscriptionTags(subscription.getQualifiedName()), - "provider", providerName - )) - ); - } - - public HermesTimer oAuthProviderLatencyTimer(String providerName) { - return HermesTimer.from( - meterRegistry.timer("oauth.token-request-latency", Tags.of("provider", providerName)) - ); - } - - public HermesCounter processedSignalsCounter(String name) { - return HermesCounters.from( - meterRegistry.counter("signals.processed", Tags.of("signal", name)) - ); - } - - public HermesCounter droppedSignalsCounter(String name) { - return HermesCounters.from( - meterRegistry.counter("signals.dropped", Tags.of("signal", name)) - ); - } + private final MeterRegistry meterRegistry; + private final GaugeRegistrar gaugeRegistrar; + + public ConsumerMetrics(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + this.gaugeRegistrar = new GaugeRegistrar(meterRegistry); + } + + public void registerQueueUtilizationGauge(T obj, String queueName, ToDoubleFunction f) { + gaugeRegistrar.registerGauge("queue." + queueName + ".utilization", obj, f); + } + + public HermesCounter queueFailuresCounter(String name) { + return HermesCounters.from(meterRegistry.counter("queue." + name + ".failures")); + } + + public void registerConsumerProcessesThreadsGauge(T obj, ToDoubleFunction f) { + gaugeRegistrar.registerGauge("consumer-processes.threads", obj, f); + } + + public void registerRunningConsumerProcessesGauge(T obj, ToDoubleFunction f) { + meterRegistry.gauge("consumer-processes.running", obj, f); + } + + public void registerDyingConsumerProcessesGauge(T obj, ToDoubleFunction f) { + meterRegistry.gauge("consumer-processes.dying", obj, f); + } + + public void registerBatchBufferTotalBytesGauge(T obj, ToDoubleFunction f) { + gaugeRegistrar.registerGauge("batch-buffer.total-bytes", obj, f); + } + + public void registerBatchBufferAvailableBytesGauge(T obj, ToDoubleFunction f) { + gaugeRegistrar.registerGauge("batch-buffer.available-bytes", obj, f); + } + + public HermesCounter oAuthSubscriptionTokenRequestCounter( + Subscription subscription, String providerName) { + return HermesCounters.from( + meterRegistry.counter( + "oauth.token-requests", + Tags.concat( + subscriptionTags(subscription.getQualifiedName()), "provider", providerName))); + } + + public HermesTimer oAuthProviderLatencyTimer(String providerName) { + return HermesTimer.from( + meterRegistry.timer("oauth.token-request-latency", Tags.of("provider", providerName))); + } + + public HermesCounter processedSignalsCounter(String name) { + return HermesCounters.from(meterRegistry.counter("signals.processed", Tags.of("signal", name))); + } + + public HermesCounter droppedSignalsCounter(String name) { + return HermesCounters.from(meterRegistry.counter("signals.dropped", Tags.of("signal", name))); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/ConsumerSenderMetrics.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/ConsumerSenderMetrics.java index d3d8a3c80e..7821e7af16 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/ConsumerSenderMetrics.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/ConsumerSenderMetrics.java @@ -1,9 +1,5 @@ package pl.allegro.tech.hermes.common.metric; -import io.micrometer.core.instrument.MeterRegistry; - -import java.util.function.ToDoubleFunction; - import static pl.allegro.tech.hermes.common.metric.Gauges.CONSUMER_SENDER_HTTP_1_BATCH_CLIENT_ACTIVE_CONNECTIONS; import static pl.allegro.tech.hermes.common.metric.Gauges.CONSUMER_SENDER_HTTP_1_BATCH_CLIENT_IDLE_CONNECTIONS; import static pl.allegro.tech.hermes.common.metric.Gauges.CONSUMER_SENDER_HTTP_1_SERIAL_CLIENT_ACTIVE_CONNECTIONS; @@ -11,53 +7,56 @@ import static pl.allegro.tech.hermes.common.metric.Gauges.CONSUMER_SENDER_HTTP_2_SERIAL_CLIENT_CONNECTIONS; import static pl.allegro.tech.hermes.common.metric.Gauges.CONSUMER_SENDER_HTTP_2_SERIAL_CLIENT_PENDING_CONNECTIONS; +import io.micrometer.core.instrument.MeterRegistry; +import java.util.function.ToDoubleFunction; + public class ConsumerSenderMetrics { - private final MeterRegistry meterRegistry; - private final GaugeRegistrar gaugeRegistrar; + private final MeterRegistry meterRegistry; + private final GaugeRegistrar gaugeRegistrar; - ConsumerSenderMetrics(MeterRegistry meterRegistry) { - this.meterRegistry = meterRegistry; - this.gaugeRegistrar = new GaugeRegistrar(meterRegistry); - } + ConsumerSenderMetrics(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + this.gaugeRegistrar = new GaugeRegistrar(meterRegistry); + } - public void registerRequestQueueSizeGauge(T obj, ToDoubleFunction f) { - meterRegistry.gauge("http-clients.request-queue-size", obj, f); - } + public void registerRequestQueueSizeGauge(T obj, ToDoubleFunction f) { + meterRegistry.gauge("http-clients.request-queue-size", obj, f); + } - public void registerHttp1SerialClientRequestQueueSizeGauge(T obj, ToDoubleFunction f) { - meterRegistry.gauge("http-clients.serial.http1.request-queue-size", obj, f); - } + public void registerHttp1SerialClientRequestQueueSizeGauge(T obj, ToDoubleFunction f) { + meterRegistry.gauge("http-clients.serial.http1.request-queue-size", obj, f); + } - public void registerHttp1BatchClientRequestQueueSizeGauge(T obj, ToDoubleFunction f) { - meterRegistry.gauge("http-clients.batch.http1.request-queue-size", obj, f); - } + public void registerHttp1BatchClientRequestQueueSizeGauge(T obj, ToDoubleFunction f) { + meterRegistry.gauge("http-clients.batch.http1.request-queue-size", obj, f); + } - public void registerHttp2RequestQueueSizeGauge(T obj, ToDoubleFunction f) { - meterRegistry.gauge("http-clients.serial.http2.request-queue-size", obj, f); - } + public void registerHttp2RequestQueueSizeGauge(T obj, ToDoubleFunction f) { + meterRegistry.gauge("http-clients.serial.http2.request-queue-size", obj, f); + } - public void registerHttp1SerialClientActiveConnectionsGauge(T obj, ToDoubleFunction f) { - gaugeRegistrar.registerGauge(CONSUMER_SENDER_HTTP_1_SERIAL_CLIENT_ACTIVE_CONNECTIONS, obj, f); - } + public void registerHttp1SerialClientActiveConnectionsGauge(T obj, ToDoubleFunction f) { + gaugeRegistrar.registerGauge(CONSUMER_SENDER_HTTP_1_SERIAL_CLIENT_ACTIVE_CONNECTIONS, obj, f); + } - public void registerHttp1SerialClientIdleConnectionsGauge(T obj, ToDoubleFunction f) { - gaugeRegistrar.registerGauge(CONSUMER_SENDER_HTTP_1_SERIAL_CLIENT_IDLE_CONNECTIONS, obj, f); - } + public void registerHttp1SerialClientIdleConnectionsGauge(T obj, ToDoubleFunction f) { + gaugeRegistrar.registerGauge(CONSUMER_SENDER_HTTP_1_SERIAL_CLIENT_IDLE_CONNECTIONS, obj, f); + } - public void registerHttp1BatchClientActiveConnectionsGauge(T obj, ToDoubleFunction f) { - gaugeRegistrar.registerGauge(CONSUMER_SENDER_HTTP_1_BATCH_CLIENT_ACTIVE_CONNECTIONS, obj, f); - } + public void registerHttp1BatchClientActiveConnectionsGauge(T obj, ToDoubleFunction f) { + gaugeRegistrar.registerGauge(CONSUMER_SENDER_HTTP_1_BATCH_CLIENT_ACTIVE_CONNECTIONS, obj, f); + } - public void registerHttp1BatchClientIdleConnectionsGauge(T obj, ToDoubleFunction f) { - gaugeRegistrar.registerGauge(CONSUMER_SENDER_HTTP_1_BATCH_CLIENT_IDLE_CONNECTIONS, obj, f); - } + public void registerHttp1BatchClientIdleConnectionsGauge(T obj, ToDoubleFunction f) { + gaugeRegistrar.registerGauge(CONSUMER_SENDER_HTTP_1_BATCH_CLIENT_IDLE_CONNECTIONS, obj, f); + } - public void registerHttp2SerialClientConnectionsGauge(T obj, ToDoubleFunction f) { - gaugeRegistrar.registerGauge(CONSUMER_SENDER_HTTP_2_SERIAL_CLIENT_CONNECTIONS, obj, f); - } + public void registerHttp2SerialClientConnectionsGauge(T obj, ToDoubleFunction f) { + gaugeRegistrar.registerGauge(CONSUMER_SENDER_HTTP_2_SERIAL_CLIENT_CONNECTIONS, obj, f); + } - public void registerHttp2SerialClientPendingConnectionsGauge(T obj, ToDoubleFunction f) { - gaugeRegistrar.registerGauge(CONSUMER_SENDER_HTTP_2_SERIAL_CLIENT_PENDING_CONNECTIONS, obj, f); - } + public void registerHttp2SerialClientPendingConnectionsGauge(T obj, ToDoubleFunction f) { + gaugeRegistrar.registerGauge(CONSUMER_SENDER_HTTP_2_SERIAL_CLIENT_PENDING_CONNECTIONS, obj, f); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/DeserializationMetrics.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/DeserializationMetrics.java index 3efe8fac71..27d6081d16 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/DeserializationMetrics.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/DeserializationMetrics.java @@ -7,75 +7,56 @@ import pl.allegro.tech.hermes.metrics.counters.HermesCounters; public class DeserializationMetrics { - private final MeterRegistry meterRegistry; + private final MeterRegistry meterRegistry; - private static final String BASE_PATH = "content.avro.deserialization"; - private static final String ERRORS_PATH = BASE_PATH + ".errors"; + private static final String BASE_PATH = "content.avro.deserialization"; + private static final String ERRORS_PATH = BASE_PATH + ".errors"; - public DeserializationMetrics(MeterRegistry meterRegistry) { - this.meterRegistry = meterRegistry; - } + public DeserializationMetrics(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } - public HermesCounter errorsForHeaderSchemaVersion() { - return HermesCounters.from( - deserializationErrorCounter("headerSchemaVersion") - ); + public HermesCounter errorsForHeaderSchemaVersion() { + return HermesCounters.from(deserializationErrorCounter("headerSchemaVersion")); + } - } + public HermesCounter errorsForHeaderSchemaId() { + return HermesCounters.from(deserializationErrorCounter("headerSchemaId")); + } - public HermesCounter errorsForHeaderSchemaId() { - return HermesCounters.from( - deserializationErrorCounter("headerSchemaId") - ); - } + public HermesCounter errorsForSchemaIdAwarePayload() { + return HermesCounters.from(deserializationErrorCounter("payloadWithSchemaId")); + } - public HermesCounter errorsForSchemaIdAwarePayload() { - return HermesCounters.from( - deserializationErrorCounter("payloadWithSchemaId") - ); - } + public HermesCounter errorsForSchemaVersionTruncation() { + return HermesCounters.from(deserializationErrorCounter("schemaVersionTruncation")); + } - public HermesCounter errorsForSchemaVersionTruncation() { - return HermesCounters.from( - deserializationErrorCounter("schemaVersionTruncation") - ); - } + private io.micrometer.core.instrument.Counter deserializationErrorCounter(String schemaSource) { + return meterRegistry.counter(ERRORS_PATH, Tags.of("deserialization_type", schemaSource)); + } - private io.micrometer.core.instrument.Counter deserializationErrorCounter(String schemaSource) { - return meterRegistry.counter(ERRORS_PATH, Tags.of("deserialization_type", schemaSource)); - } + public HermesCounter missingSchemaIdInPayload() { + return HermesCounters.from(meterRegistry.counter(BASE_PATH + ".missing_schemaIdInPayload")); + } - public HermesCounter missingSchemaIdInPayload() { - return HermesCounters.from( - meterRegistry.counter(BASE_PATH + ".missing_schemaIdInPayload") - ); - } + public HermesCounter usingHeaderSchemaVersion() { + return HermesCounters.from(deserializationAttemptCounter("headerSchemaVersion")); + } - public HermesCounter usingHeaderSchemaVersion() { - return HermesCounters.from( - deserializationAttemptCounter("headerSchemaVersion") - ); - } + public HermesCounter usingHeaderSchemaId() { + return HermesCounters.from(deserializationAttemptCounter("headerSchemaId")); + } - public HermesCounter usingHeaderSchemaId() { - return HermesCounters.from( - deserializationAttemptCounter("headerSchemaId") - ); - } + public HermesCounter usingSchemaIdAware() { + return HermesCounters.from(deserializationAttemptCounter("payloadWithSchemaId")); + } - public HermesCounter usingSchemaIdAware() { - return HermesCounters.from( - deserializationAttemptCounter("payloadWithSchemaId") - ); - } + public HermesCounter usingSchemaVersionTruncation() { + return HermesCounters.from(deserializationAttemptCounter("schemaVersionTruncation")); + } - public HermesCounter usingSchemaVersionTruncation() { - return HermesCounters.from( - deserializationAttemptCounter("schemaVersionTruncation") - ); - } - - private Counter deserializationAttemptCounter(String deserializationType) { - return meterRegistry.counter(BASE_PATH, Tags.of("deserialization_type", deserializationType)); - } + private Counter deserializationAttemptCounter(String deserializationType) { + return meterRegistry.counter(BASE_PATH, Tags.of("deserialization_type", deserializationType)); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/ExecutorMetrics.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/ExecutorMetrics.java index 8843d79829..cf59fc909b 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/ExecutorMetrics.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/ExecutorMetrics.java @@ -2,22 +2,22 @@ import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics; - import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; public class ExecutorMetrics { - private final MeterRegistry meterRegistry; + private final MeterRegistry meterRegistry; - public ExecutorMetrics(MeterRegistry meterRegistry) { - this.meterRegistry = meterRegistry; - } + public ExecutorMetrics(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } - public ExecutorService monitor(ExecutorService executorService, String executorName) { - return ExecutorServiceMetrics.monitor(meterRegistry, executorService, executorName); - } + public ExecutorService monitor(ExecutorService executorService, String executorName) { + return ExecutorServiceMetrics.monitor(meterRegistry, executorService, executorName); + } - public ScheduledExecutorService monitor(ScheduledExecutorService scheduledExecutorService, String executorName) { - return ExecutorServiceMetrics.monitor(meterRegistry, scheduledExecutorService, executorName); - } + public ScheduledExecutorService monitor( + ScheduledExecutorService scheduledExecutorService, String executorName) { + return ExecutorServiceMetrics.monitor(meterRegistry, scheduledExecutorService, executorName); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/GaugeRegistrar.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/GaugeRegistrar.java index cbcb86ba84..8e848ed77e 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/GaugeRegistrar.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/GaugeRegistrar.java @@ -3,27 +3,21 @@ import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Tag; import io.micrometer.core.instrument.Tags; - import java.util.function.ToDoubleFunction; public class GaugeRegistrar { - private final MeterRegistry meterRegistry; + private final MeterRegistry meterRegistry; - public GaugeRegistrar(MeterRegistry meterRegistry) { - this.meterRegistry = meterRegistry; - } + public GaugeRegistrar(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } - public void registerGauge(String name, - T stateObj, - ToDoubleFunction f) { - registerGauge(name, stateObj, f, Tags.empty()); - } + public void registerGauge(String name, T stateObj, ToDoubleFunction f) { + registerGauge(name, stateObj, f, Tags.empty()); + } - public void registerGauge(String name, - T stateObj, - ToDoubleFunction f, - Iterable tags) { - meterRegistry.gauge(name, tags, stateObj, f); - } + public void registerGauge( + String name, T stateObj, ToDoubleFunction f, Iterable tags) { + meterRegistry.gauge(name, tags, stateObj, f); + } } - diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/Gauges.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/Gauges.java index 5990e03c95..8b70e0b3ae 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/Gauges.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/Gauges.java @@ -2,15 +2,21 @@ public class Gauges { - public static final String INFLIGHT_REQUESTS = "inflight-requests"; - public static final String BACKUP_STORAGE_SIZE = "backup-storage.size"; + public static final String INFLIGHT_REQUESTS = "inflight-requests"; + public static final String BACKUP_STORAGE_SIZE = "backup-storage.size"; - public static final String CONSUMER_SENDER_HTTP_1_SERIAL_CLIENT_ACTIVE_CONNECTIONS = "http-clients.serial.http1.active-connections"; - public static final String CONSUMER_SENDER_HTTP_1_SERIAL_CLIENT_IDLE_CONNECTIONS = "http-clients.serial.http1.idle-connections"; + public static final String CONSUMER_SENDER_HTTP_1_SERIAL_CLIENT_ACTIVE_CONNECTIONS = + "http-clients.serial.http1.active-connections"; + public static final String CONSUMER_SENDER_HTTP_1_SERIAL_CLIENT_IDLE_CONNECTIONS = + "http-clients.serial.http1.idle-connections"; - public static final String CONSUMER_SENDER_HTTP_1_BATCH_CLIENT_ACTIVE_CONNECTIONS = "http-clients.batch.http1.active-connections"; - public static final String CONSUMER_SENDER_HTTP_1_BATCH_CLIENT_IDLE_CONNECTIONS = "http-clients.batch.http1.idle-connections"; + public static final String CONSUMER_SENDER_HTTP_1_BATCH_CLIENT_ACTIVE_CONNECTIONS = + "http-clients.batch.http1.active-connections"; + public static final String CONSUMER_SENDER_HTTP_1_BATCH_CLIENT_IDLE_CONNECTIONS = + "http-clients.batch.http1.idle-connections"; - public static final String CONSUMER_SENDER_HTTP_2_SERIAL_CLIENT_CONNECTIONS = "http-clients.serial.http2.connections"; - public static final String CONSUMER_SENDER_HTTP_2_SERIAL_CLIENT_PENDING_CONNECTIONS = "http-clients.serial.http2.pending-connections"; + public static final String CONSUMER_SENDER_HTTP_2_SERIAL_CLIENT_CONNECTIONS = + "http-clients.serial.http2.connections"; + public static final String CONSUMER_SENDER_HTTP_2_SERIAL_CLIENT_PENDING_CONNECTIONS = + "http-clients.serial.http2.pending-connections"; } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/Histograms.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/Histograms.java index 04238a427c..67a8bad658 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/Histograms.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/Histograms.java @@ -1,6 +1,6 @@ package pl.allegro.tech.hermes.common.metric; - public class Histograms { - public static final String PERSISTED_UNDELIVERED_MESSAGE_SIZE = "undelivered-messages.persisted.message-size"; + public static final String PERSISTED_UNDELIVERED_MESSAGE_SIZE = + "undelivered-messages.persisted.message-size"; } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/MaxRateMetrics.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/MaxRateMetrics.java index 18a458e52b..5f608a6072 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/MaxRateMetrics.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/MaxRateMetrics.java @@ -1,49 +1,51 @@ package pl.allegro.tech.hermes.common.metric; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static pl.allegro.tech.hermes.common.metric.SubscriptionTagsFactory.subscriptionTags; + import io.micrometer.core.instrument.MeterRegistry; +import java.util.List; +import java.util.function.ToDoubleFunction; import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.metrics.HermesCounter; import pl.allegro.tech.hermes.metrics.counters.HermesCounters; -import java.util.List; -import java.util.function.ToDoubleFunction; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static pl.allegro.tech.hermes.common.metric.SubscriptionTagsFactory.subscriptionTags; - public class MaxRateMetrics { - private final MeterRegistry meterRegistry; - - MaxRateMetrics(MeterRegistry meterRegistry) { - this.meterRegistry = meterRegistry; - } - - public void registerCalculationDurationInMillisGauge(T obj, ToDoubleFunction f) { - meterRegistry.more().timeGauge("max-rate.calculation.duration", List.of(), obj, MILLISECONDS, f); - } - - public HermesCounter historyUpdateFailuresCounter(SubscriptionName subscription) { - return HermesCounters.from( - meterRegistry.counter("max-rate.history-update.failures", subscriptionTags(subscription)) - ); - } - - public HermesCounter fetchFailuresCounter(SubscriptionName subscription) { - return HermesCounters.from( - meterRegistry.counter("max-rate.fetch.failures", subscriptionTags(subscription)) - ); - } - - public void registerCalculatedRateGauge(SubscriptionName subscription, T obj, ToDoubleFunction f) { - meterRegistry.gauge("max-rate.calculated-rate", subscriptionTags(subscription), obj, f); - } - - public void registerActualRateGauge(SubscriptionName subscription, T obj, ToDoubleFunction f) { - meterRegistry.gauge("max-rate.actual-rate", subscriptionTags(subscription), obj, f); - } - - public void registerOutputRateGauge(SubscriptionName subscription, T obj, ToDoubleFunction f) { - meterRegistry.gauge("max-rate.output-rate", subscriptionTags(subscription), obj, f); - } + private final MeterRegistry meterRegistry; + + MaxRateMetrics(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } + + public void registerCalculationDurationInMillisGauge(T obj, ToDoubleFunction f) { + meterRegistry + .more() + .timeGauge("max-rate.calculation.duration", List.of(), obj, MILLISECONDS, f); + } + + public HermesCounter historyUpdateFailuresCounter(SubscriptionName subscription) { + return HermesCounters.from( + meterRegistry.counter("max-rate.history-update.failures", subscriptionTags(subscription))); + } + + public HermesCounter fetchFailuresCounter(SubscriptionName subscription) { + return HermesCounters.from( + meterRegistry.counter("max-rate.fetch.failures", subscriptionTags(subscription))); + } + + public void registerCalculatedRateGauge( + SubscriptionName subscription, T obj, ToDoubleFunction f) { + meterRegistry.gauge("max-rate.calculated-rate", subscriptionTags(subscription), obj, f); + } + + public void registerActualRateGauge( + SubscriptionName subscription, T obj, ToDoubleFunction f) { + meterRegistry.gauge("max-rate.actual-rate", subscriptionTags(subscription), obj, f); + } + + public void registerOutputRateGauge( + SubscriptionName subscription, T obj, ToDoubleFunction f) { + meterRegistry.gauge("max-rate.output-rate", subscriptionTags(subscription), obj, f); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/Meters.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/Meters.java index 4438260971..b502e0eea1 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/Meters.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/Meters.java @@ -5,8 +5,9 @@ public class Meters { - public static final String THROUGHPUT_BYTES = "throughput"; - public static final String TOPIC_THROUGHPUT_BYTES = THROUGHPUT_BYTES + "." + GROUP + "." + TOPIC; + public static final String THROUGHPUT_BYTES = "throughput"; + public static final String TOPIC_THROUGHPUT_BYTES = THROUGHPUT_BYTES + "." + GROUP + "." + TOPIC; - public static final String PERSISTED_UNDELIVERED_MESSAGES_METER = "undelivered-messages.persisted"; + public static final String PERSISTED_UNDELIVERED_MESSAGES_METER = + "undelivered-messages.persisted"; } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/MetricsFacade.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/MetricsFacade.java index 9608763425..2f1bb31897 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/MetricsFacade.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/MetricsFacade.java @@ -1,124 +1,122 @@ package pl.allegro.tech.hermes.common.metric; +import static pl.allegro.tech.hermes.common.metric.SubscriptionTagsFactory.subscriptionTags; + import io.micrometer.core.instrument.Meter; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.search.Search; -import pl.allegro.tech.hermes.api.SubscriptionName; - import java.util.Collection; - -import static pl.allegro.tech.hermes.common.metric.SubscriptionTagsFactory.subscriptionTags; +import pl.allegro.tech.hermes.api.SubscriptionName; public class MetricsFacade { - private final MeterRegistry meterRegistry; - private final TopicMetrics topicMetrics; - private final SubscriptionMetrics subscriptionMetrics; - private final ConsumerMetrics consumerMetrics; - private final TrackerElasticSearchMetrics trackerElasticSearchMetrics; - private final PersistentBufferMetrics persistentBufferMetrics; - private final ProducerMetrics producerMetrics; - private final ExecutorMetrics executorMetrics; - private final SchemaClientMetrics schemaClientMetrics; - private final UndeliveredMessagesMetrics undeliveredMessagesMetrics; - private final DeserializationMetrics deserializationMetrics; - private final WorkloadMetrics workloadMetrics; - private final ConsumerSenderMetrics consumerSenderMetrics; - private final OffsetCommitsMetrics offsetCommitsMetrics; - private final MaxRateMetrics maxRateMetrics; - private final BrokerMetrics brokerMetrics; - private final ConsistencyMetrics consistencyMetrics; - - public MetricsFacade(MeterRegistry meterRegistry) { - this.meterRegistry = meterRegistry; - this.topicMetrics = new TopicMetrics(meterRegistry); - this.subscriptionMetrics = new SubscriptionMetrics(meterRegistry); - this.consumerMetrics = new ConsumerMetrics(meterRegistry); - this.trackerElasticSearchMetrics = new TrackerElasticSearchMetrics(meterRegistry); - this.persistentBufferMetrics = new PersistentBufferMetrics(meterRegistry); - this.producerMetrics = new ProducerMetrics(meterRegistry); - this.executorMetrics = new ExecutorMetrics(meterRegistry); - this.schemaClientMetrics = new SchemaClientMetrics(meterRegistry); - this.undeliveredMessagesMetrics = new UndeliveredMessagesMetrics(meterRegistry); - this.deserializationMetrics = new DeserializationMetrics(meterRegistry); - this.workloadMetrics = new WorkloadMetrics(meterRegistry); - this.consumerSenderMetrics = new ConsumerSenderMetrics(meterRegistry); - this.offsetCommitsMetrics = new OffsetCommitsMetrics(meterRegistry); - this.maxRateMetrics = new MaxRateMetrics(meterRegistry); - this.brokerMetrics = new BrokerMetrics(meterRegistry); - this.consistencyMetrics = new ConsistencyMetrics(meterRegistry); - } - - public TopicMetrics topics() { - return topicMetrics; - } - - public SubscriptionMetrics subscriptions() { - return subscriptionMetrics; - } - - public ConsumerMetrics consumer() { - return consumerMetrics; - } - - public TrackerElasticSearchMetrics trackerElasticSearch() { - return trackerElasticSearchMetrics; - } - - public PersistentBufferMetrics persistentBuffer() { - return persistentBufferMetrics; - } - - public ProducerMetrics producer() { - return producerMetrics; - } - - public ExecutorMetrics executor() { - return executorMetrics; - } - - public SchemaClientMetrics schemaClient() { - return schemaClientMetrics; - } - - public UndeliveredMessagesMetrics undeliveredMessages() { - return undeliveredMessagesMetrics; - } - - public DeserializationMetrics deserialization() { - return deserializationMetrics; - } - - public WorkloadMetrics workload() { - return workloadMetrics; - } - - public ConsumerSenderMetrics consumerSender() { - return consumerSenderMetrics; - } - - public OffsetCommitsMetrics offsetCommits() { - return offsetCommitsMetrics; - } - - public MaxRateMetrics maxRate() { - return maxRateMetrics; - } - - public BrokerMetrics broker() { - return brokerMetrics; - } - - public ConsistencyMetrics consistency() { - return consistencyMetrics; - } - - public void unregisterAllMetricsRelatedTo(SubscriptionName subscription) { - Collection meters = Search.in(meterRegistry) - .tags(subscriptionTags(subscription)) - .meters(); - for (Meter meter : meters) { - meterRegistry.remove(meter); - } - } + private final MeterRegistry meterRegistry; + private final TopicMetrics topicMetrics; + private final SubscriptionMetrics subscriptionMetrics; + private final ConsumerMetrics consumerMetrics; + private final TrackerElasticSearchMetrics trackerElasticSearchMetrics; + private final PersistentBufferMetrics persistentBufferMetrics; + private final ProducerMetrics producerMetrics; + private final ExecutorMetrics executorMetrics; + private final SchemaClientMetrics schemaClientMetrics; + private final UndeliveredMessagesMetrics undeliveredMessagesMetrics; + private final DeserializationMetrics deserializationMetrics; + private final WorkloadMetrics workloadMetrics; + private final ConsumerSenderMetrics consumerSenderMetrics; + private final OffsetCommitsMetrics offsetCommitsMetrics; + private final MaxRateMetrics maxRateMetrics; + private final BrokerMetrics brokerMetrics; + private final ConsistencyMetrics consistencyMetrics; + + public MetricsFacade(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + this.topicMetrics = new TopicMetrics(meterRegistry); + this.subscriptionMetrics = new SubscriptionMetrics(meterRegistry); + this.consumerMetrics = new ConsumerMetrics(meterRegistry); + this.trackerElasticSearchMetrics = new TrackerElasticSearchMetrics(meterRegistry); + this.persistentBufferMetrics = new PersistentBufferMetrics(meterRegistry); + this.producerMetrics = new ProducerMetrics(meterRegistry); + this.executorMetrics = new ExecutorMetrics(meterRegistry); + this.schemaClientMetrics = new SchemaClientMetrics(meterRegistry); + this.undeliveredMessagesMetrics = new UndeliveredMessagesMetrics(meterRegistry); + this.deserializationMetrics = new DeserializationMetrics(meterRegistry); + this.workloadMetrics = new WorkloadMetrics(meterRegistry); + this.consumerSenderMetrics = new ConsumerSenderMetrics(meterRegistry); + this.offsetCommitsMetrics = new OffsetCommitsMetrics(meterRegistry); + this.maxRateMetrics = new MaxRateMetrics(meterRegistry); + this.brokerMetrics = new BrokerMetrics(meterRegistry); + this.consistencyMetrics = new ConsistencyMetrics(meterRegistry); + } + + public TopicMetrics topics() { + return topicMetrics; + } + + public SubscriptionMetrics subscriptions() { + return subscriptionMetrics; + } + + public ConsumerMetrics consumer() { + return consumerMetrics; + } + + public TrackerElasticSearchMetrics trackerElasticSearch() { + return trackerElasticSearchMetrics; + } + + public PersistentBufferMetrics persistentBuffer() { + return persistentBufferMetrics; + } + + public ProducerMetrics producer() { + return producerMetrics; + } + + public ExecutorMetrics executor() { + return executorMetrics; + } + + public SchemaClientMetrics schemaClient() { + return schemaClientMetrics; + } + + public UndeliveredMessagesMetrics undeliveredMessages() { + return undeliveredMessagesMetrics; + } + + public DeserializationMetrics deserialization() { + return deserializationMetrics; + } + + public WorkloadMetrics workload() { + return workloadMetrics; + } + + public ConsumerSenderMetrics consumerSender() { + return consumerSenderMetrics; + } + + public OffsetCommitsMetrics offsetCommits() { + return offsetCommitsMetrics; + } + + public MaxRateMetrics maxRate() { + return maxRateMetrics; + } + + public BrokerMetrics broker() { + return brokerMetrics; + } + + public ConsistencyMetrics consistency() { + return consistencyMetrics; + } + + public void unregisterAllMetricsRelatedTo(SubscriptionName subscription) { + Collection meters = + Search.in(meterRegistry).tags(subscriptionTags(subscription)).meters(); + for (Meter meter : meters) { + meterRegistry.remove(meter); + } + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/OffsetCommitsMetrics.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/OffsetCommitsMetrics.java index b3b1376edb..bb310aca1f 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/OffsetCommitsMetrics.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/OffsetCommitsMetrics.java @@ -7,39 +7,29 @@ public class OffsetCommitsMetrics { - private final MeterRegistry meterRegistry; - - OffsetCommitsMetrics(MeterRegistry meterRegistry) { - this.meterRegistry = meterRegistry; - } - - public HermesCounter skippedCounter() { - return HermesCounters.from( - meterRegistry.counter("offset-commits.skipped") - ); - } - - public HermesCounter obsoleteCounter() { - return HermesCounters.from( - meterRegistry.counter("offset-commits.obsolete") - ); - } - - public HermesCounter committedCounter() { - return HermesCounters.from( - meterRegistry.counter("offset-commits.committed") - ); - } - - public HermesTimer duration() { - return HermesTimer.from( - meterRegistry.timer("offset-commits.duration") - ); - } - - public HermesCounter failuresCounter() { - return HermesCounters.from( - meterRegistry.counter("offset-commits.failures") - ); - } + private final MeterRegistry meterRegistry; + + OffsetCommitsMetrics(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } + + public HermesCounter skippedCounter() { + return HermesCounters.from(meterRegistry.counter("offset-commits.skipped")); + } + + public HermesCounter obsoleteCounter() { + return HermesCounters.from(meterRegistry.counter("offset-commits.obsolete")); + } + + public HermesCounter committedCounter() { + return HermesCounters.from(meterRegistry.counter("offset-commits.committed")); + } + + public HermesTimer duration() { + return HermesTimer.from(meterRegistry.timer("offset-commits.duration")); + } + + public HermesCounter failuresCounter() { + return HermesCounters.from(meterRegistry.counter("offset-commits.failures")); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/PersistentBufferMetrics.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/PersistentBufferMetrics.java index 798e3f57fd..086f44bbd1 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/PersistentBufferMetrics.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/PersistentBufferMetrics.java @@ -1,19 +1,18 @@ package pl.allegro.tech.hermes.common.metric; -import io.micrometer.core.instrument.MeterRegistry; +import static pl.allegro.tech.hermes.common.metric.Gauges.BACKUP_STORAGE_SIZE; +import io.micrometer.core.instrument.MeterRegistry; import java.util.function.ToDoubleFunction; -import static pl.allegro.tech.hermes.common.metric.Gauges.BACKUP_STORAGE_SIZE; - public class PersistentBufferMetrics { - private final MeterRegistry meterRegistry; + private final MeterRegistry meterRegistry; - public PersistentBufferMetrics(MeterRegistry meterRegistry) { - this.meterRegistry = meterRegistry; - } + public PersistentBufferMetrics(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } - public void registerBackupStorageSizeGauge(T obj, ToDoubleFunction f) { - meterRegistry.gauge(BACKUP_STORAGE_SIZE, obj, f); - } + public void registerBackupStorageSizeGauge(T obj, ToDoubleFunction f) { + meterRegistry.gauge(BACKUP_STORAGE_SIZE, obj, f); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/ProducerMetrics.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/ProducerMetrics.java index 0c1b65b913..3817ab294e 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/ProducerMetrics.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/ProducerMetrics.java @@ -1,127 +1,165 @@ package pl.allegro.tech.hermes.common.metric; +import static pl.allegro.tech.hermes.common.metric.Gauges.INFLIGHT_REQUESTS; + import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Tags; - import java.util.concurrent.TimeUnit; import java.util.function.ToDoubleFunction; -import static pl.allegro.tech.hermes.common.metric.Gauges.INFLIGHT_REQUESTS; - public class ProducerMetrics { - private final MeterRegistry meterRegistry; - private final GaugeRegistrar gaugeRegistrar; - - public ProducerMetrics(MeterRegistry meterRegistry) { - this.meterRegistry = meterRegistry; - this.gaugeRegistrar = new GaugeRegistrar(meterRegistry); - } - - public void registerAckAllTotalBytesGauge(T stateObj, ToDoubleFunction f, String sender, String datacenter) { - gaugeRegistrar.registerGauge(ACK_ALL_BUFFER_TOTAL_BYTES, stateObj, f, tags(sender, datacenter)); - } - - public void registerAckLeaderTotalBytesGauge(T stateObj, ToDoubleFunction f, String sender, String datacenter) { - gaugeRegistrar.registerGauge(ACK_LEADER_BUFFER_TOTAL_BYTES, stateObj, f, tags(sender, datacenter)); - } - - public void registerAckAllAvailableBytesGauge(T stateObj, ToDoubleFunction f, String sender, String datacenter) { - gaugeRegistrar.registerGauge(ACK_ALL_BUFFER_AVAILABLE_BYTES, stateObj, f, tags(sender, datacenter)); - } - - public void registerAckLeaderAvailableBytesGauge(T stateObj, ToDoubleFunction f, String sender, String datacenter) { - gaugeRegistrar.registerGauge(ACK_LEADER_BUFFER_AVAILABLE_BYTES, stateObj, f, tags(sender, datacenter)); - } - - public void registerAckAllCompressionRateGauge(T stateObj, ToDoubleFunction f, String sender, String datacenter) { - gaugeRegistrar.registerGauge(ACK_ALL_COMPRESSION_RATE, stateObj, f, tags(sender, datacenter)); - } - - public void registerAckLeaderCompressionRateGauge(T stateObj, ToDoubleFunction f, String sender, String datacenter) { - gaugeRegistrar.registerGauge(ACK_LEADER_COMPRESSION_RATE, stateObj, f, tags(sender, datacenter)); - } - - public void registerAckAllFailedBatchesGauge(T stateObj, ToDoubleFunction f, String sender, String datacenter) { - gaugeRegistrar.registerGauge(ACK_ALL_FAILED_BATCHES_TOTAL, stateObj, f, tags(sender, datacenter)); - } - - public void registerAckLeaderFailedBatchesGauge(T stateObj, ToDoubleFunction f, String sender, String datacenter) { - gaugeRegistrar.registerGauge(ACK_LEADER_FAILED_BATCHES_TOTAL, stateObj, f, tags(sender, datacenter)); - } - - public void registerAckAllMetadataAgeGauge(T stateObj, ToDoubleFunction f, String sender, String datacenter) { - registerTimeGauge(stateObj, f, ACK_ALL_METADATA_AGE, tags(sender, datacenter), TimeUnit.SECONDS); - } - - public void registerAckLeaderMetadataAgeGauge(T stateObj, ToDoubleFunction f, String sender, String datacenter) { - registerTimeGauge(stateObj, f, ACK_LEADER_METADATA_AGE, tags(sender, datacenter), TimeUnit.SECONDS); - } - - public void registerAckAllRecordQueueTimeMaxGauge(T stateObj, ToDoubleFunction f, String sender, String datacenter) { - registerTimeGauge(stateObj, f, ACK_ALL_RECORD_QUEUE_TIME_MAX, tags(sender, datacenter), TimeUnit.MILLISECONDS); - } - - public void registerAckLeaderRecordQueueTimeMaxGauge(T stateObj, ToDoubleFunction f, String sender, String datacenter) { - registerTimeGauge(stateObj, f, ACK_LEADER_RECORD_QUEUE_TIME_MAX, tags(sender, datacenter), TimeUnit.MILLISECONDS); - } - - public double getBufferTotalBytes() { - return meterRegistry.get(ACK_ALL_BUFFER_TOTAL_BYTES).gauge().value() - + meterRegistry.get(ACK_LEADER_BUFFER_TOTAL_BYTES).gauge().value(); - } - - public double getBufferAvailableBytes() { - return meterRegistry.get(ACK_ALL_BUFFER_AVAILABLE_BYTES).gauge().value() - + meterRegistry.get(ACK_LEADER_BUFFER_AVAILABLE_BYTES).gauge().value(); - } - - public void registerAckLeaderRecordSendCounter(T stateObj, ToDoubleFunction f, String sender, String datacenter) { - registerCounter(ACK_LEADER_RECORD_SEND_TOTAL, tags(sender, datacenter), stateObj, f); - } - - public void registerAckAllRecordSendCounter(T stateObj, ToDoubleFunction f, String sender, String datacenter) { - registerCounter(ACK_ALL_RECORD_SEND_TOTAL, tags(sender, datacenter), stateObj, f); - } - - public void registerProducerInflightRequestGauge(T stateObj, ToDoubleFunction f) { - meterRegistry.gauge(INFLIGHT_REQUESTS, stateObj, f); - } - - private static Tags tags(String sender, String datacenter) { - return Tags.of("storageDc", datacenter, "sender", sender); - } - - - private void registerTimeGauge(T stateObj, - ToDoubleFunction f, - String name, - Tags tags, - TimeUnit timeUnit) { - meterRegistry.more().timeGauge(name, tags, stateObj, timeUnit, f); - } - - private void registerCounter(String name, Tags tags, T stateObj, ToDoubleFunction f) { - meterRegistry.more().counter(name, tags, stateObj, f); - } - - private static final String KAFKA_PRODUCER = "kafka-producer."; - private static final String ACK_LEADER = "ack-leader."; - private static final String ACK_ALL = "ack-all."; - - private static final String ACK_ALL_BUFFER_TOTAL_BYTES = KAFKA_PRODUCER + ACK_ALL + "buffer-total-bytes"; - private static final String ACK_ALL_BUFFER_AVAILABLE_BYTES = KAFKA_PRODUCER + ACK_ALL + "buffer-available-bytes"; - private static final String ACK_ALL_METADATA_AGE = KAFKA_PRODUCER + ACK_ALL + "metadata-age"; - private static final String ACK_ALL_RECORD_QUEUE_TIME_MAX = KAFKA_PRODUCER + ACK_ALL + "record-queue-time-max"; - private static final String ACK_ALL_COMPRESSION_RATE = KAFKA_PRODUCER + ACK_ALL + "compression-rate-avg"; - private static final String ACK_ALL_FAILED_BATCHES_TOTAL = KAFKA_PRODUCER + ACK_ALL + "failed-batches-total"; - private static final String ACK_ALL_RECORD_SEND_TOTAL = KAFKA_PRODUCER + ACK_ALL + "record-send"; - - private static final String ACK_LEADER_FAILED_BATCHES_TOTAL = KAFKA_PRODUCER + ACK_LEADER + "failed-batches-total"; - private static final String ACK_LEADER_BUFFER_TOTAL_BYTES = KAFKA_PRODUCER + ACK_LEADER + "buffer-total-bytes"; - private static final String ACK_LEADER_METADATA_AGE = KAFKA_PRODUCER + ACK_LEADER + "metadata-age"; - private static final String ACK_LEADER_RECORD_QUEUE_TIME_MAX = KAFKA_PRODUCER + ACK_LEADER + "record-queue-time-max"; - private static final String ACK_LEADER_BUFFER_AVAILABLE_BYTES = KAFKA_PRODUCER + ACK_LEADER + "buffer-available-bytes"; - private static final String ACK_LEADER_COMPRESSION_RATE = KAFKA_PRODUCER + ACK_LEADER + "compression-rate-avg"; - private static final String ACK_LEADER_RECORD_SEND_TOTAL = KAFKA_PRODUCER + ACK_LEADER + "record-send"; - + private final MeterRegistry meterRegistry; + private final GaugeRegistrar gaugeRegistrar; + + public ProducerMetrics(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + this.gaugeRegistrar = new GaugeRegistrar(meterRegistry); + } + + public void registerAckAllTotalBytesGauge( + T stateObj, ToDoubleFunction f, String sender, String datacenter) { + gaugeRegistrar.registerGauge(ACK_ALL_BUFFER_TOTAL_BYTES, stateObj, f, tags(sender, datacenter)); + } + + public void registerAckLeaderTotalBytesGauge( + T stateObj, ToDoubleFunction f, String sender, String datacenter) { + gaugeRegistrar.registerGauge( + ACK_LEADER_BUFFER_TOTAL_BYTES, stateObj, f, tags(sender, datacenter)); + } + + public void registerAckAllAvailableBytesGauge( + T stateObj, ToDoubleFunction f, String sender, String datacenter) { + gaugeRegistrar.registerGauge( + ACK_ALL_BUFFER_AVAILABLE_BYTES, stateObj, f, tags(sender, datacenter)); + } + + public void registerAckLeaderAvailableBytesGauge( + T stateObj, ToDoubleFunction f, String sender, String datacenter) { + gaugeRegistrar.registerGauge( + ACK_LEADER_BUFFER_AVAILABLE_BYTES, stateObj, f, tags(sender, datacenter)); + } + + public void registerAckAllCompressionRateGauge( + T stateObj, ToDoubleFunction f, String sender, String datacenter) { + gaugeRegistrar.registerGauge(ACK_ALL_COMPRESSION_RATE, stateObj, f, tags(sender, datacenter)); + } + + public void registerAckLeaderCompressionRateGauge( + T stateObj, ToDoubleFunction f, String sender, String datacenter) { + gaugeRegistrar.registerGauge( + ACK_LEADER_COMPRESSION_RATE, stateObj, f, tags(sender, datacenter)); + } + + public void registerAckAllFailedBatchesGauge( + T stateObj, ToDoubleFunction f, String sender, String datacenter) { + gaugeRegistrar.registerGauge( + ACK_ALL_FAILED_BATCHES_TOTAL, stateObj, f, tags(sender, datacenter)); + } + + public void registerAckLeaderFailedBatchesGauge( + T stateObj, ToDoubleFunction f, String sender, String datacenter) { + gaugeRegistrar.registerGauge( + ACK_LEADER_FAILED_BATCHES_TOTAL, stateObj, f, tags(sender, datacenter)); + } + + public void registerAckAllMetadataAgeGauge( + T stateObj, ToDoubleFunction f, String sender, String datacenter) { + registerTimeGauge( + stateObj, f, ACK_ALL_METADATA_AGE, tags(sender, datacenter), TimeUnit.SECONDS); + } + + public void registerAckLeaderMetadataAgeGauge( + T stateObj, ToDoubleFunction f, String sender, String datacenter) { + registerTimeGauge( + stateObj, f, ACK_LEADER_METADATA_AGE, tags(sender, datacenter), TimeUnit.SECONDS); + } + + public void registerAckAllRecordQueueTimeMaxGauge( + T stateObj, ToDoubleFunction f, String sender, String datacenter) { + registerTimeGauge( + stateObj, + f, + ACK_ALL_RECORD_QUEUE_TIME_MAX, + tags(sender, datacenter), + TimeUnit.MILLISECONDS); + } + + public void registerAckLeaderRecordQueueTimeMaxGauge( + T stateObj, ToDoubleFunction f, String sender, String datacenter) { + registerTimeGauge( + stateObj, + f, + ACK_LEADER_RECORD_QUEUE_TIME_MAX, + tags(sender, datacenter), + TimeUnit.MILLISECONDS); + } + + public double getBufferTotalBytes() { + return meterRegistry.get(ACK_ALL_BUFFER_TOTAL_BYTES).gauge().value() + + meterRegistry.get(ACK_LEADER_BUFFER_TOTAL_BYTES).gauge().value(); + } + + public double getBufferAvailableBytes() { + return meterRegistry.get(ACK_ALL_BUFFER_AVAILABLE_BYTES).gauge().value() + + meterRegistry.get(ACK_LEADER_BUFFER_AVAILABLE_BYTES).gauge().value(); + } + + public void registerAckLeaderRecordSendCounter( + T stateObj, ToDoubleFunction f, String sender, String datacenter) { + registerCounter(ACK_LEADER_RECORD_SEND_TOTAL, tags(sender, datacenter), stateObj, f); + } + + public void registerAckAllRecordSendCounter( + T stateObj, ToDoubleFunction f, String sender, String datacenter) { + registerCounter(ACK_ALL_RECORD_SEND_TOTAL, tags(sender, datacenter), stateObj, f); + } + + public void registerProducerInflightRequestGauge(T stateObj, ToDoubleFunction f) { + meterRegistry.gauge(INFLIGHT_REQUESTS, stateObj, f); + } + + private static Tags tags(String sender, String datacenter) { + return Tags.of("storageDc", datacenter, "sender", sender); + } + + private void registerTimeGauge( + T stateObj, ToDoubleFunction f, String name, Tags tags, TimeUnit timeUnit) { + meterRegistry.more().timeGauge(name, tags, stateObj, timeUnit, f); + } + + private void registerCounter(String name, Tags tags, T stateObj, ToDoubleFunction f) { + meterRegistry.more().counter(name, tags, stateObj, f); + } + + private static final String KAFKA_PRODUCER = "kafka-producer."; + private static final String ACK_LEADER = "ack-leader."; + private static final String ACK_ALL = "ack-all."; + + private static final String ACK_ALL_BUFFER_TOTAL_BYTES = + KAFKA_PRODUCER + ACK_ALL + "buffer-total-bytes"; + private static final String ACK_ALL_BUFFER_AVAILABLE_BYTES = + KAFKA_PRODUCER + ACK_ALL + "buffer-available-bytes"; + private static final String ACK_ALL_METADATA_AGE = KAFKA_PRODUCER + ACK_ALL + "metadata-age"; + private static final String ACK_ALL_RECORD_QUEUE_TIME_MAX = + KAFKA_PRODUCER + ACK_ALL + "record-queue-time-max"; + private static final String ACK_ALL_COMPRESSION_RATE = + KAFKA_PRODUCER + ACK_ALL + "compression-rate-avg"; + private static final String ACK_ALL_FAILED_BATCHES_TOTAL = + KAFKA_PRODUCER + ACK_ALL + "failed-batches-total"; + private static final String ACK_ALL_RECORD_SEND_TOTAL = KAFKA_PRODUCER + ACK_ALL + "record-send"; + + private static final String ACK_LEADER_FAILED_BATCHES_TOTAL = + KAFKA_PRODUCER + ACK_LEADER + "failed-batches-total"; + private static final String ACK_LEADER_BUFFER_TOTAL_BYTES = + KAFKA_PRODUCER + ACK_LEADER + "buffer-total-bytes"; + private static final String ACK_LEADER_METADATA_AGE = + KAFKA_PRODUCER + ACK_LEADER + "metadata-age"; + private static final String ACK_LEADER_RECORD_QUEUE_TIME_MAX = + KAFKA_PRODUCER + ACK_LEADER + "record-queue-time-max"; + private static final String ACK_LEADER_BUFFER_AVAILABLE_BYTES = + KAFKA_PRODUCER + ACK_LEADER + "buffer-available-bytes"; + private static final String ACK_LEADER_COMPRESSION_RATE = + KAFKA_PRODUCER + ACK_LEADER + "compression-rate-avg"; + private static final String ACK_LEADER_RECORD_SEND_TOTAL = + KAFKA_PRODUCER + ACK_LEADER + "record-send"; } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/SchemaClientMetrics.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/SchemaClientMetrics.java index 020bffcc83..40faff467c 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/SchemaClientMetrics.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/SchemaClientMetrics.java @@ -6,26 +6,21 @@ import pl.allegro.tech.hermes.metrics.HermesTimer; public class SchemaClientMetrics { - private final MeterRegistry meterRegistry; + private final MeterRegistry meterRegistry; - public SchemaClientMetrics(MeterRegistry meterRegistry) { - this.meterRegistry = meterRegistry; - } + public SchemaClientMetrics(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } - public HermesTimer schemaTimer() { - return HermesTimer.from( - timer("schema.get-schema") - ); - } + public HermesTimer schemaTimer() { + return HermesTimer.from(timer("schema.get-schema")); + } - public HermesTimer versionsTimer() { - return HermesTimer.from( - timer("schema.get-versions") - ); - } - - private Timer timer(String name) { - return meterRegistry.timer(name, Tags.of("schema_repo_type", "schema-registry")); - } + public HermesTimer versionsTimer() { + return HermesTimer.from(timer("schema.get-versions")); + } + private Timer timer(String name) { + return meterRegistry.timer(name, Tags.of("schema_repo_type", "schema-registry")); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/SubscriptionHermesCounter.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/SubscriptionHermesCounter.java index 2d9d237b5a..beae420143 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/SubscriptionHermesCounter.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/SubscriptionHermesCounter.java @@ -6,24 +6,23 @@ public class SubscriptionHermesCounter extends DefaultHermesCounter { - private final SubscriptionName subscription; + private final SubscriptionName subscription; - private SubscriptionHermesCounter(Counter micrometerCounter, - SubscriptionName subscription) { - super(micrometerCounter); - this.subscription = subscription; - } + private SubscriptionHermesCounter(Counter micrometerCounter, SubscriptionName subscription) { + super(micrometerCounter); + this.subscription = subscription; + } - public static SubscriptionHermesCounter from(Counter micrometerCounter, SubscriptionName subscription) { - return new SubscriptionHermesCounter(micrometerCounter, subscription); - } + public static SubscriptionHermesCounter from( + Counter micrometerCounter, SubscriptionName subscription) { + return new SubscriptionHermesCounter(micrometerCounter, subscription); + } + SubscriptionName getSubscription() { + return subscription; + } - SubscriptionName getSubscription() { - return subscription; - } - - Counter getMicrometerCounter() { - return micrometerCounter; - } + Counter getMicrometerCounter() { + return micrometerCounter; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/SubscriptionMetrics.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/SubscriptionMetrics.java index ee381390d1..dbb04dda39 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/SubscriptionMetrics.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/SubscriptionMetrics.java @@ -1,135 +1,147 @@ package pl.allegro.tech.hermes.common.metric; +import static pl.allegro.tech.hermes.common.metric.SubscriptionTagsFactory.subscriptionTags; + import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.DistributionSummary; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Tags; +import java.util.function.ToDoubleFunction; import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.metrics.HermesCounter; import pl.allegro.tech.hermes.metrics.HermesHistogram; import pl.allegro.tech.hermes.metrics.HermesTimer; import pl.allegro.tech.hermes.metrics.counters.HermesCounters; -import java.util.function.ToDoubleFunction; - -import static pl.allegro.tech.hermes.common.metric.SubscriptionTagsFactory.subscriptionTags; - public class SubscriptionMetrics { - private final MeterRegistry meterRegistry; - - public SubscriptionMetrics(MeterRegistry meterRegistry) { - this.meterRegistry = meterRegistry; - } - - public SubscriptionHermesCounter throughputInBytes(SubscriptionName subscription) { - return SubscriptionHermesCounter.from( - micrometerCounter(SubscriptionMetricsNames.SUBSCRIPTION_THROUGHPUT, subscription), subscription); - } - - public HermesCounter successes(SubscriptionName subscription) { - return size -> micrometerCounter(SubscriptionMetricsNames.SUBSCRIPTION_DELIVERED, subscription).increment(size); - } - - public HermesCounter batchSuccesses(SubscriptionName subscription) { - return HermesCounters.from( - micrometerCounter(SubscriptionMetricsNames.SUBSCRIPTION_BATCHES, subscription) - ); - } - - public HermesCounter discarded(SubscriptionName subscription) { - return size -> micrometerCounter(SubscriptionMetricsNames.SUBSCRIPTION_DISCARDED, subscription).increment(size); - - } - - public HermesCounter retries(SubscriptionName subscription) { - return size -> micrometerCounter(SubscriptionMetricsNames.SUBSCRIPTION_RETRIES, subscription).increment(size); - } - - public HermesTimer latency(SubscriptionName subscription) { - return HermesTimer.from( - meterRegistry.timer(SubscriptionMetricsNames.SUBSCRIPTION_LATENCY, subscriptionTags(subscription)) - ); - } - - public HermesTimer rateLimiterAcquire(SubscriptionName subscription) { - return HermesTimer.from( - meterRegistry.timer(SubscriptionMetricsNames.SUBSCRIPTION_RATE_LIMITER_ACQUIRE, subscriptionTags(subscription)) - ); - } - - public void registerInflightGauge(SubscriptionName subscription, T obj, ToDoubleFunction f) { - meterRegistry.gauge(SubscriptionMetricsNames.SUBSCRIPTION_INFLIGHT, subscriptionTags(subscription), obj, f); - } - - public void registerPendingOffsetsGauge(SubscriptionName subscription, T obj, ToDoubleFunction f) { - meterRegistry.gauge(SubscriptionMetricsNames.SUBSCRIPTION_PENDING_OFFSETS, subscriptionTags(subscription), obj, f); - } - - public HermesTimer consumerIdleTimer(SubscriptionName subscription) { - return HermesTimer.from( - meterRegistry.timer(SubscriptionMetricsNames.SUBSCRIPTION_IDLE_DURATION, subscriptionTags(subscription)) - ); - } - - public HermesCounter filteredOutCounter(SubscriptionName subscription) { - return HermesCounters.from( - micrometerCounter(SubscriptionMetricsNames.SUBSCRIPTION_FILTERED_OUT, subscription) - ); - } - - public HermesCounter httpAnswerCounter(SubscriptionName subscription, int statusCode) { - return size -> meterRegistry.counter( + private final MeterRegistry meterRegistry; + + public SubscriptionMetrics(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } + + public SubscriptionHermesCounter throughputInBytes(SubscriptionName subscription) { + return SubscriptionHermesCounter.from( + micrometerCounter(SubscriptionMetricsNames.SUBSCRIPTION_THROUGHPUT, subscription), + subscription); + } + + public HermesCounter successes(SubscriptionName subscription) { + return size -> + micrometerCounter(SubscriptionMetricsNames.SUBSCRIPTION_DELIVERED, subscription) + .increment(size); + } + + public HermesCounter batchSuccesses(SubscriptionName subscription) { + return HermesCounters.from( + micrometerCounter(SubscriptionMetricsNames.SUBSCRIPTION_BATCHES, subscription)); + } + + public HermesCounter discarded(SubscriptionName subscription) { + return size -> + micrometerCounter(SubscriptionMetricsNames.SUBSCRIPTION_DISCARDED, subscription) + .increment(size); + } + + public HermesCounter retries(SubscriptionName subscription) { + return size -> + micrometerCounter(SubscriptionMetricsNames.SUBSCRIPTION_RETRIES, subscription) + .increment(size); + } + + public HermesTimer latency(SubscriptionName subscription) { + return HermesTimer.from( + meterRegistry.timer( + SubscriptionMetricsNames.SUBSCRIPTION_LATENCY, subscriptionTags(subscription))); + } + + public HermesTimer rateLimiterAcquire(SubscriptionName subscription) { + return HermesTimer.from( + meterRegistry.timer( + SubscriptionMetricsNames.SUBSCRIPTION_RATE_LIMITER_ACQUIRE, + subscriptionTags(subscription))); + } + + public void registerInflightGauge( + SubscriptionName subscription, T obj, ToDoubleFunction f) { + meterRegistry.gauge( + SubscriptionMetricsNames.SUBSCRIPTION_INFLIGHT, subscriptionTags(subscription), obj, f); + } + + public void registerPendingOffsetsGauge( + SubscriptionName subscription, T obj, ToDoubleFunction f) { + meterRegistry.gauge( + SubscriptionMetricsNames.SUBSCRIPTION_PENDING_OFFSETS, + subscriptionTags(subscription), + obj, + f); + } + + public HermesTimer consumerIdleTimer(SubscriptionName subscription) { + return HermesTimer.from( + meterRegistry.timer( + SubscriptionMetricsNames.SUBSCRIPTION_IDLE_DURATION, subscriptionTags(subscription))); + } + + public HermesCounter filteredOutCounter(SubscriptionName subscription) { + return HermesCounters.from( + micrometerCounter(SubscriptionMetricsNames.SUBSCRIPTION_FILTERED_OUT, subscription)); + } + + public HermesCounter httpAnswerCounter(SubscriptionName subscription, int statusCode) { + return size -> + meterRegistry + .counter( SubscriptionMetricsNames.SUBSCRIPTION_HTTP_STATUS_CODES, - Tags.concat(subscriptionTags(subscription), "status_code", String.valueOf(statusCode)) - ).increment(size); - } - - public HermesCounter timeoutsCounter(SubscriptionName subscription) { - return HermesCounters.from( - micrometerCounter(SubscriptionMetricsNames.SUBSCRIPTION_TIMEOUTS, subscription) - ); - } - - public HermesCounter otherErrorsCounter(SubscriptionName subscription) { - return HermesCounters.from( - micrometerCounter(SubscriptionMetricsNames.SUBSCRIPTION_OTHER_ERRORS, subscription) - ); - } - - public HermesCounter failuresCounter(SubscriptionName subscription) { - return HermesCounters.from( - micrometerCounter(SubscriptionMetricsNames.SUBSCRIPTION_FAILURES, subscription) - ); - } - - public HermesHistogram inflightTimeInMillisHistogram(SubscriptionName subscriptionName) { - return value -> DistributionSummary.builder(SubscriptionMetricsNames.SUBSCRIPTION_INFLIGHT_TIME) - .tags(subscriptionTags(subscriptionName)) - .register(meterRegistry) - .record(value / 1000d); - } - - private Counter micrometerCounter(String metricName, SubscriptionName subscription) { - return meterRegistry.counter(metricName, subscriptionTags(subscription)); - } - - public static class SubscriptionMetricsNames { - public static final String SUBSCRIPTION_DELIVERED = "subscription.delivered"; - public static final String SUBSCRIPTION_THROUGHPUT = "subscription.throughput-bytes"; - public static final String SUBSCRIPTION_BATCHES = "subscription.batches"; - public static final String SUBSCRIPTION_DISCARDED = "subscription.discarded"; - public static final String SUBSCRIPTION_RETRIES = "subscription.retries"; - public static final String SUBSCRIPTION_LATENCY = "subscription.latency"; - public static final String SUBSCRIPTION_RATE_LIMITER_ACQUIRE = "subscription.rate-limiter-acquire"; - public static final String SUBSCRIPTION_INFLIGHT = "subscription.inflight"; - public static final String SUBSCRIPTION_PENDING_OFFSETS = "subscription.pending-offsets"; - public static final String SUBSCRIPTION_IDLE_DURATION = "subscription.idle-duration"; - public static final String SUBSCRIPTION_FILTERED_OUT = "subscription.filtered-out"; - public static final String SUBSCRIPTION_HTTP_STATUS_CODES = "subscription.http-status-codes"; - public static final String SUBSCRIPTION_TIMEOUTS = "subscription.timeouts"; - public static final String SUBSCRIPTION_OTHER_ERRORS = "subscription.other-errors"; - public static final String SUBSCRIPTION_FAILURES = "subscription.failures"; - public static final String SUBSCRIPTION_INFLIGHT_TIME = "subscription.inflight-time-seconds"; - } - + Tags.concat( + subscriptionTags(subscription), "status_code", String.valueOf(statusCode))) + .increment(size); + } + + public HermesCounter timeoutsCounter(SubscriptionName subscription) { + return HermesCounters.from( + micrometerCounter(SubscriptionMetricsNames.SUBSCRIPTION_TIMEOUTS, subscription)); + } + + public HermesCounter otherErrorsCounter(SubscriptionName subscription) { + return HermesCounters.from( + micrometerCounter(SubscriptionMetricsNames.SUBSCRIPTION_OTHER_ERRORS, subscription)); + } + + public HermesCounter failuresCounter(SubscriptionName subscription) { + return HermesCounters.from( + micrometerCounter(SubscriptionMetricsNames.SUBSCRIPTION_FAILURES, subscription)); + } + + public HermesHistogram inflightTimeInMillisHistogram(SubscriptionName subscriptionName) { + return value -> + DistributionSummary.builder(SubscriptionMetricsNames.SUBSCRIPTION_INFLIGHT_TIME) + .tags(subscriptionTags(subscriptionName)) + .register(meterRegistry) + .record(value / 1000d); + } + + private Counter micrometerCounter(String metricName, SubscriptionName subscription) { + return meterRegistry.counter(metricName, subscriptionTags(subscription)); + } + + public static class SubscriptionMetricsNames { + public static final String SUBSCRIPTION_DELIVERED = "subscription.delivered"; + public static final String SUBSCRIPTION_THROUGHPUT = "subscription.throughput-bytes"; + public static final String SUBSCRIPTION_BATCHES = "subscription.batches"; + public static final String SUBSCRIPTION_DISCARDED = "subscription.discarded"; + public static final String SUBSCRIPTION_RETRIES = "subscription.retries"; + public static final String SUBSCRIPTION_LATENCY = "subscription.latency"; + public static final String SUBSCRIPTION_RATE_LIMITER_ACQUIRE = + "subscription.rate-limiter-acquire"; + public static final String SUBSCRIPTION_INFLIGHT = "subscription.inflight"; + public static final String SUBSCRIPTION_PENDING_OFFSETS = "subscription.pending-offsets"; + public static final String SUBSCRIPTION_IDLE_DURATION = "subscription.idle-duration"; + public static final String SUBSCRIPTION_FILTERED_OUT = "subscription.filtered-out"; + public static final String SUBSCRIPTION_HTTP_STATUS_CODES = "subscription.http-status-codes"; + public static final String SUBSCRIPTION_TIMEOUTS = "subscription.timeouts"; + public static final String SUBSCRIPTION_OTHER_ERRORS = "subscription.other-errors"; + public static final String SUBSCRIPTION_FAILURES = "subscription.failures"; + public static final String SUBSCRIPTION_INFLIGHT_TIME = "subscription.inflight-time-seconds"; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/SubscriptionTagsFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/SubscriptionTagsFactory.java index 931e21dc5f..47c8766f0e 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/SubscriptionTagsFactory.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/SubscriptionTagsFactory.java @@ -1,17 +1,15 @@ package pl.allegro.tech.hermes.common.metric; import io.micrometer.core.instrument.Tag; -import pl.allegro.tech.hermes.api.SubscriptionName; - import java.util.Set; +import pl.allegro.tech.hermes.api.SubscriptionName; class SubscriptionTagsFactory { - static Set subscriptionTags(SubscriptionName subscriptionName) { - return Set.of( - Tag.of("group", subscriptionName.getTopicName().getGroupName()), - Tag.of("topic", subscriptionName.getTopicName().getName()), - Tag.of("subscription", subscriptionName.getName()) - ); - } + static Set subscriptionTags(SubscriptionName subscriptionName) { + return Set.of( + Tag.of("group", subscriptionName.getTopicName().getGroupName()), + Tag.of("topic", subscriptionName.getTopicName().getName()), + Tag.of("subscription", subscriptionName.getName())); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/TopicMetrics.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/TopicMetrics.java index e7ae1dcf89..f359ff7799 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/TopicMetrics.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/TopicMetrics.java @@ -14,152 +14,130 @@ import pl.allegro.tech.hermes.metrics.counters.HermesCounters; public class TopicMetrics { - private final MeterRegistry meterRegistry; - - public TopicMetrics(MeterRegistry meterRegistry) { - this.meterRegistry = meterRegistry; - } - - public HermesTimer ackAllGlobalLatency() { - return HermesTimer.from( - meterRegistry.timer(TopicMetricsNames.TOPIC_ACK_ALL_GLOBAL_LATENCY) - ); - } - - public HermesTimer ackAllTopicLatency(TopicName topic) { - return HermesTimer.from( - micrometerTimer(TopicMetricsNames.TOPIC_ACK_ALL_LATENCY, topic)); - } - - public HermesTimer ackAllBrokerLatency() { - return HermesTimer.from( - meterRegistry.timer(TopicMetricsNames.TOPIC_ACK_ALL_BROKER_LATENCY)); - } - - public HermesTimer ackLeaderGlobalLatency() { - return HermesTimer.from( - meterRegistry.timer(TopicMetricsNames.TOPIC_ACK_LEADER_GLOBAL_LATENCY)); - } - - public HermesTimer ackLeaderTopicLatency(TopicName topic) { - return HermesTimer.from( - micrometerTimer(TopicMetricsNames.TOPIC_ACK_LEADER_LATENCY, topic)); - } - - public HermesTimer ackLeaderBrokerLatency() { - return HermesTimer.from( - meterRegistry.timer(TopicMetricsNames.TOPIC_ACK_LEADER_BROKER_LATENCY)); - } - - public HermesCounter topicThroughputBytes(TopicName topicName) { - return HermesCounters.from( - micrometerCounter(TopicMetricsNames.TOPIC_THROUGHPUT, topicName) - ); - } - - public HermesCounter topicGlobalThroughputBytes() { - return HermesCounters.from( - meterRegistry.counter(TopicMetricsNames.TOPIC_GLOBAL_THROUGHPUT) - ); - } - - public HermesCounter topicPublished(TopicName topicName, String datacenter) { - return HermesCounters.from( - micrometerCounter(TopicMetricsNames.TOPIC_PUBLISHED, topicName, Tag.of("storageDc", datacenter)) - ); - } - - public HermesCounter topicGlobalRequestCounter() { - return HermesCounters.from( - meterRegistry.counter(TopicMetricsNames.TOPIC_GLOBAL_REQUESTS) - ); - } - - public HermesCounter topicRequestCounter(TopicName topicName) { - return HermesCounters.from( - micrometerCounter(TopicMetricsNames.TOPIC_REQUESTS, topicName) - ); - } - - public HermesCounter topicGlobalDelayedProcessingCounter() { - return HermesCounters.from( - meterRegistry.counter(TopicMetricsNames.TOPIC_GLOBAL_DELAYED_PROCESSING) - ); - } - - public HermesCounter topicDelayedProcessingCounter(TopicName topicName) { - return HermesCounters.from( - micrometerCounter(TopicMetricsNames.TOPIC_DELAYED_PROCESSING, topicName) - ); - } - - public HermesCounter topicGlobalHttpStatusCodeCounter(int statusCode) { - return HermesCounters.from( - meterRegistry.counter(TopicMetricsNames.TOPIC_GLOBAL_HTTP_STATUS_CODES, Tags.of("status_code", String.valueOf(statusCode))) - ); - } - - public HermesCounter topicHttpStatusCodeCounter(TopicName topicName, int statusCode) { - return HermesCounters.from( - meterRegistry.counter(TopicMetricsNames.TOPIC_HTTP_STATUS_CODES, topicTags(topicName) - .and("status_code", String.valueOf(statusCode))) - ); - } - - public HermesCounter topicDuplicatedMessageCounter(TopicName topicName) { - return HermesCounters.from( - micrometerCounter(TopicMetricsNames.TOPIC_DUPLICATED_MESSAGE, topicName) - ); - } - - public HermesHistogram topicGlobalMessageContentSizeHistogram() { - return DefaultHermesHistogram.of( - DistributionSummary.builder(TopicMetricsNames.TOPIC_GLOBAL_MESSAGE_SIZE_BYTES) - .register(meterRegistry) - ); - } - - public HermesHistogram topicMessageContentSizeHistogram(TopicName topicName) { - return DefaultHermesHistogram.of( - DistributionSummary.builder(TopicMetricsNames.TOPIC_MESSAGE_SIZE_BYTES) - .tags(topicTags(topicName)) - .register(meterRegistry) - ); - } - - private Timer micrometerTimer(String metricName, TopicName topicName) { - return meterRegistry.timer(metricName, topicTags(topicName)); - } - - private Counter micrometerCounter(String metricName, TopicName topicName, Tag ... tags) { - return meterRegistry.counter(metricName, topicTags(topicName).and(tags)); - } - - private Tags topicTags(TopicName topicName) { - return Tags.of( - Tag.of("group", topicName.getGroupName()), - Tag.of("topic", topicName.getName()) - ); - } - - public static class TopicMetricsNames { - public static final String TOPIC_ACK_ALL_GLOBAL_LATENCY = "topic.ack-all.global-latency"; - public static final String TOPIC_ACK_ALL_LATENCY = "topic.ack-all.latency"; - public static final String TOPIC_ACK_ALL_BROKER_LATENCY = "topic.ack-all.broker-latency"; - public static final String TOPIC_ACK_LEADER_GLOBAL_LATENCY = "topic.ack-leader.global-latency"; - public static final String TOPIC_ACK_LEADER_LATENCY = "topic.ack-leader.latency"; - public static final String TOPIC_ACK_LEADER_BROKER_LATENCY = "topic.ack-leader.broker-latency"; - public static final String TOPIC_THROUGHPUT = "topic.throughput-bytes"; - public static final String TOPIC_GLOBAL_THROUGHPUT = "topic.global-throughput-bytes"; - public static final String TOPIC_PUBLISHED = "topic.published"; - public static final String TOPIC_GLOBAL_REQUESTS = "topic.global-requests"; - public static final String TOPIC_REQUESTS = "topic.requests"; - public static final String TOPIC_GLOBAL_DELAYED_PROCESSING = "topic-global-delayed-processing"; - public static final String TOPIC_DELAYED_PROCESSING = "topic-delayed-processing"; - public static final String TOPIC_GLOBAL_HTTP_STATUS_CODES = "topic-global-http-status-codes"; - public static final String TOPIC_HTTP_STATUS_CODES = "topic-http-status-codes"; - public static final String TOPIC_GLOBAL_MESSAGE_SIZE_BYTES = "topic-global-message-size-bytes"; - public static final String TOPIC_MESSAGE_SIZE_BYTES = "topic-message-size-bytes"; - public static final String TOPIC_DUPLICATED_MESSAGE = "topic-duplicated-message"; - } + private final MeterRegistry meterRegistry; + + public TopicMetrics(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } + + public HermesTimer ackAllGlobalLatency() { + return HermesTimer.from(meterRegistry.timer(TopicMetricsNames.TOPIC_ACK_ALL_GLOBAL_LATENCY)); + } + + public HermesTimer ackAllTopicLatency(TopicName topic) { + return HermesTimer.from(micrometerTimer(TopicMetricsNames.TOPIC_ACK_ALL_LATENCY, topic)); + } + + public HermesTimer ackAllBrokerLatency() { + return HermesTimer.from(meterRegistry.timer(TopicMetricsNames.TOPIC_ACK_ALL_BROKER_LATENCY)); + } + + public HermesTimer ackLeaderGlobalLatency() { + return HermesTimer.from(meterRegistry.timer(TopicMetricsNames.TOPIC_ACK_LEADER_GLOBAL_LATENCY)); + } + + public HermesTimer ackLeaderTopicLatency(TopicName topic) { + return HermesTimer.from(micrometerTimer(TopicMetricsNames.TOPIC_ACK_LEADER_LATENCY, topic)); + } + + public HermesTimer ackLeaderBrokerLatency() { + return HermesTimer.from(meterRegistry.timer(TopicMetricsNames.TOPIC_ACK_LEADER_BROKER_LATENCY)); + } + + public HermesCounter topicThroughputBytes(TopicName topicName) { + return HermesCounters.from(micrometerCounter(TopicMetricsNames.TOPIC_THROUGHPUT, topicName)); + } + + public HermesCounter topicGlobalThroughputBytes() { + return HermesCounters.from(meterRegistry.counter(TopicMetricsNames.TOPIC_GLOBAL_THROUGHPUT)); + } + + public HermesCounter topicPublished(TopicName topicName, String datacenter) { + return HermesCounters.from( + micrometerCounter( + TopicMetricsNames.TOPIC_PUBLISHED, topicName, Tag.of("storageDc", datacenter))); + } + + public HermesCounter topicGlobalRequestCounter() { + return HermesCounters.from(meterRegistry.counter(TopicMetricsNames.TOPIC_GLOBAL_REQUESTS)); + } + + public HermesCounter topicRequestCounter(TopicName topicName) { + return HermesCounters.from(micrometerCounter(TopicMetricsNames.TOPIC_REQUESTS, topicName)); + } + + public HermesCounter topicGlobalDelayedProcessingCounter() { + return HermesCounters.from( + meterRegistry.counter(TopicMetricsNames.TOPIC_GLOBAL_DELAYED_PROCESSING)); + } + + public HermesCounter topicDelayedProcessingCounter(TopicName topicName) { + return HermesCounters.from( + micrometerCounter(TopicMetricsNames.TOPIC_DELAYED_PROCESSING, topicName)); + } + + public HermesCounter topicGlobalHttpStatusCodeCounter(int statusCode) { + return HermesCounters.from( + meterRegistry.counter( + TopicMetricsNames.TOPIC_GLOBAL_HTTP_STATUS_CODES, + Tags.of("status_code", String.valueOf(statusCode)))); + } + + public HermesCounter topicHttpStatusCodeCounter(TopicName topicName, int statusCode) { + return HermesCounters.from( + meterRegistry.counter( + TopicMetricsNames.TOPIC_HTTP_STATUS_CODES, + topicTags(topicName).and("status_code", String.valueOf(statusCode)))); + } + + public HermesCounter topicDuplicatedMessageCounter(TopicName topicName) { + return HermesCounters.from( + micrometerCounter(TopicMetricsNames.TOPIC_DUPLICATED_MESSAGE, topicName)); + } + + public HermesHistogram topicGlobalMessageContentSizeHistogram() { + return DefaultHermesHistogram.of( + DistributionSummary.builder(TopicMetricsNames.TOPIC_GLOBAL_MESSAGE_SIZE_BYTES) + .register(meterRegistry)); + } + + public HermesHistogram topicMessageContentSizeHistogram(TopicName topicName) { + return DefaultHermesHistogram.of( + DistributionSummary.builder(TopicMetricsNames.TOPIC_MESSAGE_SIZE_BYTES) + .tags(topicTags(topicName)) + .register(meterRegistry)); + } + + private Timer micrometerTimer(String metricName, TopicName topicName) { + return meterRegistry.timer(metricName, topicTags(topicName)); + } + + private Counter micrometerCounter(String metricName, TopicName topicName, Tag... tags) { + return meterRegistry.counter(metricName, topicTags(topicName).and(tags)); + } + + private Tags topicTags(TopicName topicName) { + return Tags.of(Tag.of("group", topicName.getGroupName()), Tag.of("topic", topicName.getName())); + } + + public static class TopicMetricsNames { + public static final String TOPIC_ACK_ALL_GLOBAL_LATENCY = "topic.ack-all.global-latency"; + public static final String TOPIC_ACK_ALL_LATENCY = "topic.ack-all.latency"; + public static final String TOPIC_ACK_ALL_BROKER_LATENCY = "topic.ack-all.broker-latency"; + public static final String TOPIC_ACK_LEADER_GLOBAL_LATENCY = "topic.ack-leader.global-latency"; + public static final String TOPIC_ACK_LEADER_LATENCY = "topic.ack-leader.latency"; + public static final String TOPIC_ACK_LEADER_BROKER_LATENCY = "topic.ack-leader.broker-latency"; + public static final String TOPIC_THROUGHPUT = "topic.throughput-bytes"; + public static final String TOPIC_GLOBAL_THROUGHPUT = "topic.global-throughput-bytes"; + public static final String TOPIC_PUBLISHED = "topic.published"; + public static final String TOPIC_GLOBAL_REQUESTS = "topic.global-requests"; + public static final String TOPIC_REQUESTS = "topic.requests"; + public static final String TOPIC_GLOBAL_DELAYED_PROCESSING = "topic-global-delayed-processing"; + public static final String TOPIC_DELAYED_PROCESSING = "topic-delayed-processing"; + public static final String TOPIC_GLOBAL_HTTP_STATUS_CODES = "topic-global-http-status-codes"; + public static final String TOPIC_HTTP_STATUS_CODES = "topic-http-status-codes"; + public static final String TOPIC_GLOBAL_MESSAGE_SIZE_BYTES = "topic-global-message-size-bytes"; + public static final String TOPIC_MESSAGE_SIZE_BYTES = "topic-message-size-bytes"; + public static final String TOPIC_DUPLICATED_MESSAGE = "topic-duplicated-message"; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/TrackerElasticSearchMetrics.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/TrackerElasticSearchMetrics.java index 657fb68c1e..139700391a 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/TrackerElasticSearchMetrics.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/TrackerElasticSearchMetrics.java @@ -1,59 +1,51 @@ package pl.allegro.tech.hermes.common.metric; import io.micrometer.core.instrument.MeterRegistry; -import pl.allegro.tech.hermes.metrics.HermesTimer; - import java.util.function.ToDoubleFunction; +import pl.allegro.tech.hermes.metrics.HermesTimer; public class TrackerElasticSearchMetrics { - private final MeterRegistry meterRegistry; - private final GaugeRegistrar gaugeRegistrar; - - public TrackerElasticSearchMetrics(MeterRegistry meterRegistry) { - this.meterRegistry = meterRegistry; - this.gaugeRegistrar = new GaugeRegistrar(meterRegistry); - } - - public void registerProducerTrackerElasticSearchQueueSizeGauge(T stateObj, ToDoubleFunction f) { - gaugeRegistrar.registerGauge( - Gauges.TRACKER_ELASTICSEARCH_QUEUE_SIZE, - stateObj, f - ); - } - - public void registerProducerTrackerElasticSearchRemainingCapacity(T stateObj, ToDoubleFunction f) { - gaugeRegistrar.registerGauge( - Gauges.TRACKER_ELASTICSEARCH_REMAINING_CAPACITY, - stateObj, f - ); - } - - public void registerConsumerTrackerElasticSearchQueueSizeGauge(T stateObj, ToDoubleFunction f) { - gaugeRegistrar.registerGauge( - Gauges.TRACKER_ELASTICSEARCH_QUEUE_SIZE, - stateObj, f - ); - } - - public void registerConsumerTrackerElasticSearchRemainingCapacity(T stateObj, ToDoubleFunction f) { - gaugeRegistrar.registerGauge( - Gauges.TRACKER_ELASTICSEARCH_REMAINING_CAPACITY, - stateObj, f - ); - } - - public HermesTimer trackerElasticSearchCommitLatencyTimer() { - return HermesTimer.from( - meterRegistry.timer(Timers.ELASTICSEARCH_COMMIT_LATENCY) - ); - } - - private static class Gauges { - public static final String TRACKER_ELASTICSEARCH_QUEUE_SIZE = "tracker.elasticsearch.queue-size"; - public static final String TRACKER_ELASTICSEARCH_REMAINING_CAPACITY = "tracker.elasticsearch.remaining-capacity"; - } - - private static class Timers { - public static final String ELASTICSEARCH_COMMIT_LATENCY = "tracker.elasticsearch.commit-latency"; - } + private final MeterRegistry meterRegistry; + private final GaugeRegistrar gaugeRegistrar; + + public TrackerElasticSearchMetrics(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + this.gaugeRegistrar = new GaugeRegistrar(meterRegistry); + } + + public void registerProducerTrackerElasticSearchQueueSizeGauge( + T stateObj, ToDoubleFunction f) { + gaugeRegistrar.registerGauge(Gauges.TRACKER_ELASTICSEARCH_QUEUE_SIZE, stateObj, f); + } + + public void registerProducerTrackerElasticSearchRemainingCapacity( + T stateObj, ToDoubleFunction f) { + gaugeRegistrar.registerGauge(Gauges.TRACKER_ELASTICSEARCH_REMAINING_CAPACITY, stateObj, f); + } + + public void registerConsumerTrackerElasticSearchQueueSizeGauge( + T stateObj, ToDoubleFunction f) { + gaugeRegistrar.registerGauge(Gauges.TRACKER_ELASTICSEARCH_QUEUE_SIZE, stateObj, f); + } + + public void registerConsumerTrackerElasticSearchRemainingCapacity( + T stateObj, ToDoubleFunction f) { + gaugeRegistrar.registerGauge(Gauges.TRACKER_ELASTICSEARCH_REMAINING_CAPACITY, stateObj, f); + } + + public HermesTimer trackerElasticSearchCommitLatencyTimer() { + return HermesTimer.from(meterRegistry.timer(Timers.ELASTICSEARCH_COMMIT_LATENCY)); + } + + private static class Gauges { + public static final String TRACKER_ELASTICSEARCH_QUEUE_SIZE = + "tracker.elasticsearch.queue-size"; + public static final String TRACKER_ELASTICSEARCH_REMAINING_CAPACITY = + "tracker.elasticsearch.remaining-capacity"; + } + + private static class Timers { + public static final String ELASTICSEARCH_COMMIT_LATENCY = + "tracker.elasticsearch.commit-latency"; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/UndeliveredMessagesMetrics.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/UndeliveredMessagesMetrics.java index 3a286a0bf3..0148d9ef57 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/UndeliveredMessagesMetrics.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/UndeliveredMessagesMetrics.java @@ -8,22 +8,19 @@ import pl.allegro.tech.hermes.metrics.counters.HermesCounters; public class UndeliveredMessagesMetrics { - private final MeterRegistry meterRegistry; + private final MeterRegistry meterRegistry; - public UndeliveredMessagesMetrics(MeterRegistry meterRegistry) { - this.meterRegistry = meterRegistry; - } + public UndeliveredMessagesMetrics(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } - public HermesCounter undeliveredMessagesCounter() { - return HermesCounters.from( - meterRegistry.counter("undelivered-messages.persisted") - ); - } + public HermesCounter undeliveredMessagesCounter() { + return HermesCounters.from(meterRegistry.counter("undelivered-messages.persisted")); + } - public HermesHistogram undeliveredMessagesSizeHistogram() { - return DefaultHermesHistogram.of( - DistributionSummary.builder("undelivered-messages.persisted.message-size.bytes") - .register(meterRegistry) - ); - } + public HermesHistogram undeliveredMessagesSizeHistogram() { + return DefaultHermesHistogram.of( + DistributionSummary.builder("undelivered-messages.persisted.message-size.bytes") + .register(meterRegistry)); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/WorkloadMetrics.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/WorkloadMetrics.java index 1faa3486c3..fd14858968 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/WorkloadMetrics.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/WorkloadMetrics.java @@ -4,143 +4,109 @@ import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Tags; import io.micrometer.core.instrument.search.Search; -import pl.allegro.tech.hermes.metrics.HermesTimer; - import java.util.Collection; import java.util.Set; import java.util.function.ToDoubleFunction; +import pl.allegro.tech.hermes.metrics.HermesTimer; public class WorkloadMetrics { - private static final String CONSUMER_ID_TAG = "consumer-id"; - private static final String KAFKA_CLUSTER_TAG = "kafka-cluster"; - - private final MeterRegistry meterRegistry; - private final GaugeRegistrar gaugeRegistrar; - - WorkloadMetrics(MeterRegistry meterRegistry) { - this.meterRegistry = meterRegistry; - this.gaugeRegistrar = new GaugeRegistrar(meterRegistry); - } - - public void registerAllAssignmentsGauge(T obj, String kafkaCluster, ToDoubleFunction f) { - gaugeRegistrar.registerGauge( - "workload.all-assignments", - obj, - f, - Tags.of(KAFKA_CLUSTER_TAG, kafkaCluster) - ); - } - - public void registerMissingResourcesGauge(T obj, String kafkaCluster, ToDoubleFunction f) { - gaugeRegistrar.registerGauge( - "workload.missing-resources", - obj, - f, - Tags.of(KAFKA_CLUSTER_TAG, kafkaCluster) - ); - } - - public void registerDeletedAssignmentsGauge(T obj, String kafkaCluster, ToDoubleFunction f) { - gaugeRegistrar.registerGauge( - "workload.deleted-assignments", - obj, - f, - Tags.of(KAFKA_CLUSTER_TAG, kafkaCluster) - ); - } - - public void registerCreatedAssignmentsGauge(T obj, String kafkaCluster, ToDoubleFunction f) { - gaugeRegistrar.registerGauge( - "workload.created-assignments", - obj, - f, - Tags.of(KAFKA_CLUSTER_TAG, kafkaCluster) - ); - } - - public HermesTimer rebalanceDurationTimer(String kafkaCluster) { - return HermesTimer.from( - meterRegistry.timer("workload.rebalance-duration", Tags.of(KAFKA_CLUSTER_TAG, kafkaCluster)) - ); - } - - public void registerRunningSubscriptionsGauge(T obj, ToDoubleFunction f) { - gaugeRegistrar.registerGauge("workload.subscriptions.running", obj, f); - } - - public void registerAssignedSubscriptionsGauge(T obj, ToDoubleFunction f) { - gaugeRegistrar.registerGauge("workload.subscriptions.assigned", obj, f); - } - - public void registerMissingSubscriptionsGauge(T obj, ToDoubleFunction f) { - gaugeRegistrar.registerGauge("workload.subscriptions.missing", obj, f); - } - - public void registerOversubscribedGauge(T obj, ToDoubleFunction f) { - gaugeRegistrar.registerGauge("workload.subscriptions.oversubscribed", obj, f); - } - - public void registerOperationsPerSecondGauge(T obj, ToDoubleFunction f) { - gaugeRegistrar.registerGauge("workload.weighted.ops", obj, f); - } - - public void registerCpuUtilizationGauge(T obj, ToDoubleFunction f) { - gaugeRegistrar.registerGauge("workload.weighted.cpu-utilization", obj, f); - } - - public void registerCurrentScoreGauge(String consumerId, T obj, ToDoubleFunction f) { - gaugeRegistrar.registerGauge( - "workload.weighted.current-score", - obj, - f, - Tags.of(CONSUMER_ID_TAG, consumerId) - ); - } - - public void registerProposedErrorGauge(String consumerId, T obj, ToDoubleFunction f) { - gaugeRegistrar.registerGauge( - "workload.weighted.proposed-error", - obj, - f, - Tags.of(CONSUMER_ID_TAG, consumerId) - ); - } - - public void registerScoringErrorGauge(String consumerId, T obj, ToDoubleFunction f) { - gaugeRegistrar.registerGauge( - "workload.weighted.scoring-error", - obj, - f, - Tags.of(CONSUMER_ID_TAG, consumerId) - ); - } - - public void registerCurrentWeightGauge(String consumerId, T obj, ToDoubleFunction f) { - gaugeRegistrar.registerGauge( - "workload.weighted.current-weight.ops", - obj, - f, - Tags.of(CONSUMER_ID_TAG, consumerId) - ); - } - - public void registerProposedWeightGauge(String consumerId, T obj, ToDoubleFunction f) { - gaugeRegistrar.registerGauge( - "workload.weighted.proposed-weight.ops", - obj, - f, - Tags.of(CONSUMER_ID_TAG, consumerId) - ); - } - - public void unregisterAllWorkloadWeightedGaugesForConsumerIds(Set consumerIds) { - Collection gauges = Search.in(meterRegistry) - .tag(CONSUMER_ID_TAG, consumerIds::contains) - .name(s -> s.startsWith("workload.weighted")) - .gauges(); - for (Gauge gauge : gauges) { - meterRegistry.remove(gauge); - } - } + private static final String CONSUMER_ID_TAG = "consumer-id"; + private static final String KAFKA_CLUSTER_TAG = "kafka-cluster"; + + private final MeterRegistry meterRegistry; + private final GaugeRegistrar gaugeRegistrar; + + WorkloadMetrics(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + this.gaugeRegistrar = new GaugeRegistrar(meterRegistry); + } + + public void registerAllAssignmentsGauge(T obj, String kafkaCluster, ToDoubleFunction f) { + gaugeRegistrar.registerGauge( + "workload.all-assignments", obj, f, Tags.of(KAFKA_CLUSTER_TAG, kafkaCluster)); + } + + public void registerMissingResourcesGauge(T obj, String kafkaCluster, ToDoubleFunction f) { + gaugeRegistrar.registerGauge( + "workload.missing-resources", obj, f, Tags.of(KAFKA_CLUSTER_TAG, kafkaCluster)); + } + + public void registerDeletedAssignmentsGauge( + T obj, String kafkaCluster, ToDoubleFunction f) { + gaugeRegistrar.registerGauge( + "workload.deleted-assignments", obj, f, Tags.of(KAFKA_CLUSTER_TAG, kafkaCluster)); + } + + public void registerCreatedAssignmentsGauge( + T obj, String kafkaCluster, ToDoubleFunction f) { + gaugeRegistrar.registerGauge( + "workload.created-assignments", obj, f, Tags.of(KAFKA_CLUSTER_TAG, kafkaCluster)); + } + + public HermesTimer rebalanceDurationTimer(String kafkaCluster) { + return HermesTimer.from( + meterRegistry.timer( + "workload.rebalance-duration", Tags.of(KAFKA_CLUSTER_TAG, kafkaCluster))); + } + + public void registerRunningSubscriptionsGauge(T obj, ToDoubleFunction f) { + gaugeRegistrar.registerGauge("workload.subscriptions.running", obj, f); + } + + public void registerAssignedSubscriptionsGauge(T obj, ToDoubleFunction f) { + gaugeRegistrar.registerGauge("workload.subscriptions.assigned", obj, f); + } + + public void registerMissingSubscriptionsGauge(T obj, ToDoubleFunction f) { + gaugeRegistrar.registerGauge("workload.subscriptions.missing", obj, f); + } + + public void registerOversubscribedGauge(T obj, ToDoubleFunction f) { + gaugeRegistrar.registerGauge("workload.subscriptions.oversubscribed", obj, f); + } + + public void registerOperationsPerSecondGauge(T obj, ToDoubleFunction f) { + gaugeRegistrar.registerGauge("workload.weighted.ops", obj, f); + } + + public void registerCpuUtilizationGauge(T obj, ToDoubleFunction f) { + gaugeRegistrar.registerGauge("workload.weighted.cpu-utilization", obj, f); + } + + public void registerCurrentScoreGauge(String consumerId, T obj, ToDoubleFunction f) { + gaugeRegistrar.registerGauge( + "workload.weighted.current-score", obj, f, Tags.of(CONSUMER_ID_TAG, consumerId)); + } + + public void registerProposedErrorGauge(String consumerId, T obj, ToDoubleFunction f) { + gaugeRegistrar.registerGauge( + "workload.weighted.proposed-error", obj, f, Tags.of(CONSUMER_ID_TAG, consumerId)); + } + + public void registerScoringErrorGauge(String consumerId, T obj, ToDoubleFunction f) { + gaugeRegistrar.registerGauge( + "workload.weighted.scoring-error", obj, f, Tags.of(CONSUMER_ID_TAG, consumerId)); + } + + public void registerCurrentWeightGauge(String consumerId, T obj, ToDoubleFunction f) { + gaugeRegistrar.registerGauge( + "workload.weighted.current-weight.ops", obj, f, Tags.of(CONSUMER_ID_TAG, consumerId)); + } + + public void registerProposedWeightGauge(String consumerId, T obj, ToDoubleFunction f) { + gaugeRegistrar.registerGauge( + "workload.weighted.proposed-weight.ops", obj, f, Tags.of(CONSUMER_ID_TAG, consumerId)); + } + + public void unregisterAllWorkloadWeightedGaugesForConsumerIds(Set consumerIds) { + Collection gauges = + Search.in(meterRegistry) + .tag(CONSUMER_ID_TAG, consumerIds::contains) + .name(s -> s.startsWith("workload.weighted")) + .gauges(); + for (Gauge gauge : gauges) { + meterRegistry.remove(gauge); + } + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/counter/CounterStorage.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/counter/CounterStorage.java index 5ab6c4a078..59da3018e7 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/counter/CounterStorage.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/counter/CounterStorage.java @@ -4,17 +4,17 @@ public interface CounterStorage { - void setTopicPublishedCounter(TopicName topicName, long count); + void setTopicPublishedCounter(TopicName topicName, long count); - void setSubscriptionDeliveredCounter(TopicName topicName, String subscriptionName, long count); + void setSubscriptionDeliveredCounter(TopicName topicName, String subscriptionName, long count); - long getTopicPublishedCounter(TopicName topicName); + long getTopicPublishedCounter(TopicName topicName); - long getSubscriptionDeliveredCounter(TopicName topicName, String subscriptionName); + long getSubscriptionDeliveredCounter(TopicName topicName, String subscriptionName); - void setSubscriptionDiscardedCounter(TopicName topicName, String subscription, long value); + void setSubscriptionDiscardedCounter(TopicName topicName, String subscription, long value); - void incrementVolumeCounter(TopicName topicName, String subscriptionName, long value); + void incrementVolumeCounter(TopicName topicName, String subscriptionName, long value); - void incrementVolumeCounter(TopicName topicName, long value); + void incrementVolumeCounter(TopicName topicName, long value); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/counter/MetricsDeltaCalculator.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/counter/MetricsDeltaCalculator.java index ddaa73ed36..073ba0e2a9 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/counter/MetricsDeltaCalculator.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/counter/MetricsDeltaCalculator.java @@ -4,31 +4,32 @@ import java.util.concurrent.ConcurrentMap; /** - * Small util to calculate delta between previous and current state of metric. It helps in developing incremental metric storage, as Metrics - * always wants to push current state of Metric, not increment. + * Small util to calculate delta between previous and current state of metric. It helps in + * developing incremental metric storage, as Metrics always wants to push current state of Metric, + * not increment. */ public class MetricsDeltaCalculator { - private final ConcurrentMap previousValues = new ConcurrentHashMap<>(); + private final ConcurrentMap previousValues = new ConcurrentHashMap<>(); - public long calculateDelta(String metricName, Long currentValue) { - Long previousValue = previousValues.put(metricName, currentValue); + public long calculateDelta(String metricName, Long currentValue) { + Long previousValue = previousValues.put(metricName, currentValue); - long delta = currentValue; - if (previousValue != null) { - delta = currentValue - previousValue; - } - return delta; + long delta = currentValue; + if (previousValue != null) { + delta = currentValue - previousValue; } + return delta; + } - public void revertDelta(String metricName, Long delta) { - Long previousValue = previousValues.get(metricName); - if (previousValue != null) { - previousValues.put(metricName, previousValue - delta); - } + public void revertDelta(String metricName, Long delta) { + Long previousValue = previousValues.get(metricName); + if (previousValue != null) { + previousValues.put(metricName, previousValue - delta); } + } - public void clear() { - previousValues.clear(); - } + public void clear() { + previousValues.clear(); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/counter/zookeeper/CounterMatcher.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/counter/zookeeper/CounterMatcher.java index 7e52b5186b..cc096a488c 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/counter/zookeeper/CounterMatcher.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/counter/zookeeper/CounterMatcher.java @@ -1,92 +1,94 @@ package pl.allegro.tech.hermes.common.metric.counter.zookeeper; -import io.micrometer.core.instrument.Counter; -import pl.allegro.tech.hermes.api.TopicName; - -import java.util.Optional; - import static pl.allegro.tech.hermes.common.metric.SubscriptionMetrics.SubscriptionMetricsNames; import static pl.allegro.tech.hermes.common.metric.TopicMetrics.TopicMetricsNames; -class CounterMatcher { - - private static final String GROUP_TAG_NAME = "group"; - private static final String TOPIC_TAG_NAME = "topic"; - private static final String SUBSCRIPTION_TAG_NAME = "subscription"; +import io.micrometer.core.instrument.Counter; +import java.util.Optional; +import pl.allegro.tech.hermes.api.TopicName; - private final Counter counter; - private final String metricSearchPrefix; - private TopicName topicName; - private long value; - private Optional subscription; +class CounterMatcher { - public CounterMatcher(Counter counter, String metricSearchPrefix) { - this.counter = counter; - this.metricSearchPrefix = metricSearchPrefix; - parseCounter(this.counter); + private static final String GROUP_TAG_NAME = "group"; + private static final String TOPIC_TAG_NAME = "topic"; + private static final String SUBSCRIPTION_TAG_NAME = "subscription"; + + private final Counter counter; + private final String metricSearchPrefix; + private TopicName topicName; + private long value; + private Optional subscription; + + public CounterMatcher(Counter counter, String metricSearchPrefix) { + this.counter = counter; + this.metricSearchPrefix = metricSearchPrefix; + parseCounter(this.counter); + } + + private void parseCounter(Counter counter) { + if (isTopicPublished() || isTopicThroughput()) { + topicName = + new TopicName( + counter.getId().getTag(GROUP_TAG_NAME), counter.getId().getTag(TOPIC_TAG_NAME)); + subscription = Optional.empty(); + } else if (isSubscriptionDelivered() + || isSubscriptionThroughput() + || isSubscriptionDiscarded() + || isSubscriptionFiltered()) { + topicName = + new TopicName( + counter.getId().getTag(GROUP_TAG_NAME), counter.getId().getTag(TOPIC_TAG_NAME)); + subscription = Optional.of(counter.getId().getTag(SUBSCRIPTION_TAG_NAME)); } + value = (long) counter.count(); + } - private void parseCounter(Counter counter) { - if (isTopicPublished() || isTopicThroughput()) { - topicName = new TopicName(counter.getId().getTag(GROUP_TAG_NAME), counter.getId().getTag(TOPIC_TAG_NAME)); - subscription = Optional.empty(); - } else if ( - isSubscriptionDelivered() - || isSubscriptionThroughput() - || isSubscriptionDiscarded() - || isSubscriptionFiltered() - ) { - topicName = new TopicName(counter.getId().getTag(GROUP_TAG_NAME), counter.getId().getTag(TOPIC_TAG_NAME)); - subscription = Optional.of(counter.getId().getTag(SUBSCRIPTION_TAG_NAME)); - } - value = (long) counter.count(); - } + public boolean isTopicPublished() { + return isTopicCounter() && nameEquals(TopicMetricsNames.TOPIC_PUBLISHED); + } - public boolean isTopicPublished() { - return isTopicCounter() && nameEquals(TopicMetricsNames.TOPIC_PUBLISHED); - } + public boolean isTopicThroughput() { + return isTopicCounter() && nameEquals(TopicMetricsNames.TOPIC_THROUGHPUT); + } - public boolean isTopicThroughput() { - return isTopicCounter() && nameEquals(TopicMetricsNames.TOPIC_THROUGHPUT); - } - - public boolean isSubscriptionThroughput() { - return isSubscriptionCounter() && nameEquals(SubscriptionMetricsNames.SUBSCRIPTION_THROUGHPUT); - } + public boolean isSubscriptionThroughput() { + return isSubscriptionCounter() && nameEquals(SubscriptionMetricsNames.SUBSCRIPTION_THROUGHPUT); + } - public boolean isSubscriptionDelivered() { - return isSubscriptionCounter() && nameEquals(SubscriptionMetricsNames.SUBSCRIPTION_DELIVERED); - } + public boolean isSubscriptionDelivered() { + return isSubscriptionCounter() && nameEquals(SubscriptionMetricsNames.SUBSCRIPTION_DELIVERED); + } - public boolean isSubscriptionDiscarded() { - return isSubscriptionCounter() && nameEquals(SubscriptionMetricsNames.SUBSCRIPTION_DISCARDED); - } + public boolean isSubscriptionDiscarded() { + return isSubscriptionCounter() && nameEquals(SubscriptionMetricsNames.SUBSCRIPTION_DISCARDED); + } - public boolean isSubscriptionFiltered() { - return isSubscriptionCounter() && nameEquals(SubscriptionMetricsNames.SUBSCRIPTION_FILTERED_OUT); - } + public boolean isSubscriptionFiltered() { + return isSubscriptionCounter() + && nameEquals(SubscriptionMetricsNames.SUBSCRIPTION_FILTERED_OUT); + } - public TopicName getTopicName() { - return topicName; - } + public TopicName getTopicName() { + return topicName; + } - public String getSubscriptionName() { - return subscription.orElse(""); - } + public String getSubscriptionName() { + return subscription.orElse(""); + } - public long getValue() { - return value; - } + public long getValue() { + return value; + } - private boolean isTopicCounter() { - return counter.getId().getTag(TOPIC_TAG_NAME) != null; - } + private boolean isTopicCounter() { + return counter.getId().getTag(TOPIC_TAG_NAME) != null; + } - private boolean isSubscriptionCounter() { - return counter.getId().getTag(SUBSCRIPTION_TAG_NAME) != null; - } + private boolean isSubscriptionCounter() { + return counter.getId().getTag(SUBSCRIPTION_TAG_NAME) != null; + } - private boolean nameEquals(String name) { - return counter.getId().getName().equals(metricSearchPrefix + name); - } + private boolean nameEquals(String name) { + return counter.getId().getName().equals(metricSearchPrefix + name); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/counter/zookeeper/ZookeeperCounterReporter.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/counter/zookeeper/ZookeeperCounterReporter.java index 3293bea92b..5d31f60e7c 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/counter/zookeeper/ZookeeperCounterReporter.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/counter/zookeeper/ZookeeperCounterReporter.java @@ -4,103 +4,94 @@ import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.search.Search; +import java.util.Collection; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.TopicName; import pl.allegro.tech.hermes.common.metric.counter.CounterStorage; import pl.allegro.tech.hermes.metrics.PathsCompiler; -import java.util.Collection; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - public class ZookeeperCounterReporter { - private static final Logger logger = LoggerFactory.getLogger(ZookeeperCounterReporter.class); + private static final Logger logger = LoggerFactory.getLogger(ZookeeperCounterReporter.class); - private final CounterStorage counterStorage; - private final String metricsSearchPrefix; - private final MeterRegistry meterRegistry; + private final CounterStorage counterStorage; + private final String metricsSearchPrefix; + private final MeterRegistry meterRegistry; - private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( - new ThreadFactoryBuilder() - .setNameFormat("zookeeper-reporter-scheduled-executor-%d") - .setDaemon(true) - .build()); + private final ScheduledExecutorService scheduledExecutorService = + Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder() + .setNameFormat("zookeeper-reporter-scheduled-executor-%d") + .setDaemon(true) + .build()); - public ZookeeperCounterReporter(MeterRegistry registry, - CounterStorage counterStorage, - String metricsSearchPrefix) { - this.meterRegistry = registry; - this.counterStorage = counterStorage; - this.metricsSearchPrefix = metricsSearchPrefix; - } + public ZookeeperCounterReporter( + MeterRegistry registry, CounterStorage counterStorage, String metricsSearchPrefix) { + this.meterRegistry = registry; + this.counterStorage = counterStorage; + this.metricsSearchPrefix = metricsSearchPrefix; + } - public void start(long period, TimeUnit unit) { - scheduledExecutorService.scheduleWithFixedDelay(this::report, 0, period, unit); - } + public void start(long period, TimeUnit unit) { + scheduledExecutorService.scheduleWithFixedDelay(this::report, 0, period, unit); + } - public void report() { - try { - Collection counters = Search.in(meterRegistry).counters(); - counters.forEach(counter -> { - CounterMatcher matcher = new CounterMatcher(counter, metricsSearchPrefix); - reportCounter(matcher); - reportVolumeCounter(matcher); - }); - } catch (RuntimeException ex) { - logger.error("Error during reporting metrics to Zookeeper...", ex); - } + public void report() { + try { + Collection counters = Search.in(meterRegistry).counters(); + counters.forEach( + counter -> { + CounterMatcher matcher = new CounterMatcher(counter, metricsSearchPrefix); + reportCounter(matcher); + reportVolumeCounter(matcher); + }); + } catch (RuntimeException ex) { + logger.error("Error during reporting metrics to Zookeeper...", ex); } + } - private void reportVolumeCounter(CounterMatcher matcher) { - if (matcher.isTopicThroughput()) { - counterStorage.incrementVolumeCounter( - escapedTopicName(matcher.getTopicName()), - matcher.getValue() - ); - } else if (matcher.isSubscriptionThroughput()) { - counterStorage.incrementVolumeCounter( - escapedTopicName(matcher.getTopicName()), - escapeMetricsReplacementChar(matcher.getSubscriptionName()), - matcher.getValue() - ); - } + private void reportVolumeCounter(CounterMatcher matcher) { + if (matcher.isTopicThroughput()) { + counterStorage.incrementVolumeCounter( + escapedTopicName(matcher.getTopicName()), matcher.getValue()); + } else if (matcher.isSubscriptionThroughput()) { + counterStorage.incrementVolumeCounter( + escapedTopicName(matcher.getTopicName()), + escapeMetricsReplacementChar(matcher.getSubscriptionName()), + matcher.getValue()); } + } - private void reportCounter(CounterMatcher matcher) { - if (matcher.getValue() == 0) { - return; - } - - if (matcher.isTopicPublished()) { - counterStorage.setTopicPublishedCounter( - escapedTopicName(matcher.getTopicName()), - matcher.getValue() - ); - } else if (matcher.isSubscriptionDelivered()) { - counterStorage.setSubscriptionDeliveredCounter( - escapedTopicName(matcher.getTopicName()), - escapeMetricsReplacementChar(matcher.getSubscriptionName()), - matcher.getValue() - ); - } else if (matcher.isSubscriptionDiscarded()) { - counterStorage.setSubscriptionDiscardedCounter( - escapedTopicName(matcher.getTopicName()), - escapeMetricsReplacementChar(matcher.getSubscriptionName()), - matcher.getValue() - ); - } + private void reportCounter(CounterMatcher matcher) { + if (matcher.getValue() == 0) { + return; } - private static TopicName escapedTopicName(TopicName topicName) { - return new TopicName( - escapeMetricsReplacementChar(topicName.getGroupName()), - topicName.getName() - ); + if (matcher.isTopicPublished()) { + counterStorage.setTopicPublishedCounter( + escapedTopicName(matcher.getTopicName()), matcher.getValue()); + } else if (matcher.isSubscriptionDelivered()) { + counterStorage.setSubscriptionDeliveredCounter( + escapedTopicName(matcher.getTopicName()), + escapeMetricsReplacementChar(matcher.getSubscriptionName()), + matcher.getValue()); + } else if (matcher.isSubscriptionDiscarded()) { + counterStorage.setSubscriptionDiscardedCounter( + escapedTopicName(matcher.getTopicName()), + escapeMetricsReplacementChar(matcher.getSubscriptionName()), + matcher.getValue()); } + } - private static String escapeMetricsReplacementChar(String value) { - return value.replaceAll(PathsCompiler.REPLACEMENT_CHAR, "\\."); - } + private static TopicName escapedTopicName(TopicName topicName) { + return new TopicName( + escapeMetricsReplacementChar(topicName.getGroupName()), topicName.getName()); + } + + private static String escapeMetricsReplacementChar(String value) { + return value.replaceAll(PathsCompiler.REPLACEMENT_CHAR, "\\."); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/counter/zookeeper/ZookeeperCounterStorage.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/counter/zookeeper/ZookeeperCounterStorage.java index aae7619a1a..c5624c7624 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/counter/zookeeper/ZookeeperCounterStorage.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/counter/zookeeper/ZookeeperCounterStorage.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.common.metric.counter.zookeeper; +import static pl.allegro.tech.hermes.metrics.PathContext.pathContext; +import static pl.allegro.tech.hermes.metrics.PathsCompiler.GROUP; +import static pl.allegro.tech.hermes.metrics.PathsCompiler.SUBSCRIPTION; +import static pl.allegro.tech.hermes.metrics.PathsCompiler.TOPIC; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.TopicName; @@ -11,131 +16,163 @@ import pl.allegro.tech.hermes.metrics.PathContext; import pl.allegro.tech.hermes.metrics.PathsCompiler; -import static pl.allegro.tech.hermes.metrics.PathContext.pathContext; -import static pl.allegro.tech.hermes.metrics.PathsCompiler.GROUP; -import static pl.allegro.tech.hermes.metrics.PathsCompiler.SUBSCRIPTION; -import static pl.allegro.tech.hermes.metrics.PathsCompiler.TOPIC; - public class ZookeeperCounterStorage implements CounterStorage { - static final String TOPIC_PUBLISHED = "/groups/" + GROUP + "/topics/" + TOPIC + "/metrics/published"; - static final String TOPIC_VOLUME_COUNTER = "/groups/" + GROUP + "/topics/" + TOPIC + "/metrics/volume"; - static final String SUBSCRIPTION_DELIVERED = - "/groups/" + GROUP + "/topics/" + TOPIC + "/subscriptions/" + SUBSCRIPTION + "/metrics/delivered"; - static final String SUBSCRIPTION_DISCARDED = - "/groups/" + GROUP + "/topics/" + TOPIC + "/subscriptions/" + SUBSCRIPTION + "/metrics/discarded"; - static final String SUBSCRIPTION_VOLUME_COUNTER = - "/groups/" + GROUP + "/topics/" + TOPIC + "/subscriptions/" + SUBSCRIPTION + "/metrics/volume"; - - private static final Logger LOGGER = LoggerFactory.getLogger(ZookeeperCounterStorage.class); - - private final MetricsDeltaCalculator deltaCalculator = new MetricsDeltaCalculator(); - private final SharedCounter sharedCounter; - - private final SubscriptionRepository subscriptionRepository; - private final PathsCompiler pathsCompiler; - private final String zookeeperRoot; - - public ZookeeperCounterStorage(SharedCounter sharedCounter, - SubscriptionRepository subscriptionRepository, - PathsCompiler pathsCompiler, - String zookeeperRoot) { - this.sharedCounter = sharedCounter; - this.subscriptionRepository = subscriptionRepository; - this.pathsCompiler = pathsCompiler; - this.zookeeperRoot = zookeeperRoot; - - } - - @Override - public void setTopicPublishedCounter(TopicName topicName, long count) { - String topicPublishedCounter = topicPublishedCounter(topicName); - incrementSharedCounter(topicPublishedCounter, count); - } - - @Override - public long getTopicPublishedCounter(TopicName topicName) { - return sharedCounter.getValue(topicPublishedCounter(topicName)); - } - - @Override - public long getSubscriptionDeliveredCounter(TopicName topicName, String subscriptionName) { - return sharedCounter.getValue(subscriptionDeliveredCounter(topicName, subscriptionName)); - } - - @Override - public void setSubscriptionDeliveredCounter(TopicName topicName, String subscriptionName, long count) { - try { - subscriptionRepository.ensureSubscriptionExists(topicName, subscriptionName); - incrementSharedCounter(subscriptionDeliveredCounter(topicName, subscriptionName), count); - } catch (SubscriptionNotExistsException e) { - LOGGER.debug("Trying to report metric on not existing subscription {} {}", topicName, subscriptionName); - } - } - - @Override - public void setSubscriptionDiscardedCounter(TopicName topicName, String subscriptionName, long count) { - try { - subscriptionRepository.ensureSubscriptionExists(topicName, subscriptionName); - incrementSharedCounter(subscriptionDiscardedCounter(topicName, subscriptionName), count); - } catch (SubscriptionNotExistsException e) { - LOGGER.debug("Trying to report metric on not existing subscription {} {}", topicName, subscriptionName); - } - } - - @Override - public void incrementVolumeCounter(TopicName topicName, String subscriptionName, long value) { - try { - subscriptionRepository.ensureSubscriptionExists(topicName, subscriptionName); - incrementSharedCounter(volumeCounterPath(topicName, subscriptionName), value); - } catch (SubscriptionNotExistsException e) { - LOGGER.debug("Trying to report metric on not existing subscription {} {}", topicName, subscriptionName); - } - } - - @Override - public void incrementVolumeCounter(TopicName topicName, long value) { - incrementSharedCounter(topicVolumeCounter(topicName), value); - } - - private void incrementSharedCounter(String metricPath, long count) { - long delta = deltaCalculator.calculateDelta(metricPath, count); - - if (delta != 0 && !sharedCounter.increment(metricPath, delta)) { - deltaCalculator.revertDelta(metricPath, delta); - } - } - - private String topicPublishedCounter(TopicName topicName) { - PathContext pathContext = pathContext().withGroup(topicName.getGroupName()).withTopic(topicName.getName()).build(); - return pathsCompiler.compile(appendRootPath(TOPIC_PUBLISHED), pathContext); - } - - private String subscriptionDeliveredCounter(TopicName topicName, String subscriptionName) { - PathContext pathContext = subscriptionPathContext(topicName, subscriptionName); - return pathsCompiler.compile(appendRootPath(SUBSCRIPTION_DELIVERED), pathContext); - } - - private String subscriptionDiscardedCounter(TopicName topicName, String subscriptionName) { - PathContext pathContext = subscriptionPathContext(topicName, subscriptionName); - return pathsCompiler.compile(appendRootPath(SUBSCRIPTION_DISCARDED), pathContext); - } - - private String volumeCounterPath(TopicName topicName, String subscriptionName) { - PathContext pathContext = subscriptionPathContext(topicName, subscriptionName); - return pathsCompiler.compile(appendRootPath(SUBSCRIPTION_VOLUME_COUNTER), pathContext); - } - - private String topicVolumeCounter(TopicName topicName) { - PathContext pathContext = pathContext().withGroup(topicName.getGroupName()).withTopic(topicName.getName()).build(); - return pathsCompiler.compile(appendRootPath(TOPIC_VOLUME_COUNTER), pathContext); - } - - private PathContext subscriptionPathContext(TopicName topicName, String subscriptionName) { - return pathContext().withGroup(topicName.getGroupName()).withTopic(topicName.getName()).withSubscription(subscriptionName).build(); - } - - private String appendRootPath(String path) { - return zookeeperRoot + path; - } + static final String TOPIC_PUBLISHED = + "/groups/" + GROUP + "/topics/" + TOPIC + "/metrics/published"; + static final String TOPIC_VOLUME_COUNTER = + "/groups/" + GROUP + "/topics/" + TOPIC + "/metrics/volume"; + static final String SUBSCRIPTION_DELIVERED = + "/groups/" + + GROUP + + "/topics/" + + TOPIC + + "/subscriptions/" + + SUBSCRIPTION + + "/metrics/delivered"; + static final String SUBSCRIPTION_DISCARDED = + "/groups/" + + GROUP + + "/topics/" + + TOPIC + + "/subscriptions/" + + SUBSCRIPTION + + "/metrics/discarded"; + static final String SUBSCRIPTION_VOLUME_COUNTER = + "/groups/" + + GROUP + + "/topics/" + + TOPIC + + "/subscriptions/" + + SUBSCRIPTION + + "/metrics/volume"; + + private static final Logger LOGGER = LoggerFactory.getLogger(ZookeeperCounterStorage.class); + + private final MetricsDeltaCalculator deltaCalculator = new MetricsDeltaCalculator(); + private final SharedCounter sharedCounter; + + private final SubscriptionRepository subscriptionRepository; + private final PathsCompiler pathsCompiler; + private final String zookeeperRoot; + + public ZookeeperCounterStorage( + SharedCounter sharedCounter, + SubscriptionRepository subscriptionRepository, + PathsCompiler pathsCompiler, + String zookeeperRoot) { + this.sharedCounter = sharedCounter; + this.subscriptionRepository = subscriptionRepository; + this.pathsCompiler = pathsCompiler; + this.zookeeperRoot = zookeeperRoot; + } + + @Override + public void setTopicPublishedCounter(TopicName topicName, long count) { + String topicPublishedCounter = topicPublishedCounter(topicName); + incrementSharedCounter(topicPublishedCounter, count); + } + + @Override + public long getTopicPublishedCounter(TopicName topicName) { + return sharedCounter.getValue(topicPublishedCounter(topicName)); + } + + @Override + public long getSubscriptionDeliveredCounter(TopicName topicName, String subscriptionName) { + return sharedCounter.getValue(subscriptionDeliveredCounter(topicName, subscriptionName)); + } + + @Override + public void setSubscriptionDeliveredCounter( + TopicName topicName, String subscriptionName, long count) { + try { + subscriptionRepository.ensureSubscriptionExists(topicName, subscriptionName); + incrementSharedCounter(subscriptionDeliveredCounter(topicName, subscriptionName), count); + } catch (SubscriptionNotExistsException e) { + LOGGER.debug( + "Trying to report metric on not existing subscription {} {}", + topicName, + subscriptionName); + } + } + + @Override + public void setSubscriptionDiscardedCounter( + TopicName topicName, String subscriptionName, long count) { + try { + subscriptionRepository.ensureSubscriptionExists(topicName, subscriptionName); + incrementSharedCounter(subscriptionDiscardedCounter(topicName, subscriptionName), count); + } catch (SubscriptionNotExistsException e) { + LOGGER.debug( + "Trying to report metric on not existing subscription {} {}", + topicName, + subscriptionName); + } + } + + @Override + public void incrementVolumeCounter(TopicName topicName, String subscriptionName, long value) { + try { + subscriptionRepository.ensureSubscriptionExists(topicName, subscriptionName); + incrementSharedCounter(volumeCounterPath(topicName, subscriptionName), value); + } catch (SubscriptionNotExistsException e) { + LOGGER.debug( + "Trying to report metric on not existing subscription {} {}", + topicName, + subscriptionName); + } + } + + @Override + public void incrementVolumeCounter(TopicName topicName, long value) { + incrementSharedCounter(topicVolumeCounter(topicName), value); + } + + private void incrementSharedCounter(String metricPath, long count) { + long delta = deltaCalculator.calculateDelta(metricPath, count); + + if (delta != 0 && !sharedCounter.increment(metricPath, delta)) { + deltaCalculator.revertDelta(metricPath, delta); + } + } + + private String topicPublishedCounter(TopicName topicName) { + PathContext pathContext = + pathContext().withGroup(topicName.getGroupName()).withTopic(topicName.getName()).build(); + return pathsCompiler.compile(appendRootPath(TOPIC_PUBLISHED), pathContext); + } + + private String subscriptionDeliveredCounter(TopicName topicName, String subscriptionName) { + PathContext pathContext = subscriptionPathContext(topicName, subscriptionName); + return pathsCompiler.compile(appendRootPath(SUBSCRIPTION_DELIVERED), pathContext); + } + + private String subscriptionDiscardedCounter(TopicName topicName, String subscriptionName) { + PathContext pathContext = subscriptionPathContext(topicName, subscriptionName); + return pathsCompiler.compile(appendRootPath(SUBSCRIPTION_DISCARDED), pathContext); + } + + private String volumeCounterPath(TopicName topicName, String subscriptionName) { + PathContext pathContext = subscriptionPathContext(topicName, subscriptionName); + return pathsCompiler.compile(appendRootPath(SUBSCRIPTION_VOLUME_COUNTER), pathContext); + } + + private String topicVolumeCounter(TopicName topicName) { + PathContext pathContext = + pathContext().withGroup(topicName.getGroupName()).withTopic(topicName.getName()).build(); + return pathsCompiler.compile(appendRootPath(TOPIC_VOLUME_COUNTER), pathContext); + } + + private PathContext subscriptionPathContext(TopicName topicName, String subscriptionName) { + return pathContext() + .withGroup(topicName.getGroupName()) + .withTopic(topicName.getName()) + .withSubscription(subscriptionName) + .build(); + } + + private String appendRootPath(String path) { + return zookeeperRoot + path; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/executor/InstrumentedExecutorServiceFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/executor/InstrumentedExecutorServiceFactory.java index 08a69b7936..91a83c82e1 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/executor/InstrumentedExecutorServiceFactory.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/executor/InstrumentedExecutorServiceFactory.java @@ -1,8 +1,6 @@ package pl.allegro.tech.hermes.common.metric.executor; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import pl.allegro.tech.hermes.common.metric.MetricsFacade; - import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.RejectedExecutionHandler; @@ -11,87 +9,90 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import pl.allegro.tech.hermes.common.metric.MetricsFacade; public class InstrumentedExecutorServiceFactory { - private final MetricsFacade metricsFacade; - private final RejectedExecutionHandler rejectedExecutionHandler = new ThreadPoolExecutor.AbortPolicy(); - - public InstrumentedExecutorServiceFactory(MetricsFacade metricsFacade) { - this.metricsFacade = metricsFacade; + private final MetricsFacade metricsFacade; + private final RejectedExecutionHandler rejectedExecutionHandler = + new ThreadPoolExecutor.AbortPolicy(); + + public InstrumentedExecutorServiceFactory(MetricsFacade metricsFacade) { + this.metricsFacade = metricsFacade; + } + + public ExecutorService getExecutorService(String name, int size, boolean monitoringEnabled) { + return getExecutorService(name, size, monitoringEnabled, Integer.MAX_VALUE); + } + + public ExecutorService getExecutorService( + String name, int size, boolean monitoringEnabled, int queueCapacity) { + ThreadFactory threadFactory = + new ThreadFactoryBuilder().setNameFormat(name + "-executor-%d").build(); + ThreadPoolExecutor executor = newFixedThreadPool(name, size, threadFactory, queueCapacity); + executor.prestartAllCoreThreads(); + + return monitoringEnabled ? monitor(name, executor) : executor; + } + + public class ScheduledExecutorServiceBuilder { + final String name; + final int size; + boolean monitoringEnabled = false; + boolean removeOnCancel = false; + + public ScheduledExecutorServiceBuilder(String name, int size) { + this.name = name; + this.size = size; } - public ExecutorService getExecutorService(String name, int size, boolean monitoringEnabled) { - return getExecutorService(name, size, monitoringEnabled, Integer.MAX_VALUE); + public ScheduledExecutorServiceBuilder withMonitoringEnabled(boolean monitoringEnabled) { + this.monitoringEnabled = monitoringEnabled; + return this; } - public ExecutorService getExecutorService(String name, int size, boolean monitoringEnabled, int queueCapacity) { - ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat(name + "-executor-%d").build(); - ThreadPoolExecutor executor = newFixedThreadPool(name, size, threadFactory, queueCapacity); - executor.prestartAllCoreThreads(); - - return monitoringEnabled ? monitor(name, executor) : executor; + public ScheduledExecutorServiceBuilder withRemoveOnCancel(boolean removeOnCancel) { + this.removeOnCancel = removeOnCancel; + return this; } - - public class ScheduledExecutorServiceBuilder { - final String name; - final int size; - boolean monitoringEnabled = false; - boolean removeOnCancel = false; - - public ScheduledExecutorServiceBuilder(String name, int size) { - this.name = name; - this.size = size; - } - - public ScheduledExecutorServiceBuilder withMonitoringEnabled(boolean monitoringEnabled) { - this.monitoringEnabled = monitoringEnabled; - return this; - } - - public ScheduledExecutorServiceBuilder withRemoveOnCancel(boolean removeOnCancel) { - this.removeOnCancel = removeOnCancel; - return this; - } - - public ScheduledExecutorService create() { - ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat(name + "-scheduled-executor-%d").build(); - ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(size, threadFactory); - executor.setRemoveOnCancelPolicy(removeOnCancel); - return monitoringEnabled ? monitor(name, executor) : executor; - } - - private ScheduledExecutorService monitor(String threadPoolName, ScheduledExecutorService executor) { - return metricsFacade.executor().monitor(executor, threadPoolName); - } + public ScheduledExecutorService create() { + ThreadFactory threadFactory = + new ThreadFactoryBuilder().setNameFormat(name + "-scheduled-executor-%d").build(); + ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(size, threadFactory); + executor.setRemoveOnCancelPolicy(removeOnCancel); + return monitoringEnabled ? monitor(name, executor) : executor; } - public ScheduledExecutorServiceBuilder scheduledExecutorBuilder( - String name, int size - ) { - return new ScheduledExecutorServiceBuilder(name, size); - } - - private ExecutorService monitor(String threadPoolName, ExecutorService executor) { - return metricsFacade.executor().monitor(executor, threadPoolName); - } - - - /** - * Copy of {@link java.util.concurrent.Executors#newFixedThreadPool(int, java.util.concurrent.ThreadFactory)} - * with configurable queue capacity. - */ - private ThreadPoolExecutor newFixedThreadPool(String executorName, int size, ThreadFactory threadFactory, int queueCapacity) { - ThreadPoolExecutor executor = new ThreadPoolExecutor( - size, - size, - 0L, - TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(queueCapacity), - threadFactory, - rejectedExecutionHandler - ); - return executor; + private ScheduledExecutorService monitor( + String threadPoolName, ScheduledExecutorService executor) { + return metricsFacade.executor().monitor(executor, threadPoolName); } + } + + public ScheduledExecutorServiceBuilder scheduledExecutorBuilder(String name, int size) { + return new ScheduledExecutorServiceBuilder(name, size); + } + + private ExecutorService monitor(String threadPoolName, ExecutorService executor) { + return metricsFacade.executor().monitor(executor, threadPoolName); + } + + /** + * Copy of {@link java.util.concurrent.Executors#newFixedThreadPool(int, + * java.util.concurrent.ThreadFactory)} with configurable queue capacity. + */ + private ThreadPoolExecutor newFixedThreadPool( + String executorName, int size, ThreadFactory threadFactory, int queueCapacity) { + ThreadPoolExecutor executor = + new ThreadPoolExecutor( + size, + size, + 0L, + TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(queueCapacity), + threadFactory, + rejectedExecutionHandler); + return executor; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/timer/StartedTimersPair.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/timer/StartedTimersPair.java index aa26a97bc1..44f7ac7505 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/timer/StartedTimersPair.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/metric/timer/StartedTimersPair.java @@ -1,22 +1,21 @@ package pl.allegro.tech.hermes.common.metric.timer; -import pl.allegro.tech.hermes.metrics.HermesTimerContext; - import java.io.Closeable; +import pl.allegro.tech.hermes.metrics.HermesTimerContext; public class StartedTimersPair implements Closeable { - private final HermesTimerContext time1; - private final HermesTimerContext time2; + private final HermesTimerContext time1; + private final HermesTimerContext time2; - public StartedTimersPair(HermesTimerContext timer1, HermesTimerContext timer2) { - time1 = timer1; - time2 = timer2; - } + public StartedTimersPair(HermesTimerContext timer1, HermesTimerContext timer2) { + time1 = timer1; + time2 = timer2; + } - @Override - public void close() { - time1.close(); - time2.close(); - } + @Override + public void close() { + time1.close(); + time2.close(); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/AvroCompiledSchemaRepositoryFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/AvroCompiledSchemaRepositoryFactory.java index a3e8a49fe7..abccb0da34 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/AvroCompiledSchemaRepositoryFactory.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/AvroCompiledSchemaRepositoryFactory.java @@ -1,5 +1,6 @@ package pl.allegro.tech.hermes.common.schema; +import java.time.Duration; import org.apache.avro.Schema; import pl.allegro.tech.hermes.schema.CachedCompiledSchemaRepository; import pl.allegro.tech.hermes.schema.CompiledSchemaRepository; @@ -7,35 +8,33 @@ import pl.allegro.tech.hermes.schema.RawSchemaClient; import pl.allegro.tech.hermes.schema.SchemaCompilersFactory; -import java.time.Duration; - public class AvroCompiledSchemaRepositoryFactory { - private final RawSchemaClient rawSchemaClient; - private final int maximumSize; - private final Duration expireAfterAccess; - private final boolean cacheEnabled; + private final RawSchemaClient rawSchemaClient; + private final int maximumSize; + private final Duration expireAfterAccess; + private final boolean cacheEnabled; - public AvroCompiledSchemaRepositoryFactory(RawSchemaClient rawSchemaClient, - int maximumSize, - Duration expireAfterAccess, - boolean cacheEnabled) { - this.rawSchemaClient = rawSchemaClient; - this.maximumSize = maximumSize; - this.expireAfterAccess = expireAfterAccess; - this.cacheEnabled = cacheEnabled; - } + public AvroCompiledSchemaRepositoryFactory( + RawSchemaClient rawSchemaClient, + int maximumSize, + Duration expireAfterAccess, + boolean cacheEnabled) { + this.rawSchemaClient = rawSchemaClient; + this.maximumSize = maximumSize; + this.expireAfterAccess = expireAfterAccess; + this.cacheEnabled = cacheEnabled; + } - public CompiledSchemaRepository provide() { - CompiledSchemaRepository repository = new DirectCompiledSchemaRepository<>(rawSchemaClient, - SchemaCompilersFactory.avroSchemaCompiler()); + public CompiledSchemaRepository provide() { + CompiledSchemaRepository repository = + new DirectCompiledSchemaRepository<>( + rawSchemaClient, SchemaCompilersFactory.avroSchemaCompiler()); - if (cacheEnabled) { - return new CachedCompiledSchemaRepository<>(repository, - maximumSize, - expireAfterAccess); - } else { - return repository; - } + if (cacheEnabled) { + return new CachedCompiledSchemaRepository<>(repository, maximumSize, expireAfterAccess); + } else { + return repository; } + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/RawSchemaClientFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/RawSchemaClientFactory.java index 028d649399..a6fd3b015d 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/RawSchemaClientFactory.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/RawSchemaClientFactory.java @@ -9,46 +9,43 @@ public class RawSchemaClientFactory { - private final String kafkaNamespace; - private final String kafkaNamespaceSeparator; - private final MetricsFacade metricsFacade; - private final ObjectMapper objectMapper; - private final SchemaRepositoryInstanceResolver resolver; - private final boolean subjectSuffixEnabled; - private final boolean subjectNamespaceEnabled; + private final String kafkaNamespace; + private final String kafkaNamespaceSeparator; + private final MetricsFacade metricsFacade; + private final ObjectMapper objectMapper; + private final SchemaRepositoryInstanceResolver resolver; + private final boolean subjectSuffixEnabled; + private final boolean subjectNamespaceEnabled; - public RawSchemaClientFactory(String kafkaNamespace, - String kafkaNamespaceSeparator, - MetricsFacade metricsFacade, - ObjectMapper objectMapper, - SchemaRepositoryInstanceResolver resolver, - boolean subjectSuffixEnabled, - boolean subjectNamespaceEnabled) { - this.kafkaNamespace = kafkaNamespace; - this.kafkaNamespaceSeparator = kafkaNamespaceSeparator; - this.metricsFacade = metricsFacade; - this.objectMapper = objectMapper; - this.resolver = resolver; - this.subjectSuffixEnabled = subjectSuffixEnabled; - this.subjectNamespaceEnabled = subjectNamespaceEnabled; - } + public RawSchemaClientFactory( + String kafkaNamespace, + String kafkaNamespaceSeparator, + MetricsFacade metricsFacade, + ObjectMapper objectMapper, + SchemaRepositoryInstanceResolver resolver, + boolean subjectSuffixEnabled, + boolean subjectNamespaceEnabled) { + this.kafkaNamespace = kafkaNamespace; + this.kafkaNamespaceSeparator = kafkaNamespaceSeparator; + this.metricsFacade = metricsFacade; + this.objectMapper = objectMapper; + this.resolver = resolver; + this.subjectSuffixEnabled = subjectSuffixEnabled; + this.subjectNamespaceEnabled = subjectNamespaceEnabled; + } - public RawSchemaClient provide() { - SubjectNamingStrategy subjectNamingStrategy = SubjectNamingStrategy.qualifiedName - .withValueSuffixIf(subjectSuffixEnabled) - .withNamespacePrefixIf( - subjectNamespaceEnabled, - new SubjectNamingStrategy.Namespace( - kafkaNamespace, - kafkaNamespaceSeparator - ) - ); - return createMetricsTrackingClient( - new SchemaRegistryRawSchemaClient(resolver, objectMapper, subjectNamingStrategy) - ); - } + public RawSchemaClient provide() { + SubjectNamingStrategy subjectNamingStrategy = + SubjectNamingStrategy.qualifiedName + .withValueSuffixIf(subjectSuffixEnabled) + .withNamespacePrefixIf( + subjectNamespaceEnabled, + new SubjectNamingStrategy.Namespace(kafkaNamespace, kafkaNamespaceSeparator)); + return createMetricsTrackingClient( + new SchemaRegistryRawSchemaClient(resolver, objectMapper, subjectNamingStrategy)); + } - private RawSchemaClient createMetricsTrackingClient(RawSchemaClient rawSchemaClient) { - return new ReadMetricsTrackingRawSchemaClient(rawSchemaClient, metricsFacade); - } + private RawSchemaClient createMetricsTrackingClient(RawSchemaClient rawSchemaClient) { + return new ReadMetricsTrackingRawSchemaClient(rawSchemaClient, metricsFacade); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/ReadMetricsTrackingRawSchemaClient.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/ReadMetricsTrackingRawSchemaClient.java index 10b792c9c7..0b699d4654 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/ReadMetricsTrackingRawSchemaClient.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/ReadMetricsTrackingRawSchemaClient.java @@ -1,5 +1,8 @@ package pl.allegro.tech.hermes.common.schema; +import java.util.List; +import java.util.Optional; +import java.util.function.Supplier; import pl.allegro.tech.hermes.api.RawSchema; import pl.allegro.tech.hermes.api.RawSchemaWithMetadata; import pl.allegro.tech.hermes.api.TopicName; @@ -10,67 +13,64 @@ import pl.allegro.tech.hermes.schema.SchemaId; import pl.allegro.tech.hermes.schema.SchemaVersion; -import java.util.List; -import java.util.Optional; -import java.util.function.Supplier; - public class ReadMetricsTrackingRawSchemaClient implements RawSchemaClient { - private final RawSchemaClient rawSchemaClient; - private final MetricsFacade metricsFacade; + private final RawSchemaClient rawSchemaClient; + private final MetricsFacade metricsFacade; - public ReadMetricsTrackingRawSchemaClient( - RawSchemaClient rawSchemaClient, - MetricsFacade metricsFacade) { - this.rawSchemaClient = rawSchemaClient; - this.metricsFacade = metricsFacade; - } + public ReadMetricsTrackingRawSchemaClient( + RawSchemaClient rawSchemaClient, MetricsFacade metricsFacade) { + this.rawSchemaClient = rawSchemaClient; + this.metricsFacade = metricsFacade; + } - @Override - public Optional getRawSchemaWithMetadata(TopicName topic, SchemaVersion version) { - return timedSchema(() -> rawSchemaClient.getRawSchemaWithMetadata(topic, version)); - } + @Override + public Optional getRawSchemaWithMetadata( + TopicName topic, SchemaVersion version) { + return timedSchema(() -> rawSchemaClient.getRawSchemaWithMetadata(topic, version)); + } - @Override - public Optional getRawSchemaWithMetadata(TopicName topic, SchemaId schemaId) { - return timedSchema(() -> rawSchemaClient.getRawSchemaWithMetadata(topic, schemaId)); - } + @Override + public Optional getRawSchemaWithMetadata( + TopicName topic, SchemaId schemaId) { + return timedSchema(() -> rawSchemaClient.getRawSchemaWithMetadata(topic, schemaId)); + } - @Override - public Optional getLatestRawSchemaWithMetadata(TopicName topic) { - return timedSchema(() -> rawSchemaClient.getLatestRawSchemaWithMetadata(topic)); - } + @Override + public Optional getLatestRawSchemaWithMetadata(TopicName topic) { + return timedSchema(() -> rawSchemaClient.getLatestRawSchemaWithMetadata(topic)); + } - @Override - public List getVersions(TopicName topic) { - return timedVersions(() -> rawSchemaClient.getVersions(topic)); - } + @Override + public List getVersions(TopicName topic) { + return timedVersions(() -> rawSchemaClient.getVersions(topic)); + } - @Override - public void registerSchema(TopicName topic, RawSchema rawSchema) { - rawSchemaClient.registerSchema(topic, rawSchema); - } + @Override + public void registerSchema(TopicName topic, RawSchema rawSchema) { + rawSchemaClient.registerSchema(topic, rawSchema); + } - @Override - public void deleteAllSchemaVersions(TopicName topic) { - rawSchemaClient.deleteAllSchemaVersions(topic); - } + @Override + public void deleteAllSchemaVersions(TopicName topic) { + rawSchemaClient.deleteAllSchemaVersions(topic); + } - @Override - public void validateSchema(TopicName topic, RawSchema rawSchema) { - rawSchemaClient.validateSchema(topic, rawSchema); - } + @Override + public void validateSchema(TopicName topic, RawSchema rawSchema) { + rawSchemaClient.validateSchema(topic, rawSchema); + } - private T timedSchema(Supplier callable) { - return timed(callable, metricsFacade.schemaClient().schemaTimer()); - } + private T timedSchema(Supplier callable) { + return timed(callable, metricsFacade.schemaClient().schemaTimer()); + } - private T timedVersions(Supplier callable) { - return timed(callable, metricsFacade.schemaClient().versionsTimer()); - } + private T timedVersions(Supplier callable) { + return timed(callable, metricsFacade.schemaClient().versionsTimer()); + } - private T timed(Supplier callable, HermesTimer timer) { - try (HermesTimerContext time = timer.time()) { - return callable.get(); - } + private T timed(Supplier callable, HermesTimer timer) { + try (HermesTimerContext time = timer.time()) { + return callable.get(); } + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaCacheRefresherCallback.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaCacheRefresherCallback.java index b59baf0c0c..f44d109ac7 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaCacheRefresherCallback.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaCacheRefresherCallback.java @@ -1,5 +1,6 @@ package pl.allegro.tech.hermes.common.schema; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.ContentType; @@ -11,57 +12,61 @@ import pl.allegro.tech.hermes.schema.SchemaVersion; import pl.allegro.tech.hermes.schema.SchemaVersionsResult; -import java.util.List; - class SchemaCacheRefresherCallback implements TopicCallback { - private static final Logger logger = LoggerFactory.getLogger(SchemaVersionsRepositoryFactory.class); + private static final Logger logger = + LoggerFactory.getLogger(SchemaVersionsRepositoryFactory.class); - public static final boolean REFRESH_ONLINE = true; + public static final boolean REFRESH_ONLINE = true; - private final CachedSchemaVersionsRepository schemaVersionsRepository; - private final CachedCompiledSchemaRepository compiledSchemaRepository; + private final CachedSchemaVersionsRepository schemaVersionsRepository; + private final CachedCompiledSchemaRepository compiledSchemaRepository; - public SchemaCacheRefresherCallback(CachedSchemaVersionsRepository schemaVersionsRepository, - CachedCompiledSchemaRepository compiledSchemaRepository) { - this.schemaVersionsRepository = schemaVersionsRepository; - this.compiledSchemaRepository = compiledSchemaRepository; - } + public SchemaCacheRefresherCallback( + CachedSchemaVersionsRepository schemaVersionsRepository, + CachedCompiledSchemaRepository compiledSchemaRepository) { + this.schemaVersionsRepository = schemaVersionsRepository; + this.compiledSchemaRepository = compiledSchemaRepository; + } - @Override - public void onTopicRemoved(Topic topic) { - schemaVersionsRepository.removeFromCache(topic); - compiledSchemaRepository.removeFromCache(topic); - } + @Override + public void onTopicRemoved(Topic topic) { + schemaVersionsRepository.removeFromCache(topic); + compiledSchemaRepository.removeFromCache(topic); + } - @Override - public void onTopicCreated(Topic topic) { - refreshSchemas(topic); - } + @Override + public void onTopicCreated(Topic topic) { + refreshSchemas(topic); + } - @Override - public void onTopicChanged(Topic topic) { - refreshSchemas(topic); - } + @Override + public void onTopicChanged(Topic topic) { + refreshSchemas(topic); + } - private void refreshSchemas(Topic topic) { - if (topic.getContentType() == ContentType.AVRO) { - logger.info("Refreshing all schemas for {} topic.", topic.getQualifiedName()); - SchemaVersionsResult versions = schemaVersionsRepository.versions(topic, REFRESH_ONLINE); - if (versions.isSuccess()) { - refreshCompiledSchemas(topic, versions.get()); - } - } + private void refreshSchemas(Topic topic) { + if (topic.getContentType() == ContentType.AVRO) { + logger.info("Refreshing all schemas for {} topic.", topic.getQualifiedName()); + SchemaVersionsResult versions = schemaVersionsRepository.versions(topic, REFRESH_ONLINE); + if (versions.isSuccess()) { + refreshCompiledSchemas(topic, versions.get()); + } } + } - private void refreshCompiledSchemas(Topic topic, List schemaVersions) { - schemaVersions.forEach(schemaVersion -> { - try { - compiledSchemaRepository.getSchema(topic, schemaVersion, REFRESH_ONLINE); - } catch (CouldNotLoadSchemaException e) { - logger.warn("Schema for topic {} at version {} could not be loaded", - topic.getQualifiedName(), schemaVersion.value(), e); - } + private void refreshCompiledSchemas(Topic topic, List schemaVersions) { + schemaVersions.forEach( + schemaVersion -> { + try { + compiledSchemaRepository.getSchema(topic, schemaVersion, REFRESH_ONLINE); + } catch (CouldNotLoadSchemaException e) { + logger.warn( + "Schema for topic {} at version {} could not be loaded", + topic.getQualifiedName(), + schemaVersion.value(), + e); + } }); - } + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaRepositoryFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaRepositoryFactory.java index 4943073ea9..918273b82d 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaRepositoryFactory.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaRepositoryFactory.java @@ -7,17 +7,18 @@ public class SchemaRepositoryFactory { - private final SchemaVersionsRepository schemaVersionsRepository; + private final SchemaVersionsRepository schemaVersionsRepository; - private final CompiledSchemaRepository compiledAvroSchemaRepository; + private final CompiledSchemaRepository compiledAvroSchemaRepository; - public SchemaRepositoryFactory(SchemaVersionsRepository schemaVersionsRepository, - CompiledSchemaRepository compiledAvroSchemaRepository) { - this.schemaVersionsRepository = schemaVersionsRepository; - this.compiledAvroSchemaRepository = compiledAvroSchemaRepository; - } + public SchemaRepositoryFactory( + SchemaVersionsRepository schemaVersionsRepository, + CompiledSchemaRepository compiledAvroSchemaRepository) { + this.schemaVersionsRepository = schemaVersionsRepository; + this.compiledAvroSchemaRepository = compiledAvroSchemaRepository; + } - public SchemaRepository provide() { - return new SchemaRepository(schemaVersionsRepository, compiledAvroSchemaRepository); - } + public SchemaRepository provide() { + return new SchemaRepository(schemaVersionsRepository, compiledAvroSchemaRepository); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaVersionRepositoryParameters.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaVersionRepositoryParameters.java index 4b04ccc34c..555f70907e 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaVersionRepositoryParameters.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaVersionRepositoryParameters.java @@ -4,11 +4,11 @@ public interface SchemaVersionRepositoryParameters { - boolean isCacheEnabled(); + boolean isCacheEnabled(); - Duration getRefreshAfterWrite(); + Duration getRefreshAfterWrite(); - Duration getExpireAfterWrite(); + Duration getExpireAfterWrite(); - int getReloadThreadPoolSize(); + int getReloadThreadPoolSize(); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaVersionsRepositoryFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaVersionsRepositoryFactory.java index 90eaf20685..211b38a2a7 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaVersionsRepositoryFactory.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/schema/SchemaVersionsRepositoryFactory.java @@ -1,6 +1,8 @@ package pl.allegro.tech.hermes.common.schema; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import pl.allegro.tech.hermes.domain.notifications.InternalNotificationsBus; import pl.allegro.tech.hermes.schema.CachedCompiledSchemaRepository; import pl.allegro.tech.hermes.schema.CachedSchemaVersionsRepository; @@ -9,47 +11,46 @@ import pl.allegro.tech.hermes.schema.RawSchemaClient; import pl.allegro.tech.hermes.schema.SchemaVersionsRepository; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - public class SchemaVersionsRepositoryFactory { - private final RawSchemaClient rawSchemaClient; - private final SchemaVersionRepositoryParameters schemaVersionsRepositoryParameters; - private final InternalNotificationsBus notificationsBus; - private final CompiledSchemaRepository compiledSchemaRepository; - - public SchemaVersionsRepositoryFactory(RawSchemaClient rawSchemaClient, - SchemaVersionRepositoryParameters schemaVersionsRepositoryParameters, - InternalNotificationsBus notificationsBus, - CompiledSchemaRepository compiledSchemaRepository) { - this.rawSchemaClient = rawSchemaClient; - this.schemaVersionsRepositoryParameters = schemaVersionsRepositoryParameters; - this.notificationsBus = notificationsBus; - this.compiledSchemaRepository = compiledSchemaRepository; - } - - public SchemaVersionsRepository provide() { - if (schemaVersionsRepositoryParameters.isCacheEnabled()) { - CachedSchemaVersionsRepository cachedSchemaVersionsRepository = new CachedSchemaVersionsRepository( - rawSchemaClient, - getVersionsReloader(), - schemaVersionsRepositoryParameters.getRefreshAfterWrite(), - schemaVersionsRepositoryParameters.getExpireAfterWrite()); - - notificationsBus.registerTopicCallback( - new SchemaCacheRefresherCallback<>( - cachedSchemaVersionsRepository, - (CachedCompiledSchemaRepository) compiledSchemaRepository)); - - return cachedSchemaVersionsRepository; - } - return new DirectSchemaVersionsRepository(rawSchemaClient); - } - - private ExecutorService getVersionsReloader() { - return Executors.newFixedThreadPool( - schemaVersionsRepositoryParameters.getReloadThreadPoolSize(), - new ThreadFactoryBuilder().setNameFormat("schema-source-reloader-%d").build()); + private final RawSchemaClient rawSchemaClient; + private final SchemaVersionRepositoryParameters schemaVersionsRepositoryParameters; + private final InternalNotificationsBus notificationsBus; + private final CompiledSchemaRepository compiledSchemaRepository; + + public SchemaVersionsRepositoryFactory( + RawSchemaClient rawSchemaClient, + SchemaVersionRepositoryParameters schemaVersionsRepositoryParameters, + InternalNotificationsBus notificationsBus, + CompiledSchemaRepository compiledSchemaRepository) { + this.rawSchemaClient = rawSchemaClient; + this.schemaVersionsRepositoryParameters = schemaVersionsRepositoryParameters; + this.notificationsBus = notificationsBus; + this.compiledSchemaRepository = compiledSchemaRepository; + } + + public SchemaVersionsRepository provide() { + if (schemaVersionsRepositoryParameters.isCacheEnabled()) { + CachedSchemaVersionsRepository cachedSchemaVersionsRepository = + new CachedSchemaVersionsRepository( + rawSchemaClient, + getVersionsReloader(), + schemaVersionsRepositoryParameters.getRefreshAfterWrite(), + schemaVersionsRepositoryParameters.getExpireAfterWrite()); + + notificationsBus.registerTopicCallback( + new SchemaCacheRefresherCallback<>( + cachedSchemaVersionsRepository, + (CachedCompiledSchemaRepository) compiledSchemaRepository)); + + return cachedSchemaVersionsRepository; } + return new DirectSchemaVersionsRepository(rawSchemaClient); + } + + private ExecutorService getVersionsReloader() { + return Executors.newFixedThreadPool( + schemaVersionsRepositoryParameters.getReloadThreadPoolSize(), + new ThreadFactoryBuilder().setNameFormat("schema-source-reloader-%d").build()); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/DefaultSslContextFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/DefaultSslContextFactory.java index f7c1b97914..3425aa0f7e 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/DefaultSslContextFactory.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/DefaultSslContextFactory.java @@ -4,24 +4,28 @@ public class DefaultSslContextFactory implements SslContextFactory { - private final String protocol; - private final KeyManagersProvider keyManagersProvider; - private final TrustManagersProvider trustManagersProvider; + private final String protocol; + private final KeyManagersProvider keyManagersProvider; + private final TrustManagersProvider trustManagersProvider; - public DefaultSslContextFactory(String protocol, KeyManagersProvider keyManagersProvider, TrustManagersProvider trustManagersProvider) { - this.protocol = protocol; - this.keyManagersProvider = keyManagersProvider; - this.trustManagersProvider = trustManagersProvider; - } + public DefaultSslContextFactory( + String protocol, + KeyManagersProvider keyManagersProvider, + TrustManagersProvider trustManagersProvider) { + this.protocol = protocol; + this.keyManagersProvider = keyManagersProvider; + this.trustManagersProvider = trustManagersProvider; + } - @Override - public SSLContextHolder create() { - try { - SSLContext sslContext = SSLContext.getInstance(protocol); - sslContext.init(keyManagersProvider.getKeyManagers(), trustManagersProvider.getTrustManagers(), null); - return new SSLContextHolder(sslContext, trustManagersProvider.getTrustManagers()); - } catch (Exception e) { - throw new SslContextCreationException(e); - } + @Override + public SSLContextHolder create() { + try { + SSLContext sslContext = SSLContext.getInstance(protocol); + sslContext.init( + keyManagersProvider.getKeyManagers(), trustManagersProvider.getTrustManagers(), null); + return new SSLContextHolder(sslContext, trustManagersProvider.getTrustManagers()); + } catch (Exception e) { + throw new SslContextCreationException(e); } + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/KeyManagersProvider.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/KeyManagersProvider.java index 305e3e3194..6136684e6a 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/KeyManagersProvider.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/KeyManagersProvider.java @@ -3,5 +3,5 @@ import javax.net.ssl.KeyManager; public interface KeyManagersProvider { - KeyManager[] getKeyManagers() throws Exception; + KeyManager[] getKeyManagers() throws Exception; } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/KeystoreConfigurationException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/KeystoreConfigurationException.java index de4d3b8fca..65e9abba69 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/KeystoreConfigurationException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/KeystoreConfigurationException.java @@ -1,7 +1,7 @@ package pl.allegro.tech.hermes.common.ssl; public class KeystoreConfigurationException extends RuntimeException { - public KeystoreConfigurationException(String keystoreSource) { - super(String.format("Unknown key store source: %s", keystoreSource)); - } + public KeystoreConfigurationException(String keystoreSource) { + super(String.format("Unknown key store source: %s", keystoreSource)); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/KeystoreProperties.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/KeystoreProperties.java index 796e570acc..4cee1c9e78 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/KeystoreProperties.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/KeystoreProperties.java @@ -1,32 +1,31 @@ package pl.allegro.tech.hermes.common.ssl; - import java.net.URI; public class KeystoreProperties { - private final String location; - private final String format; - private final String password; - - public KeystoreProperties(String location, String format, String password) { - this.location = location; - this.format = format; - this.password = password; - } - - public String getLocation() { - return location; - } - - public String getFormat() { - return format; - } - - public String getPassword() { - return password; - } - - public URI getLocationAsURI() { - return URI.create(getLocation()); - } + private final String location; + private final String format; + private final String password; + + public KeystoreProperties(String location, String format, String password) { + this.location = location; + this.format = format; + this.password = password; + } + + public String getLocation() { + return location; + } + + public String getFormat() { + return format; + } + + public String getPassword() { + return password; + } + + public URI getLocationAsURI() { + return URI.create(getLocation()); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/KeystoreSource.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/KeystoreSource.java index e6b5963a58..ecab7f9927 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/KeystoreSource.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/KeystoreSource.java @@ -1,16 +1,16 @@ package pl.allegro.tech.hermes.common.ssl; public enum KeystoreSource { - JRE("jre"), - PROVIDED("provided"); + JRE("jre"), + PROVIDED("provided"); - KeystoreSource(String value) { - this.value = value; - } + KeystoreSource(String value) { + this.value = value; + } - private final String value; + private final String value; - public final String getValue() { - return value; - } + public final String getValue() { + return value; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/SSLContextHolder.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/SSLContextHolder.java index 010fe816fb..c5a9499004 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/SSLContextHolder.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/SSLContextHolder.java @@ -5,19 +5,19 @@ public final class SSLContextHolder { - private final SSLContext sslContext; - private final TrustManager[] trustManagers; + private final SSLContext sslContext; + private final TrustManager[] trustManagers; - public SSLContextHolder(SSLContext sslContext, TrustManager[] trustManagers) { - this.sslContext = sslContext; - this.trustManagers = trustManagers; - } + public SSLContextHolder(SSLContext sslContext, TrustManager[] trustManagers) { + this.sslContext = sslContext; + this.trustManagers = trustManagers; + } - public SSLContext getSslContext() { - return sslContext; - } + public SSLContext getSslContext() { + return sslContext; + } - public TrustManager[] getTrustManagers() { - return trustManagers; - } + public TrustManager[] getTrustManagers() { + return trustManagers; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/SslContextCreationException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/SslContextCreationException.java index fd4c44709d..11d173bc99 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/SslContextCreationException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/SslContextCreationException.java @@ -1,7 +1,7 @@ package pl.allegro.tech.hermes.common.ssl; class SslContextCreationException extends RuntimeException { - SslContextCreationException(Exception e) { - super(e); - } + SslContextCreationException(Exception e) { + super(e); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/SslContextFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/SslContextFactory.java index c380098827..aaa64129fc 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/SslContextFactory.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/SslContextFactory.java @@ -2,5 +2,5 @@ public interface SslContextFactory { - SSLContextHolder create(); + SSLContextHolder create(); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/TrustManagersProvider.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/TrustManagersProvider.java index 0be68ca256..edb8a15a95 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/TrustManagersProvider.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/TrustManagersProvider.java @@ -3,5 +3,5 @@ import javax.net.ssl.TrustManager; public interface TrustManagersProvider { - TrustManager[] getTrustManagers() throws Exception; + TrustManager[] getTrustManagers() throws Exception; } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/TruststoreConfigurationException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/TruststoreConfigurationException.java index 89461be73b..8a77790465 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/TruststoreConfigurationException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/TruststoreConfigurationException.java @@ -1,7 +1,7 @@ package pl.allegro.tech.hermes.common.ssl; public class TruststoreConfigurationException extends RuntimeException { - public TruststoreConfigurationException(String truststoreSource) { - super(String.format("Unknown trust store source: %s", truststoreSource)); - } + public TruststoreConfigurationException(String truststoreSource) { + super(String.format("Unknown trust store source: %s", truststoreSource)); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/jvm/JvmKeyManagersProvider.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/jvm/JvmKeyManagersProvider.java index c4432bff3f..ae8a3aa88e 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/jvm/JvmKeyManagersProvider.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/jvm/JvmKeyManagersProvider.java @@ -1,16 +1,16 @@ package pl.allegro.tech.hermes.common.ssl.jvm; -import pl.allegro.tech.hermes.common.ssl.KeyManagersProvider; - import javax.net.ssl.KeyManager; import javax.net.ssl.KeyManagerFactory; +import pl.allegro.tech.hermes.common.ssl.KeyManagersProvider; public class JvmKeyManagersProvider implements KeyManagersProvider { - @Override - public KeyManager[] getKeyManagers() throws Exception { - KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); - keyManagerFactory.init(null, null); - return keyManagerFactory.getKeyManagers(); - } + @Override + public KeyManager[] getKeyManagers() throws Exception { + KeyManagerFactory keyManagerFactory = + KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + keyManagerFactory.init(null, null); + return keyManagerFactory.getKeyManagers(); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/jvm/JvmTrustManagerProvider.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/jvm/JvmTrustManagerProvider.java index 18c31d14d1..aa99072471 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/jvm/JvmTrustManagerProvider.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/jvm/JvmTrustManagerProvider.java @@ -1,6 +1,6 @@ package pl.allegro.tech.hermes.common.ssl.jvm; -import pl.allegro.tech.hermes.common.ssl.TrustManagersProvider; +import static java.lang.String.format; import java.io.File; import java.io.FileInputStream; @@ -9,50 +9,52 @@ import java.security.KeyStore; import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManagerFactory; - -import static java.lang.String.format; +import pl.allegro.tech.hermes.common.ssl.TrustManagersProvider; public class JvmTrustManagerProvider implements TrustManagersProvider { - @Override - public TrustManager[] getTrustManagers() throws Exception { - TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); - trustManagerFactory.init(loadJvmKeyStore()); - return trustManagerFactory.getTrustManagers(); + @Override + public TrustManager[] getTrustManagers() throws Exception { + TrustManagerFactory trustManagerFactory = + TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trustManagerFactory.init(loadJvmKeyStore()); + return trustManagerFactory.getTrustManagers(); + } + + private KeyStore loadJvmKeyStore() throws Exception { + String trustStore = System.getProperty("javax.net.ssl.trustStore", ""); + String trustStorePassword = System.getProperty("javax.net.ssl.trustStorePassword"); + String trustStoreType = + System.getProperty("javax.net.ssl.trustStoreType", KeyStore.getDefaultType()); + String trustStoreProvider = System.getProperty("javax.net.ssl.trustStoreProvider", ""); + + KeyStore keyStore = + trustStoreProvider.isEmpty() + ? KeyStore.getInstance(trustStoreType) + : KeyStore.getInstance(trustStoreType, trustStoreProvider); + char[] password = trustStorePassword == null ? null : trustStorePassword.toCharArray(); + keyStore.load(trustStoreInputStream(trustStore), password); + + return keyStore; + } + + private InputStream trustStoreInputStream(String trustStore) throws FileNotFoundException { + return new FileInputStream(trustStore.isEmpty() ? defaultTrustStore() : new File(trustStore)); + } + + private File defaultTrustStore() throws FileNotFoundException { + String javaHome = System.getProperty("java.home"); + + File jssecacerts = new File(format("%s/lib/security/jssecacerts", javaHome)); + if (jssecacerts.exists()) { + return jssecacerts; } - private KeyStore loadJvmKeyStore() throws Exception { - String trustStore = System.getProperty("javax.net.ssl.trustStore", ""); - String trustStorePassword = System.getProperty("javax.net.ssl.trustStorePassword"); - String trustStoreType = System.getProperty("javax.net.ssl.trustStoreType", KeyStore.getDefaultType()); - String trustStoreProvider = System.getProperty("javax.net.ssl.trustStoreProvider", ""); - - KeyStore keyStore = trustStoreProvider.isEmpty() - ? KeyStore.getInstance(trustStoreType) - : KeyStore.getInstance(trustStoreType, trustStoreProvider); - char[] password = trustStorePassword == null ? null : trustStorePassword.toCharArray(); - keyStore.load(trustStoreInputStream(trustStore), password); - - return keyStore; + File cacerts = new File(format("%s/lib/security/cacerts", javaHome)); + if (cacerts.exists()) { + return cacerts; } - private InputStream trustStoreInputStream(String trustStore) throws FileNotFoundException { - return new FileInputStream(trustStore.isEmpty() ? defaultTrustStore() : new File(trustStore)); - } - - private File defaultTrustStore() throws FileNotFoundException { - String javaHome = System.getProperty("java.home"); - - File jssecacerts = new File(format("%s/lib/security/jssecacerts", javaHome)); - if (jssecacerts.exists()) { - return jssecacerts; - } - - File cacerts = new File(format("%s/lib/security/cacerts", javaHome)); - if (cacerts.exists()) { - return cacerts; - } - - throw new FileNotFoundException("Default trust store not found."); - } + throw new FileNotFoundException("Default trust store not found."); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/provided/ProvidedKeyManagersProvider.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/provided/ProvidedKeyManagersProvider.java index 99d4d0627d..b2933d11af 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/provided/ProvidedKeyManagersProvider.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/provided/ProvidedKeyManagersProvider.java @@ -1,29 +1,29 @@ package pl.allegro.tech.hermes.common.ssl.provided; -import pl.allegro.tech.hermes.common.ssl.KeyManagersProvider; -import pl.allegro.tech.hermes.common.ssl.KeystoreProperties; - import java.io.InputStream; import java.security.KeyStore; import javax.net.ssl.KeyManager; import javax.net.ssl.KeyManagerFactory; +import pl.allegro.tech.hermes.common.ssl.KeyManagersProvider; +import pl.allegro.tech.hermes.common.ssl.KeystoreProperties; public class ProvidedKeyManagersProvider implements KeyManagersProvider, ResourceLoader { - private final KeystoreProperties keystoreProperties; + private final KeystoreProperties keystoreProperties; - public ProvidedKeyManagersProvider(KeystoreProperties keystoreProperties) { - this.keystoreProperties = keystoreProperties; - } + public ProvidedKeyManagersProvider(KeystoreProperties keystoreProperties) { + this.keystoreProperties = keystoreProperties; + } - @Override - public KeyManager[] getKeyManagers() throws Exception { - KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); - KeyStore keyStore = KeyStore.getInstance(keystoreProperties.getFormat()); - try (InputStream stream = getResourceAsInputStream(keystoreProperties.getLocationAsURI())) { - keyStore.load(stream, keystoreProperties.getPassword().toCharArray()); - keyManagerFactory.init(keyStore, keystoreProperties.getPassword().toCharArray()); - } - return keyManagerFactory.getKeyManagers(); + @Override + public KeyManager[] getKeyManagers() throws Exception { + KeyManagerFactory keyManagerFactory = + KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + KeyStore keyStore = KeyStore.getInstance(keystoreProperties.getFormat()); + try (InputStream stream = getResourceAsInputStream(keystoreProperties.getLocationAsURI())) { + keyStore.load(stream, keystoreProperties.getPassword().toCharArray()); + keyManagerFactory.init(keyStore, keystoreProperties.getPassword().toCharArray()); } + return keyManagerFactory.getKeyManagers(); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/provided/ProvidedTrustManagersProvider.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/provided/ProvidedTrustManagersProvider.java index e8d2532d56..b93037a69d 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/provided/ProvidedTrustManagersProvider.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/provided/ProvidedTrustManagersProvider.java @@ -1,29 +1,29 @@ package pl.allegro.tech.hermes.common.ssl.provided; -import pl.allegro.tech.hermes.common.ssl.KeystoreProperties; -import pl.allegro.tech.hermes.common.ssl.TrustManagersProvider; - import java.io.InputStream; import java.security.KeyStore; import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManagerFactory; +import pl.allegro.tech.hermes.common.ssl.KeystoreProperties; +import pl.allegro.tech.hermes.common.ssl.TrustManagersProvider; public class ProvidedTrustManagersProvider implements TrustManagersProvider, ResourceLoader { - private final KeystoreProperties keystoreProperties; + private final KeystoreProperties keystoreProperties; - public ProvidedTrustManagersProvider(KeystoreProperties keystoreProperties) { - this.keystoreProperties = keystoreProperties; - } + public ProvidedTrustManagersProvider(KeystoreProperties keystoreProperties) { + this.keystoreProperties = keystoreProperties; + } - @Override - public TrustManager[] getTrustManagers() throws Exception { - TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); - KeyStore keyStore = KeyStore.getInstance(keystoreProperties.getFormat()); - try (InputStream stream = getResourceAsInputStream(keystoreProperties.getLocationAsURI())) { - keyStore.load(stream, keystoreProperties.getPassword().toCharArray()); - trustManagerFactory.init(keyStore); - } - return trustManagerFactory.getTrustManagers(); + @Override + public TrustManager[] getTrustManagers() throws Exception { + TrustManagerFactory trustManagerFactory = + TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + KeyStore keyStore = KeyStore.getInstance(keystoreProperties.getFormat()); + try (InputStream stream = getResourceAsInputStream(keystoreProperties.getLocationAsURI())) { + keyStore.load(stream, keystoreProperties.getPassword().toCharArray()); + trustManagerFactory.init(keyStore); } + return trustManagerFactory.getTrustManagers(); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/provided/ResourceLoader.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/provided/ResourceLoader.java index 42047351eb..b7177fcaac 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/provided/ResourceLoader.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/ssl/provided/ResourceLoader.java @@ -1,17 +1,18 @@ package pl.allegro.tech.hermes.common.ssl.provided; +import static com.google.common.base.Strings.isNullOrEmpty; + import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.InputStream; import java.net.URI; -import static com.google.common.base.Strings.isNullOrEmpty; - interface ResourceLoader { - default InputStream getResourceAsInputStream(URI location) throws FileNotFoundException { - if ("classpath".equalsIgnoreCase(location.getScheme())) { - return getClass().getClassLoader().getResourceAsStream(location.getSchemeSpecificPart()); - } - return new FileInputStream(isNullOrEmpty(location.getPath()) ? location.getSchemeSpecificPart() : location.getPath()); + default InputStream getResourceAsInputStream(URI location) throws FileNotFoundException { + if ("classpath".equalsIgnoreCase(location.getScheme())) { + return getClass().getClassLoader().getResourceAsStream(location.getSchemeSpecificPart()); } + return new FileInputStream( + isNullOrEmpty(location.getPath()) ? location.getSchemeSpecificPart() : location.getPath()); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/util/InetAddressInstanceIdResolver.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/util/InetAddressInstanceIdResolver.java index ef98c2c8bc..29c3c84023 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/util/InetAddressInstanceIdResolver.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/util/InetAddressInstanceIdResolver.java @@ -1,25 +1,23 @@ package pl.allegro.tech.hermes.common.util; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.net.InetAddress; import java.net.UnknownHostException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public final class InetAddressInstanceIdResolver implements InstanceIdResolver { - private static final Logger LOGGER = LoggerFactory.getLogger(InetAddressInstanceIdResolver.class); + private static final Logger LOGGER = LoggerFactory.getLogger(InetAddressInstanceIdResolver.class); - public InetAddressInstanceIdResolver() { } + public InetAddressInstanceIdResolver() {} - public String resolve() { - String hostname = "hostname-could-not-be-detected"; - try { - hostname = InetAddress.getLocalHost().getCanonicalHostName(); - } catch (UnknownHostException e) { - LOGGER.warn("Could not determine hostname"); - } - return hostname; + public String resolve() { + String hostname = "hostname-could-not-be-detected"; + try { + hostname = InetAddress.getLocalHost().getCanonicalHostName(); + } catch (UnknownHostException e) { + LOGGER.warn("Could not determine hostname"); } - + return hostname; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/util/InstanceIdResolver.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/util/InstanceIdResolver.java index 0373cec8fa..2171a491f2 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/common/util/InstanceIdResolver.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/common/util/InstanceIdResolver.java @@ -1,5 +1,5 @@ package pl.allegro.tech.hermes.common.util; public interface InstanceIdResolver { - String resolve(); + String resolve(); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/CredentialsRepository.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/CredentialsRepository.java index 7b21bf1475..7e9eb78100 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/CredentialsRepository.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/CredentialsRepository.java @@ -1,7 +1,7 @@ package pl.allegro.tech.hermes.domain; public interface CredentialsRepository { - NodePassword readAdminPassword(); + NodePassword readAdminPassword(); - void overwriteAdminPassword(String password); + void overwriteAdminPassword(String password); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/NodePassword.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/NodePassword.java index 1211ef8f59..b9a7a39504 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/NodePassword.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/NodePassword.java @@ -2,61 +2,60 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.commons.codec.digest.DigestUtils; - import java.util.Arrays; +import org.apache.commons.codec.digest.DigestUtils; public class NodePassword { - private static final int DEFAULT_STRING_LENGTH = 12; + private static final int DEFAULT_STRING_LENGTH = 12; - private final byte[] hashedPassword; + private final byte[] hashedPassword; - @JsonCreator - public NodePassword(@JsonProperty("hashedPassword") byte[] hashedPassword) { - this.hashedPassword = Arrays.copyOf(hashedPassword, hashedPassword.length); - } + @JsonCreator + public NodePassword(@JsonProperty("hashedPassword") byte[] hashedPassword) { + this.hashedPassword = Arrays.copyOf(hashedPassword, hashedPassword.length); + } - public NodePassword(String password) { - this.hashedPassword = NodePassword.hashString(password); - } + public NodePassword(String password) { + this.hashedPassword = NodePassword.hashString(password); + } - public byte[] getHashedPassword() { - return Arrays.copyOf(hashedPassword, hashedPassword.length); - } + public byte[] getHashedPassword() { + return Arrays.copyOf(hashedPassword, hashedPassword.length); + } - private static byte[] hashString(String string) { - return DigestUtils.sha256(string); - } + private static byte[] hashString(String string) { + return DigestUtils.sha256(string); + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } - if (o == null) { - return false; - } + if (o == null) { + return false; + } - if (o instanceof String) { - return this.equals(NodePassword.fromString((String) o)); - } + if (o instanceof String) { + return this.equals(NodePassword.fromString((String) o)); + } - if (getClass() != o.getClass()) { - return false; - } + if (getClass() != o.getClass()) { + return false; + } - NodePassword that = (NodePassword) o; + NodePassword that = (NodePassword) o; - return Arrays.equals(hashedPassword, that.hashedPassword); - } + return Arrays.equals(hashedPassword, that.hashedPassword); + } - @Override - public int hashCode() { - return Arrays.hashCode(hashedPassword); - } + @Override + public int hashCode() { + return Arrays.hashCode(hashedPassword); + } - public static NodePassword fromString(String string) { - return new NodePassword(string); - } + public static NodePassword fromString(String string) { + return new NodePassword(string); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/FilterableMessage.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/FilterableMessage.java index d4e8f901f2..9c769e10a8 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/FilterableMessage.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/FilterableMessage.java @@ -1,19 +1,18 @@ package pl.allegro.tech.hermes.domain.filtering; +import java.util.Map; +import java.util.Optional; import org.apache.avro.Schema; import pl.allegro.tech.hermes.api.ContentType; import pl.allegro.tech.hermes.schema.CompiledSchema; -import java.util.Map; -import java.util.Optional; - public interface FilterableMessage { - ContentType getContentType(); + ContentType getContentType(); - Map getExternalMetadata(); + Map getExternalMetadata(); - byte[] getData(); + byte[] getData(); - Optional> getSchema(); + Optional> getSchema(); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/FilteringException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/FilteringException.java index 8b6340da05..62f7da8c82 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/FilteringException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/FilteringException.java @@ -3,21 +3,21 @@ import pl.allegro.tech.hermes.common.exception.InternalProcessingException; public class FilteringException extends InternalProcessingException { - public FilteringException(Throwable throwable) { - super(throwable); - } + public FilteringException(Throwable throwable) { + super(throwable); + } - public FilteringException(String message) { - super(message); - } + public FilteringException(String message) { + super(message); + } - public FilteringException(String message, Throwable throwable) { - super(message, throwable); - } + public FilteringException(String message, Throwable throwable) { + super(message, throwable); + } - public static void check(boolean condition, String message) { - if (!condition) { - throw new FilteringException(message); - } + public static void check(boolean condition, String message) { + if (!condition) { + throw new FilteringException(message); } + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/MatchingStrategy.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/MatchingStrategy.java index b3b55ad50c..177b73a320 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/MatchingStrategy.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/MatchingStrategy.java @@ -3,17 +3,17 @@ import java.util.Optional; public enum MatchingStrategy { - ALL, - ANY; + ALL, + ANY; - public static MatchingStrategy fromString(String value, MatchingStrategy defaultValue) { - try { - return Optional.ofNullable(value) - .map(String::toUpperCase) - .map(MatchingStrategy::valueOf) - .orElse(defaultValue); - } catch (IllegalArgumentException ex) { - return defaultValue; - } + public static MatchingStrategy fromString(String value, MatchingStrategy defaultValue) { + try { + return Optional.ofNullable(value) + .map(String::toUpperCase) + .map(MatchingStrategy::valueOf) + .orElse(defaultValue); + } catch (IllegalArgumentException ex) { + return defaultValue; } + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/MessageFilter.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/MessageFilter.java index 8a03f435b0..5ce32efdcc 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/MessageFilter.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/MessageFilter.java @@ -3,20 +3,20 @@ import java.util.function.Predicate; public class MessageFilter implements Predicate { - private final String type; - private final Predicate predicate; + private final String type; + private final Predicate predicate; - public MessageFilter(String type, Predicate predicate) { - this.type = type; - this.predicate = predicate; - } + public MessageFilter(String type, Predicate predicate) { + this.type = type; + this.predicate = predicate; + } - @Override - public boolean test(FilterableMessage message) { - return predicate.test(message); - } + @Override + public boolean test(FilterableMessage message) { + return predicate.test(message); + } - public String getType() { - return type; - } + public String getType() { + return type; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/MessageFilterSource.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/MessageFilterSource.java index a946227e17..6c67564592 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/MessageFilterSource.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/MessageFilterSource.java @@ -1,11 +1,10 @@ package pl.allegro.tech.hermes.domain.filtering; -import pl.allegro.tech.hermes.api.MessageFilterSpecification; - import java.util.List; +import pl.allegro.tech.hermes.api.MessageFilterSpecification; public interface MessageFilterSource { - MessageFilter compile(MessageFilterSpecification specification); + MessageFilter compile(MessageFilterSpecification specification); - List getGlobalFilters(); + List getGlobalFilters(); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/MessageFilters.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/MessageFilters.java index c8b3f4cfae..00ed213719 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/MessageFilters.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/MessageFilters.java @@ -1,33 +1,35 @@ package pl.allegro.tech.hermes.domain.filtering; -import pl.allegro.tech.hermes.api.MessageFilterSpecification; +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toMap; import java.util.List; import java.util.Map; - -import static java.util.function.Function.identity; -import static java.util.stream.Collectors.toMap; +import pl.allegro.tech.hermes.api.MessageFilterSpecification; public class MessageFilters implements MessageFilterSource { - private final Map filters; - private final List globalFilters; + private final Map filters; + private final List globalFilters; - public MessageFilters(List globalFilters, - List subscriptionFilterCompilers) { - this.globalFilters = globalFilters; - this.filters = subscriptionFilterCompilers.stream().collect(toMap(SubscriptionMessageFilterCompiler::getType, identity())); - } + public MessageFilters( + List globalFilters, + List subscriptionFilterCompilers) { + this.globalFilters = globalFilters; + this.filters = + subscriptionFilterCompilers.stream() + .collect(toMap(SubscriptionMessageFilterCompiler::getType, identity())); + } - @Override - public MessageFilter compile(MessageFilterSpecification specification) { - if (!filters.containsKey(specification.getType())) { - throw new NoSuchFilterException(specification.getType()); - } - return filters.get(specification.getType()).getMessageFilter(specification); + @Override + public MessageFilter compile(MessageFilterSpecification specification) { + if (!filters.containsKey(specification.getType())) { + throw new NoSuchFilterException(specification.getType()); } + return filters.get(specification.getType()).getMessageFilter(specification); + } - @Override - public List getGlobalFilters() { - return globalFilters; - } + @Override + public List getGlobalFilters() { + return globalFilters; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/NoSuchFilterException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/NoSuchFilterException.java index bcc9f1cdf3..d5557d9680 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/NoSuchFilterException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/NoSuchFilterException.java @@ -1,15 +1,15 @@ package pl.allegro.tech.hermes.domain.filtering; public class NoSuchFilterException extends FilteringException { - public NoSuchFilterException(Throwable throwable) { - super(throwable); - } + public NoSuchFilterException(Throwable throwable) { + super(throwable); + } - public NoSuchFilterException(String filterType) { - super("Filter of type '" + filterType + "' not found."); - } + public NoSuchFilterException(String filterType) { + super("Filter of type '" + filterType + "' not found."); + } - public NoSuchFilterException(String message, Throwable throwable) { - super(message, throwable); - } + public NoSuchFilterException(String message, Throwable throwable) { + super(message, throwable); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/SubscriptionMessageFilterCompiler.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/SubscriptionMessageFilterCompiler.java index c4a83f2d2c..bf337662f8 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/SubscriptionMessageFilterCompiler.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/SubscriptionMessageFilterCompiler.java @@ -1,15 +1,14 @@ package pl.allegro.tech.hermes.domain.filtering; -import pl.allegro.tech.hermes.api.MessageFilterSpecification; - import java.util.function.Predicate; +import pl.allegro.tech.hermes.api.MessageFilterSpecification; public interface SubscriptionMessageFilterCompiler { - String getType(); + String getType(); - Predicate compile(MessageFilterSpecification specification); + Predicate compile(MessageFilterSpecification specification); - default MessageFilter getMessageFilter(MessageFilterSpecification specification) { - return new MessageFilter(getType(), compile(specification)); - } + default MessageFilter getMessageFilter(MessageFilterSpecification specification) { + return new MessageFilter(getType(), compile(specification)); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/UnsupportedMatchingStrategyException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/UnsupportedMatchingStrategyException.java index 9e62421ca4..5526c57920 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/UnsupportedMatchingStrategyException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/UnsupportedMatchingStrategyException.java @@ -1,7 +1,12 @@ package pl.allegro.tech.hermes.domain.filtering; public class UnsupportedMatchingStrategyException extends FilteringException { - public UnsupportedMatchingStrategyException(String filterType, MatchingStrategy strategy) { - super("Matching strategy '" + strategy + "' is not supported in filters of type '" + filterType + "'."); - } + public UnsupportedMatchingStrategyException(String filterType, MatchingStrategy strategy) { + super( + "Matching strategy '" + + strategy + + "' is not supported in filters of type '" + + filterType + + "'."); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/avro/AvroPathPredicate.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/avro/AvroPathPredicate.java index 7cd000a2e5..cfee70ac7c 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/avro/AvroPathPredicate.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/avro/AvroPathPredicate.java @@ -1,17 +1,13 @@ package pl.allegro.tech.hermes.domain.filtering.avro; -import jakarta.annotation.Nullable; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericArray; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.util.Utf8; -import pl.allegro.tech.hermes.api.ContentType; -import pl.allegro.tech.hermes.domain.filtering.FilterableMessage; -import pl.allegro.tech.hermes.domain.filtering.FilteringException; -import pl.allegro.tech.hermes.domain.filtering.MatchingStrategy; -import pl.allegro.tech.hermes.domain.filtering.UnsupportedMatchingStrategyException; -import pl.allegro.tech.hermes.schema.CompiledSchema; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyListIterator; +import static java.util.Collections.singletonList; +import static org.apache.commons.lang3.StringUtils.strip; +import static pl.allegro.tech.hermes.common.message.converter.AvroRecordToBytesConverter.bytesToRecord; +import static pl.allegro.tech.hermes.domain.filtering.FilteringException.check; +import jakarta.annotation.Nullable; import java.io.IOException; import java.util.Arrays; import java.util.HashMap; @@ -23,133 +19,145 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; - -import static java.util.Collections.emptyList; -import static java.util.Collections.emptyListIterator; -import static java.util.Collections.singletonList; -import static org.apache.commons.lang3.StringUtils.strip; -import static pl.allegro.tech.hermes.common.message.converter.AvroRecordToBytesConverter.bytesToRecord; -import static pl.allegro.tech.hermes.domain.filtering.FilteringException.check; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericArray; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.util.Utf8; +import pl.allegro.tech.hermes.api.ContentType; +import pl.allegro.tech.hermes.domain.filtering.FilterableMessage; +import pl.allegro.tech.hermes.domain.filtering.FilteringException; +import pl.allegro.tech.hermes.domain.filtering.MatchingStrategy; +import pl.allegro.tech.hermes.domain.filtering.UnsupportedMatchingStrategyException; +import pl.allegro.tech.hermes.schema.CompiledSchema; class AvroPathPredicate implements Predicate { - private static final String WILDCARD_IDX = "*"; - private static final String GROUP_SELECTOR = "selector"; - private static final String GROUP_IDX = "index"; - private static final String ARRAY_PATTERN_SELECTOR_PART = "(?<" + GROUP_SELECTOR + ">..*)"; - private static final String ARRAY_PATTERN_IDX_PART = "\\[(?<" + GROUP_IDX + ">\\" + WILDCARD_IDX + "|\\d+)]"; - private static final Pattern ARRAY_PATTERN = Pattern.compile(ARRAY_PATTERN_SELECTOR_PART + ARRAY_PATTERN_IDX_PART); - private static final String NULL_AS_STRING = "null"; - private final List path; - private final Pattern pattern; - private final MatchingStrategy matchingStrategy; - - AvroPathPredicate(String path, Pattern pattern, MatchingStrategy matchingStrategy) { - this.path = Arrays.asList(strip(path, ".").split("\\.")); - this.pattern = pattern; - this.matchingStrategy = matchingStrategy; + private static final String WILDCARD_IDX = "*"; + private static final String GROUP_SELECTOR = "selector"; + private static final String GROUP_IDX = "index"; + private static final String ARRAY_PATTERN_SELECTOR_PART = "(?<" + GROUP_SELECTOR + ">..*)"; + private static final String ARRAY_PATTERN_IDX_PART = + "\\[(?<" + GROUP_IDX + ">\\" + WILDCARD_IDX + "|\\d+)]"; + private static final Pattern ARRAY_PATTERN = + Pattern.compile(ARRAY_PATTERN_SELECTOR_PART + ARRAY_PATTERN_IDX_PART); + private static final String NULL_AS_STRING = "null"; + private final List path; + private final Pattern pattern; + private final MatchingStrategy matchingStrategy; + + AvroPathPredicate(String path, Pattern pattern, MatchingStrategy matchingStrategy) { + this.path = Arrays.asList(strip(path, ".").split("\\.")); + this.pattern = pattern; + this.matchingStrategy = matchingStrategy; + } + + @Override + public boolean test(final FilterableMessage message) { + check( + message.getContentType() == ContentType.AVRO, + "This filter supports only AVRO contentType."); + try { + List result = select(message); + Stream resultStream = result.stream().map(Object::toString); + + return !result.isEmpty() && matchResultsStream(resultStream); + } catch (Exception exception) { + throw new FilteringException(exception); } - - @Override - public boolean test(final FilterableMessage message) { - check(message.getContentType() == ContentType.AVRO, "This filter supports only AVRO contentType."); - try { - List result = select(message); - Stream resultStream = result.stream().map(Object::toString); - - return !result.isEmpty() && matchResultsStream(resultStream); - } catch (Exception exception) { - throw new FilteringException(exception); + } + + private List select(final FilterableMessage message) throws IOException { + CompiledSchema compiledSchema = message.getSchema().get(); + return select(bytesToRecord(message.getData(), compiledSchema.getSchema())); + } + + private List select(GenericRecord record) { + ListIterator iter = path.listIterator(); + return select(record, iter); + } + + private List select(Object record, ListIterator iter) { + Object current = record; + while (iter.hasNext() && isSupportedType(current)) { + if (current instanceof GenericRecord) { + GenericRecord currentRecord = (GenericRecord) current; + String selector = iter.next(); + Matcher arrayMatcher = ARRAY_PATTERN.matcher(selector); + + if (arrayMatcher.matches()) { + selector = arrayMatcher.group(GROUP_SELECTOR); + + current = recordFieldValueOrNull(selector, currentRecord); + if (!(current instanceof GenericArray)) { + return emptyList(); + } + + GenericArray currentArray = (GenericArray) current; + String idx = arrayMatcher.group(GROUP_IDX); + + if (idx.equals(WILDCARD_IDX)) { + return selectMultipleArrayItems(iter, currentArray); + } else { + current = selectSingleArrayItem(Integer.valueOf(idx), currentArray); + } + + } else { + current = recordFieldValueOrNull(selector, currentRecord); } + } else if (current instanceof HashMap) { + Map currentRecord = (HashMap) current; + Utf8 selector = new Utf8(iter.next()); + current = currentRecord.get(selector); + } } - private List select(final FilterableMessage message) throws IOException { - CompiledSchema compiledSchema = message.getSchema().get(); - return select(bytesToRecord(message.getData(), compiledSchema.getSchema())); + return iter.hasNext() ? emptyList() : singletonList(current == null ? NULL_AS_STRING : current); + } + + private boolean isSupportedType(Object record) { + return record instanceof GenericRecord || record instanceof HashMap; + } + + private List selectMultipleArrayItems( + ListIterator iter, GenericArray currentArray) { + return currentArray.stream() + .map( + item -> + select( + item, + iter.hasNext() ? path.listIterator(iter.nextIndex()) : emptyListIterator())) + .flatMap(List::stream) + .collect(Collectors.toList()); + } + + private Object selectSingleArrayItem(int idx, GenericArray currentArray) { + if (currentArray.size() <= idx) { + return null; } - private List select(GenericRecord record) { - ListIterator iter = path.listIterator(); - return select(record, iter); - } - - private List select(Object record, ListIterator iter) { - Object current = record; - while (iter.hasNext() && isSupportedType(current)) { - if (current instanceof GenericRecord) { - GenericRecord currentRecord = (GenericRecord) current; - String selector = iter.next(); - Matcher arrayMatcher = ARRAY_PATTERN.matcher(selector); - - if (arrayMatcher.matches()) { - selector = arrayMatcher.group(GROUP_SELECTOR); - - current = recordFieldValueOrNull(selector, currentRecord); - if (!(current instanceof GenericArray)) { - return emptyList(); - } - - GenericArray currentArray = (GenericArray) current; - String idx = arrayMatcher.group(GROUP_IDX); - - if (idx.equals(WILDCARD_IDX)) { - return selectMultipleArrayItems(iter, currentArray); - } else { - current = selectSingleArrayItem(Integer.valueOf(idx), currentArray); - } - - } else { - current = recordFieldValueOrNull(selector, currentRecord); - } - } else if (current instanceof HashMap) { - Map currentRecord = (HashMap) current; - Utf8 selector = new Utf8(iter.next()); - current = currentRecord.get(selector); - } - } - - return iter.hasNext() ? emptyList() : singletonList(current == null ? NULL_AS_STRING : current); + return currentArray.get(idx); + } + + private boolean matchResultsStream(Stream results) { + switch (matchingStrategy) { + case ALL: + return results.allMatch(this::matches); + case ANY: + return results.anyMatch(this::matches); + default: + throw new UnsupportedMatchingStrategyException("avropath", matchingStrategy); } + } - private boolean isSupportedType(Object record) { - return record instanceof GenericRecord || record instanceof HashMap; + @Nullable + private Object recordFieldValueOrNull(String selector, GenericRecord record) { + Schema.Field field = record.getSchema().getField(selector); + if (field == null) { + return null; } + return record.get(field.pos()); + } - private List selectMultipleArrayItems(ListIterator iter, GenericArray currentArray) { - return currentArray.stream() - .map(item -> select(item, iter.hasNext() ? path.listIterator(iter.nextIndex()) : emptyListIterator())) - .flatMap(List::stream) - .collect(Collectors.toList()); - } - - private Object selectSingleArrayItem(int idx, GenericArray currentArray) { - if (currentArray.size() <= idx) { - return null; - } - - return currentArray.get(idx); - } - - private boolean matchResultsStream(Stream results) { - switch (matchingStrategy) { - case ALL: - return results.allMatch(this::matches); - case ANY: - return results.anyMatch(this::matches); - default: - throw new UnsupportedMatchingStrategyException("avropath", matchingStrategy); - } - } - - @Nullable - private Object recordFieldValueOrNull(String selector, GenericRecord record) { - Schema.Field field = record.getSchema().getField(selector); - if (field == null) { - return null; - } - return record.get(field.pos()); - } - - private boolean matches(String value) { - return pattern.matcher(value).matches(); - } + private boolean matches(String value) { + return pattern.matcher(value).matches(); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/avro/AvroPathSubscriptionMessageFilterCompiler.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/avro/AvroPathSubscriptionMessageFilterCompiler.java index a3072bc66d..1579be965c 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/avro/AvroPathSubscriptionMessageFilterCompiler.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/avro/AvroPathSubscriptionMessageFilterCompiler.java @@ -1,26 +1,25 @@ package pl.allegro.tech.hermes.domain.filtering.avro; +import java.util.function.Predicate; +import java.util.regex.Pattern; import pl.allegro.tech.hermes.api.MessageFilterSpecification; import pl.allegro.tech.hermes.domain.filtering.FilterableMessage; import pl.allegro.tech.hermes.domain.filtering.MatchingStrategy; import pl.allegro.tech.hermes.domain.filtering.SubscriptionMessageFilterCompiler; -import java.util.function.Predicate; -import java.util.regex.Pattern; - -public class AvroPathSubscriptionMessageFilterCompiler implements SubscriptionMessageFilterCompiler { +public class AvroPathSubscriptionMessageFilterCompiler + implements SubscriptionMessageFilterCompiler { - @Override - public String getType() { - return "avropath"; - } + @Override + public String getType() { + return "avropath"; + } - @Override - public Predicate compile(MessageFilterSpecification specification) { - return new AvroPathPredicate( - specification.getPath(), - Pattern.compile(specification.getMatcher()), - MatchingStrategy.fromString(specification.getMatchingStrategy(), MatchingStrategy.ALL) - ); - } + @Override + public Predicate compile(MessageFilterSpecification specification) { + return new AvroPathPredicate( + specification.getPath(), + Pattern.compile(specification.getMatcher()), + MatchingStrategy.fromString(specification.getMatchingStrategy(), MatchingStrategy.ALL)); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/chain/FilterChain.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/chain/FilterChain.java index 815a12b9b4..15ff95c5e6 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/chain/FilterChain.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/chain/FilterChain.java @@ -1,28 +1,27 @@ package pl.allegro.tech.hermes.domain.filtering.chain; -import pl.allegro.tech.hermes.domain.filtering.FilterableMessage; -import pl.allegro.tech.hermes.domain.filtering.MessageFilter; - import java.util.ArrayList; import java.util.List; +import pl.allegro.tech.hermes.domain.filtering.FilterableMessage; +import pl.allegro.tech.hermes.domain.filtering.MessageFilter; public final class FilterChain { - private final List messageFilters; + private final List messageFilters; - FilterChain(final List messageFilters) { - this.messageFilters = new ArrayList<>(messageFilters); - } + FilterChain(final List messageFilters) { + this.messageFilters = new ArrayList<>(messageFilters); + } - public FilterResult apply(final FilterableMessage message) { - for (MessageFilter filter : messageFilters) { - try { - if (!filter.test(message)) { - return FilterResult.failed(filter.getType(), "logical"); - } - } catch (Exception ex) { - return FilterResult.failed(filter.getType(), ex); - } + public FilterResult apply(final FilterableMessage message) { + for (MessageFilter filter : messageFilters) { + try { + if (!filter.test(message)) { + return FilterResult.failed(filter.getType(), "logical"); } - return FilterResult.PASS; + } catch (Exception ex) { + return FilterResult.failed(filter.getType(), ex); + } } + return FilterResult.PASS; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/chain/FilterChainFactory.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/chain/FilterChainFactory.java index b418abd259..99f9f3fd43 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/chain/FilterChainFactory.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/chain/FilterChainFactory.java @@ -1,26 +1,24 @@ package pl.allegro.tech.hermes.domain.filtering.chain; -import pl.allegro.tech.hermes.api.MessageFilterSpecification; -import pl.allegro.tech.hermes.domain.filtering.MessageFilter; -import pl.allegro.tech.hermes.domain.filtering.MessageFilterSource; +import static java.util.stream.Stream.concat; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; - -import static java.util.stream.Stream.concat; +import pl.allegro.tech.hermes.api.MessageFilterSpecification; +import pl.allegro.tech.hermes.domain.filtering.MessageFilter; +import pl.allegro.tech.hermes.domain.filtering.MessageFilterSource; public class FilterChainFactory { - private final MessageFilterSource availableFilters; + private final MessageFilterSource availableFilters; - public FilterChainFactory(MessageFilterSource filters) { - this.availableFilters = filters; - } + public FilterChainFactory(MessageFilterSource filters) { + this.availableFilters = filters; + } - public FilterChain create(final List filters) { - Stream globalFilters = availableFilters.getGlobalFilters().stream(); - Stream subscriptionFilters = filters.stream() - .map(availableFilters::compile); - return new FilterChain(concat(globalFilters, subscriptionFilters).collect(Collectors.toList())); - } + public FilterChain create(final List filters) { + Stream globalFilters = availableFilters.getGlobalFilters().stream(); + Stream subscriptionFilters = filters.stream().map(availableFilters::compile); + return new FilterChain(concat(globalFilters, subscriptionFilters).collect(Collectors.toList())); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/chain/FilterResult.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/chain/FilterResult.java index 5a7fd4ceb2..6342456dd7 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/chain/FilterResult.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/chain/FilterResult.java @@ -1,64 +1,68 @@ package pl.allegro.tech.hermes.domain.filtering.chain; -import com.google.common.base.Joiner; - -import java.util.Optional; - import static java.lang.String.format; import static java.util.Optional.empty; +import com.google.common.base.Joiner; +import java.util.Optional; + public final class FilterResult { - private final boolean filtered; - private final Optional filterType; - private final Optional message; - private final Optional cause; + private final boolean filtered; + private final Optional filterType; + private final Optional message; + private final Optional cause; - public static final FilterResult PASS = new FilterResult(false, empty(), empty(), empty()); + public static final FilterResult PASS = new FilterResult(false, empty(), empty(), empty()); - public static FilterResult failed(String filterType, String message) { - return new FilterResult(true, Optional.of(filterType), Optional.ofNullable(message), empty()); - } + public static FilterResult failed(String filterType, String message) { + return new FilterResult(true, Optional.of(filterType), Optional.ofNullable(message), empty()); + } - public static FilterResult failed(String filterType, Exception exception) { - return new FilterResult(true, Optional.of(filterType), empty(), Optional.ofNullable(exception)); - } + public static FilterResult failed(String filterType, Exception exception) { + return new FilterResult(true, Optional.of(filterType), empty(), Optional.ofNullable(exception)); + } - private FilterResult(boolean filtered, - Optional filterType, - Optional message, - Optional cause) { - this.filtered = filtered; - this.filterType = filterType; - this.message = message; - this.cause = cause; - } + private FilterResult( + boolean filtered, + Optional filterType, + Optional message, + Optional cause) { + this.filtered = filtered; + this.filterType = filterType; + this.message = message; + this.cause = cause; + } - public boolean isFiltered() { - return filtered; - } + public boolean isFiltered() { + return filtered; + } - public Optional getFilterType() { - return filterType; - } + public Optional getFilterType() { + return filterType; + } - public Optional getMessage() { - return message; - } + public Optional getMessage() { + return message; + } - public Optional getCause() { - return cause; - } + public Optional getCause() { + return cause; + } - @Override - public String toString() { - return "[" + Joiner.on(",").skipNulls() - .join(format("%s={%s}", "filtered", filtered), - toString("filterType", filterType), - toString("message", message), - toString("cause", cause)) + "]"; - } + @Override + public String toString() { + return "[" + + Joiner.on(",") + .skipNulls() + .join( + format("%s={%s}", "filtered", filtered), + toString("filterType", filterType), + toString("message", message), + toString("cause", cause)) + + "]"; + } - private String toString(String fieldName, Optional value) { - return value.map(v -> format("%s={%s}", fieldName, v)).orElse(null); - } + private String toString(String fieldName, Optional value) { + return value.map(v -> format("%s={%s}", fieldName, v)).orElse(null); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/header/HeaderPredicate.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/header/HeaderPredicate.java index 33bd8819e2..e8e9102adc 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/header/HeaderPredicate.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/header/HeaderPredicate.java @@ -1,27 +1,25 @@ package pl.allegro.tech.hermes.domain.filtering.header; -import pl.allegro.tech.hermes.domain.filtering.FilterableMessage; - import java.util.function.Predicate; import java.util.regex.Pattern; +import pl.allegro.tech.hermes.domain.filtering.FilterableMessage; class HeaderPredicate implements Predicate { - private final String name; - private final Pattern valuePattern; + private final String name; + private final Pattern valuePattern; - HeaderPredicate(String name, Pattern valuePattern) { - this.name = name; - this.valuePattern = valuePattern; - } + HeaderPredicate(String name, Pattern valuePattern) { + this.name = name; + this.valuePattern = valuePattern; + } - @Override - public boolean test(FilterableMessage message) { - return message.getExternalMetadata() - .entrySet().stream() - .filter(h -> h.getKey().equals(name)) - .findFirst() - .filter(h -> valuePattern.matcher(h.getValue()).matches()) - .isPresent(); - } + @Override + public boolean test(FilterableMessage message) { + return message.getExternalMetadata().entrySet().stream() + .filter(h -> h.getKey().equals(name)) + .findFirst() + .filter(h -> valuePattern.matcher(h.getValue()).matches()) + .isPresent(); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/header/HeaderSubscriptionMessageFilterCompiler.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/header/HeaderSubscriptionMessageFilterCompiler.java index 4a1249a097..5b8d82ae1c 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/header/HeaderSubscriptionMessageFilterCompiler.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/header/HeaderSubscriptionMessageFilterCompiler.java @@ -1,22 +1,21 @@ package pl.allegro.tech.hermes.domain.filtering.header; +import java.util.function.Predicate; +import java.util.regex.Pattern; import pl.allegro.tech.hermes.api.MessageFilterSpecification; import pl.allegro.tech.hermes.domain.filtering.FilterableMessage; import pl.allegro.tech.hermes.domain.filtering.SubscriptionMessageFilterCompiler; -import java.util.function.Predicate; -import java.util.regex.Pattern; - public class HeaderSubscriptionMessageFilterCompiler implements SubscriptionMessageFilterCompiler { - @Override - public String getType() { - return "header"; - } - - @Override - public Predicate compile(MessageFilterSpecification specification) { - return new HeaderPredicate(specification.getHeader(), Pattern.compile(specification.getMatcher())); + @Override + public String getType() { + return "header"; + } - } + @Override + public Predicate compile(MessageFilterSpecification specification) { + return new HeaderPredicate( + specification.getHeader(), Pattern.compile(specification.getMatcher())); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/json/JsonPathPredicate.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/json/JsonPathPredicate.java index 2e90cc13c0..11793392d3 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/json/JsonPathPredicate.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/json/JsonPathPredicate.java @@ -1,59 +1,65 @@ package pl.allegro.tech.hermes.domain.filtering.json; +import static pl.allegro.tech.hermes.domain.filtering.FilteringException.check; + import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.JsonPath; +import java.io.ByteArrayInputStream; +import java.util.List; +import java.util.function.Predicate; +import java.util.regex.Pattern; +import java.util.stream.Stream; import pl.allegro.tech.hermes.api.ContentType; import pl.allegro.tech.hermes.domain.filtering.FilterableMessage; import pl.allegro.tech.hermes.domain.filtering.FilteringException; import pl.allegro.tech.hermes.domain.filtering.MatchingStrategy; import pl.allegro.tech.hermes.domain.filtering.UnsupportedMatchingStrategyException; -import java.io.ByteArrayInputStream; -import java.util.List; -import java.util.function.Predicate; -import java.util.regex.Pattern; -import java.util.stream.Stream; +class JsonPathPredicate implements Predicate { + private final Configuration configuration; + private final String path; + private final Pattern matcher; + private final MatchingStrategy matchingStrategy; -import static pl.allegro.tech.hermes.domain.filtering.FilteringException.check; + JsonPathPredicate( + String path, + Pattern matcher, + Configuration configuration, + MatchingStrategy matchingStrategy) { + this.path = path; + this.matcher = matcher; + this.configuration = configuration; + this.matchingStrategy = matchingStrategy; + } -class JsonPathPredicate implements Predicate { - private final Configuration configuration; - private final String path; - private final Pattern matcher; - private final MatchingStrategy matchingStrategy; - - JsonPathPredicate(String path, Pattern matcher, Configuration configuration, MatchingStrategy matchingStrategy) { - this.path = path; - this.matcher = matcher; - this.configuration = configuration; - this.matchingStrategy = matchingStrategy; - } + @Override + public boolean test(FilterableMessage message) { + check( + message.getContentType() == ContentType.JSON, + "This filter supports only JSON contentType."); + try { + List result = + JsonPath.parse(new ByteArrayInputStream(message.getData()), configuration).read(path); + Stream resultStream = result.stream().map(Object::toString); - @Override - public boolean test(FilterableMessage message) { - check(message.getContentType() == ContentType.JSON, "This filter supports only JSON contentType."); - try { - List result = JsonPath.parse(new ByteArrayInputStream(message.getData()), configuration).read(path); - Stream resultStream = result.stream().map(Object::toString); - - return !result.isEmpty() && matchResultsStream(resultStream); - } catch (Exception ex) { - throw new FilteringException(ex); - } + return !result.isEmpty() && matchResultsStream(resultStream); + } catch (Exception ex) { + throw new FilteringException(ex); } + } - private boolean matchResultsStream(Stream results) { - switch (matchingStrategy) { - case ALL: - return results.allMatch(this::matches); - case ANY: - return results.anyMatch(this::matches); - default: - throw new UnsupportedMatchingStrategyException("avropath", matchingStrategy); - } + private boolean matchResultsStream(Stream results) { + switch (matchingStrategy) { + case ALL: + return results.allMatch(this::matches); + case ANY: + return results.anyMatch(this::matches); + default: + throw new UnsupportedMatchingStrategyException("avropath", matchingStrategy); } + } - private boolean matches(String value) { - return matcher.matcher(value).matches(); - } + private boolean matches(String value) { + return matcher.matcher(value).matches(); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/json/JsonPathSubscriptionMessageFilterCompiler.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/json/JsonPathSubscriptionMessageFilterCompiler.java index c5fec7ca53..347b91621b 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/json/JsonPathSubscriptionMessageFilterCompiler.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/filtering/json/JsonPathSubscriptionMessageFilterCompiler.java @@ -1,32 +1,32 @@ package pl.allegro.tech.hermes.domain.filtering.json; +import static com.jayway.jsonpath.Configuration.defaultConfiguration; + import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.Option; +import java.util.function.Predicate; +import java.util.regex.Pattern; import pl.allegro.tech.hermes.api.MessageFilterSpecification; import pl.allegro.tech.hermes.domain.filtering.FilterableMessage; import pl.allegro.tech.hermes.domain.filtering.MatchingStrategy; import pl.allegro.tech.hermes.domain.filtering.SubscriptionMessageFilterCompiler; -import java.util.function.Predicate; -import java.util.regex.Pattern; - -import static com.jayway.jsonpath.Configuration.defaultConfiguration; - -public class JsonPathSubscriptionMessageFilterCompiler implements SubscriptionMessageFilterCompiler { - private final Configuration configuration = defaultConfiguration().addOptions(Option.ALWAYS_RETURN_LIST, Option.SUPPRESS_EXCEPTIONS); +public class JsonPathSubscriptionMessageFilterCompiler + implements SubscriptionMessageFilterCompiler { + private final Configuration configuration = + defaultConfiguration().addOptions(Option.ALWAYS_RETURN_LIST, Option.SUPPRESS_EXCEPTIONS); - @Override - public String getType() { - return "jsonpath"; - } + @Override + public String getType() { + return "jsonpath"; + } - @Override - public Predicate compile(MessageFilterSpecification specification) { - return new JsonPathPredicate( - specification.getPath(), - Pattern.compile(specification.getMatcher()), - configuration, - MatchingStrategy.fromString(specification.getMatchingStrategy(), MatchingStrategy.ALL) - ); - } + @Override + public Predicate compile(MessageFilterSpecification specification) { + return new JsonPathPredicate( + specification.getPath(), + Pattern.compile(specification.getMatcher()), + configuration, + MatchingStrategy.fromString(specification.getMatchingStrategy(), MatchingStrategy.ALL)); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/group/GroupAlreadyExistsException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/group/GroupAlreadyExistsException.java index 4945b8be1a..a24359d4a7 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/group/GroupAlreadyExistsException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/group/GroupAlreadyExistsException.java @@ -5,17 +5,16 @@ public class GroupAlreadyExistsException extends HermesException { - public GroupAlreadyExistsException(String groupName) { - super(String.format("Group %s already exists", groupName)); - } + public GroupAlreadyExistsException(String groupName) { + super(String.format("Group %s already exists", groupName)); + } - public GroupAlreadyExistsException(String groupName, Throwable cause) { - super(String.format("Group %s already exists", groupName), cause); - } - - @Override - public ErrorCode getCode() { - return ErrorCode.GROUP_ALREADY_EXISTS; - } + public GroupAlreadyExistsException(String groupName, Throwable cause) { + super(String.format("Group %s already exists", groupName), cause); + } + @Override + public ErrorCode getCode() { + return ErrorCode.GROUP_ALREADY_EXISTS; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/group/GroupNotEmptyException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/group/GroupNotEmptyException.java index 929f847f6d..74ad8f8530 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/group/GroupNotEmptyException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/group/GroupNotEmptyException.java @@ -5,12 +5,12 @@ public class GroupNotEmptyException extends HermesException { - public GroupNotEmptyException(String groupName) { - super(String.format("Group %s is not empty", groupName)); - } + public GroupNotEmptyException(String groupName) { + super(String.format("Group %s is not empty", groupName)); + } - @Override - public ErrorCode getCode() { - return ErrorCode.GROUP_NOT_EMPTY; - } + @Override + public ErrorCode getCode() { + return ErrorCode.GROUP_NOT_EMPTY; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/group/GroupNotExistsException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/group/GroupNotExistsException.java index 35a5e03da0..0c6d589586 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/group/GroupNotExistsException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/group/GroupNotExistsException.java @@ -5,16 +5,16 @@ public class GroupNotExistsException extends HermesException { - public GroupNotExistsException(String groupName, Exception exception) { - super(String.format("Group %s does not exist", groupName), exception); - } + public GroupNotExistsException(String groupName, Exception exception) { + super(String.format("Group %s does not exist", groupName), exception); + } - public GroupNotExistsException(String groupName) { - this(groupName, null); - } + public GroupNotExistsException(String groupName) { + this(groupName, null); + } - @Override - public ErrorCode getCode() { - return ErrorCode.GROUP_NOT_EXISTS; - } + @Override + public ErrorCode getCode() { + return ErrorCode.GROUP_NOT_EXISTS; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/group/GroupRepository.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/group/GroupRepository.java index 9dc3c65fb6..8def903fac 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/group/GroupRepository.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/group/GroupRepository.java @@ -1,25 +1,23 @@ package pl.allegro.tech.hermes.domain.group; -import pl.allegro.tech.hermes.api.Group; - import java.util.List; +import pl.allegro.tech.hermes.api.Group; public interface GroupRepository { - boolean groupExists(String groupName); - - void ensureGroupExists(String groupName); + boolean groupExists(String groupName); - void createGroup(Group group); + void ensureGroupExists(String groupName); - void updateGroup(Group group); + void createGroup(Group group); - void removeGroup(String groupName); + void updateGroup(Group group); - List listGroupNames(); + void removeGroup(String groupName); - List listGroups(); + List listGroupNames(); - Group getGroupDetails(String groupName); + List listGroups(); + Group getGroupDetails(String groupName); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/notifications/AdminCallback.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/notifications/AdminCallback.java index 3fff93f658..ae9057f5d5 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/notifications/AdminCallback.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/notifications/AdminCallback.java @@ -2,6 +2,5 @@ public interface AdminCallback { - void onAdminOperationCreated(String type, String content); - + void onAdminOperationCreated(String type, String content); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/notifications/InternalNotificationsBus.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/notifications/InternalNotificationsBus.java index 1aae720cd8..a6bf890919 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/notifications/InternalNotificationsBus.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/notifications/InternalNotificationsBus.java @@ -1,13 +1,11 @@ package pl.allegro.tech.hermes.domain.notifications; -/** - * All callbacks must be nonblocking. - */ +/** All callbacks must be nonblocking. */ public interface InternalNotificationsBus { - void registerSubscriptionCallback(SubscriptionCallback callback); + void registerSubscriptionCallback(SubscriptionCallback callback); - void registerTopicCallback(TopicCallback callback); + void registerTopicCallback(TopicCallback callback); - void registerAdminCallback(AdminCallback callback); + void registerAdminCallback(AdminCallback callback); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/notifications/SubscriptionCallback.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/notifications/SubscriptionCallback.java index f6098e4851..5e5eeeec6f 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/notifications/SubscriptionCallback.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/notifications/SubscriptionCallback.java @@ -4,13 +4,9 @@ public interface SubscriptionCallback { - default void onSubscriptionCreated(Subscription subscription) { - } + default void onSubscriptionCreated(Subscription subscription) {} - default void onSubscriptionRemoved(Subscription subscription) { - } - - default void onSubscriptionChanged(Subscription subscription) { - } + default void onSubscriptionRemoved(Subscription subscription) {} + default void onSubscriptionChanged(Subscription subscription) {} } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/notifications/TopicCallback.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/notifications/TopicCallback.java index ce8f7e140d..a8a74df733 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/notifications/TopicCallback.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/notifications/TopicCallback.java @@ -4,12 +4,9 @@ public interface TopicCallback { - default void onTopicCreated(Topic topic) { - } + default void onTopicCreated(Topic topic) {} - default void onTopicRemoved(Topic topic) { - } + default void onTopicRemoved(Topic topic) {} - default void onTopicChanged(Topic topic) { - } + default void onTopicChanged(Topic topic) {} } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/oauth/OAuthProviderAlreadyExistsException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/oauth/OAuthProviderAlreadyExistsException.java index 3657510235..ef11b4e71e 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/oauth/OAuthProviderAlreadyExistsException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/oauth/OAuthProviderAlreadyExistsException.java @@ -6,12 +6,12 @@ public class OAuthProviderAlreadyExistsException extends HermesException { - public OAuthProviderAlreadyExistsException(OAuthProvider oAuthProvider, Throwable cause) { - super(String.format("OAuth Provider %s already exists", oAuthProvider.getName()), cause); - } + public OAuthProviderAlreadyExistsException(OAuthProvider oAuthProvider, Throwable cause) { + super(String.format("OAuth Provider %s already exists", oAuthProvider.getName()), cause); + } - @Override - public ErrorCode getCode() { - return ErrorCode.OAUTH_PROVIDER_ALREADY_EXISTS; - } + @Override + public ErrorCode getCode() { + return ErrorCode.OAUTH_PROVIDER_ALREADY_EXISTS; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/oauth/OAuthProviderNotExistsException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/oauth/OAuthProviderNotExistsException.java index 2b1359cb90..b209534f54 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/oauth/OAuthProviderNotExistsException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/oauth/OAuthProviderNotExistsException.java @@ -5,12 +5,12 @@ public class OAuthProviderNotExistsException extends HermesException { - public OAuthProviderNotExistsException(String oAuthProviderName) { - super(String.format("OAuth provider %s does not exist", oAuthProviderName)); - } + public OAuthProviderNotExistsException(String oAuthProviderName) { + super(String.format("OAuth provider %s does not exist", oAuthProviderName)); + } - @Override - public ErrorCode getCode() { - return ErrorCode.OAUTH_PROVIDER_NOT_EXISTS; - } + @Override + public ErrorCode getCode() { + return ErrorCode.OAUTH_PROVIDER_NOT_EXISTS; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/oauth/OAuthProviderRepository.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/oauth/OAuthProviderRepository.java index 3d443dac9d..2fea65ef59 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/oauth/OAuthProviderRepository.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/oauth/OAuthProviderRepository.java @@ -1,24 +1,23 @@ package pl.allegro.tech.hermes.domain.oauth; -import pl.allegro.tech.hermes.api.OAuthProvider; - import java.util.List; +import pl.allegro.tech.hermes.api.OAuthProvider; public interface OAuthProviderRepository { - boolean oAuthProviderExists(String oAuthProviderName); + boolean oAuthProviderExists(String oAuthProviderName); - void ensureOAuthProviderExists(String oAuthProviderName); + void ensureOAuthProviderExists(String oAuthProviderName); - List listOAuthProviderNames(); + List listOAuthProviderNames(); - List listOAuthProviders(); + List listOAuthProviders(); - OAuthProvider getOAuthProviderDetails(String oAuthProviderName); + OAuthProvider getOAuthProviderDetails(String oAuthProviderName); - void createOAuthProvider(OAuthProvider oAuthProvider); + void createOAuthProvider(OAuthProvider oAuthProvider); - void updateOAuthProvider(OAuthProvider oAuthprovider); + void updateOAuthProvider(OAuthProvider oAuthprovider); - void removeOAuthProvider(String oAuthProviderName); + void removeOAuthProvider(String oAuthProviderName); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/readiness/DatacenterReadinessList.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/readiness/DatacenterReadinessList.java index 29758d4f4d..72736874b7 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/readiness/DatacenterReadinessList.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/readiness/DatacenterReadinessList.java @@ -2,13 +2,13 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import pl.allegro.tech.hermes.api.DatacenterReadiness; - import java.util.List; +import pl.allegro.tech.hermes.api.DatacenterReadiness; public record DatacenterReadinessList(List datacenters) { - @JsonCreator - public DatacenterReadinessList(@JsonProperty("datacenters") List datacenters) { - this.datacenters = datacenters; - } + @JsonCreator + public DatacenterReadinessList( + @JsonProperty("datacenters") List datacenters) { + this.datacenters = datacenters; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/subscription/SubscriptionAlreadyExistsException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/subscription/SubscriptionAlreadyExistsException.java index 17a23bfd20..7a1da173a8 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/subscription/SubscriptionAlreadyExistsException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/subscription/SubscriptionAlreadyExistsException.java @@ -6,21 +6,22 @@ public class SubscriptionAlreadyExistsException extends HermesException { - public SubscriptionAlreadyExistsException(Subscription subscription, Throwable cause) { - super(message(subscription), cause); - } + public SubscriptionAlreadyExistsException(Subscription subscription, Throwable cause) { + super(message(subscription), cause); + } - public SubscriptionAlreadyExistsException(Subscription subscription) { - super(message(subscription)); - } + public SubscriptionAlreadyExistsException(Subscription subscription) { + super(message(subscription)); + } - @Override - public ErrorCode getCode() { - return ErrorCode.SUBSCRIPTION_ALREADY_EXISTS; - } + @Override + public ErrorCode getCode() { + return ErrorCode.SUBSCRIPTION_ALREADY_EXISTS; + } - private static String message(Subscription subscription) { - return String.format("Subscription %s for topic %s already exists.", - subscription.getName(), subscription.getQualifiedTopicName()); - } + private static String message(Subscription subscription) { + return String.format( + "Subscription %s for topic %s already exists.", + subscription.getName(), subscription.getQualifiedTopicName()); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/subscription/SubscriptionNotExistsException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/subscription/SubscriptionNotExistsException.java index 6d1b8d5827..034c8ac5b7 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/subscription/SubscriptionNotExistsException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/subscription/SubscriptionNotExistsException.java @@ -6,12 +6,14 @@ public class SubscriptionNotExistsException extends HermesException { - public SubscriptionNotExistsException(TopicName topic, String subscription) { - super(String.format("Subscription %s for topic %s does not exist.", subscription, topic.qualifiedName())); - } + public SubscriptionNotExistsException(TopicName topic, String subscription) { + super( + String.format( + "Subscription %s for topic %s does not exist.", subscription, topic.qualifiedName())); + } - @Override - public ErrorCode getCode() { - return ErrorCode.SUBSCRIPTION_NOT_EXISTS; - } + @Override + public ErrorCode getCode() { + return ErrorCode.SUBSCRIPTION_NOT_EXISTS; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/subscription/SubscriptionRepository.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/subscription/SubscriptionRepository.java index 7aabc431a0..3c6c690d6e 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/subscription/SubscriptionRepository.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/subscription/SubscriptionRepository.java @@ -1,35 +1,35 @@ package pl.allegro.tech.hermes.domain.subscription; +import java.util.Collection; +import java.util.List; import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.api.TopicName; -import java.util.Collection; -import java.util.List; - public interface SubscriptionRepository { - boolean subscriptionExists(TopicName topicName, String subscriptionName); + boolean subscriptionExists(TopicName topicName, String subscriptionName); - void ensureSubscriptionExists(TopicName topicName, String subscriptionName); + void ensureSubscriptionExists(TopicName topicName, String subscriptionName); - void createSubscription(Subscription subscription); + void createSubscription(Subscription subscription); - void removeSubscription(TopicName topicName, String subscriptionName); + void removeSubscription(TopicName topicName, String subscriptionName); - void updateSubscription(Subscription modifiedSubscription); + void updateSubscription(Subscription modifiedSubscription); - void updateSubscriptionState(TopicName topicName, String subscriptionName, Subscription.State state); + void updateSubscriptionState( + TopicName topicName, String subscriptionName, Subscription.State state); - Subscription getSubscriptionDetails(TopicName topicName, String subscriptionName); + Subscription getSubscriptionDetails(TopicName topicName, String subscriptionName); - Subscription getSubscriptionDetails(SubscriptionName subscriptionNames); + Subscription getSubscriptionDetails(SubscriptionName subscriptionNames); - List getSubscriptionDetails(Collection subscriptionNames); + List getSubscriptionDetails(Collection subscriptionNames); - List listSubscriptionNames(TopicName topicName); + List listSubscriptionNames(TopicName topicName); - List listSubscriptions(TopicName topicName); + List listSubscriptions(TopicName topicName); - List listAllSubscriptions(); + List listAllSubscriptions(); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/TopicAlreadyExistsException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/TopicAlreadyExistsException.java index 0bb0530150..6aca9b0222 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/TopicAlreadyExistsException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/TopicAlreadyExistsException.java @@ -6,16 +6,16 @@ public class TopicAlreadyExistsException extends HermesException { - public TopicAlreadyExistsException(TopicName topicName, Throwable cause) { - super(String.format("Topic %s already exists", topicName.qualifiedName()), cause); - } + public TopicAlreadyExistsException(TopicName topicName, Throwable cause) { + super(String.format("Topic %s already exists", topicName.qualifiedName()), cause); + } - public TopicAlreadyExistsException(TopicName topicName) { - super(String.format("Topic %s already exists", topicName.qualifiedName())); - } + public TopicAlreadyExistsException(TopicName topicName) { + super(String.format("Topic %s already exists", topicName.qualifiedName())); + } - @Override - public ErrorCode getCode() { - return ErrorCode.TOPIC_ALREADY_EXISTS; - } + @Override + public ErrorCode getCode() { + return ErrorCode.TOPIC_ALREADY_EXISTS; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/TopicNotEmptyException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/TopicNotEmptyException.java index 4c00fd8888..23e26e7334 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/TopicNotEmptyException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/TopicNotEmptyException.java @@ -6,12 +6,12 @@ public class TopicNotEmptyException extends HermesException { - public TopicNotEmptyException(TopicName topicName) { - super(String.format("Topic %s is not empty", topicName.qualifiedName())); - } + public TopicNotEmptyException(TopicName topicName) { + super(String.format("Topic %s is not empty", topicName.qualifiedName())); + } - @Override - public ErrorCode getCode() { - return ErrorCode.TOPIC_NOT_EMPTY; - } + @Override + public ErrorCode getCode() { + return ErrorCode.TOPIC_NOT_EMPTY; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/TopicNotExistsException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/TopicNotExistsException.java index dc36d258aa..da889f6baf 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/TopicNotExistsException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/TopicNotExistsException.java @@ -6,16 +6,16 @@ public class TopicNotExistsException extends HermesException { - public TopicNotExistsException(TopicName topicName, Exception exception) { - super(String.format("Topic %s does not exist", topicName.qualifiedName()), exception); - } + public TopicNotExistsException(TopicName topicName, Exception exception) { + super(String.format("Topic %s does not exist", topicName.qualifiedName()), exception); + } - public TopicNotExistsException(TopicName topicName) { - this(topicName, null); - } + public TopicNotExistsException(TopicName topicName) { + this(topicName, null); + } - @Override - public ErrorCode getCode() { - return ErrorCode.TOPIC_NOT_EXISTS; - } + @Override + public ErrorCode getCode() { + return ErrorCode.TOPIC_NOT_EXISTS; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/TopicRepository.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/TopicRepository.java index 9c43d87fe9..889e3e6754 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/TopicRepository.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/TopicRepository.java @@ -1,33 +1,31 @@ package pl.allegro.tech.hermes.domain.topic; -import pl.allegro.tech.hermes.api.Topic; -import pl.allegro.tech.hermes.api.TopicName; - import java.util.Collection; import java.util.List; +import pl.allegro.tech.hermes.api.Topic; +import pl.allegro.tech.hermes.api.TopicName; public interface TopicRepository { - boolean topicExists(TopicName topicName); - - void ensureTopicExists(TopicName topicName); + boolean topicExists(TopicName topicName); - List listTopicNames(String groupName); + void ensureTopicExists(TopicName topicName); - List listTopics(String groupName); + List listTopicNames(String groupName); - void createTopic(Topic topic); + List listTopics(String groupName); - void removeTopic(TopicName topicName); + void createTopic(Topic topic); - void updateTopic(Topic topic); + void removeTopic(TopicName topicName); - void touchTopic(TopicName topicName); + void updateTopic(Topic topic); - Topic getTopicDetails(TopicName topicName); + void touchTopic(TopicName topicName); - List getTopicsDetails(Collection topicNames); + Topic getTopicDetails(TopicName topicName); - List listAllTopics(); + List getTopicsDetails(Collection topicNames); + List listAllTopics(); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/preview/MessagePreview.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/preview/MessagePreview.java index 9f9d3ce5d9..a0a7dd7649 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/preview/MessagePreview.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/preview/MessagePreview.java @@ -5,25 +5,26 @@ public class MessagePreview { - private final byte[] content; + private final byte[] content; - private final boolean truncated; + private final boolean truncated; - @JsonCreator - public MessagePreview(@JsonProperty("content") byte[] content, @JsonProperty("truncated") boolean truncated) { - this.content = content; - this.truncated = truncated; - } + @JsonCreator + public MessagePreview( + @JsonProperty("content") byte[] content, @JsonProperty("truncated") boolean truncated) { + this.content = content; + this.truncated = truncated; + } - public MessagePreview(byte[] content) { - this(content, false); - } + public MessagePreview(byte[] content) { + this(content, false); + } - public byte[] getContent() { - return content; - } + public byte[] getContent() { + return content; + } - public boolean isTruncated() { - return truncated; - } + public boolean isTruncated() { + return truncated; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/preview/MessagePreviewRepository.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/preview/MessagePreviewRepository.java index 8f06e044aa..fe8024d798 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/preview/MessagePreviewRepository.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/preview/MessagePreviewRepository.java @@ -1,13 +1,11 @@ package pl.allegro.tech.hermes.domain.topic.preview; -import pl.allegro.tech.hermes.api.TopicName; - import java.util.List; +import pl.allegro.tech.hermes.api.TopicName; public interface MessagePreviewRepository { - List loadPreview(TopicName topicName); - - void persist(TopicsMessagesPreview topicsMessagesPreview); + List loadPreview(TopicName topicName); + void persist(TopicsMessagesPreview topicsMessagesPreview); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/preview/TopicsMessagesPreview.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/preview/TopicsMessagesPreview.java index e15431b058..6ab775bf8c 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/preview/TopicsMessagesPreview.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/topic/preview/TopicsMessagesPreview.java @@ -1,27 +1,26 @@ package pl.allegro.tech.hermes.domain.topic.preview; -import pl.allegro.tech.hermes.api.TopicName; - import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import pl.allegro.tech.hermes.api.TopicName; public class TopicsMessagesPreview { - private final Map> messages = new HashMap<>(); + private final Map> messages = new HashMap<>(); - public void add(TopicName topicName, MessagePreview message) { - List messageList = messages.computeIfAbsent(topicName, k -> new ArrayList<>()); - messageList.add(message); - } + public void add(TopicName topicName, MessagePreview message) { + List messageList = messages.computeIfAbsent(topicName, k -> new ArrayList<>()); + messageList.add(message); + } - public Collection topics() { - return messages.keySet(); - } + public Collection topics() { + return messages.keySet(); + } - public List previewOf(TopicName topic) { - return messages.getOrDefault(topic, new ArrayList<>()); - } + public List previewOf(TopicName topic) { + return messages.getOrDefault(topic, new ArrayList<>()); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/workload/constraints/ConsumersWorkloadConstraints.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/workload/constraints/ConsumersWorkloadConstraints.java index 436a2c0722..4efc1ac8e9 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/workload/constraints/ConsumersWorkloadConstraints.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/workload/constraints/ConsumersWorkloadConstraints.java @@ -1,47 +1,47 @@ package pl.allegro.tech.hermes.domain.workload.constraints; import com.google.common.collect.ImmutableMap; +import java.util.Map; +import java.util.Objects; import pl.allegro.tech.hermes.api.Constraints; import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.api.TopicName; -import java.util.Map; -import java.util.Objects; - public class ConsumersWorkloadConstraints { - private final Map topicConstraints; - private final Map subscriptionConstraints; + private final Map topicConstraints; + private final Map subscriptionConstraints; - public ConsumersWorkloadConstraints(Map topicConstraints, - Map subscriptionConstraints) { - this.topicConstraints = ImmutableMap.copyOf(topicConstraints); - this.subscriptionConstraints = ImmutableMap.copyOf(subscriptionConstraints); - } + public ConsumersWorkloadConstraints( + Map topicConstraints, + Map subscriptionConstraints) { + this.topicConstraints = ImmutableMap.copyOf(topicConstraints); + this.subscriptionConstraints = ImmutableMap.copyOf(subscriptionConstraints); + } - public Map getTopicConstraints() { - return topicConstraints; - } + public Map getTopicConstraints() { + return topicConstraints; + } - public Map getSubscriptionConstraints() { - return subscriptionConstraints; - } + public Map getSubscriptionConstraints() { + return subscriptionConstraints; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ConsumersWorkloadConstraints that = (ConsumersWorkloadConstraints) o; - return Objects.equals(topicConstraints, that.topicConstraints) - && Objects.equals(subscriptionConstraints, that.subscriptionConstraints); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(topicConstraints, subscriptionConstraints); + if (o == null || getClass() != o.getClass()) { + return false; } + ConsumersWorkloadConstraints that = (ConsumersWorkloadConstraints) o; + return Objects.equals(topicConstraints, that.topicConstraints) + && Objects.equals(subscriptionConstraints, that.subscriptionConstraints); + } + + @Override + public int hashCode() { + return Objects.hash(topicConstraints, subscriptionConstraints); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/workload/constraints/SubscriptionConstraintsAlreadyExistException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/workload/constraints/SubscriptionConstraintsAlreadyExistException.java index 9deab17e7e..501a81681a 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/workload/constraints/SubscriptionConstraintsAlreadyExistException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/workload/constraints/SubscriptionConstraintsAlreadyExistException.java @@ -1,19 +1,23 @@ package pl.allegro.tech.hermes.domain.workload.constraints; +import static pl.allegro.tech.hermes.api.ErrorCode.SUBSCRIPTION_CONSTRAINTS_ALREADY_EXIST; + import pl.allegro.tech.hermes.api.ErrorCode; import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.common.exception.HermesException; -import static pl.allegro.tech.hermes.api.ErrorCode.SUBSCRIPTION_CONSTRAINTS_ALREADY_EXIST; - public class SubscriptionConstraintsAlreadyExistException extends HermesException { - public SubscriptionConstraintsAlreadyExistException(SubscriptionName subscriptionName, Throwable cause) { - super(String.format("Constraints for subscription %s already exist.", subscriptionName.getQualifiedName()), cause); - } + public SubscriptionConstraintsAlreadyExistException( + SubscriptionName subscriptionName, Throwable cause) { + super( + String.format( + "Constraints for subscription %s already exist.", subscriptionName.getQualifiedName()), + cause); + } - @Override - public ErrorCode getCode() { - return SUBSCRIPTION_CONSTRAINTS_ALREADY_EXIST; - } + @Override + public ErrorCode getCode() { + return SUBSCRIPTION_CONSTRAINTS_ALREADY_EXIST; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/workload/constraints/SubscriptionConstraintsDoNotExistException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/workload/constraints/SubscriptionConstraintsDoNotExistException.java index b1f7ca258d..133e60ff57 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/workload/constraints/SubscriptionConstraintsDoNotExistException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/workload/constraints/SubscriptionConstraintsDoNotExistException.java @@ -1,19 +1,23 @@ package pl.allegro.tech.hermes.domain.workload.constraints; +import static pl.allegro.tech.hermes.api.ErrorCode.SUBSCRIPTION_CONSTRAINTS_DO_NOT_EXIST; + import pl.allegro.tech.hermes.api.ErrorCode; import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.common.exception.HermesException; -import static pl.allegro.tech.hermes.api.ErrorCode.SUBSCRIPTION_CONSTRAINTS_DO_NOT_EXIST; - public class SubscriptionConstraintsDoNotExistException extends HermesException { - public SubscriptionConstraintsDoNotExistException(SubscriptionName subscriptionName, Throwable cause) { - super(String.format("Constraints for subscription %s do not exist.", subscriptionName.getQualifiedName()), cause); - } + public SubscriptionConstraintsDoNotExistException( + SubscriptionName subscriptionName, Throwable cause) { + super( + String.format( + "Constraints for subscription %s do not exist.", subscriptionName.getQualifiedName()), + cause); + } - @Override - public ErrorCode getCode() { - return SUBSCRIPTION_CONSTRAINTS_DO_NOT_EXIST; - } + @Override + public ErrorCode getCode() { + return SUBSCRIPTION_CONSTRAINTS_DO_NOT_EXIST; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/workload/constraints/TopicConstraintsAlreadyExistException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/workload/constraints/TopicConstraintsAlreadyExistException.java index 485eda8cfa..12196494b2 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/workload/constraints/TopicConstraintsAlreadyExistException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/workload/constraints/TopicConstraintsAlreadyExistException.java @@ -5,12 +5,13 @@ import pl.allegro.tech.hermes.common.exception.HermesException; public class TopicConstraintsAlreadyExistException extends HermesException { - public TopicConstraintsAlreadyExistException(TopicName topicName, Throwable cause) { - super(String.format("Constraints for topic %s already exist.", topicName.qualifiedName()), cause); - } + public TopicConstraintsAlreadyExistException(TopicName topicName, Throwable cause) { + super( + String.format("Constraints for topic %s already exist.", topicName.qualifiedName()), cause); + } - @Override - public ErrorCode getCode() { - return ErrorCode.TOPIC_CONSTRAINTS_ALREADY_EXIST; - } + @Override + public ErrorCode getCode() { + return ErrorCode.TOPIC_CONSTRAINTS_ALREADY_EXIST; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/workload/constraints/TopicConstraintsDoNotExistException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/workload/constraints/TopicConstraintsDoNotExistException.java index e34385ed0a..bbd7899e8a 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/workload/constraints/TopicConstraintsDoNotExistException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/workload/constraints/TopicConstraintsDoNotExistException.java @@ -1,19 +1,20 @@ package pl.allegro.tech.hermes.domain.workload.constraints; +import static pl.allegro.tech.hermes.api.ErrorCode.TOPIC_CONSTRAINTS_DO_NOT_EXIST; + import pl.allegro.tech.hermes.api.ErrorCode; import pl.allegro.tech.hermes.api.TopicName; import pl.allegro.tech.hermes.common.exception.HermesException; -import static pl.allegro.tech.hermes.api.ErrorCode.TOPIC_CONSTRAINTS_DO_NOT_EXIST; - public class TopicConstraintsDoNotExistException extends HermesException { - public TopicConstraintsDoNotExistException(TopicName topicName, Throwable cause) { - super(String.format("Constraints for topic %s do not exist.", topicName.qualifiedName()), cause); - } + public TopicConstraintsDoNotExistException(TopicName topicName, Throwable cause) { + super( + String.format("Constraints for topic %s do not exist.", topicName.qualifiedName()), cause); + } - @Override - public ErrorCode getCode() { - return TOPIC_CONSTRAINTS_DO_NOT_EXIST; - } + @Override + public ErrorCode getCode() { + return TOPIC_CONSTRAINTS_DO_NOT_EXIST; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/workload/constraints/WorkloadConstraintsRepository.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/workload/constraints/WorkloadConstraintsRepository.java index e5b153ee8a..ce1bf4aa5c 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/workload/constraints/WorkloadConstraintsRepository.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/domain/workload/constraints/WorkloadConstraintsRepository.java @@ -6,21 +6,21 @@ public interface WorkloadConstraintsRepository { - ConsumersWorkloadConstraints getConsumersWorkloadConstraints(); + ConsumersWorkloadConstraints getConsumersWorkloadConstraints(); - void createConstraints(TopicName topicName, Constraints constraints); + void createConstraints(TopicName topicName, Constraints constraints); - void createConstraints(SubscriptionName subscriptionName, Constraints constraints); + void createConstraints(SubscriptionName subscriptionName, Constraints constraints); - void updateConstraints(TopicName topicName, Constraints constraints); + void updateConstraints(TopicName topicName, Constraints constraints); - void updateConstraints(SubscriptionName subscriptionName, Constraints constraints); + void updateConstraints(SubscriptionName subscriptionName, Constraints constraints); - void deleteConstraints(TopicName topicName); + void deleteConstraints(TopicName topicName); - void deleteConstraints(SubscriptionName subscriptionName); + void deleteConstraints(SubscriptionName subscriptionName); - boolean constraintsExist(TopicName topicName); + boolean constraintsExist(TopicName topicName); - boolean constraintsExist(SubscriptionName subscriptionName); + boolean constraintsExist(SubscriptionName subscriptionName); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/MalformedDataException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/MalformedDataException.java index 62495f9acf..137bdfe42e 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/MalformedDataException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/MalformedDataException.java @@ -2,8 +2,7 @@ public class MalformedDataException extends RuntimeException { - public MalformedDataException(String path, Throwable throwable) { - super(String.format("Unable to read data from path %s", path), throwable); - } - + public MalformedDataException(String path, Throwable throwable) { + super(String.format("Unable to read data from path %s", path), throwable); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/dc/DatacenterNameProvider.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/dc/DatacenterNameProvider.java index e0ad86da8b..ede547f611 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/dc/DatacenterNameProvider.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/dc/DatacenterNameProvider.java @@ -1,5 +1,5 @@ package pl.allegro.tech.hermes.infrastructure.dc; public interface DatacenterNameProvider { - String getDatacenterName(); + String getDatacenterName(); } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/dc/DcNameProvisionException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/dc/DcNameProvisionException.java index ed7f41ccca..0a4967b028 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/dc/DcNameProvisionException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/dc/DcNameProvisionException.java @@ -2,7 +2,7 @@ public class DcNameProvisionException extends RuntimeException { - public DcNameProvisionException(String message) { - super(message); - } + public DcNameProvisionException(String message) { + super(message); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/dc/DcNameSource.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/dc/DcNameSource.java index adc1a28b66..864b838a47 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/dc/DcNameSource.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/dc/DcNameSource.java @@ -1,5 +1,5 @@ package pl.allegro.tech.hermes.infrastructure.dc; public enum DcNameSource { - ENV + ENV } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/dc/DefaultDatacenterNameProvider.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/dc/DefaultDatacenterNameProvider.java index 9da8bd9872..aa8f65af26 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/dc/DefaultDatacenterNameProvider.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/dc/DefaultDatacenterNameProvider.java @@ -5,13 +5,13 @@ public class DefaultDatacenterNameProvider implements DatacenterNameProvider { - private static final Logger logger = LoggerFactory.getLogger(DefaultDatacenterNameProvider.class); + private static final Logger logger = LoggerFactory.getLogger(DefaultDatacenterNameProvider.class); - public static final String DEFAULT_DC_NAME = "dc"; + public static final String DEFAULT_DC_NAME = "dc"; - @Override - public String getDatacenterName() { - logger.info("Providing default datacenter name: {}", DEFAULT_DC_NAME); - return DEFAULT_DC_NAME; - } + @Override + public String getDatacenterName() { + logger.info("Providing default datacenter name: {}", DEFAULT_DC_NAME); + return DEFAULT_DC_NAME; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/dc/EnvironmentVariableDatacenterNameProvider.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/dc/EnvironmentVariableDatacenterNameProvider.java index 5588b4a3f9..7f31348041 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/dc/EnvironmentVariableDatacenterNameProvider.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/dc/EnvironmentVariableDatacenterNameProvider.java @@ -4,22 +4,23 @@ import org.slf4j.LoggerFactory; public class EnvironmentVariableDatacenterNameProvider implements DatacenterNameProvider { - private static final Logger logger = LoggerFactory.getLogger(EnvironmentVariableDatacenterNameProvider.class); + private static final Logger logger = + LoggerFactory.getLogger(EnvironmentVariableDatacenterNameProvider.class); - private final String variableName; + private final String variableName; - public EnvironmentVariableDatacenterNameProvider(String variableName) { - this.variableName = variableName; - } + public EnvironmentVariableDatacenterNameProvider(String variableName) { + this.variableName = variableName; + } - @Override - public String getDatacenterName() { - String dcName = System.getenv(variableName); - if (dcName == null) { - logger.info("Undefined environment variable: " + variableName); - throw new DcNameProvisionException("Undefined environment variable: " + variableName); - } - logger.info("Providing DC name from environment variable: {}={}", variableName, dcName); - return dcName; + @Override + public String getDatacenterName() { + String dcName = System.getenv(variableName); + if (dcName == null) { + logger.info("Undefined environment variable: " + variableName); + throw new DcNameProvisionException("Undefined environment variable: " + variableName); } + logger.info("Providing DC name from environment variable: {}={}", variableName, dcName); + return dcName; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/logback/AggregatingTurboFilter.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/logback/AggregatingTurboFilter.java index 45507bb45e..6ac82a8920 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/logback/AggregatingTurboFilter.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/logback/AggregatingTurboFilter.java @@ -5,9 +5,6 @@ import ch.qos.logback.classic.turbo.TurboFilter; import ch.qos.logback.core.spi.FilterReply; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.slf4j.Marker; -import org.slf4j.MarkerFactory; - import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -20,22 +17,25 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.LongAdder; +import org.slf4j.Marker; +import org.slf4j.MarkerFactory; /** - *

This class is an implementation of {@link TurboFilter} interface that allows messages of configured loggers - * to be aggregated and logged only once with specified time interval.

+ * This class is an implementation of {@link TurboFilter} interface that allows messages of + * configured loggers to be aggregated and logged only once with specified time interval. * - *

Logged message contains a suffix "[occurrences=N]" with a number of logged events. - * The logging part is a little bit tricky, as the TurboFilter interface does not support a way of logging out of the box, - * it just tells whether a logged message should be passed further or not. - * In order to logging be possible from the TurboFilter a small trick has to be applied, the logged message is enriched - * with a custom {@link Marker} which is checked by the filter itself, so it won't filter it's own messages - * and we manage to avoid recursion.

+ *

Logged message contains a suffix "[occurrences=N]" with a number of logged events. The logging + * part is a little bit tricky, as the TurboFilter interface does not support a way of + * logging out of the box, it just tells whether a logged message should be passed further or not. + * In order to logging be possible from the TurboFilter a small trick has to be + * applied, the logged message is enriched with a custom {@link Marker} which is checked by the + * filter itself, so it won't filter it's own messages and we manage to avoid recursion. * - *

An instance of AggregatingTurboFilter starts it's own scheduled executor service with a single thread - * that logs the messages asynchronously.

+ *

An instance of AggregatingTurboFilter starts it's own scheduled executor service + * with a single thread that logs the messages asynchronously. * *

Example logback configuration: + * *

{@code
  *  
  *      ...
@@ -51,167 +51,189 @@
  */
 public class AggregatingTurboFilter extends TurboFilter {
 
-    static final Marker MARKER = MarkerFactory.getMarker("AggregatingTurboFilterMarker");
-
-    private ScheduledExecutorService executorService;
-    private final List aggregatedLogger = new ArrayList<>();
-    private long reportingIntervalMillis = 10_000;
+  static final Marker MARKER = MarkerFactory.getMarker("AggregatingTurboFilterMarker");
 
-    private final Map logAggregates = new ConcurrentHashMap<>();
+  private ScheduledExecutorService executorService;
+  private final List aggregatedLogger = new ArrayList<>();
+  private long reportingIntervalMillis = 10_000;
 
-    private static final LongAdder filterClassCounter = new LongAdder();
+  private final Map logAggregates = new ConcurrentHashMap<>();
 
-    @Override
-    public void start() {
-        super.start();
-        if (!aggregatedLogger.isEmpty()) {
-            ThreadFactory threadFactory = new ThreadFactoryBuilder()
-                    .setNameFormat("aggregating-filter-" + filterClassCounter.longValue() + "-thread-%d")
-                    .build();
-            filterClassCounter.increment();
-            executorService = Executors.newSingleThreadScheduledExecutor(threadFactory);
-            executorService.scheduleAtFixedRate(this::report, 0L, getReportingIntervalMillis(), TimeUnit.MILLISECONDS);
-        }
-    }
+  private static final LongAdder filterClassCounter = new LongAdder();
 
-    @Override
-    public void stop() {
-        executorService.shutdownNow();
-        super.stop();
+  @Override
+  public void start() {
+    super.start();
+    if (!aggregatedLogger.isEmpty()) {
+      ThreadFactory threadFactory =
+          new ThreadFactoryBuilder()
+              .setNameFormat("aggregating-filter-" + filterClassCounter.longValue() + "-thread-%d")
+              .build();
+      filterClassCounter.increment();
+      executorService = Executors.newSingleThreadScheduledExecutor(threadFactory);
+      executorService.scheduleAtFixedRate(
+          this::report, 0L, getReportingIntervalMillis(), TimeUnit.MILLISECONDS);
     }
-
-    void report() {
-        logAggregates.forEach((logger, loggerAggregates) ->
-                loggerAggregates.aggregates.keySet().forEach(entry -> {
-                    loggerAggregates.aggregates.computeIfPresent(entry, (key, summary) -> {
-                        logger.log(key.marker, Logger.FQCN, key.level,
+  }
+
+  @Override
+  public void stop() {
+    executorService.shutdownNow();
+    super.stop();
+  }
+
+  void report() {
+    logAggregates.forEach(
+        (logger, loggerAggregates) ->
+            loggerAggregates
+                .aggregates
+                .keySet()
+                .forEach(
+                    entry -> {
+                      loggerAggregates.aggregates.computeIfPresent(
+                          entry,
+                          (key, summary) -> {
+                            logger.log(
+                                key.marker,
+                                Logger.FQCN,
+                                key.level,
                                 key.message + " [occurrences=" + summary.logsCount + "]",
-                                key.params, summary.lastException);
-                        return null;
-                    });
-                }));
+                                key.params,
+                                summary.lastException);
+                            return null;
+                          });
+                    }));
+  }
+
+  @Override
+  public FilterReply decide(
+      Marker marker, Logger logger, Level level, String message, Object[] params, Throwable ex) {
+    if (isAggregatedLog(marker)) { // prevent recursion for aggregated logger events
+      return FilterReply.NEUTRAL;
     }
-
-    @Override
-    public FilterReply decide(Marker marker, Logger logger, Level level, String message, Object[] params, Throwable ex) {
-        if (isAggregatedLog(marker)) { // prevent recursion for aggregated logger events
-            return FilterReply.NEUTRAL;
-        }
-        if (!aggregatedLogger.contains(logger.getName())) {
-            return FilterReply.NEUTRAL;
-        }
-
-        if (ex == null) {
-            Optional throwable = extractLastParamThrowable(params);
-            if (throwable.isPresent()) {
-                ex = throwable.get();
-                params = Arrays.copyOfRange(params, 0, params.length - 1);
-            }
-        }
-        Throwable exception = ex;
-
-        LoggingEventKey loggingEventKey = new LoggingEventKey(message, params, level, getEnrichedMarker(marker));
-        logAggregates.computeIfAbsent(logger, l -> new LoggerAggregates())
-                .aggregates.merge(loggingEventKey, new AggregateSummary(ex),
-                (currentAggregate, emptyAggregate) -> AggregateSummary.incrementCount(currentAggregate, exception));
-
-        return FilterReply.DENY;
+    if (!aggregatedLogger.contains(logger.getName())) {
+      return FilterReply.NEUTRAL;
     }
 
-    private boolean isAggregatedLog(Marker marker) {
-        return marker != null && (marker.equals(MARKER) || marker.contains(MARKER));
+    if (ex == null) {
+      Optional throwable = extractLastParamThrowable(params);
+      if (throwable.isPresent()) {
+        ex = throwable.get();
+        params = Arrays.copyOfRange(params, 0, params.length - 1);
+      }
     }
-
-    private Optional extractLastParamThrowable(Object[] params) {
-        return Optional.ofNullable(params)
-                .map(Arrays::stream)
-                .flatMap(a -> a.skip(params.length - 1)
-                        .findFirst()
-                        .filter(o -> o instanceof Throwable)
-                        .map(Throwable.class::cast));
+    Throwable exception = ex;
+
+    LoggingEventKey loggingEventKey =
+        new LoggingEventKey(message, params, level, getEnrichedMarker(marker));
+    logAggregates
+        .computeIfAbsent(logger, l -> new LoggerAggregates())
+        .aggregates
+        .merge(
+            loggingEventKey,
+            new AggregateSummary(ex),
+            (currentAggregate, emptyAggregate) ->
+                AggregateSummary.incrementCount(currentAggregate, exception));
+
+    return FilterReply.DENY;
+  }
+
+  private boolean isAggregatedLog(Marker marker) {
+    return marker != null && (marker.equals(MARKER) || marker.contains(MARKER));
+  }
+
+  private Optional extractLastParamThrowable(Object[] params) {
+    return Optional.ofNullable(params)
+        .map(Arrays::stream)
+        .flatMap(
+            a ->
+                a.skip(params.length - 1)
+                    .findFirst()
+                    .filter(o -> o instanceof Throwable)
+                    .map(Throwable.class::cast));
+  }
+
+  private Marker getEnrichedMarker(Marker marker) {
+    if (marker == null) {
+      return MARKER;
     }
+    marker.add(MARKER);
+    return marker;
+  }
 
-    private Marker getEnrichedMarker(Marker marker) {
-        if (marker == null) {
-            return MARKER;
-        }
-        marker.add(MARKER);
-        return marker;
-    }
+  public void addAggregatedLogger(String logger) {
+    this.aggregatedLogger.add(logger);
+  }
 
-    public void addAggregatedLogger(String logger) {
-        this.aggregatedLogger.add(logger);
-    }
+  public long getReportingIntervalMillis() {
+    return reportingIntervalMillis;
+  }
 
-    public long getReportingIntervalMillis() {
-        return reportingIntervalMillis;
-    }
+  public void setReportingIntervalMillis(long reportingIntervalMillis) {
+    this.reportingIntervalMillis = reportingIntervalMillis;
+  }
 
-    public void setReportingIntervalMillis(long reportingIntervalMillis) {
-        this.reportingIntervalMillis = reportingIntervalMillis;
-    }
+  private static class LoggerAggregates {
+
+    private final Map aggregates = new ConcurrentHashMap<>();
+  }
+
+  private static class AggregateSummary {
+
+    private final int logsCount;
+    private final Throwable lastException;
 
-    private static class LoggerAggregates {
+    private AggregateSummary(Throwable lastException) {
+      this(1, lastException);
+    }
 
-        private final Map aggregates = new ConcurrentHashMap<>();
+    private AggregateSummary(int logsCount, Throwable lastException) {
+      this.logsCount = logsCount;
+      this.lastException = lastException;
     }
 
-    private static class AggregateSummary {
+    private static AggregateSummary incrementCount(
+        AggregateSummary currentAggregate, Throwable lastException) {
+      return new AggregateSummary(
+          currentAggregate.logsCount + 1,
+          Optional.ofNullable(lastException).orElse(currentAggregate.lastException));
+    }
+  }
 
-        private final int logsCount;
-        private final Throwable lastException;
+  private static class LoggingEventKey {
 
-        private AggregateSummary(Throwable lastException) {
-            this(1, lastException);
-        }
+    private final String message;
+    private final int level;
+    private final Marker marker;
+    private final Object[] params;
 
-        private AggregateSummary(int logsCount, Throwable lastException) {
-            this.logsCount = logsCount;
-            this.lastException = lastException;
-        }
+    LoggingEventKey(String message, Object[] params, Level level, Marker marker) {
+      this.message = message;
+      this.params = params;
+      this.level = Level.toLocationAwareLoggerInteger(level);
+      this.marker = marker;
+    }
 
-        private static AggregateSummary incrementCount(AggregateSummary currentAggregate, Throwable lastException) {
-            return new AggregateSummary(currentAggregate.logsCount + 1,
-                    Optional.ofNullable(lastException).orElse(currentAggregate.lastException));
-        }
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) {
+        return true;
+      }
+      if (o == null || getClass() != o.getClass()) {
+        return false;
+      }
+      LoggingEventKey that = (LoggingEventKey) o;
+      return level == that.level
+          && Objects.equals(message, that.message)
+          && Objects.equals(marker, that.marker)
+          && Arrays.equals(params, that.params);
     }
 
-    private static class LoggingEventKey {
-
-        private final String message;
-        private final int level;
-        private final Marker marker;
-        private final Object[] params;
-
-        LoggingEventKey(String message, Object[] params, Level level, Marker marker) {
-            this.message = message;
-            this.params = params;
-            this.level = Level.toLocationAwareLoggerInteger(level);
-            this.marker = marker;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) {
-                return true;
-            }
-            if (o == null || getClass() != o.getClass()) {
-                return false;
-            }
-            LoggingEventKey that = (LoggingEventKey) o;
-            return level == that.level
-                    && Objects.equals(message, that.message)
-                    && Objects.equals(marker, that.marker)
-                    && Arrays.equals(params, that.params);
-        }
-
-        @Override
-        public int hashCode() {
-            int result = Objects.hash(message, level, marker);
-            result = 31 * result + Arrays.hashCode(params);
-            return result;
-        }
+    @Override
+    public int hashCode() {
+      int result = Objects.hash(message, level, marker);
+      result = 31 * result + Arrays.hashCode(params);
+      return result;
     }
+  }
 }
-
-
diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperBasedRepository.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperBasedRepository.java
index 241fcb0260..85827afb21 100644
--- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperBasedRepository.java
+++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperBasedRepository.java
@@ -3,6 +3,13 @@
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.api.transaction.CuratorTransactionFinal;
@@ -14,194 +21,194 @@
 import pl.allegro.tech.hermes.common.exception.RepositoryNotAvailableException;
 import pl.allegro.tech.hermes.infrastructure.MalformedDataException;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Optional;
-import java.util.function.BiConsumer;
-import java.util.stream.Collectors;
-
 public abstract class ZookeeperBasedRepository {
 
-    private static final Logger logger = LoggerFactory.getLogger(ZookeeperBasedRepository.class);
-
-    private final CuratorFramework zookeeper;
-
-    private final ObjectMapper mapper;
-
-    protected final ZookeeperPaths paths;
-
-    protected ZookeeperBasedRepository(CuratorFramework zookeeper,
-                                       ObjectMapper mapper,
-                                       ZookeeperPaths paths) {
-        this.zookeeper = zookeeper;
-        this.mapper = mapper;
-        this.paths = paths;
-    }
-
-    private void ensureConnected() {
-        if (!zookeeper.getZookeeperClient().isConnected()) {
-            throw new RepositoryNotAvailableException("Could not establish connection to a Zookeeper instance");
-        }
-    }
-
-    protected void ensurePathExists(String path) {
-        ensureConnected();
-        if (!pathExists(path)) {
-            try {
-                zookeeper.create().creatingParentsIfNeeded().forPath(path);
-            } catch (Exception e) {
-                throw new InternalProcessingException(e);
-            }
-        }
-    }
-
-    protected boolean pathExists(String path) {
-        ensureConnected();
-        try {
-            Optional optionalStat = Optional.ofNullable(zookeeper.checkExists().forPath(path));
-            return optionalStat.isPresent();
-        } catch (Exception e) {
-            throw new InternalProcessingException(e);
-        }
-    }
-
-    protected List childrenOf(String path) {
-        ensureConnected();
-        try {
-            List retrievedNodes = new ArrayList<>(zookeeper.getChildren().forPath(path));
-            Collections.sort(retrievedNodes);
-            return retrievedNodes;
-        } catch (Exception ex) {
-            throw new InternalProcessingException(ex);
-        }
-    }
-
-    protected List childrenPathsOf(String path) {
-        List childNodes = childrenOf(path);
-        return childNodes.stream()
-                .map(child -> ZKPaths.makePath(path, child))
-                .collect(Collectors.toList());
-    }
-
-    @SuppressWarnings("unchecked")
-    protected byte[] readFrom(String path) {
-        return readWithStatFrom(path, bytes -> bytes, (t, stat) -> {}, false).get();
-    }
-
-    @SuppressWarnings("unchecked")
-    protected  T readFrom(String path, Class clazz) {
-        return readFrom(path, clazz, false).get();
-    }
-
-    @SuppressWarnings("unchecked")
-    protected  Optional readFrom(String path, Class clazz, boolean quiet) {
-        return readWithStatFrom(path, b -> (T) mapper.readValue(b, clazz), (t, stat) -> {}, quiet);
-    }
-
-    @SuppressWarnings("unchecked")
-    protected  Optional readFrom(String path, TypeReference type, boolean quiet) {
-        return readWithStatFrom(path, b -> (T) mapper.readValue(b, type), (t, stat) -> {}, quiet);
-    }
-
-    protected  Optional readWithStatFrom(String path, Class clazz, BiConsumer statDecorator, boolean quiet) {
-        return readWithStatFrom(path, b -> mapper.readValue(b, clazz), statDecorator, quiet);
-    }
-
-    private  Optional readWithStatFrom(String path, ThrowingReader supplier, BiConsumer statDecorator, boolean quiet) {
-        ensureConnected();
-        try {
-            Stat stat = new Stat();
-            byte[] data = zookeeper.getData().storingStatIn(stat).forPath(path);
-            if (ArrayUtils.isNotEmpty(data)) {
-                T t = supplier.read(data);
-                statDecorator.accept(t, stat);
-                return Optional.of(t);
-            }
-        } catch (JsonMappingException malformedException) {
-            logWarnOrThrowException("Unable to read data from path " + path,
-                    new MalformedDataException(path, malformedException), quiet);
-        } catch (InternalProcessingException e) {
-            throw e;
-        } catch (Exception exception) {
-            logWarnOrThrowException("Unable to read data from path " + path, new InternalProcessingException(exception),
-                    quiet);
-        }
-        return Optional.empty();
-    }
-
-    private void logWarnOrThrowException(String message, RuntimeException e, Boolean quiet) {
-        if (quiet) {
-            logger.warn(message, e);
-        } else {
-            throw e;
-        }
-    }
-
-    protected void overwrite(String path, Object value) throws Exception {
-        ensureConnected();
-        zookeeper.setData().forPath(path, mapper.writeValueAsBytes(value));
-    }
-
-    protected void overwrite(String path, byte[] value) throws Exception {
-        ensureConnected();
-        zookeeper.setData().forPath(path, value);
-    }
-
-    protected void createRecursively(String path, Object value) throws Exception {
-        ensureConnected();
-        zookeeper.create()
-                .creatingParentsIfNeeded()
-                .forPath(path, mapper.writeValueAsBytes(value));
-    }
-
-    protected void createInTransaction(String path, Object value, String childPath) throws Exception {
-        ensureConnected();
-        zookeeper.inTransaction()
-                .create().forPath(path, mapper.writeValueAsBytes(value))
-                .and()
-                .create().forPath(childPath)
-                .and()
-                .commit();
-    }
-
-    protected void deleteInTransaction(List paths) throws Exception {
-        if (paths.isEmpty()) {
-            throw new InternalProcessingException("Attempting to remove empty set of paths from ZK");
-        }
-        ensureConnected();
-        CuratorTransactionFinal transaction = zookeeper.inTransaction().delete().forPath(paths.get(0)).and();
-
-        for (int i = 1; i < paths.size(); i++) {
-            transaction = transaction.delete().forPath(paths.get(i)).and();
-        }
-
-        transaction.commit();
-    }
-
-    protected void create(String path, Object value) throws Exception {
-        ensureConnected();
-        zookeeper.create().forPath(path, mapper.writeValueAsBytes(value));
-    }
-
-    protected void create(String path, byte[] value) throws Exception {
-        ensureConnected();
-        zookeeper.create().forPath(path, value);
-    }
-
-    protected void touch(String path) throws Exception {
-        ensureConnected();
-        byte[] oldData = zookeeper.getData().forPath(path);
-        zookeeper.setData().forPath(path, oldData);
-    }
-
-    protected void remove(String path) throws Exception {
-        ensureConnected();
-        zookeeper.delete().guaranteed().deletingChildrenIfNeeded().forPath(path);
-    }
-
-    private interface ThrowingReader {
-        T read(byte[] data) throws IOException;
-    }
+  private static final Logger logger = LoggerFactory.getLogger(ZookeeperBasedRepository.class);
+
+  private final CuratorFramework zookeeper;
+
+  private final ObjectMapper mapper;
+
+  protected final ZookeeperPaths paths;
+
+  protected ZookeeperBasedRepository(
+      CuratorFramework zookeeper, ObjectMapper mapper, ZookeeperPaths paths) {
+    this.zookeeper = zookeeper;
+    this.mapper = mapper;
+    this.paths = paths;
+  }
+
+  private void ensureConnected() {
+    if (!zookeeper.getZookeeperClient().isConnected()) {
+      throw new RepositoryNotAvailableException(
+          "Could not establish connection to a Zookeeper instance");
+    }
+  }
+
+  protected void ensurePathExists(String path) {
+    ensureConnected();
+    if (!pathExists(path)) {
+      try {
+        zookeeper.create().creatingParentsIfNeeded().forPath(path);
+      } catch (Exception e) {
+        throw new InternalProcessingException(e);
+      }
+    }
+  }
+
+  protected boolean pathExists(String path) {
+    ensureConnected();
+    try {
+      Optional optionalStat = Optional.ofNullable(zookeeper.checkExists().forPath(path));
+      return optionalStat.isPresent();
+    } catch (Exception e) {
+      throw new InternalProcessingException(e);
+    }
+  }
+
+  protected List childrenOf(String path) {
+    ensureConnected();
+    try {
+      List retrievedNodes = new ArrayList<>(zookeeper.getChildren().forPath(path));
+      Collections.sort(retrievedNodes);
+      return retrievedNodes;
+    } catch (Exception ex) {
+      throw new InternalProcessingException(ex);
+    }
+  }
+
+  protected List childrenPathsOf(String path) {
+    List childNodes = childrenOf(path);
+    return childNodes.stream()
+        .map(child -> ZKPaths.makePath(path, child))
+        .collect(Collectors.toList());
+  }
+
+  @SuppressWarnings("unchecked")
+  protected byte[] readFrom(String path) {
+    return readWithStatFrom(path, bytes -> bytes, (t, stat) -> {}, false).get();
+  }
+
+  @SuppressWarnings("unchecked")
+  protected  T readFrom(String path, Class clazz) {
+    return readFrom(path, clazz, false).get();
+  }
+
+  @SuppressWarnings("unchecked")
+  protected  Optional readFrom(String path, Class clazz, boolean quiet) {
+    return readWithStatFrom(path, b -> (T) mapper.readValue(b, clazz), (t, stat) -> {}, quiet);
+  }
+
+  @SuppressWarnings("unchecked")
+  protected  Optional readFrom(String path, TypeReference type, boolean quiet) {
+    return readWithStatFrom(path, b -> (T) mapper.readValue(b, type), (t, stat) -> {}, quiet);
+  }
+
+  protected  Optional readWithStatFrom(
+      String path, Class clazz, BiConsumer statDecorator, boolean quiet) {
+    return readWithStatFrom(path, b -> mapper.readValue(b, clazz), statDecorator, quiet);
+  }
+
+  private  Optional readWithStatFrom(
+      String path, ThrowingReader supplier, BiConsumer statDecorator, boolean quiet) {
+    ensureConnected();
+    try {
+      Stat stat = new Stat();
+      byte[] data = zookeeper.getData().storingStatIn(stat).forPath(path);
+      if (ArrayUtils.isNotEmpty(data)) {
+        T t = supplier.read(data);
+        statDecorator.accept(t, stat);
+        return Optional.of(t);
+      }
+    } catch (JsonMappingException malformedException) {
+      logWarnOrThrowException(
+          "Unable to read data from path " + path,
+          new MalformedDataException(path, malformedException),
+          quiet);
+    } catch (InternalProcessingException e) {
+      throw e;
+    } catch (Exception exception) {
+      logWarnOrThrowException(
+          "Unable to read data from path " + path,
+          new InternalProcessingException(exception),
+          quiet);
+    }
+    return Optional.empty();
+  }
+
+  private void logWarnOrThrowException(String message, RuntimeException e, Boolean quiet) {
+    if (quiet) {
+      logger.warn(message, e);
+    } else {
+      throw e;
+    }
+  }
+
+  protected void overwrite(String path, Object value) throws Exception {
+    ensureConnected();
+    zookeeper.setData().forPath(path, mapper.writeValueAsBytes(value));
+  }
+
+  protected void overwrite(String path, byte[] value) throws Exception {
+    ensureConnected();
+    zookeeper.setData().forPath(path, value);
+  }
+
+  protected void createRecursively(String path, Object value) throws Exception {
+    ensureConnected();
+    zookeeper.create().creatingParentsIfNeeded().forPath(path, mapper.writeValueAsBytes(value));
+  }
+
+  protected void createInTransaction(String path, Object value, String childPath) throws Exception {
+    ensureConnected();
+    zookeeper
+        .inTransaction()
+        .create()
+        .forPath(path, mapper.writeValueAsBytes(value))
+        .and()
+        .create()
+        .forPath(childPath)
+        .and()
+        .commit();
+  }
+
+  protected void deleteInTransaction(List paths) throws Exception {
+    if (paths.isEmpty()) {
+      throw new InternalProcessingException("Attempting to remove empty set of paths from ZK");
+    }
+    ensureConnected();
+    CuratorTransactionFinal transaction =
+        zookeeper.inTransaction().delete().forPath(paths.get(0)).and();
+
+    for (int i = 1; i < paths.size(); i++) {
+      transaction = transaction.delete().forPath(paths.get(i)).and();
+    }
+
+    transaction.commit();
+  }
+
+  protected void create(String path, Object value) throws Exception {
+    ensureConnected();
+    zookeeper.create().forPath(path, mapper.writeValueAsBytes(value));
+  }
+
+  protected void create(String path, byte[] value) throws Exception {
+    ensureConnected();
+    zookeeper.create().forPath(path, value);
+  }
+
+  protected void touch(String path) throws Exception {
+    ensureConnected();
+    byte[] oldData = zookeeper.getData().forPath(path);
+    zookeeper.setData().forPath(path, oldData);
+  }
+
+  protected void remove(String path) throws Exception {
+    ensureConnected();
+    zookeeper.delete().guaranteed().deletingChildrenIfNeeded().forPath(path);
+  }
+
+  private interface ThrowingReader {
+    T read(byte[] data) throws IOException;
+  }
 }
diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperCredentialsRepository.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperCredentialsRepository.java
index bf29c41a08..a0390f5267 100644
--- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperCredentialsRepository.java
+++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperCredentialsRepository.java
@@ -6,23 +6,25 @@
 import pl.allegro.tech.hermes.domain.CredentialsRepository;
 import pl.allegro.tech.hermes.domain.NodePassword;
 
-public class ZookeeperCredentialsRepository extends ZookeeperBasedRepository implements CredentialsRepository {
+public class ZookeeperCredentialsRepository extends ZookeeperBasedRepository
+    implements CredentialsRepository {
 
-    public ZookeeperCredentialsRepository(CuratorFramework zookeeper, ObjectMapper mapper, ZookeeperPaths paths) {
-        super(zookeeper, mapper, paths);
-    }
+  public ZookeeperCredentialsRepository(
+      CuratorFramework zookeeper, ObjectMapper mapper, ZookeeperPaths paths) {
+    super(zookeeper, mapper, paths);
+  }
 
-    @Override
-    public NodePassword readAdminPassword() {
-        return readFrom(paths.groupsPath(), NodePassword.class);
-    }
+  @Override
+  public NodePassword readAdminPassword() {
+    return readFrom(paths.groupsPath(), NodePassword.class);
+  }
 
-    @Override
-    public void overwriteAdminPassword(String password) {
-        try {
-            overwrite(paths.groupsPath(), new NodePassword(password));
-        } catch (Exception e) {
-            throw new InternalProcessingException(e);
-        }
+  @Override
+  public void overwriteAdminPassword(String password) {
+    try {
+      overwrite(paths.groupsPath(), new NodePassword(password));
+    } catch (Exception e) {
+      throw new InternalProcessingException(e);
     }
+  }
 }
diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperGroupRepository.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperGroupRepository.java
index b9c745796d..67cece9fa9 100644
--- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperGroupRepository.java
+++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperGroupRepository.java
@@ -2,6 +2,9 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import jakarta.annotation.PostConstruct;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.zookeeper.KeeperException;
 import org.slf4j.Logger;
@@ -14,117 +17,108 @@
 import pl.allegro.tech.hermes.domain.group.GroupNotExistsException;
 import pl.allegro.tech.hermes.domain.group.GroupRepository;
 
-import java.util.List;
-import java.util.Optional;
-import java.util.stream.Collectors;
-
 public class ZookeeperGroupRepository extends ZookeeperBasedRepository implements GroupRepository {
 
-    private static final Logger logger = LoggerFactory.getLogger(ZookeeperGroupRepository.class);
-
-    public ZookeeperGroupRepository(CuratorFramework zookeeper,
-                                    ObjectMapper mapper,
-                                    ZookeeperPaths paths) {
-        super(zookeeper, mapper, paths);
-    }
-
-    @Override
-    public boolean groupExists(String groupName) {
-        return pathExists(paths.groupPath(groupName));
-    }
-
-    @Override
-    public void ensureGroupExists(String groupName) {
-        if (!groupExists(groupName)) {
-            throw new GroupNotExistsException(groupName);
-        }
-    }
+  private static final Logger logger = LoggerFactory.getLogger(ZookeeperGroupRepository.class);
 
-    @Override
-    public void createGroup(Group group) {
-        String groupPath = paths.groupPath(group.getGroupName());
-        logger.info("Creating group {} for path {}", group.getGroupName(), groupPath);
-
-        try {
-            createInTransaction(groupPath, group, paths.topicsPath(group.getGroupName()));
-        } catch (KeeperException.NodeExistsException ex) {
-            throw new GroupAlreadyExistsException(group.getGroupName(), ex);
-        } catch (Exception ex) {
-            throw new InternalProcessingException(ex);
-        }
-    }
+  public ZookeeperGroupRepository(
+      CuratorFramework zookeeper, ObjectMapper mapper, ZookeeperPaths paths) {
+    super(zookeeper, mapper, paths);
+  }
 
-    @Override
-    public void updateGroup(Group group) {
-        ensureGroupExists(group.getGroupName());
+  @Override
+  public boolean groupExists(String groupName) {
+    return pathExists(paths.groupPath(groupName));
+  }
 
-        logger.info("Updating group {}", group.getGroupName());
-        try {
-            overwrite(paths.groupPath(group.getGroupName()), group);
-        } catch (Exception e) {
-            throw new InternalProcessingException(e);
-        }
+  @Override
+  public void ensureGroupExists(String groupName) {
+    if (!groupExists(groupName)) {
+      throw new GroupNotExistsException(groupName);
     }
-
-    /**
-     * Atomic removal of group and group/topics
-     * nodes is required to prevent lengthy loop during removal, see:
-     * {@link pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperTopicRepository#removeTopic(TopicName)}.
-     */
-    @Override
-    public void removeGroup(String groupName) {
-        ensureGroupExists(groupName);
-        ensureGroupIsEmpty(groupName);
-
-        logger.info("Removing group: {}", groupName);
-        List pathsToDelete = List.of(
-                paths.topicsPath(groupName),
-                paths.groupPath(groupName)
-        );
-        try {
-            deleteInTransaction(pathsToDelete);
-        } catch (Exception e) {
-            throw new InternalProcessingException(e);
-        }
+  }
+
+  @Override
+  public void createGroup(Group group) {
+    String groupPath = paths.groupPath(group.getGroupName());
+    logger.info("Creating group {} for path {}", group.getGroupName(), groupPath);
+
+    try {
+      createInTransaction(groupPath, group, paths.topicsPath(group.getGroupName()));
+    } catch (KeeperException.NodeExistsException ex) {
+      throw new GroupAlreadyExistsException(group.getGroupName(), ex);
+    } catch (Exception ex) {
+      throw new InternalProcessingException(ex);
     }
+  }
 
-    private void ensureGroupIsEmpty(String groupName) {
-        if (!childrenOf(paths.topicsPath(groupName)).isEmpty()) {
-            throw new GroupNotEmptyException(groupName);
-        }
-    }
+  @Override
+  public void updateGroup(Group group) {
+    ensureGroupExists(group.getGroupName());
 
-    @Override
-    public List listGroupNames() {
-        return childrenOf(paths.groupsPath());
+    logger.info("Updating group {}", group.getGroupName());
+    try {
+      overwrite(paths.groupPath(group.getGroupName()), group);
+    } catch (Exception e) {
+      throw new InternalProcessingException(e);
     }
-
-    @Override
-    public List listGroups() {
-        return listGroupNames()
-                .stream()
-                .map(n -> getGroupDetails(n, true))
-                .filter(Optional::isPresent)
-                .map(Optional::get)
-                .collect(Collectors.toList());
-    }
-
-    @Override
-    public Group getGroupDetails(String groupName) {
-        return getGroupDetails(groupName, false).get();
-    }
-
-    private Optional getGroupDetails(String groupName, boolean quiet) {
-        ensureGroupExists(groupName);
-
-        String path = paths.groupPath(groupName);
-        return readFrom(path, Group.class, quiet);
+  }
+
+  /**
+   * Atomic removal of group and group/topics nodes is required to prevent
+   * lengthy loop during removal, see: {@link
+   * pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperTopicRepository#removeTopic(TopicName)}.
+   */
+  @Override
+  public void removeGroup(String groupName) {
+    ensureGroupExists(groupName);
+    ensureGroupIsEmpty(groupName);
+
+    logger.info("Removing group: {}", groupName);
+    List pathsToDelete = List.of(paths.topicsPath(groupName), paths.groupPath(groupName));
+    try {
+      deleteInTransaction(pathsToDelete);
+    } catch (Exception e) {
+      throw new InternalProcessingException(e);
     }
+  }
 
-    @PostConstruct
-    public void init() {
-        logger.info("Before ensuring init path exists");
-        ensurePathExists(paths.groupsPath());
-        logger.info("After ensuring init path exists");
+  private void ensureGroupIsEmpty(String groupName) {
+    if (!childrenOf(paths.topicsPath(groupName)).isEmpty()) {
+      throw new GroupNotEmptyException(groupName);
     }
+  }
+
+  @Override
+  public List listGroupNames() {
+    return childrenOf(paths.groupsPath());
+  }
+
+  @Override
+  public List listGroups() {
+    return listGroupNames().stream()
+        .map(n -> getGroupDetails(n, true))
+        .filter(Optional::isPresent)
+        .map(Optional::get)
+        .collect(Collectors.toList());
+  }
+
+  @Override
+  public Group getGroupDetails(String groupName) {
+    return getGroupDetails(groupName, false).get();
+  }
+
+  private Optional getGroupDetails(String groupName, boolean quiet) {
+    ensureGroupExists(groupName);
+
+    String path = paths.groupPath(groupName);
+    return readFrom(path, Group.class, quiet);
+  }
+
+  @PostConstruct
+  public void init() {
+    logger.info("Before ensuring init path exists");
+    ensurePathExists(paths.groupsPath());
+    logger.info("After ensuring init path exists");
+  }
 }
diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperMessagePreviewRepository.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperMessagePreviewRepository.java
index e80b574f02..d343c0b531 100644
--- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperMessagePreviewRepository.java
+++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperMessagePreviewRepository.java
@@ -1,7 +1,12 @@
 package pl.allegro.tech.hermes.infrastructure.zookeeper;
 
+import static java.lang.String.format;
+
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
 import org.apache.curator.framework.CuratorFramework;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -11,54 +16,50 @@
 import pl.allegro.tech.hermes.domain.topic.preview.MessagePreviewRepository;
 import pl.allegro.tech.hermes.domain.topic.preview.TopicsMessagesPreview;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-
-import static java.lang.String.format;
+public class ZookeeperMessagePreviewRepository extends ZookeeperBasedRepository
+    implements MessagePreviewRepository {
 
-public class ZookeeperMessagePreviewRepository extends ZookeeperBasedRepository implements MessagePreviewRepository {
+  private static final Logger logger =
+      LoggerFactory.getLogger(ZookeeperMessagePreviewRepository.class);
 
-    private static final Logger logger = LoggerFactory.getLogger(ZookeeperMessagePreviewRepository.class);
+  public ZookeeperMessagePreviewRepository(
+      CuratorFramework zookeeper, ObjectMapper mapper, ZookeeperPaths paths) {
+    super(zookeeper, mapper, paths);
+  }
 
-    public ZookeeperMessagePreviewRepository(CuratorFramework zookeeper, ObjectMapper mapper, ZookeeperPaths paths) {
-        super(zookeeper, mapper, paths);
+  @Override
+  public List loadPreview(TopicName topicName) {
+    try {
+      return Optional.of(paths.topicPreviewPath(topicName))
+          .filter(this::pathExists)
+          .flatMap(p -> readFrom(p, new TypeReference>() {}, true))
+          .orElseGet(ArrayList::new);
+    } catch (Exception e) {
+      throw new InternalProcessingException(
+          format("Could not read latest preview message for topic: %s.", topicName.qualifiedName()),
+          e);
     }
+  }
 
-    @Override
-    public List loadPreview(TopicName topicName) {
-        try {
-            return Optional.of(paths.topicPreviewPath(topicName))
-                    .filter(this::pathExists)
-                    .flatMap(p -> readFrom(p, new TypeReference>() {}, true))
-                    .orElseGet(ArrayList::new);
-        } catch (Exception e) {
-            throw new InternalProcessingException(
-                    format("Could not read latest preview message for topic: %s.", topicName.qualifiedName()), e);
-        }
-    }
-
-
-    @Override
-    public void persist(TopicsMessagesPreview topicsMessagesPreview) {
-        for (TopicName topic : topicsMessagesPreview.topics()) {
-            persistMessage(topic, topicsMessagesPreview.previewOf(topic));
-        }
+  @Override
+  public void persist(TopicsMessagesPreview topicsMessagesPreview) {
+    for (TopicName topic : topicsMessagesPreview.topics()) {
+      persistMessage(topic, topicsMessagesPreview.previewOf(topic));
     }
+  }
 
-    private void persistMessage(TopicName topic, List messages) {
-        logger.debug("Persisting {} messages for preview of topic: {}", messages.size(), topic.qualifiedName());
-        try {
-            if (pathExists(paths.topicPath(topic))) {
-                String previewPath = paths.topicPreviewPath(topic);
-                ensurePathExists(previewPath);
-                overwrite(previewPath, messages);
-            }
-        } catch (Exception exception) {
-            logger.warn(
-                    format("Could not log preview messages for topic: %s", topic.qualifiedName()),
-                    exception
-            );
-        }
+  private void persistMessage(TopicName topic, List messages) {
+    logger.debug(
+        "Persisting {} messages for preview of topic: {}", messages.size(), topic.qualifiedName());
+    try {
+      if (pathExists(paths.topicPath(topic))) {
+        String previewPath = paths.topicPreviewPath(topic);
+        ensurePathExists(previewPath);
+        overwrite(previewPath, messages);
+      }
+    } catch (Exception exception) {
+      logger.warn(
+          format("Could not log preview messages for topic: %s", topic.qualifiedName()), exception);
     }
+  }
 }
diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperOAuthProviderRepository.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperOAuthProviderRepository.java
index 4673888b4e..c84a3f6c51 100644
--- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperOAuthProviderRepository.java
+++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperOAuthProviderRepository.java
@@ -1,6 +1,9 @@
 package pl.allegro.tech.hermes.infrastructure.zookeeper;
 
+import static java.util.stream.Collectors.toList;
+
 import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.List;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.zookeeper.KeeperException;
 import org.slf4j.Logger;
@@ -11,85 +14,82 @@
 import pl.allegro.tech.hermes.domain.oauth.OAuthProviderNotExistsException;
 import pl.allegro.tech.hermes.domain.oauth.OAuthProviderRepository;
 
-import java.util.List;
-
-import static java.util.stream.Collectors.toList;
-
-public class ZookeeperOAuthProviderRepository extends ZookeeperBasedRepository implements OAuthProviderRepository {
-
-    private static final Logger logger = LoggerFactory.getLogger(ZookeeperOAuthProviderRepository.class);
-
-    public ZookeeperOAuthProviderRepository(CuratorFramework zookeeper, ObjectMapper mapper, ZookeeperPaths paths) {
-        super(zookeeper, mapper, paths);
+public class ZookeeperOAuthProviderRepository extends ZookeeperBasedRepository
+    implements OAuthProviderRepository {
+
+  private static final Logger logger =
+      LoggerFactory.getLogger(ZookeeperOAuthProviderRepository.class);
+
+  public ZookeeperOAuthProviderRepository(
+      CuratorFramework zookeeper, ObjectMapper mapper, ZookeeperPaths paths) {
+    super(zookeeper, mapper, paths);
+  }
+
+  @Override
+  public List listOAuthProviderNames() {
+    ensurePathExists(paths.oAuthProvidersPath());
+    return childrenOf(paths.oAuthProvidersPath());
+  }
+
+  @Override
+  public List listOAuthProviders() {
+    ensurePathExists(paths.oAuthProvidersPath());
+    return listOAuthProviderNames().stream().map(this::getOAuthProviderDetails).collect(toList());
+  }
+
+  @Override
+  public OAuthProvider getOAuthProviderDetails(String oAuthProviderName) {
+    ensureOAuthProviderExists(oAuthProviderName);
+    return readFrom(paths.oAuthProviderPath(oAuthProviderName), OAuthProvider.class);
+  }
+
+  @Override
+  public void createOAuthProvider(OAuthProvider oAuthProvider) {
+    String oAuthProviderPath = paths.oAuthProviderPath(oAuthProvider.getName());
+    logger.info("Creating OAuthProvider for path {}", oAuthProviderPath);
+
+    try {
+      createRecursively(oAuthProviderPath, oAuthProvider);
+    } catch (KeeperException.NodeExistsException ex) {
+      throw new OAuthProviderAlreadyExistsException(oAuthProvider, ex);
+    } catch (Exception ex) {
+      throw new InternalProcessingException(ex);
     }
+  }
 
-    @Override
-    public List listOAuthProviderNames() {
-        ensurePathExists(paths.oAuthProvidersPath());
-        return childrenOf(paths.oAuthProvidersPath());
-    }
+  @Override
+  public void updateOAuthProvider(OAuthProvider oAuthprovider) {
+    ensureOAuthProviderExists(oAuthprovider.getName());
 
-    @Override
-    public List listOAuthProviders() {
-        ensurePathExists(paths.oAuthProvidersPath());
-        return listOAuthProviderNames().stream()
-                .map(this::getOAuthProviderDetails)
-                .collect(toList());
+    logger.info("Updating OAuthProvider {}", oAuthprovider.getName());
+    try {
+      overwrite(paths.oAuthProviderPath(oAuthprovider.getName()), oAuthprovider);
+    } catch (Exception e) {
+      throw new InternalProcessingException(e);
     }
+  }
 
-    @Override
-    public OAuthProvider getOAuthProviderDetails(String oAuthProviderName) {
-        ensureOAuthProviderExists(oAuthProviderName);
-        return readFrom(paths.oAuthProviderPath(oAuthProviderName), OAuthProvider.class);
-    }
+  @Override
+  public void removeOAuthProvider(String oAuthProviderName) {
+    ensureOAuthProviderExists(oAuthProviderName);
 
-    @Override
-    public void createOAuthProvider(OAuthProvider oAuthProvider) {
-        String oAuthProviderPath = paths.oAuthProviderPath(oAuthProvider.getName());
-        logger.info("Creating OAuthProvider for path {}", oAuthProviderPath);
-
-        try {
-            createRecursively(oAuthProviderPath, oAuthProvider);
-        } catch (KeeperException.NodeExistsException ex) {
-            throw new OAuthProviderAlreadyExistsException(oAuthProvider, ex);
-        } catch (Exception ex) {
-            throw new InternalProcessingException(ex);
-        }
+    logger.info("Removing OAuthProvider {}", oAuthProviderName);
+    try {
+      remove(paths.oAuthProviderPath(oAuthProviderName));
+    } catch (Exception e) {
+      throw new InternalProcessingException(e);
     }
+  }
 
-    @Override
-    public void updateOAuthProvider(OAuthProvider oAuthprovider) {
-        ensureOAuthProviderExists(oAuthprovider.getName());
-
-        logger.info("Updating OAuthProvider {}", oAuthprovider.getName());
-        try {
-            overwrite(paths.oAuthProviderPath(oAuthprovider.getName()), oAuthprovider);
-        } catch (Exception e) {
-            throw new InternalProcessingException(e);
-        }
+  @Override
+  public void ensureOAuthProviderExists(String oAuthProviderName) {
+    if (!oAuthProviderExists(oAuthProviderName)) {
+      throw new OAuthProviderNotExistsException(oAuthProviderName);
     }
+  }
 
-    @Override
-    public void removeOAuthProvider(String oAuthProviderName) {
-        ensureOAuthProviderExists(oAuthProviderName);
-
-        logger.info("Removing OAuthProvider {}", oAuthProviderName);
-        try {
-            remove(paths.oAuthProviderPath(oAuthProviderName));
-        } catch (Exception e) {
-            throw new InternalProcessingException(e);
-        }
-    }
-
-    @Override
-    public void ensureOAuthProviderExists(String oAuthProviderName) {
-        if (!oAuthProviderExists(oAuthProviderName)) {
-            throw new OAuthProviderNotExistsException(oAuthProviderName);
-        }
-    }
-
-    @Override
-    public boolean oAuthProviderExists(String oAuthProviderName) {
-        return pathExists(paths.oAuthProviderPath(oAuthProviderName));
-    }
+  @Override
+  public boolean oAuthProviderExists(String oAuthProviderName) {
+    return pathExists(paths.oAuthProviderPath(oAuthProviderName));
+  }
 }
diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperPaths.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperPaths.java
index be3cb23393..e6d44c5830 100644
--- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperPaths.java
+++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperPaths.java
@@ -9,169 +9,180 @@
 
 public class ZookeeperPaths {
 
-    public static final String TOPICS_PATH = "topics";
-    public static final String GROUPS_PATH = "groups";
-    public static final String SUBSCRIPTIONS_PATH = "subscriptions";
-    public static final String KAFKA_TOPICS_PATH = "kafka_topics";
-    public static final String URL_SEPARATOR = "/";
-    public static final String CONSUMERS_WORKLOAD_PATH = "consumers-workload";
-    public static final String CONSUMER_LOAD_PATH = "consumer-load";
-    public static final String SUBSCRIPTION_PROFILES_PATH = "subscription-profiles";
-    public static final String CONSUMERS_WORKLOAD_CONSTRAINTS_PATH = "consumers-workload-constraints";
-    public static final String CONSUMERS_RATE_PATH = "consumers-rate";
-    public static final String METRICS_PATH = "metrics";
-    public static final String ADMIN_PATH = "admin";
-    public static final String PREVIEW_PATH = "preview";
-    public static final String OAUTH_PROVIDERS_PATH = "oauth-providers";
-    public static final String BLACKLIST_PATH = "blacklist";
-    public static final String MAX_RATE_PATH = "max-rate";
-    public static final String MAX_RATE_HISTORY_PATH = "history";
-    public static final String STORAGE_HEALTH_PATH = "storage-health";
-    public static final String DATACENTER_READINESS_PATH = "datacenter-readiness";
-    public static final String OFFLINE_RETRANSMISSION_PATH = "offline-retransmission";
-    public static final String OFFLINE_RETRANSMISSION_TASKS_PATH = "tasks";
-
-    private final String basePath;
-
-    public ZookeeperPaths(String basePath) {
-        this.basePath = basePath;
-    }
-
-    public String basePath() {
-        return basePath;
-    }
-
-    public String extractChildNode(String fullChildPath, String prefixPath) {
-        return StringUtils.removeStart(fullChildPath, prefixPath + URL_SEPARATOR);
-    }
-
-    public String adminPath() {
-        return Joiner.on(URL_SEPARATOR).join(basePath, ADMIN_PATH);
-    }
-
-    public String adminOperationPath(String operation) {
-        return Joiner.on(URL_SEPARATOR).join(adminPath(), operation);
-    }
-
-    public String groupsPath() {
-        return Joiner.on(URL_SEPARATOR).join(basePath, GROUPS_PATH);
-    }
-
-    public String groupPath(String groupName) {
-        return Joiner.on(URL_SEPARATOR).join(groupsPath(), groupName);
-    }
-
-    public String topicsPath(String groupName) {
-        return Joiner.on(URL_SEPARATOR).join(groupPath(groupName), TOPICS_PATH);
-    }
-
-    public String topicMetricPath(TopicName topicName, String metricName) {
-        return topicPath(topicName, "metrics", metricName);
-    }
-
-    public String subscriptionsPath(TopicName topicName) {
-        return Joiner.on(URL_SEPARATOR).join(topicPath(topicName), SUBSCRIPTIONS_PATH);
-    }
-
-    public String topicPath(TopicName topicName, String... tail) {
-        return Joiner.on(URL_SEPARATOR).join(topicsPath(topicName.getGroupName()), topicName.getName(), (Object[]) tail);
-    }
-
-    public String topicPreviewPath(TopicName topicName) {
-        return topicPath(topicName, ZookeeperPaths.PREVIEW_PATH);
-    }
-
-    public String topicMetricsPath(TopicName topicName) {
-        return topicPath(topicName, METRICS_PATH);
-    }
-
-    public String subscriptionPath(TopicName topicName, String subscriptionName, String... tail) {
-        return Joiner.on(URL_SEPARATOR).join(subscriptionsPath(topicName), subscriptionName, (Object[]) tail);
-    }
-
-    public String subscriptionPath(Subscription subscription) {
-        return subscriptionPath(subscription.getTopicName(), subscription.getName());
-    }
-
-    public String subscriptionMetricsPath(TopicName topicName, String subscriptionName) {
-        return subscriptionPath(topicName, subscriptionName, METRICS_PATH);
-    }
-
-    public String subscriptionMetricPath(TopicName topicName, String subscriptionName, String metricName) {
-        return subscriptionPath(topicName, subscriptionName, METRICS_PATH, metricName);
-    }
-
-    public String subscriptionMetricPath(SubscriptionName subscriptionName, String metricName) {
-        return subscriptionPath(subscriptionName.getTopicName(), subscriptionName.getName(), METRICS_PATH, metricName);
-    }
-
-    public String offsetPath(TopicName topicName,
-                             String subscriptionName,
-                             KafkaTopicName kafkaTopicName,
-                             String brokersClusterName,
-                             int partitionId) {
-        return Joiner.on(URL_SEPARATOR).join(
-                offsetsPath(topicName, subscriptionName, kafkaTopicName, brokersClusterName),
-                partitionId);
-    }
-
-    public String offsetsPath(TopicName topicName,
-                              String subscriptionName,
-                              KafkaTopicName kafkaTopicName,
-                              String brokersClusterName) {
-        return Joiner.on(URL_SEPARATOR).join(
-                subscribedKafkaTopicsPath(topicName, subscriptionName),
-                kafkaTopicName.asString(),
-                "offset",
-                brokersClusterName);
-    }
-
-    public String subscribedKafkaTopicsPath(TopicName topicName, String subscriptionName) {
-        return Joiner.on(URL_SEPARATOR).join(subscriptionPath(topicName, subscriptionName), KAFKA_TOPICS_PATH);
-    }
-
-    public String consumersWorkloadConstraintsPath() {
-        return Joiner.on(URL_SEPARATOR).join(basePath, CONSUMERS_WORKLOAD_CONSTRAINTS_PATH);
-    }
-
-    public String consumersWorkloadConstraintsPath(String constraintsPath) {
-        return Joiner.on(URL_SEPARATOR).join(consumersWorkloadConstraintsPath(), constraintsPath);
-    }
-
-    public String topicsBlacklistPath() {
-        return Joiner.on(URL_SEPARATOR).join(basePath, BLACKLIST_PATH, TOPICS_PATH);
-    }
-
-    public String blacklistedTopicPath(String qualifiedTopicName) {
-        return Joiner.on(URL_SEPARATOR).join(topicsBlacklistPath(), qualifiedTopicName);
-    }
-
-    public String oAuthProvidersPath() {
-        return Joiner.on(URL_SEPARATOR).join(basePath, OAUTH_PROVIDERS_PATH);
-    }
-
-    public String oAuthProviderPath(String oAuthProviderName) {
-        return Joiner.on(URL_SEPARATOR).join(oAuthProvidersPath(), oAuthProviderName);
-    }
-
-    public String nodeHealthPathForManagementHost(String host, String port) {
-        return Joiner.on(URL_SEPARATOR).join(basePath, STORAGE_HEALTH_PATH, String.format("%s_%s", host, port));
-    }
-
-    public String datacenterReadinessPath() {
-        return Joiner.on(URL_SEPARATOR).join(basePath, DATACENTER_READINESS_PATH);
-    }
-
-    public String offlineRetransmissionPath() {
-        return Joiner.on(URL_SEPARATOR).join(basePath, OFFLINE_RETRANSMISSION_PATH, OFFLINE_RETRANSMISSION_TASKS_PATH);
-    }
-
-    public String offlineRetransmissionPath(String taskId) {
-        return Joiner.on(URL_SEPARATOR)
-                .join(basePath, OFFLINE_RETRANSMISSION_PATH, OFFLINE_RETRANSMISSION_TASKS_PATH, taskId);
-    }
-
-    public String join(String... parts) {
-        return Joiner.on(URL_SEPARATOR).join(parts);
-    }
+  public static final String TOPICS_PATH = "topics";
+  public static final String GROUPS_PATH = "groups";
+  public static final String SUBSCRIPTIONS_PATH = "subscriptions";
+  public static final String KAFKA_TOPICS_PATH = "kafka_topics";
+  public static final String URL_SEPARATOR = "/";
+  public static final String CONSUMERS_WORKLOAD_PATH = "consumers-workload";
+  public static final String CONSUMER_LOAD_PATH = "consumer-load";
+  public static final String SUBSCRIPTION_PROFILES_PATH = "subscription-profiles";
+  public static final String CONSUMERS_WORKLOAD_CONSTRAINTS_PATH = "consumers-workload-constraints";
+  public static final String CONSUMERS_RATE_PATH = "consumers-rate";
+  public static final String METRICS_PATH = "metrics";
+  public static final String ADMIN_PATH = "admin";
+  public static final String PREVIEW_PATH = "preview";
+  public static final String OAUTH_PROVIDERS_PATH = "oauth-providers";
+  public static final String BLACKLIST_PATH = "blacklist";
+  public static final String MAX_RATE_PATH = "max-rate";
+  public static final String MAX_RATE_HISTORY_PATH = "history";
+  public static final String STORAGE_HEALTH_PATH = "storage-health";
+  public static final String DATACENTER_READINESS_PATH = "datacenter-readiness";
+  public static final String OFFLINE_RETRANSMISSION_PATH = "offline-retransmission";
+  public static final String OFFLINE_RETRANSMISSION_TASKS_PATH = "tasks";
+
+  private final String basePath;
+
+  public ZookeeperPaths(String basePath) {
+    this.basePath = basePath;
+  }
+
+  public String basePath() {
+    return basePath;
+  }
+
+  public String extractChildNode(String fullChildPath, String prefixPath) {
+    return StringUtils.removeStart(fullChildPath, prefixPath + URL_SEPARATOR);
+  }
+
+  public String adminPath() {
+    return Joiner.on(URL_SEPARATOR).join(basePath, ADMIN_PATH);
+  }
+
+  public String adminOperationPath(String operation) {
+    return Joiner.on(URL_SEPARATOR).join(adminPath(), operation);
+  }
+
+  public String groupsPath() {
+    return Joiner.on(URL_SEPARATOR).join(basePath, GROUPS_PATH);
+  }
+
+  public String groupPath(String groupName) {
+    return Joiner.on(URL_SEPARATOR).join(groupsPath(), groupName);
+  }
+
+  public String topicsPath(String groupName) {
+    return Joiner.on(URL_SEPARATOR).join(groupPath(groupName), TOPICS_PATH);
+  }
+
+  public String topicMetricPath(TopicName topicName, String metricName) {
+    return topicPath(topicName, "metrics", metricName);
+  }
+
+  public String subscriptionsPath(TopicName topicName) {
+    return Joiner.on(URL_SEPARATOR).join(topicPath(topicName), SUBSCRIPTIONS_PATH);
+  }
+
+  public String topicPath(TopicName topicName, String... tail) {
+    return Joiner.on(URL_SEPARATOR)
+        .join(topicsPath(topicName.getGroupName()), topicName.getName(), (Object[]) tail);
+  }
+
+  public String topicPreviewPath(TopicName topicName) {
+    return topicPath(topicName, ZookeeperPaths.PREVIEW_PATH);
+  }
+
+  public String topicMetricsPath(TopicName topicName) {
+    return topicPath(topicName, METRICS_PATH);
+  }
+
+  public String subscriptionPath(TopicName topicName, String subscriptionName, String... tail) {
+    return Joiner.on(URL_SEPARATOR)
+        .join(subscriptionsPath(topicName), subscriptionName, (Object[]) tail);
+  }
+
+  public String subscriptionPath(Subscription subscription) {
+    return subscriptionPath(subscription.getTopicName(), subscription.getName());
+  }
+
+  public String subscriptionMetricsPath(TopicName topicName, String subscriptionName) {
+    return subscriptionPath(topicName, subscriptionName, METRICS_PATH);
+  }
+
+  public String subscriptionMetricPath(
+      TopicName topicName, String subscriptionName, String metricName) {
+    return subscriptionPath(topicName, subscriptionName, METRICS_PATH, metricName);
+  }
+
+  public String subscriptionMetricPath(SubscriptionName subscriptionName, String metricName) {
+    return subscriptionPath(
+        subscriptionName.getTopicName(), subscriptionName.getName(), METRICS_PATH, metricName);
+  }
+
+  public String offsetPath(
+      TopicName topicName,
+      String subscriptionName,
+      KafkaTopicName kafkaTopicName,
+      String brokersClusterName,
+      int partitionId) {
+    return Joiner.on(URL_SEPARATOR)
+        .join(
+            offsetsPath(topicName, subscriptionName, kafkaTopicName, brokersClusterName),
+            partitionId);
+  }
+
+  public String offsetsPath(
+      TopicName topicName,
+      String subscriptionName,
+      KafkaTopicName kafkaTopicName,
+      String brokersClusterName) {
+    return Joiner.on(URL_SEPARATOR)
+        .join(
+            subscribedKafkaTopicsPath(topicName, subscriptionName),
+            kafkaTopicName.asString(),
+            "offset",
+            brokersClusterName);
+  }
+
+  public String subscribedKafkaTopicsPath(TopicName topicName, String subscriptionName) {
+    return Joiner.on(URL_SEPARATOR)
+        .join(subscriptionPath(topicName, subscriptionName), KAFKA_TOPICS_PATH);
+  }
+
+  public String consumersWorkloadConstraintsPath() {
+    return Joiner.on(URL_SEPARATOR).join(basePath, CONSUMERS_WORKLOAD_CONSTRAINTS_PATH);
+  }
+
+  public String consumersWorkloadConstraintsPath(String constraintsPath) {
+    return Joiner.on(URL_SEPARATOR).join(consumersWorkloadConstraintsPath(), constraintsPath);
+  }
+
+  public String topicsBlacklistPath() {
+    return Joiner.on(URL_SEPARATOR).join(basePath, BLACKLIST_PATH, TOPICS_PATH);
+  }
+
+  public String blacklistedTopicPath(String qualifiedTopicName) {
+    return Joiner.on(URL_SEPARATOR).join(topicsBlacklistPath(), qualifiedTopicName);
+  }
+
+  public String oAuthProvidersPath() {
+    return Joiner.on(URL_SEPARATOR).join(basePath, OAUTH_PROVIDERS_PATH);
+  }
+
+  public String oAuthProviderPath(String oAuthProviderName) {
+    return Joiner.on(URL_SEPARATOR).join(oAuthProvidersPath(), oAuthProviderName);
+  }
+
+  public String nodeHealthPathForManagementHost(String host, String port) {
+    return Joiner.on(URL_SEPARATOR)
+        .join(basePath, STORAGE_HEALTH_PATH, String.format("%s_%s", host, port));
+  }
+
+  public String datacenterReadinessPath() {
+    return Joiner.on(URL_SEPARATOR).join(basePath, DATACENTER_READINESS_PATH);
+  }
+
+  public String offlineRetransmissionPath() {
+    return Joiner.on(URL_SEPARATOR)
+        .join(basePath, OFFLINE_RETRANSMISSION_PATH, OFFLINE_RETRANSMISSION_TASKS_PATH);
+  }
+
+  public String offlineRetransmissionPath(String taskId) {
+    return Joiner.on(URL_SEPARATOR)
+        .join(basePath, OFFLINE_RETRANSMISSION_PATH, OFFLINE_RETRANSMISSION_TASKS_PATH, taskId);
+  }
+
+  public String join(String... parts) {
+    return Joiner.on(URL_SEPARATOR).join(parts);
+  }
 }
diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperSubscriptionOffsetChangeIndicator.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperSubscriptionOffsetChangeIndicator.java
index 633992a8c1..0ea7cf0c18 100644
--- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperSubscriptionOffsetChangeIndicator.java
+++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperSubscriptionOffsetChangeIndicator.java
@@ -1,5 +1,7 @@
 package pl.allegro.tech.hermes.infrastructure.zookeeper;
 
+import java.nio.charset.StandardCharsets;
+import java.util.List;
 import org.apache.curator.framework.CuratorFramework;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -12,149 +14,165 @@
 import pl.allegro.tech.hermes.common.kafka.offset.SubscriptionOffsetChangeIndicator;
 import pl.allegro.tech.hermes.domain.subscription.SubscriptionRepository;
 
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-
-public class ZookeeperSubscriptionOffsetChangeIndicator implements SubscriptionOffsetChangeIndicator {
-
-    private static final Logger logger = LoggerFactory.getLogger(ZookeeperSubscriptionOffsetChangeIndicator.class);
-
-    private final CuratorFramework zookeeper;
-
-    private final ZookeeperPaths paths;
-
-    private final SubscriptionRepository subscriptionRepository;
-
-    public ZookeeperSubscriptionOffsetChangeIndicator(
-            CuratorFramework zookeeper, ZookeeperPaths paths, SubscriptionRepository repository) {
-
-        this.zookeeper = zookeeper;
-        this.paths = paths;
-        this.subscriptionRepository = repository;
-    }
-
-    @Override
-    public void setSubscriptionOffset(TopicName topicName,
-                                      String subscriptionName,
-                                      String brokersClusterName,
-                                      PartitionOffset partitionOffset) {
-        subscriptionRepository.ensureSubscriptionExists(topicName, subscriptionName);
-
-        String offsetPath = paths.offsetPath(
-                topicName,
-                subscriptionName,
-                partitionOffset.getTopic(),
-                brokersClusterName,
-                partitionOffset.getPartition());
-        try {
-            byte[] offset = String.valueOf(partitionOffset.getOffset()).getBytes(StandardCharsets.UTF_8);
-            if (zookeeper.checkExists().forPath(offsetPath) == null) {
-                zookeeper.create()
-                        .creatingParentsIfNeeded()
-                        .forPath(offsetPath, offset);
-            } else {
-                zookeeper.setData().forPath(offsetPath, offset);
-            }
-        } catch (Exception e) {
-            throw new InternalProcessingException(e);
-        }
-    }
-
-    @Override
-    public PartitionOffsets getSubscriptionOffsets(TopicName topic, String subscriptionName, String brokersClusterName) {
-        subscriptionRepository.ensureSubscriptionExists(topic, subscriptionName);
-        String kafkaTopicsPath = paths.subscribedKafkaTopicsPath(topic, subscriptionName);
-
-        PartitionOffsets allOffsets = new PartitionOffsets();
-        getZookeeperChildrenForPath(kafkaTopicsPath).stream().map(KafkaTopicName::valueOf).forEach(kafkaTopic ->
-                allOffsets.addAll(getOffsetsForKafkaTopic(topic, kafkaTopic, subscriptionName, brokersClusterName))
-        );
-        return allOffsets;
-    }
-
-    @Override
-    public boolean areOffsetsMoved(TopicName topicName,
-                                   String subscriptionName,
-                                   String brokersClusterName,
-                                   KafkaTopic kafkaTopic,
-                                   List partitionIds) {
-        return partitionIds.stream()
-                .allMatch(partitionId -> offsetDoesNotExist(topicName, subscriptionName, brokersClusterName, partitionId, kafkaTopic));
+public class ZookeeperSubscriptionOffsetChangeIndicator
+    implements SubscriptionOffsetChangeIndicator {
+
+  private static final Logger logger =
+      LoggerFactory.getLogger(ZookeeperSubscriptionOffsetChangeIndicator.class);
+
+  private final CuratorFramework zookeeper;
+
+  private final ZookeeperPaths paths;
+
+  private final SubscriptionRepository subscriptionRepository;
+
+  public ZookeeperSubscriptionOffsetChangeIndicator(
+      CuratorFramework zookeeper, ZookeeperPaths paths, SubscriptionRepository repository) {
+
+    this.zookeeper = zookeeper;
+    this.paths = paths;
+    this.subscriptionRepository = repository;
+  }
+
+  @Override
+  public void setSubscriptionOffset(
+      TopicName topicName,
+      String subscriptionName,
+      String brokersClusterName,
+      PartitionOffset partitionOffset) {
+    subscriptionRepository.ensureSubscriptionExists(topicName, subscriptionName);
+
+    String offsetPath =
+        paths.offsetPath(
+            topicName,
+            subscriptionName,
+            partitionOffset.getTopic(),
+            brokersClusterName,
+            partitionOffset.getPartition());
+    try {
+      byte[] offset = String.valueOf(partitionOffset.getOffset()).getBytes(StandardCharsets.UTF_8);
+      if (zookeeper.checkExists().forPath(offsetPath) == null) {
+        zookeeper.create().creatingParentsIfNeeded().forPath(offsetPath, offset);
+      } else {
+        zookeeper.setData().forPath(offsetPath, offset);
+      }
+    } catch (Exception e) {
+      throw new InternalProcessingException(e);
     }
-
-    @Override
-    public void removeOffset(TopicName topicName,
-                             String subscriptionName,
-                             String brokersClusterName,
-                             KafkaTopicName kafkaTopicName,
-                             int partitionId) {
-        String offsetPath = paths.offsetPath(topicName, subscriptionName, kafkaTopicName, brokersClusterName, partitionId);
-
-        try {
-            zookeeper.delete().guaranteed().deletingChildrenIfNeeded().forPath(offsetPath);
-        } catch (Exception e) {
-            throw new InternalProcessingException(e);
-        }
+  }
+
+  @Override
+  public PartitionOffsets getSubscriptionOffsets(
+      TopicName topic, String subscriptionName, String brokersClusterName) {
+    subscriptionRepository.ensureSubscriptionExists(topic, subscriptionName);
+    String kafkaTopicsPath = paths.subscribedKafkaTopicsPath(topic, subscriptionName);
+
+    PartitionOffsets allOffsets = new PartitionOffsets();
+    getZookeeperChildrenForPath(kafkaTopicsPath).stream()
+        .map(KafkaTopicName::valueOf)
+        .forEach(
+            kafkaTopic ->
+                allOffsets.addAll(
+                    getOffsetsForKafkaTopic(
+                        topic, kafkaTopic, subscriptionName, brokersClusterName)));
+    return allOffsets;
+  }
+
+  @Override
+  public boolean areOffsetsMoved(
+      TopicName topicName,
+      String subscriptionName,
+      String brokersClusterName,
+      KafkaTopic kafkaTopic,
+      List partitionIds) {
+    return partitionIds.stream()
+        .allMatch(
+            partitionId ->
+                offsetDoesNotExist(
+                    topicName, subscriptionName, brokersClusterName, partitionId, kafkaTopic));
+  }
+
+  @Override
+  public void removeOffset(
+      TopicName topicName,
+      String subscriptionName,
+      String brokersClusterName,
+      KafkaTopicName kafkaTopicName,
+      int partitionId) {
+    String offsetPath =
+        paths.offsetPath(
+            topicName, subscriptionName, kafkaTopicName, brokersClusterName, partitionId);
+
+    try {
+      zookeeper.delete().guaranteed().deletingChildrenIfNeeded().forPath(offsetPath);
+    } catch (Exception e) {
+      throw new InternalProcessingException(e);
     }
-
-    private boolean offsetDoesNotExist(TopicName topicName,
-                                       String subscriptionName,
-                                       String brokersClusterName,
-                                       Integer partitionId,
-                                       KafkaTopic kafkaTopic) {
-        String offsetPath = paths.offsetPath(topicName, subscriptionName, kafkaTopic.name(), brokersClusterName, partitionId);
-        try {
-            boolean result = zookeeper.checkExists().forPath(offsetPath) == null;
-            if (!result) {
-                logger.info("Leftover on path {}", offsetPath);
-            }
-            return result;
-        } catch (Exception e) {
-            throw new InternalProcessingException(e);
-        }
+  }
+
+  private boolean offsetDoesNotExist(
+      TopicName topicName,
+      String subscriptionName,
+      String brokersClusterName,
+      Integer partitionId,
+      KafkaTopic kafkaTopic) {
+    String offsetPath =
+        paths.offsetPath(
+            topicName, subscriptionName, kafkaTopic.name(), brokersClusterName, partitionId);
+    try {
+      boolean result = zookeeper.checkExists().forPath(offsetPath) == null;
+      if (!result) {
+        logger.info("Leftover on path {}", offsetPath);
+      }
+      return result;
+    } catch (Exception e) {
+      throw new InternalProcessingException(e);
     }
-
-    private PartitionOffsets getOffsetsForKafkaTopic(TopicName topic,
-                                                     KafkaTopicName kafkaTopicName,
-                                                     String subscriptionName,
-                                                     String brokersClusterName) {
-        String offsetsPath = paths.offsetsPath(topic, subscriptionName, kafkaTopicName, brokersClusterName);
-
-        PartitionOffsets offsets = new PartitionOffsets();
-        for (String partitionAsString : getZookeeperChildrenForPath(offsetsPath)) {
-            Integer partition = Integer.valueOf(partitionAsString);
-            offsets.add(new PartitionOffset(
-                    kafkaTopicName,
-                    getOffsetForPartition(topic, kafkaTopicName, subscriptionName, brokersClusterName, partition),
-                    partition
-            ));
-        }
-        return offsets;
+  }
+
+  private PartitionOffsets getOffsetsForKafkaTopic(
+      TopicName topic,
+      KafkaTopicName kafkaTopicName,
+      String subscriptionName,
+      String brokersClusterName) {
+    String offsetsPath =
+        paths.offsetsPath(topic, subscriptionName, kafkaTopicName, brokersClusterName);
+
+    PartitionOffsets offsets = new PartitionOffsets();
+    for (String partitionAsString : getZookeeperChildrenForPath(offsetsPath)) {
+      Integer partition = Integer.valueOf(partitionAsString);
+      offsets.add(
+          new PartitionOffset(
+              kafkaTopicName,
+              getOffsetForPartition(
+                  topic, kafkaTopicName, subscriptionName, brokersClusterName, partition),
+              partition));
     }
-
-    private List getZookeeperChildrenForPath(String path) {
-        try {
-            return zookeeper.getChildren().forPath(path);
-        } catch (Exception e) {
-            throw new InternalProcessingException(e);
-        }
+    return offsets;
+  }
+
+  private List getZookeeperChildrenForPath(String path) {
+    try {
+      return zookeeper.getChildren().forPath(path);
+    } catch (Exception e) {
+      throw new InternalProcessingException(e);
     }
-
-    private Long getOffsetForPartition(TopicName topic,
-                                       KafkaTopicName kafkaTopicName,
-                                       String subscriptionName,
-                                       String brokersClusterName,
-                                       int partitionId) {
-        try {
-            String offsetPath = paths.offsetPath(topic,
-                    subscriptionName,
-                    kafkaTopicName,
-                    brokersClusterName,
-                    partitionId);
-            return Long.valueOf(new String(zookeeper.getData().forPath(offsetPath), StandardCharsets.UTF_8));
-        } catch (Exception e) {
-            throw new InternalProcessingException(e);
-        }
+  }
+
+  private Long getOffsetForPartition(
+      TopicName topic,
+      KafkaTopicName kafkaTopicName,
+      String subscriptionName,
+      String brokersClusterName,
+      int partitionId) {
+    try {
+      String offsetPath =
+          paths.offsetPath(
+              topic, subscriptionName, kafkaTopicName, brokersClusterName, partitionId);
+      return Long.valueOf(
+          new String(zookeeper.getData().forPath(offsetPath), StandardCharsets.UTF_8));
+    } catch (Exception e) {
+      throw new InternalProcessingException(e);
     }
-
+  }
 }
diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperSubscriptionRepository.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperSubscriptionRepository.java
index 104fabb23d..59221b04e6 100644
--- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperSubscriptionRepository.java
+++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperSubscriptionRepository.java
@@ -1,6 +1,10 @@
 package pl.allegro.tech.hermes.infrastructure.zookeeper;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.Collection;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.zookeeper.KeeperException;
 import org.slf4j.Logger;
@@ -14,142 +18,146 @@
 import pl.allegro.tech.hermes.domain.subscription.SubscriptionRepository;
 import pl.allegro.tech.hermes.domain.topic.TopicRepository;
 
-import java.util.Collection;
-import java.util.List;
-import java.util.Optional;
-import java.util.stream.Collectors;
-
-public class ZookeeperSubscriptionRepository extends ZookeeperBasedRepository implements SubscriptionRepository {
-
-    private static final Logger logger = LoggerFactory.getLogger(ZookeeperSubscriptionRepository.class);
-
-    private final TopicRepository topicRepository;
-
-    public ZookeeperSubscriptionRepository(CuratorFramework zookeeper,
-                                           ObjectMapper mapper,
-                                           ZookeeperPaths paths,
-                                           TopicRepository topicRepository) {
-        super(zookeeper, mapper, paths);
-        this.topicRepository = topicRepository;
-    }
-
-    @Override
-    public boolean subscriptionExists(TopicName topicName, String subscriptionName) {
-        return pathExists(paths.subscriptionPath(topicName, subscriptionName));
-    }
-
-    @Override
-    public void ensureSubscriptionExists(TopicName topicName, String subscriptionName) {
-        if (!subscriptionExists(topicName, subscriptionName)) {
-            throw new SubscriptionNotExistsException(topicName, subscriptionName);
-        }
-    }
-
-    @Override
-    public void createSubscription(Subscription subscription) {
-        topicRepository.ensureTopicExists(subscription.getTopicName());
-
-        String subscriptionPath = paths.subscriptionPath(subscription);
-        logger.info("Creating subscription {}", subscription.getQualifiedName());
-
-        try {
-            create(subscriptionPath, subscription);
-        } catch (KeeperException.NodeExistsException ex) {
-            throw new SubscriptionAlreadyExistsException(subscription, ex);
-        } catch (Exception ex) {
-            throw new InternalProcessingException(ex);
-        }
-    }
-
-    @Override
-    public void removeSubscription(TopicName topicName, String subscriptionName) {
-        ensureSubscriptionExists(topicName, subscriptionName);
-        logger.info("Removing subscription {}", new SubscriptionName(subscriptionName, topicName).getQualifiedName());
-
-        try {
-            remove(paths.subscriptionPath(topicName, subscriptionName));
-        } catch (Exception e) {
-            throw new InternalProcessingException(e);
-        }
-    }
+public class ZookeeperSubscriptionRepository extends ZookeeperBasedRepository
+    implements SubscriptionRepository {
 
-    @Override
-    public void updateSubscription(Subscription modifiedSubscription) {
-        ensureSubscriptionExists(modifiedSubscription.getTopicName(), modifiedSubscription.getName());
-        logger.info("Updating subscription {}", modifiedSubscription.getQualifiedName());
-        try {
-            overwrite(paths.subscriptionPath(modifiedSubscription), modifiedSubscription);
-        } catch (Exception e) {
-            throw new InternalProcessingException(e);
-        }
-    }
+  private static final Logger logger =
+      LoggerFactory.getLogger(ZookeeperSubscriptionRepository.class);
 
-    @Override
-    public void updateSubscriptionState(TopicName topicName, String subscriptionName, Subscription.State state) {
-        ensureSubscriptionExists(topicName, subscriptionName);
+  private final TopicRepository topicRepository;
 
-        logger.info("Changing subscription {} state to {}",
-                new SubscriptionName(subscriptionName, topicName).getQualifiedName(), state.toString());
+  public ZookeeperSubscriptionRepository(
+      CuratorFramework zookeeper,
+      ObjectMapper mapper,
+      ZookeeperPaths paths,
+      TopicRepository topicRepository) {
+    super(zookeeper, mapper, paths);
+    this.topicRepository = topicRepository;
+  }
 
-        Subscription modifiedSubscription = getSubscriptionDetails(topicName, subscriptionName);
-        if (!modifiedSubscription.getState().equals(state)) {
-            modifiedSubscription.setState(state);
-            updateSubscription(modifiedSubscription);
-        }
-    }
+  @Override
+  public boolean subscriptionExists(TopicName topicName, String subscriptionName) {
+    return pathExists(paths.subscriptionPath(topicName, subscriptionName));
+  }
 
-    @Override
-    public Subscription getSubscriptionDetails(TopicName topicName, String subscriptionName) {
-        ensureSubscriptionExists(topicName, subscriptionName);
-        return readWithStatFrom(
-                paths.subscriptionPath(topicName, subscriptionName),
-                Subscription.class,
-                (sub, stat) -> {
-                    sub.setCreatedAt(stat.getCtime());
-                    sub.setModifiedAt(stat.getMtime());
-                },
-                false
-        ).get();
+  @Override
+  public void ensureSubscriptionExists(TopicName topicName, String subscriptionName) {
+    if (!subscriptionExists(topicName, subscriptionName)) {
+      throw new SubscriptionNotExistsException(topicName, subscriptionName);
     }
+  }
 
-    private Optional getSubscriptionDetails(TopicName topicName, String subscriptionName, boolean quiet) {
-        ensureSubscriptionExists(topicName, subscriptionName);
-        return readFrom(paths.subscriptionPath(topicName, subscriptionName), Subscription.class, quiet);
-    }
+  @Override
+  public void createSubscription(Subscription subscription) {
+    topicRepository.ensureTopicExists(subscription.getTopicName());
 
-    @Override
-    public Subscription getSubscriptionDetails(SubscriptionName name) {
-        return getSubscriptionDetails(name.getTopicName(), name.getName());
-    }
+    String subscriptionPath = paths.subscriptionPath(subscription);
+    logger.info("Creating subscription {}", subscription.getQualifiedName());
 
-    @Override
-    public List getSubscriptionDetails(Collection subscriptionNames) {
-        return subscriptionNames.stream()
-                .map(n -> getSubscriptionDetails(n.getTopicName(), n.getName()))
-                .collect(Collectors.toList());
+    try {
+      create(subscriptionPath, subscription);
+    } catch (KeeperException.NodeExistsException ex) {
+      throw new SubscriptionAlreadyExistsException(subscription, ex);
+    } catch (Exception ex) {
+      throw new InternalProcessingException(ex);
     }
-
-    @Override
-    public List listSubscriptionNames(TopicName topicName) {
-        topicRepository.ensureTopicExists(topicName);
-
-        return childrenOf(paths.subscriptionsPath(topicName));
+  }
+
+  @Override
+  public void removeSubscription(TopicName topicName, String subscriptionName) {
+    ensureSubscriptionExists(topicName, subscriptionName);
+    logger.info(
+        "Removing subscription {}",
+        new SubscriptionName(subscriptionName, topicName).getQualifiedName());
+
+    try {
+      remove(paths.subscriptionPath(topicName, subscriptionName));
+    } catch (Exception e) {
+      throw new InternalProcessingException(e);
     }
-
-    @Override
-    public List listSubscriptions(TopicName topicName) {
-        return listSubscriptionNames(topicName).stream()
-                .map(subscription -> getSubscriptionDetails(topicName, subscription, true))
-                .filter(Optional::isPresent)
-                .map(Optional::get)
-                .collect(Collectors.toList());
+  }
+
+  @Override
+  public void updateSubscription(Subscription modifiedSubscription) {
+    ensureSubscriptionExists(modifiedSubscription.getTopicName(), modifiedSubscription.getName());
+    logger.info("Updating subscription {}", modifiedSubscription.getQualifiedName());
+    try {
+      overwrite(paths.subscriptionPath(modifiedSubscription), modifiedSubscription);
+    } catch (Exception e) {
+      throw new InternalProcessingException(e);
     }
-
-    @Override
-    public List listAllSubscriptions() {
-        return topicRepository.listAllTopics().stream()
-                .map(topic -> listSubscriptions(topic.getName()))
-                .flatMap(Collection::stream)
-                .collect(Collectors.toList());
+  }
+
+  @Override
+  public void updateSubscriptionState(
+      TopicName topicName, String subscriptionName, Subscription.State state) {
+    ensureSubscriptionExists(topicName, subscriptionName);
+
+    logger.info(
+        "Changing subscription {} state to {}",
+        new SubscriptionName(subscriptionName, topicName).getQualifiedName(),
+        state.toString());
+
+    Subscription modifiedSubscription = getSubscriptionDetails(topicName, subscriptionName);
+    if (!modifiedSubscription.getState().equals(state)) {
+      modifiedSubscription.setState(state);
+      updateSubscription(modifiedSubscription);
     }
+  }
+
+  @Override
+  public Subscription getSubscriptionDetails(TopicName topicName, String subscriptionName) {
+    ensureSubscriptionExists(topicName, subscriptionName);
+    return readWithStatFrom(
+            paths.subscriptionPath(topicName, subscriptionName),
+            Subscription.class,
+            (sub, stat) -> {
+              sub.setCreatedAt(stat.getCtime());
+              sub.setModifiedAt(stat.getMtime());
+            },
+            false)
+        .get();
+  }
+
+  private Optional getSubscriptionDetails(
+      TopicName topicName, String subscriptionName, boolean quiet) {
+    ensureSubscriptionExists(topicName, subscriptionName);
+    return readFrom(paths.subscriptionPath(topicName, subscriptionName), Subscription.class, quiet);
+  }
+
+  @Override
+  public Subscription getSubscriptionDetails(SubscriptionName name) {
+    return getSubscriptionDetails(name.getTopicName(), name.getName());
+  }
+
+  @Override
+  public List getSubscriptionDetails(Collection subscriptionNames) {
+    return subscriptionNames.stream()
+        .map(n -> getSubscriptionDetails(n.getTopicName(), n.getName()))
+        .collect(Collectors.toList());
+  }
+
+  @Override
+  public List listSubscriptionNames(TopicName topicName) {
+    topicRepository.ensureTopicExists(topicName);
+
+    return childrenOf(paths.subscriptionsPath(topicName));
+  }
+
+  @Override
+  public List listSubscriptions(TopicName topicName) {
+    return listSubscriptionNames(topicName).stream()
+        .map(subscription -> getSubscriptionDetails(topicName, subscription, true))
+        .filter(Optional::isPresent)
+        .map(Optional::get)
+        .collect(Collectors.toList());
+  }
+
+  @Override
+  public List listAllSubscriptions() {
+    return topicRepository.listAllTopics().stream()
+        .map(topic -> listSubscriptions(topic.getName()))
+        .flatMap(Collection::stream)
+        .collect(Collectors.toList());
+  }
 }
diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperTopicRepository.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperTopicRepository.java
index 3642677d37..7e840b4aed 100644
--- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperTopicRepository.java
+++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperTopicRepository.java
@@ -1,6 +1,11 @@
 package pl.allegro.tech.hermes.infrastructure.zookeeper;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.zookeeper.KeeperException;
 import org.slf4j.Logger;
@@ -13,194 +18,194 @@
 import pl.allegro.tech.hermes.domain.topic.TopicNotExistsException;
 import pl.allegro.tech.hermes.domain.topic.TopicRepository;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Optional;
-import java.util.stream.Collectors;
-
 public class ZookeeperTopicRepository extends ZookeeperBasedRepository implements TopicRepository {
 
-    private static final Logger logger = LoggerFactory.getLogger(ZookeeperTopicRepository.class);
-
-    private final GroupRepository groupRepository;
-
-    public ZookeeperTopicRepository(CuratorFramework zookeeper,
-                                    ObjectMapper mapper,
-                                    ZookeeperPaths paths,
-                                    GroupRepository groupRepository) {
-        super(zookeeper, mapper, paths);
-        this.groupRepository = groupRepository;
-    }
-
+  private static final Logger logger = LoggerFactory.getLogger(ZookeeperTopicRepository.class);
 
-    @Override
-    public boolean topicExists(TopicName topicName) {
-        return pathExists(paths.topicPath(topicName));
-    }
+  private final GroupRepository groupRepository;
 
-    @Override
-    public void ensureTopicExists(TopicName topicName) {
-        if (!topicExists(topicName)) {
-            throw new TopicNotExistsException(topicName);
-        }
-    }
+  public ZookeeperTopicRepository(
+      CuratorFramework zookeeper,
+      ObjectMapper mapper,
+      ZookeeperPaths paths,
+      GroupRepository groupRepository) {
+    super(zookeeper, mapper, paths);
+    this.groupRepository = groupRepository;
+  }
 
-    @Override
-    public List listTopicNames(String groupName) {
-        groupRepository.ensureGroupExists(groupName);
+  @Override
+  public boolean topicExists(TopicName topicName) {
+    return pathExists(paths.topicPath(topicName));
+  }
 
-        return childrenOf(paths.topicsPath(groupName));
+  @Override
+  public void ensureTopicExists(TopicName topicName) {
+    if (!topicExists(topicName)) {
+      throw new TopicNotExistsException(topicName);
     }
-
-    @Override
-    public List listTopics(String groupName) {
-        return listTopicNames(groupName).stream()
-                .map(name -> getTopicDetails(new TopicName(groupName, name), true))
-                .filter(Optional::isPresent)
-                .map(Optional::get)
-                .collect(Collectors.toList());
+  }
+
+  @Override
+  public List listTopicNames(String groupName) {
+    groupRepository.ensureGroupExists(groupName);
+
+    return childrenOf(paths.topicsPath(groupName));
+  }
+
+  @Override
+  public List listTopics(String groupName) {
+    return listTopicNames(groupName).stream()
+        .map(name -> getTopicDetails(new TopicName(groupName, name), true))
+        .filter(Optional::isPresent)
+        .map(Optional::get)
+        .collect(Collectors.toList());
+  }
+
+  @Override
+  public void createTopic(Topic topic) {
+    groupRepository.ensureGroupExists(topic.getName().getGroupName());
+
+    String topicPath = paths.topicPath(topic.getName());
+    logger.info("Creating topic for path {}", topicPath);
+
+    try {
+      createInTransaction(topicPath, topic, paths.subscriptionsPath(topic.getName()));
+    } catch (KeeperException.NodeExistsException ex) {
+      throw new TopicAlreadyExistsException(topic.getName(), ex);
+    } catch (Exception ex) {
+      throw new InternalProcessingException(ex);
     }
-
-    @Override
-    public void createTopic(Topic topic) {
-        groupRepository.ensureGroupExists(topic.getName().getGroupName());
-
-        String topicPath = paths.topicPath(topic.getName());
-        logger.info("Creating topic for path {}", topicPath);
-
-        try {
-            createInTransaction(topicPath, topic, paths.subscriptionsPath(topic.getName()));
-        } catch (KeeperException.NodeExistsException ex) {
-            throw new TopicAlreadyExistsException(topic.getName(), ex);
-        } catch (Exception ex) {
-            throw new InternalProcessingException(ex);
-        }
-    }
-
-    /**
-     * To remove topic node, we must remove topic node and its children. The tree looks like this:
-     * 
    - *
  • - topic - *
  • ----- /subscriptions (required) - *
  • ----- /preview (optional) - *
  • ----- /metrics (optional) - *
  • --------------- /volume - *
  • --------------- /published - *
- * - *

One way to remove the whole tree for topic that would be to use deletingChildrenIfNeeded(): - * e.g. zookeeper.delete().deletingChildrenIfNeeded().forPath(topicPath). - * However, deletingChildrenIfNeeded is not atomic. It first tries to remove the node topic - * and upon receiving KeeperException.NotEmptyException it tries to remove children recursively - * and then retries the node removal. This means that there is a potentially large time gap between - * removal of topic/subscriptions node and topic node, especially when topic removal is being done - * in remote DC. - * - *

It turns out that PathChildrenCache used by HierarchicalCacheLevel in - * Consumers and Frontend listens for topics/subscriptions changes and recreates that node when deleted. - * If the recreation happens between the topic/subscriptions and topic node removal - * than the whole removal process must be repeated resulting in a lengthy loop that may even result in StackOverflowException. - * Example of that scenario would be - *

    - *
  1. DELETE topic - issued by management, fails with KeeperException.NotEmptyException - *
  2. DELETE topic/subscriptions - issued by management, succeeds - *
  3. CREATE topic/subscriptions - issued by frontend, succeeds - *
  4. DELETE topic - issued by management, fails with KeeperException.NotEmptyException - *
  5. [...] - *
- * - *

To solve this we must remove topic and topic/subscriptions atomically. However, we must also remove - * other topic children. Transaction API does not allow for optional deletes so we: - *

    - *
  1. find all children paths - *
  2. delete all children in one transaction - *
- */ - @Override - public void removeTopic(TopicName topicName) { - ensureTopicExists(topicName); - logger.info("Removing topic: " + topicName); - - List pathsForRemoval = new ArrayList<>(); - String topicMetricsPath = paths.topicMetricsPath(topicName); - if (pathExists(topicMetricsPath)) { - pathsForRemoval.addAll(childrenPathsOf(topicMetricsPath)); - pathsForRemoval.add(topicMetricsPath); - } - - String topicPreviewPath = paths.topicPreviewPath(topicName); - if (pathExists(topicPreviewPath)) { - pathsForRemoval.add(topicPreviewPath); - } - - pathsForRemoval.add(paths.subscriptionsPath(topicName)); - pathsForRemoval.add(paths.topicPath(topicName)); - - try { - deleteInTransaction(pathsForRemoval); - } catch (Exception e) { - throw new InternalProcessingException(e); - } + } + + /** + * To remove topic node, we must remove topic node and its children. The tree looks like this: + * + *
    + *
  • - topic + *
  • ----- /subscriptions (required) + *
  • ----- /preview (optional) + *
  • ----- /metrics (optional) + *
  • --------------- /volume + *
  • --------------- /published + *
+ * + *

One way to remove the whole tree for topic that would be to use + * deletingChildrenIfNeeded(): e.g. + * zookeeper.delete().deletingChildrenIfNeeded().forPath(topicPath). However, + * deletingChildrenIfNeeded is not atomic. It first tries to remove the node topic + * and upon receiving KeeperException.NotEmptyException it tries to remove + * children recursively and then retries the node removal. This means that there is a potentially + * large time gap between removal of topic/subscriptions node and topic + * node, especially when topic removal is being done in remote DC. + * + *

It turns out that PathChildrenCache used by HierarchicalCacheLevel + * in Consumers and Frontend listens for topics/subscriptions changes and recreates + * that node when deleted. If the recreation happens between the topic/subscriptions + * and topic node removal than the whole removal process must be repeated resulting + * in a lengthy loop that may even result in StackOverflowException. Example of that + * scenario would be + * + *

    + *
  1. DELETE topic - issued by management, fails with + * KeeperException.NotEmptyException + *
  2. DELETE topic/subscriptions - issued by management, succeeds + *
  3. CREATE topic/subscriptions - issued by frontend, succeeds + *
  4. DELETE topic - issued by management, fails with + * KeeperException.NotEmptyException + *
  5. [...] + *
+ * + *

To solve this we must remove topic and topic/subscriptions + * atomically. However, we must also remove other topic children. Transaction API + * does not allow for optional deletes so we: + * + *

    + *
  1. find all children paths + *
  2. delete all children in one transaction + *
+ */ + @Override + public void removeTopic(TopicName topicName) { + ensureTopicExists(topicName); + logger.info("Removing topic: " + topicName); + + List pathsForRemoval = new ArrayList<>(); + String topicMetricsPath = paths.topicMetricsPath(topicName); + if (pathExists(topicMetricsPath)) { + pathsForRemoval.addAll(childrenPathsOf(topicMetricsPath)); + pathsForRemoval.add(topicMetricsPath); } - @Override - public void updateTopic(Topic topic) { - ensureTopicExists(topic.getName()); - - logger.info("Updating topic: " + topic.getName()); - try { - overwrite(paths.topicPath(topic.getName()), topic); - } catch (Exception e) { - throw new InternalProcessingException(e); - } + String topicPreviewPath = paths.topicPreviewPath(topicName); + if (pathExists(topicPreviewPath)) { + pathsForRemoval.add(topicPreviewPath); } - @Override - public void touchTopic(TopicName topicName) { - ensureTopicExists(topicName); + pathsForRemoval.add(paths.subscriptionsPath(topicName)); + pathsForRemoval.add(paths.topicPath(topicName)); - logger.info("Touching topic: " + topicName.qualifiedName()); - try { - touch(paths.topicPath(topicName)); - } catch (Exception ex) { - throw new InternalProcessingException(ex); - } + try { + deleteInTransaction(pathsForRemoval); + } catch (Exception e) { + throw new InternalProcessingException(e); } + } - @Override - public Topic getTopicDetails(TopicName topicName) { - return getTopicDetails(topicName, false).get(); - } + @Override + public void updateTopic(Topic topic) { + ensureTopicExists(topic.getName()); - private Optional getTopicDetails(TopicName topicName, boolean quiet) { - ensureTopicExists(topicName); - return readWithStatFrom( - paths.topicPath(topicName), - Topic.class, - (topic, stat) -> { - topic.setCreatedAt(stat.getCtime()); - topic.setModifiedAt(stat.getMtime()); - }, - quiet - ); + logger.info("Updating topic: " + topic.getName()); + try { + overwrite(paths.topicPath(topic.getName()), topic); + } catch (Exception e) { + throw new InternalProcessingException(e); } + } - @Override - public List getTopicsDetails(Collection topicNames) { - return topicNames.stream() - .map(topicName -> getTopicDetails(topicName, true)) - .filter(Optional::isPresent) - .map(Optional::get) - .collect(Collectors.toList()); - } + @Override + public void touchTopic(TopicName topicName) { + ensureTopicExists(topicName); - @Override - public List listAllTopics() { - return groupRepository.listGroupNames() - .stream() - .map(this::listTopics) - .flatMap(List::stream) - .collect(Collectors.toList()); + logger.info("Touching topic: " + topicName.qualifiedName()); + try { + touch(paths.topicPath(topicName)); + } catch (Exception ex) { + throw new InternalProcessingException(ex); } + } + + @Override + public Topic getTopicDetails(TopicName topicName) { + return getTopicDetails(topicName, false).get(); + } + + private Optional getTopicDetails(TopicName topicName, boolean quiet) { + ensureTopicExists(topicName); + return readWithStatFrom( + paths.topicPath(topicName), + Topic.class, + (topic, stat) -> { + topic.setCreatedAt(stat.getCtime()); + topic.setModifiedAt(stat.getMtime()); + }, + quiet); + } + + @Override + public List getTopicsDetails(Collection topicNames) { + return topicNames.stream() + .map(topicName -> getTopicDetails(topicName, true)) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } + + @Override + public List listAllTopics() { + return groupRepository.listGroupNames().stream() + .map(this::listTopics) + .flatMap(List::stream) + .collect(Collectors.toList()); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperWorkloadConstraintsCache.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperWorkloadConstraintsCache.java index b3d7c6ce33..6f6a821981 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperWorkloadConstraintsCache.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperWorkloadConstraintsCache.java @@ -1,6 +1,9 @@ package pl.allegro.tech.hermes.infrastructure.zookeeper; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCache; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; @@ -12,83 +15,87 @@ import pl.allegro.tech.hermes.api.TopicName; import pl.allegro.tech.hermes.domain.workload.constraints.ConsumersWorkloadConstraints; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; +class ZookeeperWorkloadConstraintsCache extends PathChildrenCache + implements PathChildrenCacheListener { -class ZookeeperWorkloadConstraintsCache extends PathChildrenCache implements PathChildrenCacheListener { + private static final Logger logger = + LoggerFactory.getLogger(ZookeeperWorkloadConstraintsCache.class); - private static final Logger logger = LoggerFactory.getLogger(ZookeeperWorkloadConstraintsCache.class); + private final Map topicConstraintsCache = new ConcurrentHashMap<>(); + private final Map subscriptionConstraintsCache = + new ConcurrentHashMap<>(); + private final ObjectMapper objectMapper; + private final ZookeeperPaths paths; - private final Map topicConstraintsCache = new ConcurrentHashMap<>(); - private final Map subscriptionConstraintsCache = new ConcurrentHashMap<>(); - private final ObjectMapper objectMapper; - private final ZookeeperPaths paths; + ZookeeperWorkloadConstraintsCache( + CuratorFramework curatorFramework, ObjectMapper objectMapper, ZookeeperPaths paths) { + super(curatorFramework, paths.consumersWorkloadConstraintsPath(), true); + this.objectMapper = objectMapper; + this.paths = paths; + getListenable().addListener(this); + } - ZookeeperWorkloadConstraintsCache(CuratorFramework curatorFramework, ObjectMapper objectMapper, ZookeeperPaths paths) { - super(curatorFramework, paths.consumersWorkloadConstraintsPath(), true); - this.objectMapper = objectMapper; - this.paths = paths; - getListenable().addListener(this); - } + ConsumersWorkloadConstraints getConsumersWorkloadConstraints() { + return new ConsumersWorkloadConstraints(topicConstraintsCache, subscriptionConstraintsCache); + } - ConsumersWorkloadConstraints getConsumersWorkloadConstraints() { - return new ConsumersWorkloadConstraints(topicConstraintsCache, subscriptionConstraintsCache); + @Override + public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) { + switch (event.getType()) { + case CHILD_ADDED: + updateCache(event.getData().getPath(), event.getData().getData()); + break; + case CHILD_REMOVED: + removeFromCache(event.getData().getPath()); + break; + case CHILD_UPDATED: + updateCache(event.getData().getPath(), event.getData().getData()); + break; + default: + break; } + } - @Override - public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) { - switch (event.getType()) { - case CHILD_ADDED: - updateCache(event.getData().getPath(), event.getData().getData()); - break; - case CHILD_REMOVED: - removeFromCache(event.getData().getPath()); - break; - case CHILD_UPDATED: - updateCache(event.getData().getPath(), event.getData().getData()); - break; - default: - break; - } + private void updateCache(String path, byte[] bytes) { + Optional constraints = bytesToConstraints(bytes, path); + if (!constraints.isPresent()) { + return; } - - private void updateCache(String path, byte[] bytes) { - Optional constraints = bytesToConstraints(bytes, path); - if (!constraints.isPresent()) { - return; - } - if (isSubscription(path)) { - subscriptionConstraintsCache.put( - SubscriptionName.fromString(paths.extractChildNode(path, paths.consumersWorkloadConstraintsPath())), - constraints.get()); - } else { - topicConstraintsCache.put( - TopicName.fromQualifiedName(paths.extractChildNode(path, paths.consumersWorkloadConstraintsPath())), - constraints.get()); - } + if (isSubscription(path)) { + subscriptionConstraintsCache.put( + SubscriptionName.fromString( + paths.extractChildNode(path, paths.consumersWorkloadConstraintsPath())), + constraints.get()); + } else { + topicConstraintsCache.put( + TopicName.fromQualifiedName( + paths.extractChildNode(path, paths.consumersWorkloadConstraintsPath())), + constraints.get()); } + } - private Optional bytesToConstraints(byte[] bytes, String path) { - try { - return Optional.ofNullable(objectMapper.readValue(bytes, Constraints.class)); - } catch (Exception e) { - logger.error("Cannot read data from node: {}", path, e); - return Optional.empty(); - } + private Optional bytesToConstraints(byte[] bytes, String path) { + try { + return Optional.ofNullable(objectMapper.readValue(bytes, Constraints.class)); + } catch (Exception e) { + logger.error("Cannot read data from node: {}", path, e); + return Optional.empty(); } + } - private void removeFromCache(String path) { - if (isSubscription(path)) { - subscriptionConstraintsCache.remove( - SubscriptionName.fromString(paths.extractChildNode(path, paths.consumersWorkloadConstraintsPath()))); - } else { - topicConstraintsCache.remove( - TopicName.fromQualifiedName(paths.extractChildNode(path, paths.consumersWorkloadConstraintsPath()))); - } + private void removeFromCache(String path) { + if (isSubscription(path)) { + subscriptionConstraintsCache.remove( + SubscriptionName.fromString( + paths.extractChildNode(path, paths.consumersWorkloadConstraintsPath()))); + } else { + topicConstraintsCache.remove( + TopicName.fromQualifiedName( + paths.extractChildNode(path, paths.consumersWorkloadConstraintsPath()))); } + } - private boolean isSubscription(String path) { - return path.contains("$"); - } + private boolean isSubscription(String path) { + return path.contains("$"); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperWorkloadConstraintsRepository.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperWorkloadConstraintsRepository.java index c1a0ad6f5f..6f1ba031e7 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperWorkloadConstraintsRepository.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperWorkloadConstraintsRepository.java @@ -16,123 +16,130 @@ import pl.allegro.tech.hermes.domain.workload.constraints.TopicConstraintsDoNotExistException; import pl.allegro.tech.hermes.domain.workload.constraints.WorkloadConstraintsRepository; -public class ZookeeperWorkloadConstraintsRepository extends ZookeeperBasedRepository implements WorkloadConstraintsRepository { - - private static final Logger logger = LoggerFactory.getLogger(ZookeeperWorkloadConstraintsRepository.class); - - private final ZookeeperWorkloadConstraintsCache pathChildrenCache; - - public ZookeeperWorkloadConstraintsRepository(CuratorFramework curator, ObjectMapper mapper, ZookeeperPaths paths) { - this(curator, mapper, paths, new ZookeeperWorkloadConstraintsCache(curator, mapper, paths)); +public class ZookeeperWorkloadConstraintsRepository extends ZookeeperBasedRepository + implements WorkloadConstraintsRepository { + + private static final Logger logger = + LoggerFactory.getLogger(ZookeeperWorkloadConstraintsRepository.class); + + private final ZookeeperWorkloadConstraintsCache pathChildrenCache; + + public ZookeeperWorkloadConstraintsRepository( + CuratorFramework curator, ObjectMapper mapper, ZookeeperPaths paths) { + this(curator, mapper, paths, new ZookeeperWorkloadConstraintsCache(curator, mapper, paths)); + } + + ZookeeperWorkloadConstraintsRepository( + CuratorFramework curator, + ObjectMapper mapper, + ZookeeperPaths paths, + ZookeeperWorkloadConstraintsCache pathChildrenCache) { + super(curator, mapper, paths); + this.pathChildrenCache = pathChildrenCache; + try { + this.pathChildrenCache.start(); + } catch (Exception e) { + throw new InternalProcessingException("ZookeeperWorkloadConstraintsCache cannot start.", e); } - - ZookeeperWorkloadConstraintsRepository(CuratorFramework curator, ObjectMapper mapper, ZookeeperPaths paths, - ZookeeperWorkloadConstraintsCache pathChildrenCache) { - super(curator, mapper, paths); - this.pathChildrenCache = pathChildrenCache; - try { - this.pathChildrenCache.start(); - } catch (Exception e) { - throw new InternalProcessingException("ZookeeperWorkloadConstraintsCache cannot start.", e); - } + } + + @Override + public ConsumersWorkloadConstraints getConsumersWorkloadConstraints() { + return pathChildrenCache.getConsumersWorkloadConstraints(); + } + + @Override + public void createConstraints(TopicName topicName, Constraints constraints) { + logger.info("Creating constraints for topic {}", topicName.qualifiedName()); + String path = paths.consumersWorkloadConstraintsPath(topicName.qualifiedName()); + try { + createConstraints(path, constraints); + } catch (KeeperException.NodeExistsException e) { + throw new TopicConstraintsAlreadyExistException(topicName, e); } - - @Override - public ConsumersWorkloadConstraints getConsumersWorkloadConstraints() { - return pathChildrenCache.getConsumersWorkloadConstraints(); + } + + @Override + public void createConstraints(SubscriptionName subscriptionName, Constraints constraints) { + logger.info("Creating constraints for subscription {}", subscriptionName.getQualifiedName()); + String path = paths.consumersWorkloadConstraintsPath(subscriptionName.getQualifiedName()); + try { + createConstraints(path, constraints); + } catch (KeeperException.NodeExistsException e) { + throw new SubscriptionConstraintsAlreadyExistException(subscriptionName, e); } - - @Override - public void createConstraints(TopicName topicName, Constraints constraints) { - logger.info("Creating constraints for topic {}", topicName.qualifiedName()); - String path = paths.consumersWorkloadConstraintsPath(topicName.qualifiedName()); - try { - createConstraints(path, constraints); - } catch (KeeperException.NodeExistsException e) { - throw new TopicConstraintsAlreadyExistException(topicName, e); - } + } + + private void createConstraints(String path, Constraints constraints) + throws KeeperException.NodeExistsException { + try { + createRecursively(path, constraints); + } catch (KeeperException.NodeExistsException e) { + throw e; + } catch (Exception e) { + throw new InternalProcessingException(e); } - - @Override - public void createConstraints(SubscriptionName subscriptionName, Constraints constraints) { - logger.info("Creating constraints for subscription {}", subscriptionName.getQualifiedName()); - String path = paths.consumersWorkloadConstraintsPath(subscriptionName.getQualifiedName()); - try { - createConstraints(path, constraints); - } catch (KeeperException.NodeExistsException e) { - throw new SubscriptionConstraintsAlreadyExistException(subscriptionName, e); - } + } + + @Override + public void updateConstraints(TopicName topicName, Constraints constraints) { + logger.info("Updating constraints for topic {}", topicName.qualifiedName()); + String path = paths.consumersWorkloadConstraintsPath(topicName.qualifiedName()); + try { + overwrite(path, constraints); + } catch (KeeperException.NoNodeException e) { + throw new TopicConstraintsDoNotExistException(topicName, e); + } catch (Exception e) { + throw new InternalProcessingException(e); } - - private void createConstraints(String path, Constraints constraints) throws KeeperException.NodeExistsException { - try { - createRecursively(path, constraints); - } catch (KeeperException.NodeExistsException e) { - throw e; - } catch (Exception e) { - throw new InternalProcessingException(e); - } + } + + @Override + public void updateConstraints(SubscriptionName subscriptionName, Constraints constraints) { + logger.info("Updating constraints for subscription {}", subscriptionName.getQualifiedName()); + String path = paths.consumersWorkloadConstraintsPath(subscriptionName.getQualifiedName()); + try { + overwrite(path, constraints); + } catch (KeeperException.NoNodeException e) { + throw new SubscriptionConstraintsDoNotExistException(subscriptionName, e); + } catch (Exception e) { + throw new InternalProcessingException(e); } - - @Override - public void updateConstraints(TopicName topicName, Constraints constraints) { - logger.info("Updating constraints for topic {}", topicName.qualifiedName()); - String path = paths.consumersWorkloadConstraintsPath(topicName.qualifiedName()); - try { - overwrite(path, constraints); - } catch (KeeperException.NoNodeException e) { - throw new TopicConstraintsDoNotExistException(topicName, e); - } catch (Exception e) { - throw new InternalProcessingException(e); - } - } - - @Override - public void updateConstraints(SubscriptionName subscriptionName, Constraints constraints) { - logger.info("Updating constraints for subscription {}", subscriptionName.getQualifiedName()); - String path = paths.consumersWorkloadConstraintsPath(subscriptionName.getQualifiedName()); - try { - overwrite(path, constraints); - } catch (KeeperException.NoNodeException e) { - throw new SubscriptionConstraintsDoNotExistException(subscriptionName, e); - } catch (Exception e) { - throw new InternalProcessingException(e); - } - } - - @Override - public void deleteConstraints(TopicName topicName) { - logger.info("Deleting constraints for topic {}", topicName.qualifiedName()); - String path = paths.consumersWorkloadConstraintsPath(topicName.qualifiedName()); - deleteConstraints(path); - } - - @Override - public void deleteConstraints(SubscriptionName subscriptionName) { - logger.info("Deleting constraints for subscription {}", subscriptionName.getQualifiedName()); - String path = paths.consumersWorkloadConstraintsPath(subscriptionName.getQualifiedName()); - deleteConstraints(path); - } - - private void deleteConstraints(String path) { - try { - remove(path); - } catch (KeeperException.NoNodeException e) { - // ignore - it's ok - } catch (Exception e) { - throw new InternalProcessingException(e); - } - } - - @Override - public boolean constraintsExist(TopicName topicName) { - String path = paths.consumersWorkloadConstraintsPath(topicName.qualifiedName()); - return pathExists(path); - } - - @Override - public boolean constraintsExist(SubscriptionName subscriptionName) { - String path = paths.consumersWorkloadConstraintsPath(subscriptionName.getQualifiedName()); - return pathExists(path); + } + + @Override + public void deleteConstraints(TopicName topicName) { + logger.info("Deleting constraints for topic {}", topicName.qualifiedName()); + String path = paths.consumersWorkloadConstraintsPath(topicName.qualifiedName()); + deleteConstraints(path); + } + + @Override + public void deleteConstraints(SubscriptionName subscriptionName) { + logger.info("Deleting constraints for subscription {}", subscriptionName.getQualifiedName()); + String path = paths.consumersWorkloadConstraintsPath(subscriptionName.getQualifiedName()); + deleteConstraints(path); + } + + private void deleteConstraints(String path) { + try { + remove(path); + } catch (KeeperException.NoNodeException e) { + // ignore - it's ok + } catch (Exception e) { + throw new InternalProcessingException(e); } + } + + @Override + public boolean constraintsExist(TopicName topicName) { + String path = paths.consumersWorkloadConstraintsPath(topicName.qualifiedName()); + return pathExists(path); + } + + @Override + public boolean constraintsExist(SubscriptionName subscriptionName) { + String path = paths.consumersWorkloadConstraintsPath(subscriptionName.getQualifiedName()); + return pathExists(path); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/cache/CacheListeners.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/cache/CacheListeners.java index 69d0292b64..fc3c15bf33 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/cache/CacheListeners.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/cache/CacheListeners.java @@ -1,31 +1,33 @@ package pl.allegro.tech.hermes.infrastructure.zookeeper.cache; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.function.Consumer; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; class CacheListeners { - private static final Logger logger = LoggerFactory.getLogger(CacheListeners.class); + private static final Logger logger = LoggerFactory.getLogger(CacheListeners.class); - private final Queue> callbacks = new ConcurrentLinkedQueue<>(); + private final Queue> callbacks = new ConcurrentLinkedQueue<>(); - void addListener(Consumer callback) { - callbacks.add(callback); - } + void addListener(Consumer callback) { + callbacks.add(callback); + } - void call(PathChildrenCacheEvent event) { - for (Consumer callback : callbacks) { - try { - callback.accept(event); - } catch (Exception exception) { - logger.error("Failed to run callback action {} for event with data: {}", - callback.getClass().getSimpleName(), event.getData(), exception); - } - } + void call(PathChildrenCacheEvent event) { + for (Consumer callback : callbacks) { + try { + callback.accept(event); + } catch (Exception exception) { + logger.error( + "Failed to run callback action {} for event with data: {}", + callback.getClass().getSimpleName(), + event.getData(), + exception); + } } + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/cache/HierarchicalCache.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/cache/HierarchicalCache.java index 23a7952f79..4f79d10549 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/cache/HierarchicalCache.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/cache/HierarchicalCache.java @@ -1,102 +1,109 @@ package pl.allegro.tech.hermes.infrastructure.zookeeper.cache; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; -import org.apache.zookeeper.KeeperException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import pl.allegro.tech.hermes.common.exception.InternalProcessingException; - import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.function.BiFunction; import java.util.function.Consumer; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import pl.allegro.tech.hermes.common.exception.InternalProcessingException; public class HierarchicalCache { - private static final Logger logger = LoggerFactory.getLogger(HierarchicalCache.class); + private static final Logger logger = LoggerFactory.getLogger(HierarchicalCache.class); - private final CuratorFramework curatorFramework; + private final CuratorFramework curatorFramework; - private final ExecutorService executorService; + private final ExecutorService executorService; - private final String basePath; - private final boolean removeNodesWithNoData; + private final String basePath; + private final boolean removeNodesWithNoData; - private final List levelPrefixes = new ArrayList<>(); + private final List levelPrefixes = new ArrayList<>(); - private final int maxDepth; + private final int maxDepth; - private final List levelCallbacks = new ArrayList<>(); + private final List levelCallbacks = new ArrayList<>(); - private HierarchicalCacheLevel rootCache; + private HierarchicalCacheLevel rootCache; - public HierarchicalCache(CuratorFramework curatorFramework, - ExecutorService executorService, - String basePath, - int maxDepth, - List levelPrefixes, - boolean removeNodesWithNoData) { - this.curatorFramework = curatorFramework; - this.executorService = executorService; - this.basePath = basePath; - this.removeNodesWithNoData = removeNodesWithNoData; - this.levelPrefixes.addAll(levelPrefixes); - this.maxDepth = maxDepth; + public HierarchicalCache( + CuratorFramework curatorFramework, + ExecutorService executorService, + String basePath, + int maxDepth, + List levelPrefixes, + boolean removeNodesWithNoData) { + this.curatorFramework = curatorFramework; + this.executorService = executorService; + this.basePath = basePath; + this.removeNodesWithNoData = removeNodesWithNoData; + this.levelPrefixes.addAll(levelPrefixes); + this.maxDepth = maxDepth; - for (int i = 0; i < maxDepth; ++i) { - levelCallbacks.add(new CacheListeners()); - } - } - - public void start() throws Exception { - ensureBasePath(); - rootCache = createLevelCache(0, basePath); - } - - public void stop() throws Exception { - rootCache.stop(); - } - - public void registerCallback(int depth, Consumer callback) { - levelCallbacks.get(depth).addListener(callback); - } - - private HierarchicalCacheLevel createLevelCache(int depth, String path) { - BiFunction function = depth + 1 < maxDepth ? this::createLevelCache : null; - HierarchicalCacheLevel levelCache = new HierarchicalCacheLevel(curatorFramework, - executorService, - path(depth, path), - depth, - levelCallbacks.get(depth), - Optional.ofNullable(function), - removeNodesWithNoData); - try { - logger.debug("Starting hierarchical cache level for path {} and depth {}", path, depth); - levelCache.start(); - } catch (Exception e) { - logger.error("Failed to start hierarchical cache level for path {} and depth {}", path(depth, path), depth, e); - } - return levelCache; + for (int i = 0; i < maxDepth; ++i) { + levelCallbacks.add(new CacheListeners()); } - - private String path(int depth, String basePath) { - return basePath + (levelPrefixes.size() > depth ? "/" + levelPrefixes.get(depth) : ""); + } + + public void start() throws Exception { + ensureBasePath(); + rootCache = createLevelCache(0, basePath); + } + + public void stop() throws Exception { + rootCache.stop(); + } + + public void registerCallback(int depth, Consumer callback) { + levelCallbacks.get(depth).addListener(callback); + } + + private HierarchicalCacheLevel createLevelCache(int depth, String path) { + BiFunction function = + depth + 1 < maxDepth ? this::createLevelCache : null; + HierarchicalCacheLevel levelCache = + new HierarchicalCacheLevel( + curatorFramework, + executorService, + path(depth, path), + depth, + levelCallbacks.get(depth), + Optional.ofNullable(function), + removeNodesWithNoData); + try { + logger.debug("Starting hierarchical cache level for path {} and depth {}", path, depth); + levelCache.start(); + } catch (Exception e) { + logger.error( + "Failed to start hierarchical cache level for path {} and depth {}", + path(depth, path), + depth, + e); } - - private void ensureBasePath() { - try { - try { - if (curatorFramework.checkExists().forPath(basePath) == null) { - curatorFramework.create().creatingParentsIfNeeded().forPath(basePath); - } - } catch (KeeperException.NodeExistsException e) { - // ignore - } - } catch (Exception e) { - throw new InternalProcessingException(e); + return levelCache; + } + + private String path(int depth, String basePath) { + return basePath + (levelPrefixes.size() > depth ? "/" + levelPrefixes.get(depth) : ""); + } + + private void ensureBasePath() { + try { + try { + if (curatorFramework.checkExists().forPath(basePath) == null) { + curatorFramework.create().creatingParentsIfNeeded().forPath(basePath); } + } catch (KeeperException.NodeExistsException e) { + // ignore + } + } catch (Exception e) { + throw new InternalProcessingException(e); } + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/cache/HierarchicalCacheLevel.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/cache/HierarchicalCacheLevel.java index 2e1a9795b3..51377f9e18 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/cache/HierarchicalCacheLevel.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/cache/HierarchicalCacheLevel.java @@ -1,14 +1,5 @@ package pl.allegro.tech.hermes.infrastructure.zookeeper.cache; -import org.apache.commons.lang3.ArrayUtils; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.recipes.cache.PathChildrenCache; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; -import org.apache.zookeeper.KeeperException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.util.HashMap; import java.util.List; @@ -20,156 +11,183 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.BiFunction; import java.util.stream.Collectors; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.recipes.cache.PathChildrenCache; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; class HierarchicalCacheLevel extends PathChildrenCache implements PathChildrenCacheListener { - private static Logger logger = LoggerFactory.getLogger(HierarchicalCacheLevel.class); - - private final ReadWriteLock subcacheLock = new ReentrantReadWriteLock(true); - - private final CacheListeners consumer; - - private final CuratorFramework curatorClient; - private final int currentDepth; - - private final Optional> nextLevelFactory; - private final boolean removeNodesWithNoData; - - private final Map subcacheMap = new HashMap<>(); - - HierarchicalCacheLevel(CuratorFramework curatorClient, - ExecutorService executorService, - String path, - int depth, - CacheListeners eventConsumer, - Optional> nextLevelFactory, - boolean removeNodesWithNoData) { - super(curatorClient, path, true, false, executorService); - this.curatorClient = curatorClient; - this.currentDepth = depth; - this.consumer = eventConsumer; - this.nextLevelFactory = nextLevelFactory; - this.removeNodesWithNoData = removeNodesWithNoData; - getListenable().addListener(this); + private static Logger logger = LoggerFactory.getLogger(HierarchicalCacheLevel.class); + + private final ReadWriteLock subcacheLock = new ReentrantReadWriteLock(true); + + private final CacheListeners consumer; + + private final CuratorFramework curatorClient; + private final int currentDepth; + + private final Optional> nextLevelFactory; + private final boolean removeNodesWithNoData; + + private final Map subcacheMap = new HashMap<>(); + + HierarchicalCacheLevel( + CuratorFramework curatorClient, + ExecutorService executorService, + String path, + int depth, + CacheListeners eventConsumer, + Optional> nextLevelFactory, + boolean removeNodesWithNoData) { + super(curatorClient, path, true, false, executorService); + this.curatorClient = curatorClient; + this.currentDepth = depth; + this.consumer = eventConsumer; + this.nextLevelFactory = nextLevelFactory; + this.removeNodesWithNoData = removeNodesWithNoData; + getListenable().addListener(this); + } + + @Override + public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception { + if (event.getData() == null) { + return; } - @Override - public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception { - if (event.getData() == null) { - return; - } - - String path = event.getData().getPath(); - String cacheName = cacheNameFromPath(path); - logger.debug("Got {} event for path {}", event.getType(), path); - - switch (event.getType()) { - case CHILD_ADDED: - addSubcache(path, cacheName, event.getData().getData()); - break; - case CHILD_REMOVED: - removeSubcache(path, cacheName); - break; - default: - break; - } - - consumer.call(event); + String path = event.getData().getPath(); + String cacheName = cacheNameFromPath(path); + logger.debug("Got {} event for path {}", event.getType(), path); + + switch (event.getType()) { + case CHILD_ADDED: + addSubcache(path, cacheName, event.getData().getData()); + break; + case CHILD_REMOVED: + removeSubcache(path, cacheName); + break; + default: + break; } - void stop() throws IOException { - Lock writeLock = subcacheLock.writeLock(); - writeLock.lock(); - try { - for (HierarchicalCacheLevel subcache : subcacheMap.values()) { - subcache.stop(); - } - subcacheMap.clear(); - this.close(); - } finally { - writeLock.unlock(); - } + consumer.call(event); + } + + void stop() throws IOException { + Lock writeLock = subcacheLock.writeLock(); + writeLock.lock(); + try { + for (HierarchicalCacheLevel subcache : subcacheMap.values()) { + subcache.stop(); + } + subcacheMap.clear(); + this.close(); + } finally { + writeLock.unlock(); } - - private void addSubcache(String path, String cacheName, byte[] data) { - Lock writeLock = subcacheLock.writeLock(); - writeLock.lock(); - try { - logger.debug("Adding cache for path {}; Cache name: {}; Depth: {}; InstanceId: {}", - path, cacheName, currentDepth, Integer.toHexString(hashCode())); - - if (ArrayUtils.isEmpty(data) && removeNodesWithNoData) { - logger.warn("Removing path {} due to no data in the znode", path); - printOrphanedChildren(path); - removeNodeRecursively(path); - return; - } - if (subcacheMap.containsKey(cacheName)) { - logger.debug("Possible duplicate of new entry for {}, ignoring", cacheName); - return; - } - nextLevelFactory.ifPresent(f -> subcacheMap.put(cacheName, f.apply(currentDepth + 1, path))); - } finally { - writeLock.unlock(); - } + } + + private void addSubcache(String path, String cacheName, byte[] data) { + Lock writeLock = subcacheLock.writeLock(); + writeLock.lock(); + try { + logger.debug( + "Adding cache for path {}; Cache name: {}; Depth: {}; InstanceId: {}", + path, + cacheName, + currentDepth, + Integer.toHexString(hashCode())); + + if (ArrayUtils.isEmpty(data) && removeNodesWithNoData) { + logger.warn("Removing path {} due to no data in the znode", path); + printOrphanedChildren(path); + removeNodeRecursively(path); + return; + } + if (subcacheMap.containsKey(cacheName)) { + logger.debug("Possible duplicate of new entry for {}, ignoring", cacheName); + return; + } + nextLevelFactory.ifPresent(f -> subcacheMap.put(cacheName, f.apply(currentDepth + 1, path))); + } finally { + writeLock.unlock(); } - - private void printOrphanedChildren(String path) { - try { - List children = curatorClient.getChildren().forPath(path); - logger.warn("Nodes with empty parent {}: {}", - path, children.stream().map(Object::toString).collect(Collectors.joining(","))); - printChildrenWithEmptyParentRecursively(path, children); - } catch (KeeperException.NoNodeException e) { - logger.info("Could not receive list of children for path {} as the path does not exist", path); - } catch (Exception e) { - logger.warn("Could not receive list of children for path {} due to error", path, e); - } + } + + private void printOrphanedChildren(String path) { + try { + List children = curatorClient.getChildren().forPath(path); + logger.warn( + "Nodes with empty parent {}: {}", + path, + children.stream().map(Object::toString).collect(Collectors.joining(","))); + printChildrenWithEmptyParentRecursively(path, children); + } catch (KeeperException.NoNodeException e) { + logger.info( + "Could not receive list of children for path {} as the path does not exist", path); + } catch (Exception e) { + logger.warn("Could not receive list of children for path {} due to error", path, e); } - - private void printChildrenWithEmptyParentRecursively(String pathWithEmptyParent, List children) { - children.forEach(c -> { - try { - String nextPath = pathWithEmptyParent + "/" + c; - logger.warn("Node with empty parent: {}", nextPath); - List nextChildren = curatorClient.getChildren().forPath(nextPath); - - printChildrenWithEmptyParentRecursively(nextPath, nextChildren); - } catch (KeeperException.NoNodeException e) { - logger.info("Could not receive list of children for path {} as the path does not exist", pathWithEmptyParent); - } catch (Exception e) { - logger.warn("Could not receive list of children for path {} due to error", pathWithEmptyParent, e); - } + } + + private void printChildrenWithEmptyParentRecursively( + String pathWithEmptyParent, List children) { + children.forEach( + c -> { + try { + String nextPath = pathWithEmptyParent + "/" + c; + logger.warn("Node with empty parent: {}", nextPath); + List nextChildren = curatorClient.getChildren().forPath(nextPath); + + printChildrenWithEmptyParentRecursively(nextPath, nextChildren); + } catch (KeeperException.NoNodeException e) { + logger.info( + "Could not receive list of children for path {} as the path does not exist", + pathWithEmptyParent); + } catch (Exception e) { + logger.warn( + "Could not receive list of children for path {} due to error", + pathWithEmptyParent, + e); + } }); + } + + private void removeNodeRecursively(String path) { + try { + curatorClient.delete().deletingChildrenIfNeeded().forPath(path); + logger.warn("Removed recursively path {}", path); + } catch (Exception e) { + logger.warn("Error while deleting recursively path {}", path, e); } - - private void removeNodeRecursively(String path) { - try { - curatorClient.delete().deletingChildrenIfNeeded().forPath(path); - logger.warn("Removed recursively path {}", path); - } catch (Exception e) { - logger.warn("Error while deleting recursively path {}", path, e); - } + } + + private void removeSubcache(String path, String cacheName) throws Exception { + Lock writeLock = subcacheLock.writeLock(); + writeLock.lock(); + logger.debug( + "Removing cache for path {}; Cache name: {}; Depth {}; InstanceId: {}", + path, + cacheName, + currentDepth, + Integer.toHexString(hashCode())); + try { + HierarchicalCacheLevel subcache = subcacheMap.remove(cacheName); + if (subcache == null) { + logger.debug("Possible duplicate of removed entry for {}, ignoring", cacheName); + return; + } + subcache.close(); + } finally { + writeLock.unlock(); } + } - private void removeSubcache(String path, String cacheName) throws Exception { - Lock writeLock = subcacheLock.writeLock(); - writeLock.lock(); - logger.debug("Removing cache for path {}; Cache name: {}; Depth {}; InstanceId: {}", - path, cacheName, currentDepth, Integer.toHexString(hashCode())); - try { - HierarchicalCacheLevel subcache = subcacheMap.remove(cacheName); - if (subcache == null) { - logger.debug("Possible duplicate of removed entry for {}, ignoring", cacheName); - return; - } - subcache.close(); - } finally { - writeLock.unlock(); - } - } - - private String cacheNameFromPath(String path) { - return path.substring(path.lastIndexOf('/') + 1); - } + private String cacheNameFromPath(String path) { + return path.substring(path.lastIndexOf('/') + 1); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/cache/ModelAwareZookeeperNotifyingCache.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/cache/ModelAwareZookeeperNotifyingCache.java index c42921f558..ea16b86317 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/cache/ModelAwareZookeeperNotifyingCache.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/cache/ModelAwareZookeeperNotifyingCache.java @@ -1,67 +1,62 @@ package pl.allegro.tech.hermes.infrastructure.zookeeper.cache; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.function.Consumer; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.function.Consumer; - public class ModelAwareZookeeperNotifyingCache { - private static final Logger logger = LoggerFactory.getLogger(ModelAwareZookeeperNotifyingCache.class); + private static final Logger logger = + LoggerFactory.getLogger(ModelAwareZookeeperNotifyingCache.class); - private static final int GROUP_LEVEL = 0; + private static final int GROUP_LEVEL = 0; - private static final int TOPIC_LEVEL = 1; + private static final int TOPIC_LEVEL = 1; - private static final int SUBSCRIPTION_LEVEL = 2; + private static final int SUBSCRIPTION_LEVEL = 2; - private final HierarchicalCache cache; - private final ExecutorService executor; + private final HierarchicalCache cache; + private final ExecutorService executor; - public ModelAwareZookeeperNotifyingCache(CuratorFramework curator, ExecutorService executor, String rootPath) { - List levelPrefixes = Arrays.asList( - ZookeeperPaths.GROUPS_PATH, ZookeeperPaths.TOPICS_PATH, ZookeeperPaths.SUBSCRIPTIONS_PATH - ); - this.executor = executor; - this.cache = new HierarchicalCache( - curator, - executor, - rootPath, - 3, - levelPrefixes, - true - ); - } + public ModelAwareZookeeperNotifyingCache( + CuratorFramework curator, ExecutorService executor, String rootPath) { + List levelPrefixes = + Arrays.asList( + ZookeeperPaths.GROUPS_PATH, + ZookeeperPaths.TOPICS_PATH, + ZookeeperPaths.SUBSCRIPTIONS_PATH); + this.executor = executor; + this.cache = new HierarchicalCache(curator, executor, rootPath, 3, levelPrefixes, true); + } - public void start() throws Exception { - cache.start(); - } + public void start() throws Exception { + cache.start(); + } - public void stop() { - try { - cache.stop(); - executor.shutdownNow(); - } catch (Exception e) { - logger.warn("Failed to stop Zookeeper cache", e); - } + public void stop() { + try { + cache.stop(); + executor.shutdownNow(); + } catch (Exception e) { + logger.warn("Failed to stop Zookeeper cache", e); } + } - public void registerGroupCallback(Consumer callback) { - cache.registerCallback(GROUP_LEVEL, callback); - } + public void registerGroupCallback(Consumer callback) { + cache.registerCallback(GROUP_LEVEL, callback); + } + public void registerTopicCallback(Consumer callback) { + cache.registerCallback(TOPIC_LEVEL, callback); + } - public void registerTopicCallback(Consumer callback) { - cache.registerCallback(TOPIC_LEVEL, callback); - } - - public void registerSubscriptionCallback(Consumer callback) { - cache.registerCallback(SUBSCRIPTION_LEVEL, callback); - } + public void registerSubscriptionCallback(Consumer callback) { + cache.registerCallback(SUBSCRIPTION_LEVEL, callback); + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/counter/SharedCounter.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/counter/SharedCounter.java index 4a051053ac..498389cde6 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/counter/SharedCounter.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/counter/SharedCounter.java @@ -3,58 +3,63 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; +import java.time.Duration; +import java.util.concurrent.TimeUnit; import org.apache.curator.RetryPolicy; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong; import org.apache.curator.retry.ExponentialBackoffRetry; -import java.time.Duration; -import java.util.concurrent.TimeUnit; - public class SharedCounter { - private final LoadingCache distributedAtomicLongs; + private final LoadingCache distributedAtomicLongs; - public SharedCounter(CuratorFramework curatorClient, Duration expireAfter, - Duration distributedLoaderBackoff, int distributedLoaderRetries) { - distributedAtomicLongs = CacheBuilder.newBuilder() - .expireAfterAccess(expireAfter.toHours(), TimeUnit.HOURS) - .build(new DistributedAtomicLongLoader( - curatorClient, - new ExponentialBackoffRetry((int) distributedLoaderBackoff.toMillis(), distributedLoaderRetries)) - ); - } + public SharedCounter( + CuratorFramework curatorClient, + Duration expireAfter, + Duration distributedLoaderBackoff, + int distributedLoaderRetries) { + distributedAtomicLongs = + CacheBuilder.newBuilder() + .expireAfterAccess(expireAfter.toHours(), TimeUnit.HOURS) + .build( + new DistributedAtomicLongLoader( + curatorClient, + new ExponentialBackoffRetry( + (int) distributedLoaderBackoff.toMillis(), distributedLoaderRetries))); + } - public boolean increment(String path, long count) { - try { - return distributedAtomicLongs.get(path).add(count).succeeded(); - } catch (Exception e) { - throw new ZookeeperCounterException(path, e); - } + public boolean increment(String path, long count) { + try { + return distributedAtomicLongs.get(path).add(count).succeeded(); + } catch (Exception e) { + throw new ZookeeperCounterException(path, e); } + } - public long getValue(String path) { - try { - return distributedAtomicLongs.get(path).get().preValue(); - } catch (Exception e) { - throw new ZookeeperCounterException(path, e); - } + public long getValue(String path) { + try { + return distributedAtomicLongs.get(path).get().preValue(); + } catch (Exception e) { + throw new ZookeeperCounterException(path, e); } + } - private static final class DistributedAtomicLongLoader extends CacheLoader { + private static final class DistributedAtomicLongLoader + extends CacheLoader { - private final CuratorFramework client; + private final CuratorFramework client; - private final RetryPolicy retryPolicy; + private final RetryPolicy retryPolicy; - DistributedAtomicLongLoader(CuratorFramework client, RetryPolicy retryPolicy) { - this.client = client; - this.retryPolicy = retryPolicy; - } + DistributedAtomicLongLoader(CuratorFramework client, RetryPolicy retryPolicy) { + this.client = client; + this.retryPolicy = retryPolicy; + } - @Override - public DistributedAtomicLong load(String key) throws Exception { - return new DistributedAtomicLong(client, key, retryPolicy); - } + @Override + public DistributedAtomicLong load(String key) throws Exception { + return new DistributedAtomicLong(client, key, retryPolicy); } + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/counter/ZookeeperCounterException.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/counter/ZookeeperCounterException.java index 8d0fdfa0b8..06b792be86 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/counter/ZookeeperCounterException.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/counter/ZookeeperCounterException.java @@ -6,16 +6,16 @@ @SuppressWarnings("serial") public class ZookeeperCounterException extends HermesException { - public ZookeeperCounterException(String key, String message) { - super("Exception while trying to access counter " + key + " via Zookeeper. " + message); - } + public ZookeeperCounterException(String key, String message) { + super("Exception while trying to access counter " + key + " via Zookeeper. " + message); + } - public ZookeeperCounterException(String key, Throwable cause) { - super("Exception while trying to access counter " + key + " via Zookeeper.", cause); - } + public ZookeeperCounterException(String key, Throwable cause) { + super("Exception while trying to access counter " + key + " via Zookeeper.", cause); + } - @Override - public ErrorCode getCode() { - return ErrorCode.INTERNAL_ERROR; - } + @Override + public ErrorCode getCode() { + return ErrorCode.INTERNAL_ERROR; + } } diff --git a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/notifications/ZookeeperInternalNotificationBus.java b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/notifications/ZookeeperInternalNotificationBus.java index ccab8542d7..0ba00581db 100644 --- a/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/notifications/ZookeeperInternalNotificationBus.java +++ b/hermes-common/src/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/notifications/ZookeeperInternalNotificationBus.java @@ -1,6 +1,11 @@ package pl.allegro.tech.hermes.infrastructure.zookeeper.notifications; +import static java.util.Optional.empty; +import static java.util.Optional.of; + import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.util.Optional; import org.apache.commons.lang3.ArrayUtils; import org.apache.curator.framework.recipes.cache.ChildData; import org.slf4j.Logger; @@ -13,78 +18,79 @@ import pl.allegro.tech.hermes.domain.notifications.TopicCallback; import pl.allegro.tech.hermes.infrastructure.zookeeper.cache.ModelAwareZookeeperNotifyingCache; -import java.io.IOException; -import java.util.Optional; - -import static java.util.Optional.empty; -import static java.util.Optional.of; - public class ZookeeperInternalNotificationBus implements InternalNotificationsBus { - private static final Logger logger = LoggerFactory.getLogger(ZookeeperInternalNotificationBus.class); + private static final Logger logger = + LoggerFactory.getLogger(ZookeeperInternalNotificationBus.class); - private final ObjectMapper objectMapper; + private final ObjectMapper objectMapper; - private final ModelAwareZookeeperNotifyingCache modelNotifyingCache; + private final ModelAwareZookeeperNotifyingCache modelNotifyingCache; - public ZookeeperInternalNotificationBus(ObjectMapper objectMapper, ModelAwareZookeeperNotifyingCache modelNotifyingCache) { - this.objectMapper = objectMapper; - this.modelNotifyingCache = modelNotifyingCache; - } + public ZookeeperInternalNotificationBus( + ObjectMapper objectMapper, ModelAwareZookeeperNotifyingCache modelNotifyingCache) { + this.objectMapper = objectMapper; + this.modelNotifyingCache = modelNotifyingCache; + } - @Override - public void registerSubscriptionCallback(SubscriptionCallback callback) { - modelNotifyingCache.registerSubscriptionCallback((e) -> { - switch (e.getType()) { - case CHILD_ADDED: - readSilently(e.getData(), Subscription.class).ifPresent(callback::onSubscriptionCreated); - break; - case CHILD_UPDATED: - readSilently(e.getData(), Subscription.class).ifPresent(callback::onSubscriptionChanged); - break; - case CHILD_REMOVED: - readSilently(e.getData(), Subscription.class).ifPresent(callback::onSubscriptionRemoved); - break; - default: - break; - } + @Override + public void registerSubscriptionCallback(SubscriptionCallback callback) { + modelNotifyingCache.registerSubscriptionCallback( + (e) -> { + switch (e.getType()) { + case CHILD_ADDED: + readSilently(e.getData(), Subscription.class) + .ifPresent(callback::onSubscriptionCreated); + break; + case CHILD_UPDATED: + readSilently(e.getData(), Subscription.class) + .ifPresent(callback::onSubscriptionChanged); + break; + case CHILD_REMOVED: + readSilently(e.getData(), Subscription.class) + .ifPresent(callback::onSubscriptionRemoved); + break; + default: + break; + } }); - } + } - @Override - public void registerTopicCallback(TopicCallback callback) { - modelNotifyingCache.registerTopicCallback((e) -> { - switch (e.getType()) { - case CHILD_ADDED: - readSilently(e.getData(), Topic.class).ifPresent(callback::onTopicCreated); - break; - case CHILD_UPDATED: - readSilently(e.getData(), Topic.class).ifPresent(callback::onTopicChanged); - break; - case CHILD_REMOVED: - readSilently(e.getData(), Topic.class).ifPresent(callback::onTopicRemoved); - break; - default: - break; - } + @Override + public void registerTopicCallback(TopicCallback callback) { + modelNotifyingCache.registerTopicCallback( + (e) -> { + switch (e.getType()) { + case CHILD_ADDED: + readSilently(e.getData(), Topic.class).ifPresent(callback::onTopicCreated); + break; + case CHILD_UPDATED: + readSilently(e.getData(), Topic.class).ifPresent(callback::onTopicChanged); + break; + case CHILD_REMOVED: + readSilently(e.getData(), Topic.class).ifPresent(callback::onTopicRemoved); + break; + default: + break; + } }); - } + } - @Override - public void registerAdminCallback(AdminCallback callback) { - // TODO we should move admin callbacks here in favor of AdminTool - } + @Override + public void registerAdminCallback(AdminCallback callback) { + // TODO we should move admin callbacks here in favor of AdminTool + } - private Optional readSilently(ChildData data, Class clazz) { - if (ArrayUtils.isEmpty(data.getData())) { - logger.warn("No data at path {}", data.getPath()); - return empty(); - } - try { - return of(objectMapper.readValue(data.getData(), clazz)); - } catch (IOException exception) { - logger.warn("Failed to parse object at path {}", data.getPath(), exception); - return empty(); - } + private Optional readSilently(ChildData data, Class clazz) { + if (ArrayUtils.isEmpty(data.getData())) { + logger.warn("No data at path {}", data.getPath()); + return empty(); + } + try { + return of(objectMapper.readValue(data.getData(), clazz)); + } catch (IOException exception) { + logger.warn("Failed to parse object at path {}", data.getPath(), exception); + return empty(); } + } } diff --git a/hermes-common/src/test/java/pl/allegro/tech/hermes/common/di/ObjectMapperFactoryTest.java b/hermes-common/src/test/java/pl/allegro/tech/hermes/common/di/ObjectMapperFactoryTest.java index 7b10a31a30..bfa508d545 100644 --- a/hermes-common/src/test/java/pl/allegro/tech/hermes/common/di/ObjectMapperFactoryTest.java +++ b/hermes-common/src/test/java/pl/allegro/tech/hermes/common/di/ObjectMapperFactoryTest.java @@ -1,63 +1,62 @@ package pl.allegro.tech.hermes.common.di; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertEquals; + import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.Before; import org.junit.Test; import pl.allegro.tech.hermes.common.di.factories.ObjectMapperFactory; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.assertEquals; - public class ObjectMapperFactoryTest { - ObjectMapper mapper; + ObjectMapper mapper; - @Before - public void init() { - ObjectMapperFactory factory = new ObjectMapperFactory(false, false); - mapper = factory.provide(); - } + @Before + public void init() { + ObjectMapperFactory factory = new ObjectMapperFactory(false, false); + mapper = factory.provide(); + } - @Test - public void shouldDeserializeClassWithUnknownFields() throws Exception { - //given - String json = "{\"unknownProperty\": \"value\", \"name\":\"Charles\"}"; + @Test + public void shouldDeserializeClassWithUnknownFields() throws Exception { + // given + String json = "{\"unknownProperty\": \"value\", \"name\":\"Charles\"}"; - //when - DummyUser subscription = mapper.readValue(json, DummyUser.class); + // when + DummyUser subscription = mapper.readValue(json, DummyUser.class); - //then - assertEquals("Charles", subscription.name); - } + // then + assertEquals("Charles", subscription.name); + } - @Test - public void shouldSerializeObjectWithoutNullPropertiesInclusion() throws Exception { - //given - DummyUser object = new DummyUser(null); + @Test + public void shouldSerializeObjectWithoutNullPropertiesInclusion() throws Exception { + // given + DummyUser object = new DummyUser(null); - //when - final String jsonValue = mapper.writeValueAsString(object); + // when + final String jsonValue = mapper.writeValueAsString(object); - //then - assertThat(jsonValue).doesNotContain("name").doesNotContain("null"); - } + // then + assertThat(jsonValue).doesNotContain("name").doesNotContain("null"); + } - private static final class DummyUser { - private String name; + private static final class DummyUser { + private String name; - private DummyUser() { - } + private DummyUser() {} - private DummyUser(String name) { - this.name = name; - } + private DummyUser(String name) { + this.name = name; + } - public void setName(String name) { - this.name = name; - } + public void setName(String name) { + this.name = name; + } - public String getName() { - return name; - } + public String getName() { + return name; } + } } diff --git a/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/undelivered/ZookeeperUndeliveredMessageLogTest.java b/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/undelivered/ZookeeperUndeliveredMessageLogTest.java index 648f61134e..f04e72631a 100644 --- a/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/undelivered/ZookeeperUndeliveredMessageLogTest.java +++ b/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/undelivered/ZookeeperUndeliveredMessageLogTest.java @@ -1,11 +1,18 @@ package pl.allegro.tech.hermes.common.message.undelivered; +import static org.assertj.core.api.Assertions.assertThat; +import static pl.allegro.tech.hermes.api.SentMessageTrace.Builder.undeliveredMessage; +import static pl.allegro.tech.hermes.common.metric.Histograms.PERSISTED_UNDELIVERED_MESSAGE_SIZE; +import static pl.allegro.tech.hermes.common.metric.Meters.PERSISTED_UNDELIVERED_MESSAGES_METER; +import static pl.allegro.tech.hermes.test.helper.metrics.MicrometerUtils.metricValue; + import com.fasterxml.jackson.databind.ObjectMapper; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.DistributionSummary; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.search.Search; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import java.util.Optional; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -15,116 +22,117 @@ import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; import pl.allegro.tech.hermes.test.helper.zookeeper.ZookeeperBaseTest; -import java.util.Optional; - -import static org.assertj.core.api.Assertions.assertThat; -import static pl.allegro.tech.hermes.api.SentMessageTrace.Builder.undeliveredMessage; -import static pl.allegro.tech.hermes.common.metric.Histograms.PERSISTED_UNDELIVERED_MESSAGE_SIZE; -import static pl.allegro.tech.hermes.common.metric.Meters.PERSISTED_UNDELIVERED_MESSAGES_METER; -import static pl.allegro.tech.hermes.test.helper.metrics.MicrometerUtils.metricValue; - public class ZookeeperUndeliveredMessageLogTest extends ZookeeperBaseTest { - private static final TopicName TOPIC = new TopicName("undeliveredMessageLogGroup", "topic"); - - private static final String SUBSCRIPTION = "subscription"; - - private final ZookeeperPaths paths = new ZookeeperPaths("/hermes"); - - private final MeterRegistry meterRegistry = new SimpleMeterRegistry(); - private final MetricsFacade metricsFacade = new MetricsFacade(meterRegistry); - - private final ZookeeperUndeliveredMessageLog log = new ZookeeperUndeliveredMessageLog( - zookeeperClient, - paths, - new ObjectMapper(), - metricsFacade - ); - - private final ZookeeperLastUndeliveredMessageReader reader = new ZookeeperLastUndeliveredMessageReader( - zookeeperClient, - paths, - new ObjectMapper() - ); - - @Before - public void setUp() throws Exception { - zookeeperClient.create().creatingParentsIfNeeded().forPath(paths.subscriptionPath(TOPIC, SUBSCRIPTION)); - } - - @After - public void cleanUp() throws Exception { - deleteData(paths.basePath()); - } - - @Test - public void shouldAddUndeliveredMessageToLog() throws Exception { - // given when - log.add(createUndeliveredMessage(SUBSCRIPTION, "message")); - log.persist(); - - // then - SentMessageTrace lastMessage = reader.last(TOPIC, "subscription").get(); - assertThat(lastMessage.getMessage()).isEqualTo("message"); - assertThatMetricsHaveBeenReported(1); - } - - @Test - public void shouldReturnLatestUndeliveredMessage() throws Exception { - // given - log.add(createUndeliveredMessage(SUBSCRIPTION, "old message")); - log.add(createUndeliveredMessage(SUBSCRIPTION, "new message")); - log.persist(); - - // when - SentMessageTrace lastMessage = reader.last(TOPIC, "subscription").get(); - - // then - assertThat(lastMessage.getMessage()).isEqualTo("new message"); - assertThatMetricsHaveBeenReported(1); - } - - @Test - public void shouldReturnAbsentIfThereAreNoUndeliveredMessagesForGivenSubscription() { - // when - Optional result = reader.last(new TopicName("unknown", "topic"), "subscription"); - - // then - assertThat(result.isPresent()).isFalse(); - assertThatMetricsHaveBeenReported(0); - } - - @Test - public void shouldNotAddUndeliveredMessageLogToNonExistingSubscriptionPath() { - // given - log.add(createUndeliveredMessage("unknownSubscription", "message")); - log.persist(); - - // when - Optional result = reader.last(TOPIC, "unknownSubscription"); - - // then - assertThat(result.isPresent()).isFalse(); - assertThatMetricsHaveBeenReported(0); - } - - private SentMessageTrace createUndeliveredMessage(String subscription, String message) { - return undeliveredMessage() - .withTopicName(TOPIC.qualifiedName()) - .withSubscription(subscription) - .withMessage(message) - .withReason(new IllegalArgumentException().getMessage()) - .withTimestamp(1L) - .withPartition(1) - .withOffset(1L) - .withCluster("cluster") - .build(); - } - - private void assertThatMetricsHaveBeenReported(int persistedMessageCount) { - assertThat(metricValue(meterRegistry, PERSISTED_UNDELIVERED_MESSAGES_METER, Search::counter, Counter::count).orElse(0.0d)) - .isEqualTo(persistedMessageCount); - assertThat(metricValue(meterRegistry, PERSISTED_UNDELIVERED_MESSAGE_SIZE + ".bytes", Search::summary, DistributionSummary::count) - .orElse(0L)).isEqualTo(persistedMessageCount); - } + private static final TopicName TOPIC = new TopicName("undeliveredMessageLogGroup", "topic"); + + private static final String SUBSCRIPTION = "subscription"; + + private final ZookeeperPaths paths = new ZookeeperPaths("/hermes"); + + private final MeterRegistry meterRegistry = new SimpleMeterRegistry(); + private final MetricsFacade metricsFacade = new MetricsFacade(meterRegistry); + + private final ZookeeperUndeliveredMessageLog log = + new ZookeeperUndeliveredMessageLog(zookeeperClient, paths, new ObjectMapper(), metricsFacade); + + private final ZookeeperLastUndeliveredMessageReader reader = + new ZookeeperLastUndeliveredMessageReader(zookeeperClient, paths, new ObjectMapper()); + + @Before + public void setUp() throws Exception { + zookeeperClient + .create() + .creatingParentsIfNeeded() + .forPath(paths.subscriptionPath(TOPIC, SUBSCRIPTION)); + } + + @After + public void cleanUp() throws Exception { + deleteData(paths.basePath()); + } + + @Test + public void shouldAddUndeliveredMessageToLog() throws Exception { + // given when + log.add(createUndeliveredMessage(SUBSCRIPTION, "message")); + log.persist(); + + // then + SentMessageTrace lastMessage = reader.last(TOPIC, "subscription").get(); + assertThat(lastMessage.getMessage()).isEqualTo("message"); + assertThatMetricsHaveBeenReported(1); + } + + @Test + public void shouldReturnLatestUndeliveredMessage() throws Exception { + // given + log.add(createUndeliveredMessage(SUBSCRIPTION, "old message")); + log.add(createUndeliveredMessage(SUBSCRIPTION, "new message")); + log.persist(); + + // when + SentMessageTrace lastMessage = reader.last(TOPIC, "subscription").get(); + + // then + assertThat(lastMessage.getMessage()).isEqualTo("new message"); + assertThatMetricsHaveBeenReported(1); + } + + @Test + public void shouldReturnAbsentIfThereAreNoUndeliveredMessagesForGivenSubscription() { + // when + Optional result = + reader.last(new TopicName("unknown", "topic"), "subscription"); + + // then + assertThat(result.isPresent()).isFalse(); + assertThatMetricsHaveBeenReported(0); + } + + @Test + public void shouldNotAddUndeliveredMessageLogToNonExistingSubscriptionPath() { + // given + log.add(createUndeliveredMessage("unknownSubscription", "message")); + log.persist(); + + // when + Optional result = reader.last(TOPIC, "unknownSubscription"); + + // then + assertThat(result.isPresent()).isFalse(); + assertThatMetricsHaveBeenReported(0); + } + + private SentMessageTrace createUndeliveredMessage(String subscription, String message) { + return undeliveredMessage() + .withTopicName(TOPIC.qualifiedName()) + .withSubscription(subscription) + .withMessage(message) + .withReason(new IllegalArgumentException().getMessage()) + .withTimestamp(1L) + .withPartition(1) + .withOffset(1L) + .withCluster("cluster") + .build(); + } + + private void assertThatMetricsHaveBeenReported(int persistedMessageCount) { + assertThat( + metricValue( + meterRegistry, + PERSISTED_UNDELIVERED_MESSAGES_METER, + Search::counter, + Counter::count) + .orElse(0.0d)) + .isEqualTo(persistedMessageCount); + assertThat( + metricValue( + meterRegistry, + PERSISTED_UNDELIVERED_MESSAGE_SIZE + ".bytes", + Search::summary, + DistributionSummary::count) + .orElse(0L)) + .isEqualTo(persistedMessageCount); + } } diff --git a/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageContentWrapperTest.java b/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageContentWrapperTest.java index 4d7599ee11..bf018d6511 100644 --- a/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageContentWrapperTest.java +++ b/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/wrapper/AvroMessageContentWrapperTest.java @@ -1,12 +1,12 @@ package pl.allegro.tech.hermes.common.message.wrapper; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.util.Utf8; -import org.apache.commons.io.IOUtils; -import org.junit.Before; -import org.junit.Test; -import pl.allegro.tech.hermes.test.helper.avro.AvroUser; +import static java.lang.Long.valueOf; +import static org.assertj.core.api.Assertions.assertThat; +import static pl.allegro.tech.hermes.common.message.converter.AvroRecordToBytesConverter.bytesToRecord; +import static pl.allegro.tech.hermes.common.message.converter.AvroRecordToBytesConverter.recordToBytes; +import static pl.allegro.tech.hermes.common.message.wrapper.AvroMetadataMarker.METADATA_MARKER; +import static pl.allegro.tech.hermes.common.message.wrapper.AvroMetadataMarker.METADATA_MESSAGE_ID_KEY; +import static pl.allegro.tech.hermes.common.message.wrapper.AvroMetadataMarker.METADATA_TIMESTAMP_KEY; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -15,121 +15,127 @@ import java.util.HashMap; import java.util.Map; import java.util.UUID; - -import static java.lang.Long.valueOf; -import static org.assertj.core.api.Assertions.assertThat; -import static pl.allegro.tech.hermes.common.message.converter.AvroRecordToBytesConverter.bytesToRecord; -import static pl.allegro.tech.hermes.common.message.converter.AvroRecordToBytesConverter.recordToBytes; -import static pl.allegro.tech.hermes.common.message.wrapper.AvroMetadataMarker.METADATA_MARKER; -import static pl.allegro.tech.hermes.common.message.wrapper.AvroMetadataMarker.METADATA_MESSAGE_ID_KEY; -import static pl.allegro.tech.hermes.common.message.wrapper.AvroMetadataMarker.METADATA_TIMESTAMP_KEY; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.util.Utf8; +import org.apache.commons.io.IOUtils; +import org.junit.Before; +import org.junit.Test; +import pl.allegro.tech.hermes.test.helper.avro.AvroUser; public class AvroMessageContentWrapperTest { - private AvroMessageContentWrapper avroMessageContentWrapper; - private AvroUser avroUser; - private byte[] content; - - private final String id = UUID.randomUUID().toString(); - private final Long timestamp = System.currentTimeMillis(); - - @Before - public void setup() throws IOException { - avroUser = new AvroUser("Bob", 10, "red"); - content = avroUser.asBytes(); - avroMessageContentWrapper = new AvroMessageContentWrapper(Clock.systemDefaultZone()); - } - - @Test - public void shouldWrapAndUnwrapAvroMessageWithMetadata() { - // when - byte[] wrappedMessage = - avroMessageContentWrapper.wrapContent(content, id, timestamp, avroUser.getSchema(), Collections.emptyMap()); - UnwrappedMessageContent unwrappedMessageContent = - avroMessageContentWrapper.unwrapContent(wrappedMessage, avroUser.getCompiledSchema()); - - // then - assertThat(unwrappedMessageContent.getMessageMetadata().getId()).isEqualTo(id); - assertThat(unwrappedMessageContent.getMessageMetadata().getTimestamp()).isEqualTo(timestamp); - assertThat(unwrappedMessageContent.getContent()).contains(content); - } - - @Test - @SuppressWarnings("unchecked") - public void shouldWrappedMessageContainsMetadata() { - // when - byte[] wrappedMessage = avroMessageContentWrapper.wrapContent(content, id, timestamp, avroUser.getSchema(), Collections.emptyMap()); - - // then - GenericRecord messageWithMetadata = bytesToRecord(wrappedMessage, avroUser.getSchema()); - Map metadata = (Map) messageWithMetadata.get(METADATA_MARKER); - assertThat(metadata.get(METADATA_MESSAGE_ID_KEY).toString()).isEqualTo(id); - assertThat(valueOf(metadata.get(METADATA_TIMESTAMP_KEY).toString())).isEqualTo(timestamp); - assertThat(wrappedMessage).contains(content); - } - - @Test - public void shouldWrappedMessageBeUnchangedWhenSchemaNotContainsMetadata() throws IOException { - // when - Schema schemaWithoutMetadata = new Schema.Parser().parse( - IOUtils.toString( - getClass().getResourceAsStream("/schema/user_no_metadata.avsc"), - StandardCharsets.UTF_8 - ) - ); - byte[] wrappedMessage = - avroMessageContentWrapper.wrapContent(content, id, timestamp, schemaWithoutMetadata, Collections.emptyMap()); - - // then - GenericRecord wrappedMessageAsRecord = bytesToRecord(wrappedMessage, avroUser.getSchema()); - assertThat(wrappedMessageAsRecord.get(METADATA_MARKER)).isNull(); - } - - @Test - public void shouldUnwrapAvroMessageAndGenerateMetadataWhenNotExists() throws Throwable { - //given - byte [] wrappedMessage = wrapContentWithoutMetadata(content, avroUser.getSchema()); - - //when - UnwrappedMessageContent unwrappedMessage = - avroMessageContentWrapper.unwrapContent(wrappedMessage, avroUser.getCompiledSchema()); - - //then - assertThat(unwrappedMessage.getMessageMetadata().getId()).isEmpty(); - assertThat(unwrappedMessage.getMessageMetadata().getTimestamp()).isNotNull(); - assertThat(unwrappedMessage.getContent()).startsWith(content); - } - - @Test - public void shouldUnwrapAvroMessageAndSetEmptyMessageIdWhenNotGivenInMetadata() throws Throwable { - // given - byte [] wrappedMessage = wrapContentWithoutMessageIdInMetadata(content, avroUser.getSchema()); - - //when - UnwrappedMessageContent unwrappedMessage = avroMessageContentWrapper.unwrapContent(wrappedMessage, avroUser.getCompiledSchema()); - - // then - assertThat(unwrappedMessage.getMessageMetadata().getId()).isEmpty(); - assertThat(unwrappedMessage.getMessageMetadata().getTimestamp()).isNotNull(); - assertThat(unwrappedMessage.getContent()).contains(content); - } - - private byte[] wrapContentWithoutMetadata(byte[] message, Schema schema) throws Exception { - return wrapContent(message, schema, null); - } - - private byte[] wrapContentWithoutMessageIdInMetadata(byte[] message, Schema schema) throws Exception { - return wrapContent(message, schema, metadataMapWithoutMessageId(timestamp)); - } - - private byte[] wrapContent(byte[] message, Schema schema, Map metadata) throws Exception { - GenericRecord genericRecord = bytesToRecord(message, schema); - genericRecord.put(METADATA_MARKER, metadata); - return recordToBytes(genericRecord, schema); - } - - private Map metadataMapWithoutMessageId(long timestamp) { - Map metadata = new HashMap<>(); - metadata.put(METADATA_TIMESTAMP_KEY, new Utf8(Long.toString(timestamp))); - return metadata; - } + private AvroMessageContentWrapper avroMessageContentWrapper; + private AvroUser avroUser; + private byte[] content; + + private final String id = UUID.randomUUID().toString(); + private final Long timestamp = System.currentTimeMillis(); + + @Before + public void setup() throws IOException { + avroUser = new AvroUser("Bob", 10, "red"); + content = avroUser.asBytes(); + avroMessageContentWrapper = new AvroMessageContentWrapper(Clock.systemDefaultZone()); + } + + @Test + public void shouldWrapAndUnwrapAvroMessageWithMetadata() { + // when + byte[] wrappedMessage = + avroMessageContentWrapper.wrapContent( + content, id, timestamp, avroUser.getSchema(), Collections.emptyMap()); + UnwrappedMessageContent unwrappedMessageContent = + avroMessageContentWrapper.unwrapContent(wrappedMessage, avroUser.getCompiledSchema()); + + // then + assertThat(unwrappedMessageContent.getMessageMetadata().getId()).isEqualTo(id); + assertThat(unwrappedMessageContent.getMessageMetadata().getTimestamp()).isEqualTo(timestamp); + assertThat(unwrappedMessageContent.getContent()).contains(content); + } + + @Test + @SuppressWarnings("unchecked") + public void shouldWrappedMessageContainsMetadata() { + // when + byte[] wrappedMessage = + avroMessageContentWrapper.wrapContent( + content, id, timestamp, avroUser.getSchema(), Collections.emptyMap()); + + // then + GenericRecord messageWithMetadata = bytesToRecord(wrappedMessage, avroUser.getSchema()); + Map metadata = (Map) messageWithMetadata.get(METADATA_MARKER); + assertThat(metadata.get(METADATA_MESSAGE_ID_KEY).toString()).isEqualTo(id); + assertThat(valueOf(metadata.get(METADATA_TIMESTAMP_KEY).toString())).isEqualTo(timestamp); + assertThat(wrappedMessage).contains(content); + } + + @Test + public void shouldWrappedMessageBeUnchangedWhenSchemaNotContainsMetadata() throws IOException { + // when + Schema schemaWithoutMetadata = + new Schema.Parser() + .parse( + IOUtils.toString( + getClass().getResourceAsStream("/schema/user_no_metadata.avsc"), + StandardCharsets.UTF_8)); + byte[] wrappedMessage = + avroMessageContentWrapper.wrapContent( + content, id, timestamp, schemaWithoutMetadata, Collections.emptyMap()); + + // then + GenericRecord wrappedMessageAsRecord = bytesToRecord(wrappedMessage, avroUser.getSchema()); + assertThat(wrappedMessageAsRecord.get(METADATA_MARKER)).isNull(); + } + + @Test + public void shouldUnwrapAvroMessageAndGenerateMetadataWhenNotExists() throws Throwable { + // given + byte[] wrappedMessage = wrapContentWithoutMetadata(content, avroUser.getSchema()); + + // when + UnwrappedMessageContent unwrappedMessage = + avroMessageContentWrapper.unwrapContent(wrappedMessage, avroUser.getCompiledSchema()); + + // then + assertThat(unwrappedMessage.getMessageMetadata().getId()).isEmpty(); + assertThat(unwrappedMessage.getMessageMetadata().getTimestamp()).isNotNull(); + assertThat(unwrappedMessage.getContent()).startsWith(content); + } + + @Test + public void shouldUnwrapAvroMessageAndSetEmptyMessageIdWhenNotGivenInMetadata() throws Throwable { + // given + byte[] wrappedMessage = wrapContentWithoutMessageIdInMetadata(content, avroUser.getSchema()); + + // when + UnwrappedMessageContent unwrappedMessage = + avroMessageContentWrapper.unwrapContent(wrappedMessage, avroUser.getCompiledSchema()); + + // then + assertThat(unwrappedMessage.getMessageMetadata().getId()).isEmpty(); + assertThat(unwrappedMessage.getMessageMetadata().getTimestamp()).isNotNull(); + assertThat(unwrappedMessage.getContent()).contains(content); + } + + private byte[] wrapContentWithoutMetadata(byte[] message, Schema schema) throws Exception { + return wrapContent(message, schema, null); + } + + private byte[] wrapContentWithoutMessageIdInMetadata(byte[] message, Schema schema) + throws Exception { + return wrapContent(message, schema, metadataMapWithoutMessageId(timestamp)); + } + + private byte[] wrapContent(byte[] message, Schema schema, Map metadata) + throws Exception { + GenericRecord genericRecord = bytesToRecord(message, schema); + genericRecord.put(METADATA_MARKER, metadata); + return recordToBytes(genericRecord, schema); + } + + private Map metadataMapWithoutMessageId(long timestamp) { + Map metadata = new HashMap<>(); + metadata.put(METADATA_TIMESTAMP_KEY, new Utf8(Long.toString(timestamp))); + return metadata; + } } diff --git a/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/wrapper/JsonMessageContentWrapperTest.java b/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/wrapper/JsonMessageContentWrapperTest.java index c699023402..e6d332bffe 100644 --- a/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/wrapper/JsonMessageContentWrapperTest.java +++ b/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/wrapper/JsonMessageContentWrapperTest.java @@ -1,77 +1,87 @@ package pl.allegro.tech.hermes.common.message.wrapper; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.entry; + import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; -import org.assertj.core.data.MapEntry; -import org.junit.Ignore; -import org.junit.Test; - import java.io.IOException; import java.util.Collections; import java.util.Map; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.entry; +import org.assertj.core.data.MapEntry; +import org.junit.Ignore; +import org.junit.Test; public class JsonMessageContentWrapperTest { - private final byte[] testContent = "{\"key\":\"value\"}".getBytes(); - private final ObjectMapper mapper = new ObjectMapper(); - private final MessageMetadata metadata = - new MessageMetadata(System.currentTimeMillis(), "14cf17ea-f1ea-a464-6bd6478615bb", ImmutableMap.of()); - private final Map metadataAsMap = - ImmutableMap.of("timestamp", metadata.getTimestamp(), "id", metadata.getId(), "externalMetadata", Collections.emptyMap()); - private final MapEntry unwrappingMarker = entry("_w", true); - private final MapEntry content = entry("message", readMap(testContent)); + private final byte[] testContent = "{\"key\":\"value\"}".getBytes(); + private final ObjectMapper mapper = new ObjectMapper(); + private final MessageMetadata metadata = + new MessageMetadata( + System.currentTimeMillis(), "14cf17ea-f1ea-a464-6bd6478615bb", ImmutableMap.of()); + private final Map metadataAsMap = + ImmutableMap.of( + "timestamp", + metadata.getTimestamp(), + "id", + metadata.getId(), + "externalMetadata", + Collections.emptyMap()); + private final MapEntry unwrappingMarker = entry("_w", true); + private final MapEntry content = entry("message", readMap(testContent)); - private final JsonMessageContentWrapper contentWrapper = new JsonMessageContentWrapper(content.key.toString(), "metadata", mapper); - private Map externalMetadata = ImmutableMap.of(); + private final JsonMessageContentWrapper contentWrapper = + new JsonMessageContentWrapper(content.key.toString(), "metadata", mapper); + private Map externalMetadata = ImmutableMap.of(); - @Test - @SuppressWarnings("unchecked") - public void shouldWrapJsonWithMetadata() { - //when - byte[] result = contentWrapper.wrapContent(testContent, metadata.getId(), metadata.getTimestamp(), externalMetadata); + @Test + @SuppressWarnings("unchecked") + public void shouldWrapJsonWithMetadata() { + // when + byte[] result = + contentWrapper.wrapContent( + testContent, metadata.getId(), metadata.getTimestamp(), externalMetadata); - //then - assertThat(readMap(result)).containsExactly(unwrappingMarker, entry("metadata", metadataAsMap), content); - } + // then + assertThat(readMap(result)) + .containsExactly(unwrappingMarker, entry("metadata", metadataAsMap), content); + } - @Test - public void shouldUnwrapMessageWithMetadata() { - //when - UnwrappedMessageContent result = contentWrapper.unwrapContent( - contentWrapper.wrapContent(testContent, metadata.getId(), metadata.getTimestamp(), externalMetadata) - ); + @Test + public void shouldUnwrapMessageWithMetadata() { + // when + UnwrappedMessageContent result = + contentWrapper.unwrapContent( + contentWrapper.wrapContent( + testContent, metadata.getId(), metadata.getTimestamp(), externalMetadata)); - //then - assertThat(result.getContent()).isEqualTo(testContent); - assertThat(result.getMessageMetadata()).isEqualTo(metadata); - } + // then + assertThat(result.getContent()).isEqualTo(testContent); + assertThat(result.getMessageMetadata()).isEqualTo(metadata); + } - @Test - public void shouldTolerateUnwrappingUnwrappedMessage() { - //when - UnwrappedMessageContent result = contentWrapper.unwrapContent(testContent); + @Test + public void shouldTolerateUnwrappingUnwrappedMessage() { + // when + UnwrappedMessageContent result = contentWrapper.unwrapContent(testContent); - //then - assertThat(result.getMessageMetadata().getId()).isNotEmpty(); - assertThat(result.getMessageMetadata().getTimestamp()).isEqualTo(1L); - } + // then + assertThat(result.getMessageMetadata().getId()).isNotEmpty(); + assertThat(result.getMessageMetadata().getTimestamp()).isEqualTo(1L); + } - @Ignore - @Test(expected = UnwrappingException.class) - public void shouldThrowExceptionWhenMetadataNotFound() { - contentWrapper.unwrapContent(testContent); - } + @Ignore + @Test(expected = UnwrappingException.class) + public void shouldThrowExceptionWhenMetadataNotFound() { + contentWrapper.unwrapContent(testContent); + } - private Map readMap(byte[] result) { - try { - return mapper.readValue(new String(result), new TypeReference>(){}); - } catch (IOException e) { - throw new IllegalStateException("Error while reading map", e); - } + private Map readMap(byte[] result) { + try { + return mapper.readValue(new String(result), new TypeReference>() {}); + } catch (IOException e) { + throw new IllegalStateException("Error while reading map", e); } - + } } diff --git a/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/wrapper/MessageContentWrapperTest.java b/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/wrapper/MessageContentWrapperTest.java index af5b937bed..2f8507021f 100644 --- a/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/wrapper/MessageContentWrapperTest.java +++ b/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/wrapper/MessageContentWrapperTest.java @@ -1,11 +1,19 @@ package pl.allegro.tech.hermes.common.message.wrapper; +import static java.util.Arrays.asList; +import static org.assertj.core.api.Assertions.assertThat; +import static pl.allegro.tech.hermes.test.helper.avro.AvroUserSchemaLoader.load; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topic; + import com.fasterxml.jackson.databind.ObjectMapper; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Tags; import io.micrometer.core.instrument.search.Search; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import java.nio.ByteBuffer; +import java.time.Clock; +import java.util.List; import org.apache.avro.Schema; import org.apache.commons.collections4.map.HashedMap; import org.junit.Test; @@ -21,366 +29,418 @@ import pl.allegro.tech.hermes.test.helper.avro.AvroUser; import pl.allegro.tech.hermes.test.helper.metrics.MicrometerUtils; -import java.nio.ByteBuffer; -import java.time.Clock; -import java.util.List; - -import static java.util.Arrays.asList; -import static org.assertj.core.api.Assertions.assertThat; -import static pl.allegro.tech.hermes.test.helper.avro.AvroUserSchemaLoader.load; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topic; - public class MessageContentWrapperTest { - private static final Integer NO_VERSION_IN_HEADER = null; - private static final Integer NO_ID_IN_HEADER = null; - private static final int MESSAGE_TIMESTAMP = 1582029457; - private static final HashedMap NO_EXTERNAL_METADATA = new HashedMap<>(); - private static final String MESSAGE_ID = "messageId-1"; - private static final int VERSION_ONE = 1; - private static final int VERSION_TWO = 2; - private static final int VERSION_THREE = 3; - private static final int ID_ONE = 1; - private static final int ID_THREE = 3; - private static final int ID_FIVE = 5; - - private final MeterRegistry meterRegistry = new SimpleMeterRegistry(); - private final MetricsFacade metricsFacade = new MetricsFacade(meterRegistry); - private final JsonMessageContentWrapper jsonWrapper = new JsonMessageContentWrapper("message", "metadata", new ObjectMapper()); - private final AvroMessageContentWrapper avroWrapper = new AvroMessageContentWrapper(Clock.systemDefaultZone()); - - - private final AvroMessageHeaderSchemaVersionContentWrapper headerSchemaVersionWrapper = - new AvroMessageHeaderSchemaVersionContentWrapper(schemaRepository, avroWrapper, metricsFacade); - private final AvroMessageSchemaIdAwareContentWrapper schemaAwareWrapper = - new AvroMessageSchemaIdAwareContentWrapper(schemaRepository, avroWrapper, metricsFacade); - - private static final CompiledSchema schema1 = CompiledSchema.of(load("/schema/user.avsc"), ID_ONE, VERSION_ONE); - private static final CompiledSchema schema2 = CompiledSchema.of(load("/schema/user_v2.avsc"), ID_THREE, VERSION_TWO); - private static final CompiledSchema schema3 = CompiledSchema.of(load("/schema/user_v3.avsc"), ID_FIVE, VERSION_THREE); - - private CompositeMessageContentWrapper createMessageContentWrapper( - boolean schemaHeaderIdEnabled, - boolean schemaVersionTruncationEnabled - ) { - - final AvroMessageHeaderSchemaIdContentWrapper headerSchemaIdWrapper = - new AvroMessageHeaderSchemaIdContentWrapper(schemaRepository, avroWrapper, metricsFacade, schemaHeaderIdEnabled); - - final AvroMessageSchemaVersionTruncationContentWrapper schemaIdAndHeaderContentWrapper = - new AvroMessageSchemaVersionTruncationContentWrapper( - schemaRepository, - avroWrapper, - metricsFacade, - schemaVersionTruncationEnabled - ); - - return new CompositeMessageContentWrapper(jsonWrapper, avroWrapper, schemaAwareWrapper, - headerSchemaVersionWrapper, headerSchemaIdWrapper, schemaIdAndHeaderContentWrapper); - } - - private final CompositeMessageContentWrapper compositeMessageContentWrapper = createMessageContentWrapper(false, true); - - static SchemaVersionsRepository schemaVersionsRepository = new SchemaVersionsRepository() { + private static final Integer NO_VERSION_IN_HEADER = null; + private static final Integer NO_ID_IN_HEADER = null; + private static final int MESSAGE_TIMESTAMP = 1582029457; + private static final HashedMap NO_EXTERNAL_METADATA = new HashedMap<>(); + private static final String MESSAGE_ID = "messageId-1"; + private static final int VERSION_ONE = 1; + private static final int VERSION_TWO = 2; + private static final int VERSION_THREE = 3; + private static final int ID_ONE = 1; + private static final int ID_THREE = 3; + private static final int ID_FIVE = 5; + + private final MeterRegistry meterRegistry = new SimpleMeterRegistry(); + private final MetricsFacade metricsFacade = new MetricsFacade(meterRegistry); + private final JsonMessageContentWrapper jsonWrapper = + new JsonMessageContentWrapper("message", "metadata", new ObjectMapper()); + private final AvroMessageContentWrapper avroWrapper = + new AvroMessageContentWrapper(Clock.systemDefaultZone()); + + private final AvroMessageHeaderSchemaVersionContentWrapper headerSchemaVersionWrapper = + new AvroMessageHeaderSchemaVersionContentWrapper( + schemaRepository, avroWrapper, metricsFacade); + private final AvroMessageSchemaIdAwareContentWrapper schemaAwareWrapper = + new AvroMessageSchemaIdAwareContentWrapper(schemaRepository, avroWrapper, metricsFacade); + + private static final CompiledSchema schema1 = + CompiledSchema.of(load("/schema/user.avsc"), ID_ONE, VERSION_ONE); + private static final CompiledSchema schema2 = + CompiledSchema.of(load("/schema/user_v2.avsc"), ID_THREE, VERSION_TWO); + private static final CompiledSchema schema3 = + CompiledSchema.of(load("/schema/user_v3.avsc"), ID_FIVE, VERSION_THREE); + + private CompositeMessageContentWrapper createMessageContentWrapper( + boolean schemaHeaderIdEnabled, boolean schemaVersionTruncationEnabled) { + + final AvroMessageHeaderSchemaIdContentWrapper headerSchemaIdWrapper = + new AvroMessageHeaderSchemaIdContentWrapper( + schemaRepository, avroWrapper, metricsFacade, schemaHeaderIdEnabled); + + final AvroMessageSchemaVersionTruncationContentWrapper schemaIdAndHeaderContentWrapper = + new AvroMessageSchemaVersionTruncationContentWrapper( + schemaRepository, avroWrapper, metricsFacade, schemaVersionTruncationEnabled); + + return new CompositeMessageContentWrapper( + jsonWrapper, + avroWrapper, + schemaAwareWrapper, + headerSchemaVersionWrapper, + headerSchemaIdWrapper, + schemaIdAndHeaderContentWrapper); + } + + private final CompositeMessageContentWrapper compositeMessageContentWrapper = + createMessageContentWrapper(false, true); + + static SchemaVersionsRepository schemaVersionsRepository = + new SchemaVersionsRepository() { @Override public SchemaVersionsResult versions(Topic topic, boolean online) { - List onlineVersions = asList(schema3.getVersion(), schema2.getVersion(), schema1.getVersion()); - List cachedVersions = asList(schema2.getVersion(), schema1.getVersion()); - return online - ? SchemaVersionsResult.succeeded(onlineVersions) - : SchemaVersionsResult.succeeded(cachedVersions); + List onlineVersions = + asList(schema3.getVersion(), schema2.getVersion(), schema1.getVersion()); + List cachedVersions = asList(schema2.getVersion(), schema1.getVersion()); + return online + ? SchemaVersionsResult.succeeded(onlineVersions) + : SchemaVersionsResult.succeeded(cachedVersions); } @Override - public void close() { - } - }; + public void close() {} + }; - static CompiledSchemaRepository compiledSchemaRepository = new CompiledSchemaRepository() { + static CompiledSchemaRepository compiledSchemaRepository = + new CompiledSchemaRepository() { @Override - public CompiledSchema getSchema(Topic topic, SchemaVersion version, boolean online) { - switch (version.value()) { - case VERSION_ONE: - return schema1; - case VERSION_TWO: - return schema2; - case VERSION_THREE: - return schema3; - default: - throw new RuntimeException("sry"); - } + public CompiledSchema getSchema( + Topic topic, SchemaVersion version, boolean online) { + switch (version.value()) { + case VERSION_ONE: + return schema1; + case VERSION_TWO: + return schema2; + case VERSION_THREE: + return schema3; + default: + throw new RuntimeException("sry"); + } } @Override public CompiledSchema getSchema(Topic topic, SchemaId id) { - switch (id.value()) { - case ID_ONE: - return schema1; - case ID_THREE: - return schema2; - case ID_FIVE: - return schema3; - default: - throw new RuntimeException("sry"); - } + switch (id.value()) { + case ID_ONE: + return schema1; + case ID_THREE: + return schema2; + case ID_FIVE: + return schema3; + default: + throw new RuntimeException("sry"); + } } - }; - - static SchemaRepository schemaRepository = new SchemaRepository(schemaVersionsRepository, compiledSchemaRepository); - - - @Test - public void shouldUnwrapMessageUsingSchemaIdFromPayload() { - // given - String messageId = MESSAGE_ID; - int messageTimestamp = MESSAGE_TIMESTAMP; - - SchemaId schemaId = createSchemaId(ID_FIVE); - Topic topic = createTopicWithSchemaIdAwarePayload(); - AvroUser user = createAvroUser(schemaId, topic); - - byte[] wrapped = compositeMessageContentWrapper.wrapAvro( - user.asBytes(), - messageId, - messageTimestamp, - topic, - user.getCompiledSchema(), - NO_EXTERNAL_METADATA - ); - - // when - UnwrappedMessageContent unwrappedMessageContent = - compositeMessageContentWrapper.unwrapAvro(wrapped, topic, NO_ID_IN_HEADER, NO_VERSION_IN_HEADER); - - // then - assertResult(unwrappedMessageContent, schemaId, user.asBytes(), messageId, messageTimestamp); - assertMetrics(0, 0, 0, 0, 0, 1, 0, 0, 0); - } - - @Test - public void shouldUnwrapUsingHeaderSchemaVersionIfHeaderPresent() { - // given - String messageId = MESSAGE_ID; - int messageTimestamp = MESSAGE_TIMESTAMP; - - SchemaVersion schemaVersion = createSchemaVersion(VERSION_TWO); - Topic topic = createTopic(); - AvroUser user = createAvroUser(schemaVersion, topic); - - byte[] wrapped = compositeMessageContentWrapper.wrapAvro( - user.asBytes(), - messageId, - messageTimestamp, - topic, - user.getCompiledSchema(), - NO_EXTERNAL_METADATA - ); - - // when - UnwrappedMessageContent unwrappedMessageContent = - compositeMessageContentWrapper.unwrapAvro(wrapped, topic, NO_ID_IN_HEADER, schemaVersion.value()); - - // then - assertResult(unwrappedMessageContent, schemaVersion, user.asBytes(), messageId, messageTimestamp); - assertMetrics(0, 0, 0, 0, 0, 0, 1, 0, 0); - } - - @Test - public void shouldUnwrapUsingHeaderSchemaIdIfHeaderPresent() { - // given - CompositeMessageContentWrapper compositeMessageContentWrapperWithHeaderEnabled = createMessageContentWrapper(true, false); - String messageId = MESSAGE_ID; - int messageTimestamp = MESSAGE_TIMESTAMP; - - SchemaId schemaId = createSchemaId(ID_THREE); - Topic topic = createTopic(); - AvroUser user = createAvroUser(schemaId, topic); - - byte[] wrapped = compositeMessageContentWrapperWithHeaderEnabled - .wrapAvro(user.asBytes(), messageId, messageTimestamp, topic, user.getCompiledSchema(), NO_EXTERNAL_METADATA); - - // when - UnwrappedMessageContent unwrappedMessageContent = compositeMessageContentWrapperWithHeaderEnabled - .unwrapAvro(wrapped, topic, schemaId.value(), NO_VERSION_IN_HEADER); - - // then - assertResult(unwrappedMessageContent, schemaId, user.asBytes(), messageId, messageTimestamp); - assertMetrics(0, 0, 0, 0, 0, 0, 0, 1, 0); - } - - @Test - public void shouldUnwrapUsingSchemaIdAwareIfVersionAndIdInSchemaPresentDespiteServiceHeaderPresent() { - // given - String messageId = MESSAGE_ID; - int messageTimestamp = MESSAGE_TIMESTAMP; - - SchemaId schemaId = createSchemaId(ID_THREE); - Topic topic = createTopicWithSchemaIdAwarePayload(); - AvroUser user = createAvroUser(schemaId, topic); - CompiledSchema schema = user.getCompiledSchema(); - - byte[] wrapped = - compositeMessageContentWrapper.wrapAvro(user.asBytes(), messageId, messageTimestamp, topic, schema, NO_EXTERNAL_METADATA); - - // when - UnwrappedMessageContent unwrappedMessageContent = - compositeMessageContentWrapper.unwrapAvro(wrapped, topic, schema.getId().value(), schema.getVersion().value()); - - // then - assertResult(unwrappedMessageContent, schema.getVersion(), user.asBytes(), messageId, messageTimestamp); - assertMetrics(0, 0, 0, 0, 0, 1, 0, 0, 0); - } - - @Test - public void shouldUnwrapUsingHeaderSchemaVersionIfHeaderPresentAndNoMagicByte() { - // given - String messageId = MESSAGE_ID; - int messageTimestamp = MESSAGE_TIMESTAMP; - - SchemaVersion schemaVersion = createSchemaVersion(VERSION_TWO); // no magic byte - Topic topicToWrap = createTopic(); - AvroUser user = createAvroUser(schemaVersion, topicToWrap); - - byte[] wrapped = - compositeMessageContentWrapper - .wrapAvro(user.asBytes(), messageId, messageTimestamp, topicToWrap, user.getCompiledSchema(), NO_EXTERNAL_METADATA); - - Topic topicToUnwrap = createTopicWithSchemaIdAwarePayload(); - - // when - UnwrappedMessageContent unwrappedMessageContent = - compositeMessageContentWrapper.unwrapAvro(wrapped, topicToUnwrap, NO_ID_IN_HEADER, schemaVersion.value()); - - // then - assertResult(unwrappedMessageContent, schemaVersion, user.asBytes(), messageId, messageTimestamp); - - // missedSchemaVersionInPayload == no magic byte - assertMetrics(1, 0, 0, 0, 0, 0, 1, 0, 0); - } - - @Test - public void shouldTrimSchemaVersionFromMessageWhenSchemaVersionPresentInHeader() { - // given - String messageId = MESSAGE_ID; - int messageTimestamp = MESSAGE_TIMESTAMP; - - SchemaVersion schemaVersion = createSchemaVersion(VERSION_ONE); - Topic topic = createTopic(); - AvroUser user = createAvroUser(schemaVersion, topic); - - byte[] message = - compositeMessageContentWrapper - .wrapAvro(user.asBytes(), messageId, messageTimestamp, topic, user.getCompiledSchema(), NO_EXTERNAL_METADATA); - byte[] wrapped = serializeWithSchemaVersionInPayload(schemaVersion, message); - - // when - UnwrappedMessageContent unwrappedMessageContent = - compositeMessageContentWrapper.unwrapAvro(wrapped, topic, NO_ID_IN_HEADER, VERSION_ONE); - - // then - assertResult(unwrappedMessageContent, schemaVersion, user.asBytes(), messageId, messageTimestamp); - assertMetrics(0, 0, 0, 0, 0, 0, 0, 0, 1); - } - - private void assertResult(UnwrappedMessageContent result, - SchemaVersion schemaVersion, - byte[] recordBytes, - String messageId, - int timestamp) { - assertThat(result.getSchema().get().getVersion()).isEqualTo(schemaVersion); - assertResult(result, recordBytes, messageId, timestamp); - } - - private void assertResult(UnwrappedMessageContent result, - SchemaId schemaId, - byte[] recordBytes, - String messageId, - int timestamp) { - assertThat(result.getSchema().get().getId()).isEqualTo(schemaId); - assertResult(result, recordBytes, messageId, timestamp); - } - - private void assertResult(UnwrappedMessageContent result, byte[] recordBytes, String messageId, int timestamp) { - assertThat(result.getContent()).contains(recordBytes); - assertThat(result.getMessageMetadata().getId()).isEqualTo(messageId); - assertThat(result.getMessageMetadata().getTimestamp()).isEqualTo(timestamp); - } - - private void assertMetrics(int missedSchemaIdInPayload, - int errorsForPayloadWithSchemaId, - int errorsForHeaderSchemaVersion, - int errorsForHeaderSchemaId, - int errorsWithSchemaVersionTruncation, - int usingSchemaIdAware, - int usingHeaderSchemaVersion, - int usingHeaderSchemaId, - int usingSchemaVersionTruncation) { - final String basePath = "content.avro.deserialization"; - assertThat(meterRegistryCounterValue(basePath + ".missing_schemaIdInPayload", Tags.empty())) - .isEqualTo(missedSchemaIdInPayload); - - assertThat(meterRegistryCounterValue(basePath + ".errors", Tags.of("deserialization_type", "payloadWithSchemaId"))) - .isEqualTo(errorsForPayloadWithSchemaId); - - assertThat(meterRegistryCounterValue(basePath + ".errors", Tags.of("deserialization_type", "headerSchemaVersion"))) - .isEqualTo(errorsForHeaderSchemaVersion); - - assertThat(meterRegistryCounterValue(basePath + ".errors", Tags.of("deserialization_type", "headerSchemaId"))) - .isEqualTo(errorsForHeaderSchemaId); - - assertThat(meterRegistryCounterValue(basePath + ".errors", Tags.of("deserialization_type", "schemaVersionTruncation"))) - .isEqualTo(errorsWithSchemaVersionTruncation); - - assertThat(meterRegistryCounterValue(basePath, Tags.of("deserialization_type", "payloadWithSchemaId"))) - .isEqualTo(usingSchemaIdAware); - - assertThat(meterRegistryCounterValue(basePath, Tags.of("deserialization_type", "headerSchemaVersion"))) - .isEqualTo(usingHeaderSchemaVersion); - - assertThat(meterRegistryCounterValue(basePath, Tags.of("deserialization_type", "headerSchemaId"))).isEqualTo(usingHeaderSchemaId); - - assertThat(meterRegistryCounterValue(basePath, Tags.of("deserialization_type", "schemaVersionTruncation"))) - .isEqualTo(usingSchemaVersionTruncation); - } - - private int meterRegistryCounterValue(String metricName, Tags tags) { - return MicrometerUtils.metricValue(meterRegistry, metricName, tags, Search::counter, Counter::count).orElse(0.0d).intValue(); - } - - - private Topic createTopic() { - return topic("group", "topic").build(); - } - - private Topic createTopicWithSchemaIdAwarePayload() { - return topic("group", "topic-idAware").withSchemaIdAwareSerialization().build(); - } - - private AvroUser createAvroUser(SchemaVersion schemaVersion, Topic topic) { - CompiledSchema schema = schemaRepository.getKnownAvroSchemaVersion(topic, schemaVersion); - return new AvroUser(schema, "user-1", 15, "colour-1"); - } - - private AvroUser createAvroUser(SchemaId id, Topic topic) { - CompiledSchema schema = schemaRepository.getAvroSchema(topic, id); - return new AvroUser(schema, "user-1", 15, "colour-1"); - } - - private static SchemaVersion createSchemaVersion(int schemaVersionValue) { - return SchemaVersion.valueOf(schemaVersionValue); - } - - private static SchemaId createSchemaId(int schemaIdValue) { - return SchemaId.valueOf(schemaIdValue); - } - - private static byte[] serializeWithSchemaVersionInPayload(SchemaVersion schemaVersion, byte[] data) { - final int HEADER_SIZE = 5; - final byte MAGIC_BYTE_VALUE = 0; - ByteBuffer buffer = ByteBuffer.allocate(HEADER_SIZE + data.length); - buffer.put(MAGIC_BYTE_VALUE); - buffer.putInt(schemaVersion.value()); - buffer.put(data); - return buffer.array(); - } + }; + + static SchemaRepository schemaRepository = + new SchemaRepository(schemaVersionsRepository, compiledSchemaRepository); + + @Test + public void shouldUnwrapMessageUsingSchemaIdFromPayload() { + // given + String messageId = MESSAGE_ID; + int messageTimestamp = MESSAGE_TIMESTAMP; + + SchemaId schemaId = createSchemaId(ID_FIVE); + Topic topic = createTopicWithSchemaIdAwarePayload(); + AvroUser user = createAvroUser(schemaId, topic); + + byte[] wrapped = + compositeMessageContentWrapper.wrapAvro( + user.asBytes(), + messageId, + messageTimestamp, + topic, + user.getCompiledSchema(), + NO_EXTERNAL_METADATA); + + // when + UnwrappedMessageContent unwrappedMessageContent = + compositeMessageContentWrapper.unwrapAvro( + wrapped, topic, NO_ID_IN_HEADER, NO_VERSION_IN_HEADER); + + // then + assertResult(unwrappedMessageContent, schemaId, user.asBytes(), messageId, messageTimestamp); + assertMetrics(0, 0, 0, 0, 0, 1, 0, 0, 0); + } + + @Test + public void shouldUnwrapUsingHeaderSchemaVersionIfHeaderPresent() { + // given + String messageId = MESSAGE_ID; + int messageTimestamp = MESSAGE_TIMESTAMP; + + SchemaVersion schemaVersion = createSchemaVersion(VERSION_TWO); + Topic topic = createTopic(); + AvroUser user = createAvroUser(schemaVersion, topic); + + byte[] wrapped = + compositeMessageContentWrapper.wrapAvro( + user.asBytes(), + messageId, + messageTimestamp, + topic, + user.getCompiledSchema(), + NO_EXTERNAL_METADATA); + + // when + UnwrappedMessageContent unwrappedMessageContent = + compositeMessageContentWrapper.unwrapAvro( + wrapped, topic, NO_ID_IN_HEADER, schemaVersion.value()); + + // then + assertResult( + unwrappedMessageContent, schemaVersion, user.asBytes(), messageId, messageTimestamp); + assertMetrics(0, 0, 0, 0, 0, 0, 1, 0, 0); + } + + @Test + public void shouldUnwrapUsingHeaderSchemaIdIfHeaderPresent() { + // given + CompositeMessageContentWrapper compositeMessageContentWrapperWithHeaderEnabled = + createMessageContentWrapper(true, false); + String messageId = MESSAGE_ID; + int messageTimestamp = MESSAGE_TIMESTAMP; + + SchemaId schemaId = createSchemaId(ID_THREE); + Topic topic = createTopic(); + AvroUser user = createAvroUser(schemaId, topic); + + byte[] wrapped = + compositeMessageContentWrapperWithHeaderEnabled.wrapAvro( + user.asBytes(), + messageId, + messageTimestamp, + topic, + user.getCompiledSchema(), + NO_EXTERNAL_METADATA); + + // when + UnwrappedMessageContent unwrappedMessageContent = + compositeMessageContentWrapperWithHeaderEnabled.unwrapAvro( + wrapped, topic, schemaId.value(), NO_VERSION_IN_HEADER); + + // then + assertResult(unwrappedMessageContent, schemaId, user.asBytes(), messageId, messageTimestamp); + assertMetrics(0, 0, 0, 0, 0, 0, 0, 1, 0); + } + + @Test + public void + shouldUnwrapUsingSchemaIdAwareIfVersionAndIdInSchemaPresentDespiteServiceHeaderPresent() { + // given + String messageId = MESSAGE_ID; + int messageTimestamp = MESSAGE_TIMESTAMP; + + SchemaId schemaId = createSchemaId(ID_THREE); + Topic topic = createTopicWithSchemaIdAwarePayload(); + AvroUser user = createAvroUser(schemaId, topic); + CompiledSchema schema = user.getCompiledSchema(); + + byte[] wrapped = + compositeMessageContentWrapper.wrapAvro( + user.asBytes(), messageId, messageTimestamp, topic, schema, NO_EXTERNAL_METADATA); + + // when + UnwrappedMessageContent unwrappedMessageContent = + compositeMessageContentWrapper.unwrapAvro( + wrapped, topic, schema.getId().value(), schema.getVersion().value()); + + // then + assertResult( + unwrappedMessageContent, schema.getVersion(), user.asBytes(), messageId, messageTimestamp); + assertMetrics(0, 0, 0, 0, 0, 1, 0, 0, 0); + } + + @Test + public void shouldUnwrapUsingHeaderSchemaVersionIfHeaderPresentAndNoMagicByte() { + // given + String messageId = MESSAGE_ID; + int messageTimestamp = MESSAGE_TIMESTAMP; + + SchemaVersion schemaVersion = createSchemaVersion(VERSION_TWO); // no magic byte + Topic topicToWrap = createTopic(); + AvroUser user = createAvroUser(schemaVersion, topicToWrap); + + byte[] wrapped = + compositeMessageContentWrapper.wrapAvro( + user.asBytes(), + messageId, + messageTimestamp, + topicToWrap, + user.getCompiledSchema(), + NO_EXTERNAL_METADATA); + + Topic topicToUnwrap = createTopicWithSchemaIdAwarePayload(); + + // when + UnwrappedMessageContent unwrappedMessageContent = + compositeMessageContentWrapper.unwrapAvro( + wrapped, topicToUnwrap, NO_ID_IN_HEADER, schemaVersion.value()); + + // then + assertResult( + unwrappedMessageContent, schemaVersion, user.asBytes(), messageId, messageTimestamp); + + // missedSchemaVersionInPayload == no magic byte + assertMetrics(1, 0, 0, 0, 0, 0, 1, 0, 0); + } + + @Test + public void shouldTrimSchemaVersionFromMessageWhenSchemaVersionPresentInHeader() { + // given + String messageId = MESSAGE_ID; + int messageTimestamp = MESSAGE_TIMESTAMP; + + SchemaVersion schemaVersion = createSchemaVersion(VERSION_ONE); + Topic topic = createTopic(); + AvroUser user = createAvroUser(schemaVersion, topic); + + byte[] message = + compositeMessageContentWrapper.wrapAvro( + user.asBytes(), + messageId, + messageTimestamp, + topic, + user.getCompiledSchema(), + NO_EXTERNAL_METADATA); + byte[] wrapped = serializeWithSchemaVersionInPayload(schemaVersion, message); + + // when + UnwrappedMessageContent unwrappedMessageContent = + compositeMessageContentWrapper.unwrapAvro(wrapped, topic, NO_ID_IN_HEADER, VERSION_ONE); + + // then + assertResult( + unwrappedMessageContent, schemaVersion, user.asBytes(), messageId, messageTimestamp); + assertMetrics(0, 0, 0, 0, 0, 0, 0, 0, 1); + } + + private void assertResult( + UnwrappedMessageContent result, + SchemaVersion schemaVersion, + byte[] recordBytes, + String messageId, + int timestamp) { + assertThat(result.getSchema().get().getVersion()).isEqualTo(schemaVersion); + assertResult(result, recordBytes, messageId, timestamp); + } + + private void assertResult( + UnwrappedMessageContent result, + SchemaId schemaId, + byte[] recordBytes, + String messageId, + int timestamp) { + assertThat(result.getSchema().get().getId()).isEqualTo(schemaId); + assertResult(result, recordBytes, messageId, timestamp); + } + + private void assertResult( + UnwrappedMessageContent result, byte[] recordBytes, String messageId, int timestamp) { + assertThat(result.getContent()).contains(recordBytes); + assertThat(result.getMessageMetadata().getId()).isEqualTo(messageId); + assertThat(result.getMessageMetadata().getTimestamp()).isEqualTo(timestamp); + } + + private void assertMetrics( + int missedSchemaIdInPayload, + int errorsForPayloadWithSchemaId, + int errorsForHeaderSchemaVersion, + int errorsForHeaderSchemaId, + int errorsWithSchemaVersionTruncation, + int usingSchemaIdAware, + int usingHeaderSchemaVersion, + int usingHeaderSchemaId, + int usingSchemaVersionTruncation) { + final String basePath = "content.avro.deserialization"; + assertThat(meterRegistryCounterValue(basePath + ".missing_schemaIdInPayload", Tags.empty())) + .isEqualTo(missedSchemaIdInPayload); + + assertThat( + meterRegistryCounterValue( + basePath + ".errors", Tags.of("deserialization_type", "payloadWithSchemaId"))) + .isEqualTo(errorsForPayloadWithSchemaId); + + assertThat( + meterRegistryCounterValue( + basePath + ".errors", Tags.of("deserialization_type", "headerSchemaVersion"))) + .isEqualTo(errorsForHeaderSchemaVersion); + + assertThat( + meterRegistryCounterValue( + basePath + ".errors", Tags.of("deserialization_type", "headerSchemaId"))) + .isEqualTo(errorsForHeaderSchemaId); + + assertThat( + meterRegistryCounterValue( + basePath + ".errors", Tags.of("deserialization_type", "schemaVersionTruncation"))) + .isEqualTo(errorsWithSchemaVersionTruncation); + + assertThat( + meterRegistryCounterValue( + basePath, Tags.of("deserialization_type", "payloadWithSchemaId"))) + .isEqualTo(usingSchemaIdAware); + + assertThat( + meterRegistryCounterValue( + basePath, Tags.of("deserialization_type", "headerSchemaVersion"))) + .isEqualTo(usingHeaderSchemaVersion); + + assertThat( + meterRegistryCounterValue(basePath, Tags.of("deserialization_type", "headerSchemaId"))) + .isEqualTo(usingHeaderSchemaId); + + assertThat( + meterRegistryCounterValue( + basePath, Tags.of("deserialization_type", "schemaVersionTruncation"))) + .isEqualTo(usingSchemaVersionTruncation); + } + + private int meterRegistryCounterValue(String metricName, Tags tags) { + return MicrometerUtils.metricValue( + meterRegistry, metricName, tags, Search::counter, Counter::count) + .orElse(0.0d) + .intValue(); + } + + private Topic createTopic() { + return topic("group", "topic").build(); + } + + private Topic createTopicWithSchemaIdAwarePayload() { + return topic("group", "topic-idAware").withSchemaIdAwareSerialization().build(); + } + + private AvroUser createAvroUser(SchemaVersion schemaVersion, Topic topic) { + CompiledSchema schema = + schemaRepository.getKnownAvroSchemaVersion(topic, schemaVersion); + return new AvroUser(schema, "user-1", 15, "colour-1"); + } + + private AvroUser createAvroUser(SchemaId id, Topic topic) { + CompiledSchema schema = schemaRepository.getAvroSchema(topic, id); + return new AvroUser(schema, "user-1", 15, "colour-1"); + } + + private static SchemaVersion createSchemaVersion(int schemaVersionValue) { + return SchemaVersion.valueOf(schemaVersionValue); + } + + private static SchemaId createSchemaId(int schemaIdValue) { + return SchemaId.valueOf(schemaIdValue); + } + + private static byte[] serializeWithSchemaVersionInPayload( + SchemaVersion schemaVersion, byte[] data) { + final int HEADER_SIZE = 5; + final byte MAGIC_BYTE_VALUE = 0; + ByteBuffer buffer = ByteBuffer.allocate(HEADER_SIZE + data.length); + buffer.put(MAGIC_BYTE_VALUE); + buffer.putInt(schemaVersion.value()); + buffer.put(data); + return buffer.array(); + } } diff --git a/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaAwareSerDeTest.java b/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaAwareSerDeTest.java index a11f0b8833..8efa9d7212 100644 --- a/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaAwareSerDeTest.java +++ b/hermes-common/src/test/java/pl/allegro/tech/hermes/common/message/wrapper/SchemaAwareSerDeTest.java @@ -1,43 +1,44 @@ package pl.allegro.tech.hermes.common.message.wrapper; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertThrows; + import org.junit.Test; import pl.allegro.tech.hermes.schema.SchemaId; import pl.allegro.tech.hermes.test.helper.avro.AvroUser; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.assertThrows; - public class SchemaAwareSerDeTest { - static final AvroUser avro = new AvroUser("bob", 10, "red"); - - @Test - public void shouldSerialize() { - // given - SchemaId id = SchemaId.valueOf(8); - - // when - byte[] serialized = SchemaAwareSerDe.serialize(id, avro.asBytes()); - - // then - assertThat(serialized).startsWith((byte) 0); - } - - @Test - public void shouldDeserialize() { - // given - byte[] serialized = SchemaAwareSerDe.serialize(SchemaId.valueOf(8), avro.asBytes()); - - // when - SchemaAwarePayload deserialized = SchemaAwareSerDe.deserialize(serialized); - - // then - assertThat(deserialized.getSchemaId().value()).isEqualTo(8); - assertThat(deserialized.getPayload()).isEqualTo(avro.asBytes()); - } - - @Test - public void shouldThrowExceptionWhenDeserializingWithoutMagicByte() { - // when - assertThrows(DeserializationException.class, () -> SchemaAwareSerDe.deserialize(new byte[]{1, 2, 3})); - } + static final AvroUser avro = new AvroUser("bob", 10, "red"); + + @Test + public void shouldSerialize() { + // given + SchemaId id = SchemaId.valueOf(8); + + // when + byte[] serialized = SchemaAwareSerDe.serialize(id, avro.asBytes()); + + // then + assertThat(serialized).startsWith((byte) 0); + } + + @Test + public void shouldDeserialize() { + // given + byte[] serialized = SchemaAwareSerDe.serialize(SchemaId.valueOf(8), avro.asBytes()); + + // when + SchemaAwarePayload deserialized = SchemaAwareSerDe.deserialize(serialized); + + // then + assertThat(deserialized.getSchemaId().value()).isEqualTo(8); + assertThat(deserialized.getPayload()).isEqualTo(avro.asBytes()); + } + + @Test + public void shouldThrowExceptionWhenDeserializingWithoutMagicByte() { + // when + assertThrows( + DeserializationException.class, () -> SchemaAwareSerDe.deserialize(new byte[] {1, 2, 3})); + } } diff --git a/hermes-common/src/test/java/pl/allegro/tech/hermes/common/metric/counter/MetricsDeltaCalculatorTest.java b/hermes-common/src/test/java/pl/allegro/tech/hermes/common/metric/counter/MetricsDeltaCalculatorTest.java index 57a40c7988..cffce230aa 100644 --- a/hermes-common/src/test/java/pl/allegro/tech/hermes/common/metric/counter/MetricsDeltaCalculatorTest.java +++ b/hermes-common/src/test/java/pl/allegro/tech/hermes/common/metric/counter/MetricsDeltaCalculatorTest.java @@ -1,53 +1,53 @@ package pl.allegro.tech.hermes.common.metric.counter; -import org.junit.Test; - import static org.assertj.core.api.Assertions.assertThat; +import org.junit.Test; + public class MetricsDeltaCalculatorTest { - private final MetricsDeltaCalculator calculator = new MetricsDeltaCalculator(); + private final MetricsDeltaCalculator calculator = new MetricsDeltaCalculator(); - @Test - public void shouldReturnCurrentValueWhenThereIsNoPreviousStateForMetric() { - // given when - long delta = calculator.calculateDelta("metricWithoutHistory", 13L); + @Test + public void shouldReturnCurrentValueWhenThereIsNoPreviousStateForMetric() { + // given when + long delta = calculator.calculateDelta("metricWithoutHistory", 13L); - // then - assertThat(delta).isEqualTo(13); - } + // then + assertThat(delta).isEqualTo(13); + } - @Test - public void shouldReturnDeltaBetweenCurrentAndPreviousState() { - // given - calculator.calculateDelta("metric", 13L); + @Test + public void shouldReturnDeltaBetweenCurrentAndPreviousState() { + // given + calculator.calculateDelta("metric", 13L); - // when - long delta = calculator.calculateDelta("metric", 20L); + // when + long delta = calculator.calculateDelta("metric", 20L); - // then - assertThat(delta).isEqualTo(7); - } + // then + assertThat(delta).isEqualTo(7); + } - @Test - public void shouldRevertDelta() { - //given - String metricName = "metricToRevert"; - calculator.calculateDelta(metricName, 15L); - long delta = calculator.calculateDelta(metricName, 20L); + @Test + public void shouldRevertDelta() { + // given + String metricName = "metricToRevert"; + calculator.calculateDelta(metricName, 15L); + long delta = calculator.calculateDelta(metricName, 20L); - //when - calculator.revertDelta(metricName, delta); + // when + calculator.revertDelta(metricName, delta); - assertThat(calculator.calculateDelta(metricName, 20L)).isEqualTo(5L); - } + assertThat(calculator.calculateDelta(metricName, 20L)).isEqualTo(5L); + } - @Test - public void shouldNotRevertDeltaForNonExistingMetric() { - // when - calculator.revertDelta("emptyMetric", 10L); + @Test + public void shouldNotRevertDeltaForNonExistingMetric() { + // when + calculator.revertDelta("emptyMetric", 10L); - // then - assertThat(calculator.calculateDelta("emptyMetric", 10L)).isEqualTo(10L); - } + // then + assertThat(calculator.calculateDelta("emptyMetric", 10L)).isEqualTo(10L); + } } diff --git a/hermes-common/src/test/java/pl/allegro/tech/hermes/common/metric/counter/zookeeper/ZookeeperCounterReporterTest.java b/hermes-common/src/test/java/pl/allegro/tech/hermes/common/metric/counter/zookeeper/ZookeeperCounterReporterTest.java index a974a283b0..8e009bd91d 100644 --- a/hermes-common/src/test/java/pl/allegro/tech/hermes/common/metric/counter/zookeeper/ZookeeperCounterReporterTest.java +++ b/hermes-common/src/test/java/pl/allegro/tech/hermes/common/metric/counter/zookeeper/ZookeeperCounterReporterTest.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.common.metric.counter.zookeeper; +import static org.mockito.Mockito.verify; + import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import org.junit.Before; @@ -13,92 +15,89 @@ import pl.allegro.tech.hermes.common.metric.counter.CounterStorage; import pl.allegro.tech.hermes.common.util.InstanceIdResolver; -import static org.mockito.Mockito.verify; - @RunWith(MockitoJUnitRunner.class) public class ZookeeperCounterReporterTest { - public static final String GROUP_NAME = "pl.allegro.tech.skylab"; - public static final String TOPIC_NAME_UNDERSCORE = "topic_1"; - public static final String SUBSCRIPTION_NAME = "subscription.name"; - public static final TopicName topic = new TopicName(GROUP_NAME, TOPIC_NAME_UNDERSCORE); - public static final SubscriptionName subscription = new SubscriptionName(SUBSCRIPTION_NAME, topic); + public static final String GROUP_NAME = "pl.allegro.tech.skylab"; + public static final String TOPIC_NAME_UNDERSCORE = "topic_1"; + public static final String SUBSCRIPTION_NAME = "subscription.name"; + public static final TopicName topic = new TopicName(GROUP_NAME, TOPIC_NAME_UNDERSCORE); + public static final SubscriptionName subscription = + new SubscriptionName(SUBSCRIPTION_NAME, topic); - public static final long COUNT = 100L; + public static final long COUNT = 100L; - @Mock - private CounterStorage counterStorage; + @Mock private CounterStorage counterStorage; - private final MeterRegistry meterRegistry = new SimpleMeterRegistry(); + private final MeterRegistry meterRegistry = new SimpleMeterRegistry(); - private final MetricsFacade metricsFacade = new MetricsFacade(meterRegistry); + private final MetricsFacade metricsFacade = new MetricsFacade(meterRegistry); - @Mock - private InstanceIdResolver instanceIdResolver; + @Mock private InstanceIdResolver instanceIdResolver; - private ZookeeperCounterReporter zookeeperCounterReporter; + private ZookeeperCounterReporter zookeeperCounterReporter; - @Before - public void before() { - zookeeperCounterReporter = new ZookeeperCounterReporter(meterRegistry, counterStorage, ""); - } + @Before + public void before() { + zookeeperCounterReporter = new ZookeeperCounterReporter(meterRegistry, counterStorage, ""); + } - @Test - public void shouldReportPublishedMessages() { - // given - metricsFacade.topics().topicPublished(topic, "dc1").increment(COUNT); + @Test + public void shouldReportPublishedMessages() { + // given + metricsFacade.topics().topicPublished(topic, "dc1").increment(COUNT); - // when - zookeeperCounterReporter.report(); + // when + zookeeperCounterReporter.report(); - // then - verify(counterStorage).setTopicPublishedCounter(topic, COUNT); - } + // then + verify(counterStorage).setTopicPublishedCounter(topic, COUNT); + } - @Test - public void shouldReportDeliveredMessages() { - // given - metricsFacade.subscriptions().successes(subscription).increment(COUNT); + @Test + public void shouldReportDeliveredMessages() { + // given + metricsFacade.subscriptions().successes(subscription).increment(COUNT); - // when - zookeeperCounterReporter.report(); + // when + zookeeperCounterReporter.report(); - // then - verify(counterStorage).setSubscriptionDeliveredCounter(topic, SUBSCRIPTION_NAME, COUNT); - } + // then + verify(counterStorage).setSubscriptionDeliveredCounter(topic, SUBSCRIPTION_NAME, COUNT); + } - @Test - public void shouldReportDiscardedMessages() { - // given - metricsFacade.subscriptions().discarded(subscription).increment(COUNT); + @Test + public void shouldReportDiscardedMessages() { + // given + metricsFacade.subscriptions().discarded(subscription).increment(COUNT); - // when - zookeeperCounterReporter.report(); + // when + zookeeperCounterReporter.report(); - // then - verify(counterStorage).setSubscriptionDiscardedCounter(topic, SUBSCRIPTION_NAME, COUNT); - } + // then + verify(counterStorage).setSubscriptionDiscardedCounter(topic, SUBSCRIPTION_NAME, COUNT); + } - @Test - public void shouldReportSubscriptionVolumeCounter() { - // given - metricsFacade.subscriptions().throughputInBytes(subscription).increment(COUNT); + @Test + public void shouldReportSubscriptionVolumeCounter() { + // given + metricsFacade.subscriptions().throughputInBytes(subscription).increment(COUNT); - // when - zookeeperCounterReporter.report(); + // when + zookeeperCounterReporter.report(); - // then - verify(counterStorage).incrementVolumeCounter(topic, SUBSCRIPTION_NAME, COUNT); - } + // then + verify(counterStorage).incrementVolumeCounter(topic, SUBSCRIPTION_NAME, COUNT); + } - @Test - public void shouldReportTopicVolumeCounter() { - // given - metricsFacade.topics().topicThroughputBytes(topic).increment(COUNT); + @Test + public void shouldReportTopicVolumeCounter() { + // given + metricsFacade.topics().topicThroughputBytes(topic).increment(COUNT); - // when - zookeeperCounterReporter.report(); + // when + zookeeperCounterReporter.report(); - // then - verify(counterStorage).incrementVolumeCounter(topic, COUNT); - } -} \ No newline at end of file + // then + verify(counterStorage).incrementVolumeCounter(topic, COUNT); + } +} diff --git a/hermes-common/src/test/java/pl/allegro/tech/hermes/common/metric/counter/zookeeper/ZookeeperCounterStorageTest.java b/hermes-common/src/test/java/pl/allegro/tech/hermes/common/metric/counter/zookeeper/ZookeeperCounterStorageTest.java index 2d92eec12a..78bc2faa65 100644 --- a/hermes-common/src/test/java/pl/allegro/tech/hermes/common/metric/counter/zookeeper/ZookeeperCounterStorageTest.java +++ b/hermes-common/src/test/java/pl/allegro/tech/hermes/common/metric/counter/zookeeper/ZookeeperCounterStorageTest.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.common.metric.counter.zookeeper; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -12,82 +17,82 @@ import pl.allegro.tech.hermes.infrastructure.zookeeper.counter.SharedCounter; import pl.allegro.tech.hermes.metrics.PathsCompiler; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - @RunWith(MockitoJUnitRunner.class) public class ZookeeperCounterStorageTest { - @Mock - private SharedCounter sharedCounter; - - @Mock - private SubscriptionRepository subscriptionRepository; - - private ZookeeperCounterStorage storage; - - @Before - public void initialize() { - PathsCompiler pathCompiler = new PathsCompiler("my-host-example.net"); - storage = new ZookeeperCounterStorage(sharedCounter, subscriptionRepository, pathCompiler, "/hermes"); - } - - @Test - public void shouldIncrementTopicMetricUsingSharedCounter() { - //when - storage.setTopicPublishedCounter(TopicName.fromQualifiedName("test.topic"), 10); - - // then - verify(sharedCounter).increment("/hermes/groups/test/topics/topic/metrics/published", 10); - } - - @Test - public void shouldReadValueFromTopicMetric() { - // given - when(sharedCounter.getValue("/hermes/groups/test/topics/topic/metrics/published")).thenReturn(10L); - - // when - long value = storage.getTopicPublishedCounter(TopicName.fromQualifiedName("test.topic")); - - // then - assertThat(value).isEqualTo(10); - } - - @Test - public void shouldIncrementSubscriptionMetricUsingSharedCounter() { - // given when - storage.setSubscriptionDeliveredCounter(TopicName.fromQualifiedName("test.topic"), "sub", 10); - - // then - verify(sharedCounter).increment("/hermes/groups/test/topics/topic/subscriptions/sub/metrics/delivered", 10); - } - - @Test - public void shouldReadValueFromSubscriptionMetric() { - // given - when(sharedCounter.getValue("/hermes/groups/test/topics/topic/subscriptions/sub/metrics/delivered")).thenReturn(10L); - - // when - long value = storage.getSubscriptionDeliveredCounter(TopicName.fromQualifiedName("test.topic"), "sub"); - - // then - assertThat(value).isEqualTo(10); - } - - @Test - public void shouldNotIncrementSharedCounterForNonExistingSubscription() { - //given - TopicName topicName = TopicName.fromQualifiedName("test.topic"); - String subscriptionName = "sub"; - doThrow(new SubscriptionNotExistsException(topicName, subscriptionName)) - .when(subscriptionRepository).ensureSubscriptionExists(topicName, subscriptionName); - - //when - storage.setSubscriptionDeliveredCounter(topicName, subscriptionName, 1L); - - //then - Mockito.verifyNoInteractions(sharedCounter); - } -} \ No newline at end of file + @Mock private SharedCounter sharedCounter; + + @Mock private SubscriptionRepository subscriptionRepository; + + private ZookeeperCounterStorage storage; + + @Before + public void initialize() { + PathsCompiler pathCompiler = new PathsCompiler("my-host-example.net"); + storage = + new ZookeeperCounterStorage(sharedCounter, subscriptionRepository, pathCompiler, "/hermes"); + } + + @Test + public void shouldIncrementTopicMetricUsingSharedCounter() { + // when + storage.setTopicPublishedCounter(TopicName.fromQualifiedName("test.topic"), 10); + + // then + verify(sharedCounter).increment("/hermes/groups/test/topics/topic/metrics/published", 10); + } + + @Test + public void shouldReadValueFromTopicMetric() { + // given + when(sharedCounter.getValue("/hermes/groups/test/topics/topic/metrics/published")) + .thenReturn(10L); + + // when + long value = storage.getTopicPublishedCounter(TopicName.fromQualifiedName("test.topic")); + + // then + assertThat(value).isEqualTo(10); + } + + @Test + public void shouldIncrementSubscriptionMetricUsingSharedCounter() { + // given when + storage.setSubscriptionDeliveredCounter(TopicName.fromQualifiedName("test.topic"), "sub", 10); + + // then + verify(sharedCounter) + .increment("/hermes/groups/test/topics/topic/subscriptions/sub/metrics/delivered", 10); + } + + @Test + public void shouldReadValueFromSubscriptionMetric() { + // given + when(sharedCounter.getValue( + "/hermes/groups/test/topics/topic/subscriptions/sub/metrics/delivered")) + .thenReturn(10L); + + // when + long value = + storage.getSubscriptionDeliveredCounter(TopicName.fromQualifiedName("test.topic"), "sub"); + + // then + assertThat(value).isEqualTo(10); + } + + @Test + public void shouldNotIncrementSharedCounterForNonExistingSubscription() { + // given + TopicName topicName = TopicName.fromQualifiedName("test.topic"); + String subscriptionName = "sub"; + doThrow(new SubscriptionNotExistsException(topicName, subscriptionName)) + .when(subscriptionRepository) + .ensureSubscriptionExists(topicName, subscriptionName); + + // when + storage.setSubscriptionDeliveredCounter(topicName, subscriptionName, 1L); + + // then + Mockito.verifyNoInteractions(sharedCounter); + } +} diff --git a/hermes-common/src/test/java/pl/allegro/tech/hermes/infrastructure/zookeeper/counter/SharedCounterTest.java b/hermes-common/src/test/java/pl/allegro/tech/hermes/infrastructure/zookeeper/counter/SharedCounterTest.java index f3ab02804f..4f2996f757 100644 --- a/hermes-common/src/test/java/pl/allegro/tech/hermes/infrastructure/zookeeper/counter/SharedCounterTest.java +++ b/hermes-common/src/test/java/pl/allegro/tech/hermes/infrastructure/zookeeper/counter/SharedCounterTest.java @@ -1,44 +1,44 @@ package pl.allegro.tech.hermes.infrastructure.zookeeper.counter; +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Duration; import org.junit.Before; import org.junit.Test; import pl.allegro.tech.hermes.test.helper.zookeeper.ZookeeperBaseTest; -import java.time.Duration; - -import static org.assertj.core.api.Assertions.assertThat; - public class SharedCounterTest extends ZookeeperBaseTest { - private SharedCounter counter; - - @Before - public void initialize() { - this.counter = new SharedCounter(zookeeperClient, Duration.ofHours(72), Duration.ofSeconds(1), 3); - } - - @Test - public void shouldIncrementAndRetrieveCounterForGivenPath() { - // given when - counter.increment("/increment", 10); - wait.untilZookeeperPathIsCreated("/increment"); - - // then - assertThat(counter.getValue("/increment")).isEqualTo(10); - } - - @Test - public void shouldIncrementCounterAtomicallyWhenIncrementedConcurrently() { - // given - SharedCounter otherCounter = new SharedCounter(zookeeperClient, Duration.ofHours(72), Duration.ofSeconds(1), 3); - - // when - counter.increment("/sharedIncrement", 10); - otherCounter.increment("/sharedIncrement", 15); - wait.untilZookeeperPathIsCreated("/sharedIncrement"); - - // then - assertThat(counter.getValue("/sharedIncrement")).isEqualTo(25); - } - + private SharedCounter counter; + + @Before + public void initialize() { + this.counter = + new SharedCounter(zookeeperClient, Duration.ofHours(72), Duration.ofSeconds(1), 3); + } + + @Test + public void shouldIncrementAndRetrieveCounterForGivenPath() { + // given when + counter.increment("/increment", 10); + wait.untilZookeeperPathIsCreated("/increment"); + + // then + assertThat(counter.getValue("/increment")).isEqualTo(10); + } + + @Test + public void shouldIncrementCounterAtomicallyWhenIncrementedConcurrently() { + // given + SharedCounter otherCounter = + new SharedCounter(zookeeperClient, Duration.ofHours(72), Duration.ofSeconds(1), 3); + + // when + counter.increment("/sharedIncrement", 10); + otherCounter.increment("/sharedIncrement", 15); + wait.untilZookeeperPathIsCreated("/sharedIncrement"); + + // then + assertThat(counter.getValue("/sharedIncrement")).isEqualTo(25); + } } diff --git a/hermes-console/.gitignore b/hermes-console/.gitignore index 3c0efc83c8..b583b23b8b 100644 --- a/hermes-console/.gitignore +++ b/hermes-console/.gitignore @@ -21,7 +21,6 @@ coverage # Editor directories and files .vscode/* !.vscode/extensions.json -.idea *.suo *.ntvs* *.njsproj diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/CommonConsumerParameters.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/CommonConsumerParameters.java index 4d5a99f7db..4f6ff03778 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/CommonConsumerParameters.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/CommonConsumerParameters.java @@ -5,11 +5,11 @@ public interface CommonConsumerParameters { - SupervisorParameters getBackgroundSupervisor(); + SupervisorParameters getBackgroundSupervisor(); - SerialConsumerParameters getSerialConsumer(); + SerialConsumerParameters getSerialConsumer(); - int getSignalProcessingQueueSize(); + int getSignalProcessingQueueSize(); - boolean isUseTopicMessageSizeEnabled(); + boolean isUseTopicMessageSizeEnabled(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/ConsumerEndpoint.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/ConsumerEndpoint.java index f66e1fca9a..9513575a71 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/ConsumerEndpoint.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/ConsumerEndpoint.java @@ -1,28 +1,26 @@ package pl.allegro.tech.hermes.consumers; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; + import jakarta.ws.rs.GET; import jakarta.ws.rs.Path; import jakarta.ws.rs.Produces; -import pl.allegro.tech.hermes.consumers.supervisor.process.RunningSubscriptionStatus; - import java.util.List; - -import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; +import pl.allegro.tech.hermes.consumers.supervisor.process.RunningSubscriptionStatus; @Path("status") public interface ConsumerEndpoint { - @GET - @Produces(APPLICATION_JSON) - @Path("/subscriptions") - List listSubscriptions(); - - @GET - @Path("/subscriptionsCount") - Integer countSubscriptions(); + @GET + @Produces(APPLICATION_JSON) + @Path("/subscriptions") + List listSubscriptions(); - @GET - @Path("/health") - String health(); + @GET + @Path("/subscriptionsCount") + Integer countSubscriptions(); + @GET + @Path("/health") + String health(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/HermesConsumers.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/HermesConsumers.java index 43ad374ae7..0c0fdf46cc 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/HermesConsumers.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/HermesConsumers.java @@ -7,9 +7,9 @@ @SpringBootApplication public class HermesConsumers { - public static void main(String[] args) { - SpringApplication application = new SpringApplication(HermesConsumers.class); - application.setWebApplicationType(WebApplicationType.NONE); - application.run(args); - } + public static void main(String[] args) { + SpringApplication application = new SpringApplication(HermesConsumers.class); + application.setWebApplicationType(WebApplicationType.NONE); + application.run(args); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/BatchProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/BatchProperties.java index 02a461b7a5..6d8f907a70 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/BatchProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/BatchProperties.java @@ -2,27 +2,26 @@ import org.springframework.boot.context.properties.ConfigurationProperties; - @ConfigurationProperties(prefix = "consumer.batch") public class BatchProperties { - private int poolableSize = 1024; + private int poolableSize = 1024; - private int maxPoolSize = 64 * 1024 * 1024; + private int maxPoolSize = 64 * 1024 * 1024; - public int getPoolableSize() { - return poolableSize; - } + public int getPoolableSize() { + return poolableSize; + } - public void setPoolableSize(int poolableSize) { - this.poolableSize = poolableSize; - } + public void setPoolableSize(int poolableSize) { + this.poolableSize = poolableSize; + } - public int getMaxPoolSize() { - return maxPoolSize; - } + public int getMaxPoolSize() { + return maxPoolSize; + } - public void setMaxPoolSize(int maxPoolSize) { - this.maxPoolSize = maxPoolSize; - } + public void setMaxPoolSize(int maxPoolSize) { + this.maxPoolSize = maxPoolSize; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/CommitOffsetProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/CommitOffsetProperties.java index 266dc21bf6..0162051c83 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/CommitOffsetProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/CommitOffsetProperties.java @@ -1,29 +1,28 @@ package pl.allegro.tech.hermes.consumers.config; -import org.springframework.boot.context.properties.ConfigurationProperties; - import java.time.Duration; +import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties(prefix = "consumer.commit.offset") public class CommitOffsetProperties { - private Duration period = Duration.ofSeconds(60); + private Duration period = Duration.ofSeconds(60); - private int queuesSize = 200_000; + private int queuesSize = 200_000; - public Duration getPeriod() { - return period; - } + public Duration getPeriod() { + return period; + } - public void setPeriod(Duration period) { - this.period = period; - } + public void setPeriod(Duration period) { + this.period = period; + } - public int getQueuesSize() { - return queuesSize; - } + public int getQueuesSize() { + return queuesSize; + } - public void setQueuesSize(int queuesSize) { - this.queuesSize = queuesSize; - } + public void setQueuesSize(int queuesSize) { + this.queuesSize = queuesSize; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/CommonConfiguration.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/CommonConfiguration.java index 725e4108cb..23a7c78965 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/CommonConfiguration.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/CommonConfiguration.java @@ -1,11 +1,17 @@ package pl.allegro.tech.hermes.consumers.config; +import static io.micrometer.core.instrument.Clock.SYSTEM; +import static java.util.Collections.emptyList; + import com.fasterxml.jackson.databind.ObjectMapper; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.composite.CompositeMeterRegistry; import io.micrometer.prometheus.PrometheusConfig; import io.micrometer.prometheus.PrometheusMeterRegistry; import jakarta.inject.Named; +import java.time.Clock; +import java.util.Arrays; +import java.util.List; import org.apache.curator.framework.CuratorFramework; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; @@ -68,252 +74,274 @@ import pl.allegro.tech.hermes.metrics.PathsCompiler; import pl.allegro.tech.hermes.schema.SchemaRepository; -import java.time.Clock; -import java.util.Arrays; -import java.util.List; - -import static io.micrometer.core.instrument.Clock.SYSTEM; -import static java.util.Collections.emptyList; - @Configuration @EnableConfigurationProperties({ - MetricsProperties.class, - MicrometerRegistryProperties.class, - PrometheusProperties.class, - SchemaProperties.class, - ZookeeperClustersProperties.class, - ContentRootProperties.class, - DatacenterNameProperties.class + MetricsProperties.class, + MicrometerRegistryProperties.class, + PrometheusProperties.class, + SchemaProperties.class, + ZookeeperClustersProperties.class, + ContentRootProperties.class, + DatacenterNameProperties.class }) public class CommonConfiguration { - @Bean - public DatacenterNameProvider dcNameProvider(DatacenterNameProperties datacenterNameProperties) { - if (datacenterNameProperties.getSource() == DcNameSource.ENV) { - return new EnvironmentVariableDatacenterNameProvider(datacenterNameProperties.getEnv()); - } else { - return new DefaultDatacenterNameProvider(); - } - } - - @Bean - public SubscriptionRepository subscriptionRepository(CuratorFramework zookeeper, - ZookeeperPaths paths, - ObjectMapper mapper, - TopicRepository topicRepository) { - return new ZookeeperSubscriptionRepository(zookeeper, mapper, paths, topicRepository); - } - - @Bean - public OAuthProviderRepository oAuthProviderRepository(CuratorFramework zookeeper, - ZookeeperPaths paths, - ObjectMapper mapper) { - return new ZookeeperOAuthProviderRepository(zookeeper, mapper, paths); - } - - @Bean - public TopicRepository topicRepository(CuratorFramework zookeeper, - ZookeeperPaths paths, - ObjectMapper mapper, - GroupRepository groupRepository) { - return new ZookeeperTopicRepository(zookeeper, mapper, paths, groupRepository); - } - - @Bean - public GroupRepository groupRepository(CuratorFramework zookeeper, ZookeeperPaths paths, ObjectMapper mapper) { - return new ZookeeperGroupRepository(zookeeper, mapper, paths); - } - - @Bean(destroyMethod = "close") - public CuratorFramework hermesCurator(ZookeeperClustersProperties zookeeperClustersProperties, - CuratorClientFactory curatorClientFactory, DatacenterNameProvider datacenterNameProvider) { - ZookeeperProperties zookeeperProperties = zookeeperClustersProperties.toZookeeperProperties(datacenterNameProvider); - return new HermesCuratorClientFactory(zookeeperProperties, curatorClientFactory).provide(); - } - - @Bean - public CuratorClientFactory curatorClientFactory(ZookeeperClustersProperties zookeeperClustersProperties, - DatacenterNameProvider datacenterNameProvider) { - ZookeeperProperties zookeeperProperties = zookeeperClustersProperties.toZookeeperProperties(datacenterNameProvider); - return new CuratorClientFactory(zookeeperProperties); - } - - @Bean - public InternalNotificationsBus zookeeperInternalNotificationBus(ObjectMapper objectMapper, - ModelAwareZookeeperNotifyingCache modelNotifyingCache) { - return new ZookeeperInternalNotificationBus(objectMapper, modelNotifyingCache); - } - - @Bean(destroyMethod = "stop") - public ModelAwareZookeeperNotifyingCache modelAwareZookeeperNotifyingCache(CuratorFramework curator, - MetricsFacade metricsFacade, - ZookeeperClustersProperties zookeeperClustersProperties, - DatacenterNameProvider datacenterNameProvider) { - ZookeeperProperties zookeeperProperties = zookeeperClustersProperties.toZookeeperProperties(datacenterNameProvider); - return new ModelAwareZookeeperNotifyingCacheFactory(curator, metricsFacade, zookeeperProperties).provide(); - } - - @Bean - public UndeliveredMessageLog undeliveredMessageLog(CuratorFramework zookeeper, - ZookeeperPaths paths, - ObjectMapper mapper, - MetricsFacade metricsFacade) { - return new ZookeeperUndeliveredMessageLog(zookeeper, paths, mapper, metricsFacade); - } - - @Bean - public InstrumentedExecutorServiceFactory instrumentedExecutorServiceFactory(MetricsFacade metricsFacade) { - return new InstrumentedExecutorServiceFactory(metricsFacade); - } - - @Bean - public ZookeeperAdminCache zookeeperAdminCache(ZookeeperPaths zookeeperPaths, - CuratorFramework client, - ObjectMapper objectMapper, - Clock clock) { - return new ZookeeperAdminCache(zookeeperPaths, client, objectMapper, clock); - } - - @Bean - public ObjectMapper objectMapper(SchemaProperties schemaProperties) { - return new ObjectMapperFactory( - schemaProperties.isIdSerializationEnabled(), - /* fallbackToRemoteDatacenter is frontend specific property, we so don't expose consumer side property for it */ - false - ).provide(); - } - - - @Bean - public CompositeMessageContentWrapper messageContentWrapper(ObjectMapper mapper, - Clock clock, - SchemaRepository schemaRepository, - MetricsFacade metricsFacade, - SchemaProperties schemaProperties, - ContentRootProperties contentRootProperties) { - AvroMessageContentWrapper avroMessageContentWrapper = new AvroMessageContentWrapper(clock); - - return new CompositeMessageContentWrapper( - new JsonMessageContentWrapper(contentRootProperties.getMessage(), contentRootProperties.getMetadata(), mapper), - avroMessageContentWrapper, - new AvroMessageSchemaIdAwareContentWrapper(schemaRepository, avroMessageContentWrapper, - metricsFacade), - new AvroMessageHeaderSchemaVersionContentWrapper(schemaRepository, avroMessageContentWrapper, - metricsFacade), - new AvroMessageHeaderSchemaIdContentWrapper(schemaRepository, avroMessageContentWrapper, - metricsFacade, schemaProperties.isIdHeaderEnabled()), - new AvroMessageSchemaVersionTruncationContentWrapper(schemaRepository, avroMessageContentWrapper, - metricsFacade, schemaProperties.isVersionTruncationEnabled()) - ); - } - - @Bean - public KafkaNamesMapper prodKafkaNamesMapper(KafkaClustersProperties kafkaClustersProperties) { - return new NamespaceKafkaNamesMapper(kafkaClustersProperties.getNamespace(), kafkaClustersProperties.getNamespaceSeparator()); - } - - @Bean - public Clock clock() { - return new ClockFactory().provide(); - } - - @Bean - public ZookeeperPaths zookeeperPaths(ZookeeperClustersProperties zookeeperClustersProperties, - DatacenterNameProvider datacenterNameProvider) { - ZookeeperProperties zookeeperProperties = zookeeperClustersProperties.toZookeeperProperties(datacenterNameProvider); - return new ZookeeperPaths(zookeeperProperties.getRoot()); - } - - @Bean - public WorkloadConstraintsRepository workloadConstraintsRepository(CuratorFramework curator, - ObjectMapper mapper, - ZookeeperPaths paths) { - return new ZookeeperWorkloadConstraintsRepository(curator, mapper, paths); - } - - @Bean - public MetricsFacade metricsFacade(MeterRegistry meterRegistry) { - return new MetricsFacade(meterRegistry); - } - - @Bean - PrometheusConfig prometheusConfig(PrometheusProperties properties) { - return new PrometheusConfigAdapter(properties); - } - - @Bean - public PrometheusMeterRegistry micrometerRegistry(MicrometerRegistryParameters micrometerRegistryParameters, - PrometheusConfig prometheusConfig, - CounterStorage counterStorage) { - return new PrometheusMeterRegistryFactory(micrometerRegistryParameters, - prometheusConfig, counterStorage, "hermes-consumers").provide(); - } - - @Bean - @Primary - public MeterRegistry compositeMeterRegistry(List registries) { - return new CompositeMeterRegistry(SYSTEM, registries); - } - - @Bean - @Named("moduleName") - public String moduleName() { - return "consumer"; - } - - @Bean - public PathsCompiler pathsCompiler(InstanceIdResolver instanceIdResolver) { - return new PathsCompiler(instanceIdResolver.resolve()); - } - - @Bean - public CounterStorage zookeeperCounterStorage(SharedCounter sharedCounter, - SubscriptionRepository subscriptionRepository, - PathsCompiler pathsCompiler, - ZookeeperClustersProperties zookeeperClustersProperties, - DatacenterNameProvider datacenterNameProvider) { - ZookeeperProperties zookeeperProperties = zookeeperClustersProperties.toZookeeperProperties(datacenterNameProvider); - return new ZookeeperCounterStorage(sharedCounter, subscriptionRepository, pathsCompiler, zookeeperProperties.getRoot()); - } - - @Bean - public SharedCounter sharedCounter(CuratorFramework zookeeper, - ZookeeperClustersProperties zookeeperClustersProperties, - MetricsProperties metricsProperties, - DatacenterNameProvider datacenterNameProvider) { - ZookeeperProperties zookeeperProperties = zookeeperClustersProperties.toZookeeperProperties(datacenterNameProvider); - return new SharedCounter(zookeeper, - metricsProperties.getCounterExpireAfterAccess(), - zookeeperProperties.getBaseSleepTime(), - zookeeperProperties.getMaxRetries() - ); - } - - @Bean - public InstanceIdResolver instanceIdResolver() { - return new InetAddressInstanceIdResolver(); - } - - @Bean - public SubscriptionOffsetChangeIndicator subscriptionOffsetChangeIndicatorFactory(CuratorFramework zookeeper, - ZookeeperPaths paths, - SubscriptionRepository subscriptionRepository) { - return new ZookeeperSubscriptionOffsetChangeIndicator(zookeeper, paths, subscriptionRepository); - } - - @Bean - FilterChainFactory filterChainFactory() { - List subscriptionFilterCompilers = Arrays.asList( - new AvroPathSubscriptionMessageFilterCompiler(), - new JsonPathSubscriptionMessageFilterCompiler(), - new HeaderSubscriptionMessageFilterCompiler() - ); - MessageFilters messageFilters = new MessageFilters(emptyList(), subscriptionFilterCompilers); - return new FilterChainFactory(messageFilters); - } - - @Bean - public ExecutorServiceFactory executorServiceFactory() { - return new DefaultExecutorServiceFactory(); + @Bean + public DatacenterNameProvider dcNameProvider(DatacenterNameProperties datacenterNameProperties) { + if (datacenterNameProperties.getSource() == DcNameSource.ENV) { + return new EnvironmentVariableDatacenterNameProvider(datacenterNameProperties.getEnv()); + } else { + return new DefaultDatacenterNameProvider(); } + } + + @Bean + public SubscriptionRepository subscriptionRepository( + CuratorFramework zookeeper, + ZookeeperPaths paths, + ObjectMapper mapper, + TopicRepository topicRepository) { + return new ZookeeperSubscriptionRepository(zookeeper, mapper, paths, topicRepository); + } + + @Bean + public OAuthProviderRepository oAuthProviderRepository( + CuratorFramework zookeeper, ZookeeperPaths paths, ObjectMapper mapper) { + return new ZookeeperOAuthProviderRepository(zookeeper, mapper, paths); + } + + @Bean + public TopicRepository topicRepository( + CuratorFramework zookeeper, + ZookeeperPaths paths, + ObjectMapper mapper, + GroupRepository groupRepository) { + return new ZookeeperTopicRepository(zookeeper, mapper, paths, groupRepository); + } + + @Bean + public GroupRepository groupRepository( + CuratorFramework zookeeper, ZookeeperPaths paths, ObjectMapper mapper) { + return new ZookeeperGroupRepository(zookeeper, mapper, paths); + } + + @Bean(destroyMethod = "close") + public CuratorFramework hermesCurator( + ZookeeperClustersProperties zookeeperClustersProperties, + CuratorClientFactory curatorClientFactory, + DatacenterNameProvider datacenterNameProvider) { + ZookeeperProperties zookeeperProperties = + zookeeperClustersProperties.toZookeeperProperties(datacenterNameProvider); + return new HermesCuratorClientFactory(zookeeperProperties, curatorClientFactory).provide(); + } + + @Bean + public CuratorClientFactory curatorClientFactory( + ZookeeperClustersProperties zookeeperClustersProperties, + DatacenterNameProvider datacenterNameProvider) { + ZookeeperProperties zookeeperProperties = + zookeeperClustersProperties.toZookeeperProperties(datacenterNameProvider); + return new CuratorClientFactory(zookeeperProperties); + } + + @Bean + public InternalNotificationsBus zookeeperInternalNotificationBus( + ObjectMapper objectMapper, ModelAwareZookeeperNotifyingCache modelNotifyingCache) { + return new ZookeeperInternalNotificationBus(objectMapper, modelNotifyingCache); + } + + @Bean(destroyMethod = "stop") + public ModelAwareZookeeperNotifyingCache modelAwareZookeeperNotifyingCache( + CuratorFramework curator, + MetricsFacade metricsFacade, + ZookeeperClustersProperties zookeeperClustersProperties, + DatacenterNameProvider datacenterNameProvider) { + ZookeeperProperties zookeeperProperties = + zookeeperClustersProperties.toZookeeperProperties(datacenterNameProvider); + return new ModelAwareZookeeperNotifyingCacheFactory(curator, metricsFacade, zookeeperProperties) + .provide(); + } + + @Bean + public UndeliveredMessageLog undeliveredMessageLog( + CuratorFramework zookeeper, + ZookeeperPaths paths, + ObjectMapper mapper, + MetricsFacade metricsFacade) { + return new ZookeeperUndeliveredMessageLog(zookeeper, paths, mapper, metricsFacade); + } + + @Bean + public InstrumentedExecutorServiceFactory instrumentedExecutorServiceFactory( + MetricsFacade metricsFacade) { + return new InstrumentedExecutorServiceFactory(metricsFacade); + } + + @Bean + public ZookeeperAdminCache zookeeperAdminCache( + ZookeeperPaths zookeeperPaths, + CuratorFramework client, + ObjectMapper objectMapper, + Clock clock) { + return new ZookeeperAdminCache(zookeeperPaths, client, objectMapper, clock); + } + + @Bean + public ObjectMapper objectMapper(SchemaProperties schemaProperties) { + return new ObjectMapperFactory( + schemaProperties.isIdSerializationEnabled(), + /* fallbackToRemoteDatacenter is frontend specific property, we so don't expose consumer side property for it */ + false) + .provide(); + } + + @Bean + public CompositeMessageContentWrapper messageContentWrapper( + ObjectMapper mapper, + Clock clock, + SchemaRepository schemaRepository, + MetricsFacade metricsFacade, + SchemaProperties schemaProperties, + ContentRootProperties contentRootProperties) { + AvroMessageContentWrapper avroMessageContentWrapper = new AvroMessageContentWrapper(clock); + + return new CompositeMessageContentWrapper( + new JsonMessageContentWrapper( + contentRootProperties.getMessage(), contentRootProperties.getMetadata(), mapper), + avroMessageContentWrapper, + new AvroMessageSchemaIdAwareContentWrapper( + schemaRepository, avroMessageContentWrapper, metricsFacade), + new AvroMessageHeaderSchemaVersionContentWrapper( + schemaRepository, avroMessageContentWrapper, metricsFacade), + new AvroMessageHeaderSchemaIdContentWrapper( + schemaRepository, + avroMessageContentWrapper, + metricsFacade, + schemaProperties.isIdHeaderEnabled()), + new AvroMessageSchemaVersionTruncationContentWrapper( + schemaRepository, + avroMessageContentWrapper, + metricsFacade, + schemaProperties.isVersionTruncationEnabled())); + } + + @Bean + public KafkaNamesMapper prodKafkaNamesMapper(KafkaClustersProperties kafkaClustersProperties) { + return new NamespaceKafkaNamesMapper( + kafkaClustersProperties.getNamespace(), kafkaClustersProperties.getNamespaceSeparator()); + } + + @Bean + public Clock clock() { + return new ClockFactory().provide(); + } + + @Bean + public ZookeeperPaths zookeeperPaths( + ZookeeperClustersProperties zookeeperClustersProperties, + DatacenterNameProvider datacenterNameProvider) { + ZookeeperProperties zookeeperProperties = + zookeeperClustersProperties.toZookeeperProperties(datacenterNameProvider); + return new ZookeeperPaths(zookeeperProperties.getRoot()); + } + + @Bean + public WorkloadConstraintsRepository workloadConstraintsRepository( + CuratorFramework curator, ObjectMapper mapper, ZookeeperPaths paths) { + return new ZookeeperWorkloadConstraintsRepository(curator, mapper, paths); + } + + @Bean + public MetricsFacade metricsFacade(MeterRegistry meterRegistry) { + return new MetricsFacade(meterRegistry); + } + + @Bean + PrometheusConfig prometheusConfig(PrometheusProperties properties) { + return new PrometheusConfigAdapter(properties); + } + + @Bean + public PrometheusMeterRegistry micrometerRegistry( + MicrometerRegistryParameters micrometerRegistryParameters, + PrometheusConfig prometheusConfig, + CounterStorage counterStorage) { + return new PrometheusMeterRegistryFactory( + micrometerRegistryParameters, prometheusConfig, counterStorage, "hermes-consumers") + .provide(); + } + + @Bean + @Primary + public MeterRegistry compositeMeterRegistry(List registries) { + return new CompositeMeterRegistry(SYSTEM, registries); + } + + @Bean + @Named("moduleName") + public String moduleName() { + return "consumer"; + } + + @Bean + public PathsCompiler pathsCompiler(InstanceIdResolver instanceIdResolver) { + return new PathsCompiler(instanceIdResolver.resolve()); + } + + @Bean + public CounterStorage zookeeperCounterStorage( + SharedCounter sharedCounter, + SubscriptionRepository subscriptionRepository, + PathsCompiler pathsCompiler, + ZookeeperClustersProperties zookeeperClustersProperties, + DatacenterNameProvider datacenterNameProvider) { + ZookeeperProperties zookeeperProperties = + zookeeperClustersProperties.toZookeeperProperties(datacenterNameProvider); + return new ZookeeperCounterStorage( + sharedCounter, subscriptionRepository, pathsCompiler, zookeeperProperties.getRoot()); + } + + @Bean + public SharedCounter sharedCounter( + CuratorFramework zookeeper, + ZookeeperClustersProperties zookeeperClustersProperties, + MetricsProperties metricsProperties, + DatacenterNameProvider datacenterNameProvider) { + ZookeeperProperties zookeeperProperties = + zookeeperClustersProperties.toZookeeperProperties(datacenterNameProvider); + return new SharedCounter( + zookeeper, + metricsProperties.getCounterExpireAfterAccess(), + zookeeperProperties.getBaseSleepTime(), + zookeeperProperties.getMaxRetries()); + } + + @Bean + public InstanceIdResolver instanceIdResolver() { + return new InetAddressInstanceIdResolver(); + } + + @Bean + public SubscriptionOffsetChangeIndicator subscriptionOffsetChangeIndicatorFactory( + CuratorFramework zookeeper, + ZookeeperPaths paths, + SubscriptionRepository subscriptionRepository) { + return new ZookeeperSubscriptionOffsetChangeIndicator(zookeeper, paths, subscriptionRepository); + } + + @Bean + FilterChainFactory filterChainFactory() { + List subscriptionFilterCompilers = + Arrays.asList( + new AvroPathSubscriptionMessageFilterCompiler(), + new JsonPathSubscriptionMessageFilterCompiler(), + new HeaderSubscriptionMessageFilterCompiler()); + MessageFilters messageFilters = new MessageFilters(emptyList(), subscriptionFilterCompilers); + return new FilterChainFactory(messageFilters); + } + + @Bean + public ExecutorServiceFactory executorServiceFactory() { + return new DefaultExecutorServiceFactory(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/CommonConsumerProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/CommonConsumerProperties.java index 2b7df3a2c2..e91544ef26 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/CommonConsumerProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/CommonConsumerProperties.java @@ -1,157 +1,158 @@ package pl.allegro.tech.hermes.consumers.config; +import java.time.Duration; import org.springframework.boot.context.properties.ConfigurationProperties; import pl.allegro.tech.hermes.consumers.CommonConsumerParameters; import pl.allegro.tech.hermes.consumers.consumer.SerialConsumerParameters; import pl.allegro.tech.hermes.consumers.supervisor.SupervisorParameters; -import java.time.Duration; - @ConfigurationProperties(prefix = "consumer") public class CommonConsumerProperties implements CommonConsumerParameters { - private int threadPoolSize = 500; + private int threadPoolSize = 500; - private int healthCheckPort = 8000; + private int healthCheckPort = 8000; - private Duration subscriptionIdsCacheRemovedExpireAfterAccess = Duration.ofSeconds(60); + private Duration subscriptionIdsCacheRemovedExpireAfterAccess = Duration.ofSeconds(60); - private SupervisorParameters backgroundSupervisor = new BackgroundSupervisor(); + private SupervisorParameters backgroundSupervisor = new BackgroundSupervisor(); - private SerialConsumerParameters serialConsumer = new SerialConsumer(); + private SerialConsumerParameters serialConsumer = new SerialConsumer(); - private int signalProcessingQueueSize = 5_000; + private int signalProcessingQueueSize = 5_000; - private boolean useTopicMessageSizeEnabled = false; + private boolean useTopicMessageSizeEnabled = false; - private Duration undeliveredMessageLogPersistPeriod = Duration.ofSeconds(5); + private Duration undeliveredMessageLogPersistPeriod = Duration.ofSeconds(5); - public int getThreadPoolSize() { - return threadPoolSize; - } + public int getThreadPoolSize() { + return threadPoolSize; + } - public void setThreadPoolSize(int threadPoolSize) { - this.threadPoolSize = threadPoolSize; - } + public void setThreadPoolSize(int threadPoolSize) { + this.threadPoolSize = threadPoolSize; + } - public int getHealthCheckPort() { - return healthCheckPort; - } + public int getHealthCheckPort() { + return healthCheckPort; + } - public void setHealthCheckPort(int healthCheckPort) { - this.healthCheckPort = healthCheckPort; - } + public void setHealthCheckPort(int healthCheckPort) { + this.healthCheckPort = healthCheckPort; + } - public Duration getSubscriptionIdsCacheRemovedExpireAfterAccess() { - return subscriptionIdsCacheRemovedExpireAfterAccess; - } + public Duration getSubscriptionIdsCacheRemovedExpireAfterAccess() { + return subscriptionIdsCacheRemovedExpireAfterAccess; + } - public void setSubscriptionIdsCacheRemovedExpireAfterAccess(Duration subscriptionIdsCacheRemovedExpireAfterAccess) { - this.subscriptionIdsCacheRemovedExpireAfterAccess = subscriptionIdsCacheRemovedExpireAfterAccess; - } + public void setSubscriptionIdsCacheRemovedExpireAfterAccess( + Duration subscriptionIdsCacheRemovedExpireAfterAccess) { + this.subscriptionIdsCacheRemovedExpireAfterAccess = + subscriptionIdsCacheRemovedExpireAfterAccess; + } - @Override - public SupervisorParameters getBackgroundSupervisor() { - return backgroundSupervisor; - } + @Override + public SupervisorParameters getBackgroundSupervisor() { + return backgroundSupervisor; + } - public void setBackgroundSupervisor(SupervisorParameters backgroundSupervisor) { - this.backgroundSupervisor = backgroundSupervisor; - } + public void setBackgroundSupervisor(SupervisorParameters backgroundSupervisor) { + this.backgroundSupervisor = backgroundSupervisor; + } - @Override - public SerialConsumerParameters getSerialConsumer() { - return serialConsumer; - } + @Override + public SerialConsumerParameters getSerialConsumer() { + return serialConsumer; + } - public void setSerialConsumer(SerialConsumerParameters serialConsumer) { - this.serialConsumer = serialConsumer; - } + public void setSerialConsumer(SerialConsumerParameters serialConsumer) { + this.serialConsumer = serialConsumer; + } - @Override - public int getSignalProcessingQueueSize() { - return signalProcessingQueueSize; - } + @Override + public int getSignalProcessingQueueSize() { + return signalProcessingQueueSize; + } - public void setSignalProcessingQueueSize(int signalProcessingQueueSize) { - this.signalProcessingQueueSize = signalProcessingQueueSize; - } + public void setSignalProcessingQueueSize(int signalProcessingQueueSize) { + this.signalProcessingQueueSize = signalProcessingQueueSize; + } - @Override - public boolean isUseTopicMessageSizeEnabled() { - return useTopicMessageSizeEnabled; - } + @Override + public boolean isUseTopicMessageSizeEnabled() { + return useTopicMessageSizeEnabled; + } - public void setUseTopicMessageSizeEnabled(boolean useTopicMessageSizeEnabled) { - this.useTopicMessageSizeEnabled = useTopicMessageSizeEnabled; - } + public void setUseTopicMessageSizeEnabled(boolean useTopicMessageSizeEnabled) { + this.useTopicMessageSizeEnabled = useTopicMessageSizeEnabled; + } - public Duration getUndeliveredMessageLogPersistPeriod() { - return undeliveredMessageLogPersistPeriod; - } + public Duration getUndeliveredMessageLogPersistPeriod() { + return undeliveredMessageLogPersistPeriod; + } - public void setUndeliveredMessageLogPersistPeriod(Duration undeliveredMessageLogPersistPeriod) { - this.undeliveredMessageLogPersistPeriod = undeliveredMessageLogPersistPeriod; - } + public void setUndeliveredMessageLogPersistPeriod(Duration undeliveredMessageLogPersistPeriod) { + this.undeliveredMessageLogPersistPeriod = undeliveredMessageLogPersistPeriod; + } - public static final class SerialConsumer implements SerialConsumerParameters { + public static final class SerialConsumer implements SerialConsumerParameters { - private Duration signalProcessingInterval = Duration.ofMillis(5_000); + private Duration signalProcessingInterval = Duration.ofMillis(5_000); - private int inflightSize = 100; + private int inflightSize = 100; - @Override - public Duration getSignalProcessingInterval() { - return signalProcessingInterval; - } + @Override + public Duration getSignalProcessingInterval() { + return signalProcessingInterval; + } - public void setSignalProcessingInterval(Duration signalProcessingInterval) { - this.signalProcessingInterval = signalProcessingInterval; - } + public void setSignalProcessingInterval(Duration signalProcessingInterval) { + this.signalProcessingInterval = signalProcessingInterval; + } - @Override - public int getInflightSize() { - return inflightSize; - } + @Override + public int getInflightSize() { + return inflightSize; + } - public void setInflightSize(int inflightSize) { - this.inflightSize = inflightSize; - } + public void setInflightSize(int inflightSize) { + this.inflightSize = inflightSize; } + } - public static final class BackgroundSupervisor implements SupervisorParameters { + public static final class BackgroundSupervisor implements SupervisorParameters { - private Duration interval = Duration.ofMillis(20_000); + private Duration interval = Duration.ofMillis(20_000); - private Duration unhealthyAfter = Duration.ofMillis(600_000); + private Duration unhealthyAfter = Duration.ofMillis(600_000); - private Duration killAfter = Duration.ofMillis(300_000); + private Duration killAfter = Duration.ofMillis(300_000); - @Override - public Duration getInterval() { - return interval; - } + @Override + public Duration getInterval() { + return interval; + } - public void setInterval(Duration interval) { - this.interval = interval; - } + public void setInterval(Duration interval) { + this.interval = interval; + } - @Override - public Duration getUnhealthyAfter() { - return unhealthyAfter; - } + @Override + public Duration getUnhealthyAfter() { + return unhealthyAfter; + } - public void setUnhealthyAfter(Duration unhealthyAfter) { - this.unhealthyAfter = unhealthyAfter; - } + public void setUnhealthyAfter(Duration unhealthyAfter) { + this.unhealthyAfter = unhealthyAfter; + } - @Override - public Duration getKillAfter() { - return killAfter; - } + @Override + public Duration getKillAfter() { + return killAfter; + } - public void setKillAfter(Duration killAfter) { - this.killAfter = killAfter; - } + public void setKillAfter(Duration killAfter) { + this.killAfter = killAfter; } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ConsumerConfiguration.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ConsumerConfiguration.java index 28c8b0548f..f4d1c7bb25 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ConsumerConfiguration.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ConsumerConfiguration.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.consumers.config; +import java.time.Clock; +import java.util.List; import org.apache.curator.framework.CuratorFramework; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; @@ -36,152 +38,154 @@ import pl.allegro.tech.hermes.tracker.consumers.LogRepository; import pl.allegro.tech.hermes.tracker.consumers.Trackers; -import java.time.Clock; -import java.util.List; - @Configuration @EnableConfigurationProperties({ - CommitOffsetProperties.class, - SenderAsyncTimeoutProperties.class, - RateProperties.class, - BatchProperties.class, - KafkaClustersProperties.class, - WorkloadProperties.class, - MaxRateProperties.class + CommitOffsetProperties.class, + SenderAsyncTimeoutProperties.class, + RateProperties.class, + BatchProperties.class, + KafkaClustersProperties.class, + WorkloadProperties.class, + MaxRateProperties.class }) public class ConsumerConfiguration { - @Bean - public MaxRatePathSerializer maxRatePathSerializer() { - return new MaxRatePathSerializer(); - } - - @Bean - public NoOperationMessageConverter noOperationMessageConverter() { - return new NoOperationMessageConverter(); - } - - @Bean - public ConsumerPartitionAssignmentState consumerPartitionAssignmentState() { - return new ConsumerPartitionAssignmentState(); - } - - @Bean - public MaxRateRegistry maxRateRegistry(MaxRateProperties maxRateProperties, - KafkaClustersProperties kafkaClustersProperties, - WorkloadProperties workloadProperties, - CuratorFramework curator, - ZookeeperPaths zookeeperPaths, - SubscriptionIds subscriptionIds, - ConsumerAssignmentCache assignmentCache, - ClusterAssignmentCache clusterAssignmentCache, - DatacenterNameProvider datacenterNameProvider) { - KafkaProperties kafkaProperties = kafkaClustersProperties.toKafkaProperties(datacenterNameProvider); - return new MaxRateRegistry( - maxRateProperties.getRegistryBinaryEncoder().getHistoryBufferSizeBytes(), - maxRateProperties.getRegistryBinaryEncoder().getMaxRateBufferSizeBytes(), - workloadProperties.getNodeId(), - kafkaProperties.getClusterName(), - clusterAssignmentCache, - assignmentCache, - curator, - zookeeperPaths, - subscriptionIds - ); - } - - @Bean(initMethod = "start", destroyMethod = "stop") - public MaxRateSupervisor maxRateSupervisor(MaxRateProperties maxRateProperties, - ClusterAssignmentCache clusterAssignmentCache, - MaxRateRegistry maxRateRegistry, - ConsumerNodesRegistry consumerNodesRegistry, - SubscriptionsCache subscriptionsCache, - MetricsFacade metrics, - Clock clock) { - return new MaxRateSupervisor( - maxRateProperties, - clusterAssignmentCache, - maxRateRegistry, - consumerNodesRegistry, - subscriptionsCache, - metrics, - clock - ); - } - - @Bean - public ConsumerRateLimitSupervisor consumerRateLimitSupervisor(RateProperties rateProperties) { - return new ConsumerRateLimitSupervisor(rateProperties.getLimiterSupervisorPeriod()); - } - - @Bean - public MaxRateProviderFactory maxRateProviderFactory(MaxRateProperties maxRateProperties, - MaxRateRegistry maxRateRegistry, - MaxRateSupervisor maxRateSupervisor, - WorkloadProperties workloadProperties) { - return new MaxRateProviderFactory(maxRateProperties, workloadProperties.getNodeId(), maxRateRegistry, maxRateSupervisor); - } - - @Bean - public AvroToJsonMessageConverter avroToJsonMessageConverter() { - return new AvroToJsonMessageConverter(); - } - - @Bean - public OutputRateCalculatorFactory outputRateCalculatorFactory(RateProperties rateProperties, - MaxRateProviderFactory maxRateProviderFactory) { - return new OutputRateCalculatorFactory(rateProperties, maxRateProviderFactory); - } - - @Bean - public MessageBatchFactory messageBatchFactory(MetricsFacade metrics, - Clock clock, - BatchProperties batchProperties) { - return new ByteBufferMessageBatchFactory(batchProperties.getPoolableSize(), batchProperties.getMaxPoolSize(), clock, metrics); - } - - @Bean - public MessageConverterResolver defaultMessageConverterResolver(AvroToJsonMessageConverter avroToJsonMessageConverter, - NoOperationMessageConverter noOperationMessageConverter) { - return new DefaultMessageConverterResolver(avroToJsonMessageConverter, noOperationMessageConverter); - } - - - - @Bean - public ConsumerMessageSenderFactory consumerMessageSenderFactory(KafkaClustersProperties kafkaClustersProperties, - MessageSenderFactory messageSenderFactory, - Trackers trackers, - FutureAsyncTimeout futureAsyncTimeout, - UndeliveredMessageLog undeliveredMessageLog, Clock clock, - InstrumentedExecutorServiceFactory instrumentedExecutorServiceFactory, - ConsumerAuthorizationHandler consumerAuthorizationHandler, - SenderAsyncTimeoutProperties senderAsyncTimeoutProperties, - RateProperties rateProperties, - DatacenterNameProvider datacenterNameProvider) { - KafkaProperties kafkaProperties = kafkaClustersProperties.toKafkaProperties(datacenterNameProvider); - return new ConsumerMessageSenderFactory( - kafkaProperties.getClusterName(), - messageSenderFactory, - trackers, - futureAsyncTimeout, - undeliveredMessageLog, - clock, - instrumentedExecutorServiceFactory, - consumerAuthorizationHandler, - senderAsyncTimeoutProperties.getMilliseconds(), - rateProperties.getLimiterReportingThreadPoolSize(), - rateProperties.isLimiterReportingThreadMonitoringEnabled() - ); - } - - @Bean - public UriInterpolator messageBodyInterpolator() { - return new MessageBodyInterpolator(); - } - - @Bean(destroyMethod = "close") - public Trackers trackers(List repositories) { - return new Trackers(repositories); - } + @Bean + public MaxRatePathSerializer maxRatePathSerializer() { + return new MaxRatePathSerializer(); + } + + @Bean + public NoOperationMessageConverter noOperationMessageConverter() { + return new NoOperationMessageConverter(); + } + + @Bean + public ConsumerPartitionAssignmentState consumerPartitionAssignmentState() { + return new ConsumerPartitionAssignmentState(); + } + + @Bean + public MaxRateRegistry maxRateRegistry( + MaxRateProperties maxRateProperties, + KafkaClustersProperties kafkaClustersProperties, + WorkloadProperties workloadProperties, + CuratorFramework curator, + ZookeeperPaths zookeeperPaths, + SubscriptionIds subscriptionIds, + ConsumerAssignmentCache assignmentCache, + ClusterAssignmentCache clusterAssignmentCache, + DatacenterNameProvider datacenterNameProvider) { + KafkaProperties kafkaProperties = + kafkaClustersProperties.toKafkaProperties(datacenterNameProvider); + return new MaxRateRegistry( + maxRateProperties.getRegistryBinaryEncoder().getHistoryBufferSizeBytes(), + maxRateProperties.getRegistryBinaryEncoder().getMaxRateBufferSizeBytes(), + workloadProperties.getNodeId(), + kafkaProperties.getClusterName(), + clusterAssignmentCache, + assignmentCache, + curator, + zookeeperPaths, + subscriptionIds); + } + + @Bean(initMethod = "start", destroyMethod = "stop") + public MaxRateSupervisor maxRateSupervisor( + MaxRateProperties maxRateProperties, + ClusterAssignmentCache clusterAssignmentCache, + MaxRateRegistry maxRateRegistry, + ConsumerNodesRegistry consumerNodesRegistry, + SubscriptionsCache subscriptionsCache, + MetricsFacade metrics, + Clock clock) { + return new MaxRateSupervisor( + maxRateProperties, + clusterAssignmentCache, + maxRateRegistry, + consumerNodesRegistry, + subscriptionsCache, + metrics, + clock); + } + + @Bean + public ConsumerRateLimitSupervisor consumerRateLimitSupervisor(RateProperties rateProperties) { + return new ConsumerRateLimitSupervisor(rateProperties.getLimiterSupervisorPeriod()); + } + + @Bean + public MaxRateProviderFactory maxRateProviderFactory( + MaxRateProperties maxRateProperties, + MaxRateRegistry maxRateRegistry, + MaxRateSupervisor maxRateSupervisor, + WorkloadProperties workloadProperties) { + return new MaxRateProviderFactory( + maxRateProperties, workloadProperties.getNodeId(), maxRateRegistry, maxRateSupervisor); + } + + @Bean + public AvroToJsonMessageConverter avroToJsonMessageConverter() { + return new AvroToJsonMessageConverter(); + } + + @Bean + public OutputRateCalculatorFactory outputRateCalculatorFactory( + RateProperties rateProperties, MaxRateProviderFactory maxRateProviderFactory) { + return new OutputRateCalculatorFactory(rateProperties, maxRateProviderFactory); + } + + @Bean + public MessageBatchFactory messageBatchFactory( + MetricsFacade metrics, Clock clock, BatchProperties batchProperties) { + return new ByteBufferMessageBatchFactory( + batchProperties.getPoolableSize(), batchProperties.getMaxPoolSize(), clock, metrics); + } + + @Bean + public MessageConverterResolver defaultMessageConverterResolver( + AvroToJsonMessageConverter avroToJsonMessageConverter, + NoOperationMessageConverter noOperationMessageConverter) { + return new DefaultMessageConverterResolver( + avroToJsonMessageConverter, noOperationMessageConverter); + } + + @Bean + public ConsumerMessageSenderFactory consumerMessageSenderFactory( + KafkaClustersProperties kafkaClustersProperties, + MessageSenderFactory messageSenderFactory, + Trackers trackers, + FutureAsyncTimeout futureAsyncTimeout, + UndeliveredMessageLog undeliveredMessageLog, + Clock clock, + InstrumentedExecutorServiceFactory instrumentedExecutorServiceFactory, + ConsumerAuthorizationHandler consumerAuthorizationHandler, + SenderAsyncTimeoutProperties senderAsyncTimeoutProperties, + RateProperties rateProperties, + DatacenterNameProvider datacenterNameProvider) { + KafkaProperties kafkaProperties = + kafkaClustersProperties.toKafkaProperties(datacenterNameProvider); + return new ConsumerMessageSenderFactory( + kafkaProperties.getClusterName(), + messageSenderFactory, + trackers, + futureAsyncTimeout, + undeliveredMessageLog, + clock, + instrumentedExecutorServiceFactory, + consumerAuthorizationHandler, + senderAsyncTimeoutProperties.getMilliseconds(), + rateProperties.getLimiterReportingThreadPoolSize(), + rateProperties.isLimiterReportingThreadMonitoringEnabled()); + } + + @Bean + public UriInterpolator messageBodyInterpolator() { + return new MessageBodyInterpolator(); + } + + @Bean(destroyMethod = "close") + public Trackers trackers(List repositories) { + return new Trackers(repositories); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ConsumerHTTPHeadersPropagationAsKafkaHeadersProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ConsumerHTTPHeadersPropagationAsKafkaHeadersProperties.java index 50eb9b40da..82fab79e87 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ConsumerHTTPHeadersPropagationAsKafkaHeadersProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ConsumerHTTPHeadersPropagationAsKafkaHeadersProperties.java @@ -4,26 +4,27 @@ import pl.allegro.tech.hermes.common.kafka.HTTPHeadersPropagationAsKafkaHeadersProperties; @ConfigurationProperties(prefix = "consumer.http.headers.propagation-as-kafka-headers") -public class ConsumerHTTPHeadersPropagationAsKafkaHeadersProperties implements HTTPHeadersPropagationAsKafkaHeadersProperties { +public class ConsumerHTTPHeadersPropagationAsKafkaHeadersProperties + implements HTTPHeadersPropagationAsKafkaHeadersProperties { - public boolean enabled = true; - public String prefix = "h-"; + public boolean enabled = true; + public String prefix = "h-"; - @Override - public boolean isEnabled() { - return enabled; - } + @Override + public boolean isEnabled() { + return enabled; + } - @Override - public String getPrefix() { - return prefix; - } + @Override + public String getPrefix() { + return prefix; + } - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } - public void setPrefix(String prefix) { - this.prefix = prefix; - } + public void setPrefix(String prefix) { + this.prefix = prefix; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ConsumerReceiverConfiguration.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ConsumerReceiverConfiguration.java index 09fa41cf9b..b4c701d5c3 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ConsumerReceiverConfiguration.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ConsumerReceiverConfiguration.java @@ -1,5 +1,6 @@ package pl.allegro.tech.hermes.consumers.config; +import java.time.Clock; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -19,67 +20,70 @@ import pl.allegro.tech.hermes.schema.SchemaRepository; import pl.allegro.tech.hermes.tracker.consumers.Trackers; -import java.time.Clock; - @Configuration @EnableConfigurationProperties({ - ConsumerReceiverProperties.class, - KafkaConsumerProperties.class, - KafkaClustersProperties.class, - CommonConsumerProperties.class, - KafkaHeaderNameProperties.class, - ConsumerHTTPHeadersPropagationAsKafkaHeadersProperties.class + ConsumerReceiverProperties.class, + KafkaConsumerProperties.class, + KafkaClustersProperties.class, + CommonConsumerProperties.class, + KafkaHeaderNameProperties.class, + ConsumerHTTPHeadersPropagationAsKafkaHeadersProperties.class }) public class ConsumerReceiverConfiguration { - @Bean - public ReceiverFactory kafkaMessageReceiverFactory(CommonConsumerProperties commonConsumerProperties, - ConsumerReceiverProperties consumerReceiverProperties, - KafkaConsumerProperties kafkaConsumerProperties, - KafkaClustersProperties kafkaClustersProperties, - KafkaConsumerRecordToMessageConverterFactory messageConverterFactory, - MetricsFacade metrics, - KafkaNamesMapper kafkaNamesMapper, - FilterChainFactory filterChainFactory, - Trackers trackers, - ConsumerPartitionAssignmentState consumerPartitionAssignmentState, - DatacenterNameProvider datacenterNameProvider) { - KafkaProperties kafkaProperties = kafkaClustersProperties.toKafkaProperties(datacenterNameProvider); - + @Bean + public ReceiverFactory kafkaMessageReceiverFactory( + CommonConsumerProperties commonConsumerProperties, + ConsumerReceiverProperties consumerReceiverProperties, + KafkaConsumerProperties kafkaConsumerProperties, + KafkaClustersProperties kafkaClustersProperties, + KafkaConsumerRecordToMessageConverterFactory messageConverterFactory, + MetricsFacade metrics, + KafkaNamesMapper kafkaNamesMapper, + FilterChainFactory filterChainFactory, + Trackers trackers, + ConsumerPartitionAssignmentState consumerPartitionAssignmentState, + DatacenterNameProvider datacenterNameProvider) { + KafkaProperties kafkaProperties = + kafkaClustersProperties.toKafkaProperties(datacenterNameProvider); - return new KafkaMessageReceiverFactory( - commonConsumerProperties, - consumerReceiverProperties, - kafkaConsumerProperties, - kafkaProperties, - messageConverterFactory, - metrics, - kafkaNamesMapper, - filterChainFactory, - trackers, - consumerPartitionAssignmentState - ); - } + return new KafkaMessageReceiverFactory( + commonConsumerProperties, + consumerReceiverProperties, + kafkaConsumerProperties, + kafkaProperties, + messageConverterFactory, + metrics, + kafkaNamesMapper, + filterChainFactory, + trackers, + consumerPartitionAssignmentState); + } - @Bean - public KafkaConsumerRecordToMessageConverterFactory kafkaMessageConverterFactory( - MessageContentReaderFactory messageContentReaderFactory, - KafkaHeaderExtractor kafkaHeaderExtractor, - Clock clock) { - return new KafkaConsumerRecordToMessageConverterFactory(messageContentReaderFactory, kafkaHeaderExtractor, clock); - } + @Bean + public KafkaConsumerRecordToMessageConverterFactory kafkaMessageConverterFactory( + MessageContentReaderFactory messageContentReaderFactory, + KafkaHeaderExtractor kafkaHeaderExtractor, + Clock clock) { + return new KafkaConsumerRecordToMessageConverterFactory( + messageContentReaderFactory, kafkaHeaderExtractor, clock); + } - @Bean - public MessageContentReaderFactory messageContentReaderFactory(CompositeMessageContentWrapper compositeMessageContentWrapper, - KafkaHeaderExtractor kafkaHeaderExtractor, - SchemaRepository schemaRepository) { - return new BasicMessageContentReaderFactory(compositeMessageContentWrapper, kafkaHeaderExtractor, schemaRepository); - } + @Bean + public MessageContentReaderFactory messageContentReaderFactory( + CompositeMessageContentWrapper compositeMessageContentWrapper, + KafkaHeaderExtractor kafkaHeaderExtractor, + SchemaRepository schemaRepository) { + return new BasicMessageContentReaderFactory( + compositeMessageContentWrapper, kafkaHeaderExtractor, schemaRepository); + } - @Bean - public KafkaHeaderExtractor kafkaHeaderExtractor( - KafkaHeaderNameProperties kafkaHeaderNameProperties, - HTTPHeadersPropagationAsKafkaHeadersProperties httpHeadersPropagationAsKafkaHeadersProperties) { - return new KafkaHeaderExtractor(kafkaHeaderNameProperties, httpHeadersPropagationAsKafkaHeadersProperties); - } + @Bean + public KafkaHeaderExtractor kafkaHeaderExtractor( + KafkaHeaderNameProperties kafkaHeaderNameProperties, + HTTPHeadersPropagationAsKafkaHeadersProperties + httpHeadersPropagationAsKafkaHeadersProperties) { + return new KafkaHeaderExtractor( + kafkaHeaderNameProperties, httpHeadersPropagationAsKafkaHeadersProperties); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ConsumerReceiverProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ConsumerReceiverProperties.java index ea48d0223c..26b38c6d2b 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ConsumerReceiverProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ConsumerReceiverProperties.java @@ -1,99 +1,98 @@ package pl.allegro.tech.hermes.consumers.config; +import java.time.Duration; import org.springframework.boot.context.properties.ConfigurationProperties; import pl.allegro.tech.hermes.common.util.InetAddressInstanceIdResolver; import pl.allegro.tech.hermes.consumers.consumer.receiver.kafka.KafkaReceiverParameters; -import java.time.Duration; - @ConfigurationProperties(prefix = "consumer.receiver") public class ConsumerReceiverProperties implements KafkaReceiverParameters { - private Duration poolTimeout = Duration.ofMillis(30); + private Duration poolTimeout = Duration.ofMillis(30); - private int readQueueCapacity = 1000; + private int readQueueCapacity = 1000; - private boolean waitBetweenUnsuccessfulPolls = true; + private boolean waitBetweenUnsuccessfulPolls = true; - private Duration initialIdleTime = Duration.ofMillis(10); + private Duration initialIdleTime = Duration.ofMillis(10); - private Duration maxIdleTime = Duration.ofMillis(1000); + private Duration maxIdleTime = Duration.ofMillis(1000); - private String clientId = new InetAddressInstanceIdResolver().resolve(); + private String clientId = new InetAddressInstanceIdResolver().resolve(); - private boolean filteringRateLimiterEnabled = true; + private boolean filteringRateLimiterEnabled = true; - private boolean filteringEnabled = true; + private boolean filteringEnabled = true; - @Override - public Duration getPoolTimeout() { - return poolTimeout; - } + @Override + public Duration getPoolTimeout() { + return poolTimeout; + } - public void setPoolTimeout(Duration poolTimeout) { - this.poolTimeout = poolTimeout; - } + public void setPoolTimeout(Duration poolTimeout) { + this.poolTimeout = poolTimeout; + } - @Override - public int getReadQueueCapacity() { - return readQueueCapacity; - } + @Override + public int getReadQueueCapacity() { + return readQueueCapacity; + } - public void setReadQueueCapacity(int readQueueCapacity) { - this.readQueueCapacity = readQueueCapacity; - } + public void setReadQueueCapacity(int readQueueCapacity) { + this.readQueueCapacity = readQueueCapacity; + } - @Override - public boolean isWaitBetweenUnsuccessfulPolls() { - return waitBetweenUnsuccessfulPolls; - } + @Override + public boolean isWaitBetweenUnsuccessfulPolls() { + return waitBetweenUnsuccessfulPolls; + } - public void setWaitBetweenUnsuccessfulPolls(boolean waitBetweenUnsuccessfulPolls) { - this.waitBetweenUnsuccessfulPolls = waitBetweenUnsuccessfulPolls; - } + public void setWaitBetweenUnsuccessfulPolls(boolean waitBetweenUnsuccessfulPolls) { + this.waitBetweenUnsuccessfulPolls = waitBetweenUnsuccessfulPolls; + } - @Override - public Duration getInitialIdleTime() { - return initialIdleTime; - } + @Override + public Duration getInitialIdleTime() { + return initialIdleTime; + } - public void setInitialIdleTime(Duration initialIdleTime) { - this.initialIdleTime = initialIdleTime; - } + public void setInitialIdleTime(Duration initialIdleTime) { + this.initialIdleTime = initialIdleTime; + } - @Override - public Duration getMaxIdleTime() { - return maxIdleTime; - } + @Override + public Duration getMaxIdleTime() { + return maxIdleTime; + } - public void setMaxIdleTime(Duration maxIdleTime) { - this.maxIdleTime = maxIdleTime; - } + public void setMaxIdleTime(Duration maxIdleTime) { + this.maxIdleTime = maxIdleTime; + } - @Override - public String getClientId() { - return clientId; - } + @Override + public String getClientId() { + return clientId; + } - public void setClientId(String clientId) { - this.clientId = clientId; - } + public void setClientId(String clientId) { + this.clientId = clientId; + } - @Override - public boolean isFilteringRateLimiterEnabled() { - return filteringRateLimiterEnabled; - } + @Override + public boolean isFilteringRateLimiterEnabled() { + return filteringRateLimiterEnabled; + } - public void setFilteringRateLimiterEnabled(boolean filteringRateLimiterEnabled) { - this.filteringRateLimiterEnabled = filteringRateLimiterEnabled; - } + public void setFilteringRateLimiterEnabled(boolean filteringRateLimiterEnabled) { + this.filteringRateLimiterEnabled = filteringRateLimiterEnabled; + } - @Override - public boolean isFilteringEnabled() { - return filteringEnabled; - } + @Override + public boolean isFilteringEnabled() { + return filteringEnabled; + } - public void setFilteringEnabled(boolean filteringEnabled) { - this.filteringEnabled = filteringEnabled; - } + public void setFilteringEnabled(boolean filteringEnabled) { + this.filteringEnabled = filteringEnabled; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ConsumerSenderConfiguration.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ConsumerSenderConfiguration.java index 50be00f5e7..499bd0f3aa 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ConsumerSenderConfiguration.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ConsumerSenderConfiguration.java @@ -10,6 +10,11 @@ import com.google.auth.oauth2.GoogleCredentials; import com.google.common.collect.ImmutableSet; import jakarta.inject.Named; +import java.io.IOException; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ScheduledExecutorService; +import javax.jms.Message; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.Request; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; @@ -53,220 +58,221 @@ import pl.allegro.tech.hermes.consumers.consumer.sender.timeout.FutureAsyncTimeout; import pl.allegro.tech.hermes.consumers.consumer.trace.MetadataAppender; -import java.io.IOException; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ScheduledExecutorService; -import javax.jms.Message; - @Configuration @EnableConfigurationProperties({ - SslContextProperties.class, - HttpClientsMonitoringProperties.class, - SenderAsyncTimeoutProperties.class, - BatchProperties.class + SslContextProperties.class, + HttpClientsMonitoringProperties.class, + SenderAsyncTimeoutProperties.class, + BatchProperties.class }) public class ConsumerSenderConfiguration { - @Bean(name = "http1-serial-client-parameters") - @ConfigurationProperties(prefix = "consumer.http-client.serial.http1") - public Http1ClientProperties http1SerialClientProperties() { - return new Http1ClientProperties(); - } + @Bean(name = "http1-serial-client-parameters") + @ConfigurationProperties(prefix = "consumer.http-client.serial.http1") + public Http1ClientProperties http1SerialClientProperties() { + return new Http1ClientProperties(); + } - @Bean(name = "http1-serial-client") - public HttpClient http1SerialClient( - HttpClientsFactory httpClientsFactory, - @Named("http1-serial-client-parameters") Http1ClientParameters http1ClientParameters) { - return httpClientsFactory.createClientForHttp1("jetty-http1-serial-client", http1ClientParameters); - } + @Bean(name = "http1-serial-client") + public HttpClient http1SerialClient( + HttpClientsFactory httpClientsFactory, + @Named("http1-serial-client-parameters") Http1ClientParameters http1ClientParameters) { + return httpClientsFactory.createClientForHttp1( + "jetty-http1-serial-client", http1ClientParameters); + } - @Bean(name = "http2-serial-client-parameters") - @ConfigurationProperties(prefix = "consumer.http-client.serial.http2") - public Http2ClientProperties http2SerialClientProperties() { - return new Http2ClientProperties(); - } + @Bean(name = "http2-serial-client-parameters") + @ConfigurationProperties(prefix = "consumer.http-client.serial.http2") + public Http2ClientProperties http2SerialClientProperties() { + return new Http2ClientProperties(); + } - @Bean - public Http2ClientHolder http2ClientHolder( - HttpClientsFactory httpClientsFactory, - @Named("http2-serial-client-parameters") Http2ClientProperties http2ClientProperties) { - if (!http2ClientProperties.isEnabled()) { - return new Http2ClientHolder(null); - } else { - return new Http2ClientHolder(httpClientsFactory.createClientForHttp2("jetty-http2-serial-client", http2ClientProperties)); - } + @Bean + public Http2ClientHolder http2ClientHolder( + HttpClientsFactory httpClientsFactory, + @Named("http2-serial-client-parameters") Http2ClientProperties http2ClientProperties) { + if (!http2ClientProperties.isEnabled()) { + return new Http2ClientHolder(null); + } else { + return new Http2ClientHolder( + httpClientsFactory.createClientForHttp2( + "jetty-http2-serial-client", http2ClientProperties)); } + } - @Bean(name = "http1-batch-client-parameters") - @ConfigurationProperties(prefix = "consumer.http-client.batch.http1") - public Http1ClientProperties http1BatchClientProperties() { - return new Http1ClientProperties(); - } - - @Bean(name = "http1-batch-client") - public HttpClient http1BatchClient( - HttpClientsFactory httpClientsFactory, - @Named("http1-batch-client-parameters") Http1ClientParameters http1ClientParameters) { - return httpClientsFactory.createClientForHttp1("jetty-http1-batch-client", http1ClientParameters); - } - - @Bean(name = "oauth-http-client") - public HttpClient oauthHttpClient(HttpClientsFactory httpClientsFactory, - @Named("http1-serial-client-parameters") Http1ClientParameters http1ClientParameters) { - return httpClientsFactory.createClientForHttp1("jetty-http-oauthclient", http1ClientParameters); - } + @Bean(name = "http1-batch-client-parameters") + @ConfigurationProperties(prefix = "consumer.http-client.batch.http1") + public Http1ClientProperties http1BatchClientProperties() { + return new Http1ClientProperties(); + } + @Bean(name = "http1-batch-client") + public HttpClient http1BatchClient( + HttpClientsFactory httpClientsFactory, + @Named("http1-batch-client-parameters") Http1ClientParameters http1ClientParameters) { + return httpClientsFactory.createClientForHttp1( + "jetty-http1-batch-client", http1ClientParameters); + } - @Bean(destroyMethod = "stop") - public BatchHttpRequestFactory batchHttpRequestFactory(@Named("http1-batch-client") HttpClient httpClient) { - return new DefaultBatchHttpRequestFactory(httpClient); - } + @Bean(name = "oauth-http-client") + public HttpClient oauthHttpClient( + HttpClientsFactory httpClientsFactory, + @Named("http1-serial-client-parameters") Http1ClientParameters http1ClientParameters) { + return httpClientsFactory.createClientForHttp1("jetty-http-oauthclient", http1ClientParameters); + } - @Bean - public MessageBatchSenderFactory httpMessageBatchSenderFactory(SendingResultHandlers resultHandlers, - BatchHttpRequestFactory batchHttpRequestFactory - ) { - return new HttpMessageBatchSenderFactory( - resultHandlers, - batchHttpRequestFactory); - } + @Bean(destroyMethod = "stop") + public BatchHttpRequestFactory batchHttpRequestFactory( + @Named("http1-batch-client") HttpClient httpClient) { + return new DefaultBatchHttpRequestFactory(httpClient); + } - @Bean(initMethod = "start") - public HttpClientsWorkloadReporter httpClientsWorkloadReporter(MetricsFacade metrics, - @Named("http1-serial-client") HttpClient http1SerialClient, - @Named("http1-batch-client") HttpClient http1BatchClient, - Http2ClientHolder http2ClientHolder, - HttpClientsMonitoringProperties monitoringProperties) { - return new HttpClientsWorkloadReporter( - metrics, - http1SerialClient, - http1BatchClient, - http2ClientHolder, - monitoringProperties.isRequestQueueMonitoringEnabled(), - monitoringProperties.isConnectionPoolMonitoringEnabled()); - } + @Bean + public MessageBatchSenderFactory httpMessageBatchSenderFactory( + SendingResultHandlers resultHandlers, BatchHttpRequestFactory batchHttpRequestFactory) { + return new HttpMessageBatchSenderFactory(resultHandlers, batchHttpRequestFactory); + } - @Bean(destroyMethod = "closeProviders") - public MessageSenderFactory messageSenderFactory(List providers) { - return new MessageSenderFactory(providers); - } + @Bean(initMethod = "start") + public HttpClientsWorkloadReporter httpClientsWorkloadReporter( + MetricsFacade metrics, + @Named("http1-serial-client") HttpClient http1SerialClient, + @Named("http1-batch-client") HttpClient http1BatchClient, + Http2ClientHolder http2ClientHolder, + HttpClientsMonitoringProperties monitoringProperties) { + return new HttpClientsWorkloadReporter( + metrics, + http1SerialClient, + http1BatchClient, + http2ClientHolder, + monitoringProperties.isRequestQueueMonitoringEnabled(), + monitoringProperties.isConnectionPoolMonitoringEnabled()); + } - @Bean(name = "defaultHttpMessageSenderProvider") - public ProtocolMessageSenderProvider jettyHttpMessageSenderProvider(@Named("http1-serial-client") HttpClient httpClient, - Http2ClientHolder http2ClientHolder, - EndpointAddressResolver endpointAddressResolver, - MetadataAppender metadataAppender, - HttpAuthorizationProviderFactory authorizationProviderFactory, - HttpHeadersProvidersFactory httpHeadersProviderFactory, - SendingResultHandlers sendingResultHandlers, - HttpRequestFactoryProvider requestFactoryProvider) { - return new JettyHttpMessageSenderProvider( - httpClient, - http2ClientHolder, - endpointAddressResolver, - metadataAppender, - authorizationProviderFactory, - httpHeadersProviderFactory, - sendingResultHandlers, - requestFactoryProvider, - ImmutableSet.of("http", "https") - ); - } + @Bean(destroyMethod = "closeProviders") + public MessageSenderFactory messageSenderFactory(List providers) { + return new MessageSenderFactory(providers); + } - @Bean - public MetadataAppender defaultHttpMetadataAppender() { - return new DefaultHttpMetadataAppender(); - } + @Bean(name = "defaultHttpMessageSenderProvider") + public ProtocolMessageSenderProvider jettyHttpMessageSenderProvider( + @Named("http1-serial-client") HttpClient httpClient, + Http2ClientHolder http2ClientHolder, + EndpointAddressResolver endpointAddressResolver, + MetadataAppender metadataAppender, + HttpAuthorizationProviderFactory authorizationProviderFactory, + HttpHeadersProvidersFactory httpHeadersProviderFactory, + SendingResultHandlers sendingResultHandlers, + HttpRequestFactoryProvider requestFactoryProvider) { + return new JettyHttpMessageSenderProvider( + httpClient, + http2ClientHolder, + endpointAddressResolver, + metadataAppender, + authorizationProviderFactory, + httpHeadersProviderFactory, + sendingResultHandlers, + requestFactoryProvider, + ImmutableSet.of("http", "https")); + } - @Bean - public HttpRequestFactoryProvider defaultHttpRequestFactoryProvider() { - return new DefaultHttpRequestFactoryProvider(); - } + @Bean + public MetadataAppender defaultHttpMetadataAppender() { + return new DefaultHttpMetadataAppender(); + } - @Bean - public SendingResultHandlers defaultSendingResultHandlers() { - return new DefaultSendingResultHandlers(); - } + @Bean + public HttpRequestFactoryProvider defaultHttpRequestFactoryProvider() { + return new DefaultHttpRequestFactoryProvider(); + } - @Bean - public HttpHeadersProvidersFactory emptyHttpHeadersProvidersFactory() { - return new EmptyHttpHeadersProvidersFactory(); - } + @Bean + public SendingResultHandlers defaultSendingResultHandlers() { + return new DefaultSendingResultHandlers(); + } + @Bean + public HttpHeadersProvidersFactory emptyHttpHeadersProvidersFactory() { + return new EmptyHttpHeadersProvidersFactory(); + } - @Bean - public HttpClientsFactory httpClientsFactory(InstrumentedExecutorServiceFactory executorFactory, - SslContextFactoryProvider sslContextFactoryProvider) { - return new HttpClientsFactory(executorFactory, sslContextFactoryProvider); - } + @Bean + public HttpClientsFactory httpClientsFactory( + InstrumentedExecutorServiceFactory executorFactory, + SslContextFactoryProvider sslContextFactoryProvider) { + return new HttpClientsFactory(executorFactory, sslContextFactoryProvider); + } - @Bean - public SslContextFactoryProvider sslContextFactoryProvider(Optional sslContextFactory, - SslContextProperties sslContextProperties) { - return new SslContextFactoryProvider(sslContextFactory.orElse(null), sslContextProperties); - } + @Bean + public SslContextFactoryProvider sslContextFactoryProvider( + Optional sslContextFactory, SslContextProperties sslContextProperties) { + return new SslContextFactoryProvider(sslContextFactory.orElse(null), sslContextProperties); + } - @Bean - public HttpAuthorizationProviderFactory httpAuthorizationProviderFactory(OAuthAccessTokens accessTokens) { - return new HttpAuthorizationProviderFactory(accessTokens); - } + @Bean + public HttpAuthorizationProviderFactory httpAuthorizationProviderFactory( + OAuthAccessTokens accessTokens) { + return new HttpAuthorizationProviderFactory(accessTokens); + } - @Bean(name = "defaultJmsMessageSenderProvider") - public ProtocolMessageSenderProvider jmsHornetQMessageSenderProvider(MetadataAppender metadataAppender) { - return new JmsHornetQMessageSenderProvider(metadataAppender); - } + @Bean(name = "defaultJmsMessageSenderProvider") + public ProtocolMessageSenderProvider jmsHornetQMessageSenderProvider( + MetadataAppender metadataAppender) { + return new JmsHornetQMessageSenderProvider(metadataAppender); + } - @Bean - public MetadataAppender jmsMetadataAppender() { - return new JmsMetadataAppender(); - } + @Bean + public MetadataAppender jmsMetadataAppender() { + return new JmsMetadataAppender(); + } - @Bean(name = "defaultPubSubMessageSenderProvider") - public ProtocolMessageSenderProvider pubSubMessageSenderProvider( - GooglePubSubSenderTargetResolver targetResolver, - CredentialsProvider credentialsProvider, - ExecutorProvider executorProvider, - RetrySettings retrySettings, - BatchingSettings batchingSettings, - GooglePubSubMessageTransformerCreator googlePubSubMessageTransformerCreator, - TransportChannelProvider transportChannelProvider) { - return new GooglePubSubMessageSenderProvider( - targetResolver, - credentialsProvider, - executorProvider, - retrySettings, - batchingSettings, - transportChannelProvider, - googlePubSubMessageTransformerCreator - ); - } + @Bean(name = "defaultPubSubMessageSenderProvider") + public ProtocolMessageSenderProvider pubSubMessageSenderProvider( + GooglePubSubSenderTargetResolver targetResolver, + CredentialsProvider credentialsProvider, + ExecutorProvider executorProvider, + RetrySettings retrySettings, + BatchingSettings batchingSettings, + GooglePubSubMessageTransformerCreator googlePubSubMessageTransformerCreator, + TransportChannelProvider transportChannelProvider) { + return new GooglePubSubMessageSenderProvider( + targetResolver, + credentialsProvider, + executorProvider, + retrySettings, + batchingSettings, + transportChannelProvider, + googlePubSubMessageTransformerCreator); + } - @Bean - @Conditional(OnGoogleDefaultCredentials.class) - public CredentialsProvider applicationDefaultCredentialsProvider() throws IOException { - return FixedCredentialsProvider.create(GoogleCredentials.getApplicationDefault()); - } + @Bean + @Conditional(OnGoogleDefaultCredentials.class) + public CredentialsProvider applicationDefaultCredentialsProvider() throws IOException { + return FixedCredentialsProvider.create(GoogleCredentials.getApplicationDefault()); + } - @Bean - @ConditionalOnMissingBean(CredentialsProvider.class) - public CredentialsProvider noCredentialsProvider() { - return NoCredentialsProvider.create(); - } + @Bean + @ConditionalOnMissingBean(CredentialsProvider.class) + public CredentialsProvider noCredentialsProvider() { + return NoCredentialsProvider.create(); + } - @Bean - public EndpointAddressResolver interpolatingEndpointAddressResolver(UriInterpolator interpolator) { - return new InterpolatingEndpointAddressResolver(interpolator); - } + @Bean + public EndpointAddressResolver interpolatingEndpointAddressResolver( + UriInterpolator interpolator) { + return new InterpolatingEndpointAddressResolver(interpolator); + } - @Bean - public FutureAsyncTimeout futureAsyncTimeoutFactory(InstrumentedExecutorServiceFactory executorFactory, - SenderAsyncTimeoutProperties senderAsyncTimeoutProperties) { - ScheduledExecutorService timeoutExecutorService = executorFactory.scheduledExecutorBuilder( - "async-timeout", - senderAsyncTimeoutProperties.getThreadPoolSize() - ).withMonitoringEnabled(senderAsyncTimeoutProperties.isThreadPoolMonitoringEnabled()) - .create(); - return new FutureAsyncTimeout(timeoutExecutorService); - } + @Bean + public FutureAsyncTimeout futureAsyncTimeoutFactory( + InstrumentedExecutorServiceFactory executorFactory, + SenderAsyncTimeoutProperties senderAsyncTimeoutProperties) { + ScheduledExecutorService timeoutExecutorService = + executorFactory + .scheduledExecutorBuilder( + "async-timeout", senderAsyncTimeoutProperties.getThreadPoolSize()) + .withMonitoringEnabled(senderAsyncTimeoutProperties.isThreadPoolMonitoringEnabled()) + .create(); + return new FutureAsyncTimeout(timeoutExecutorService); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ContentRootProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ContentRootProperties.java index fa45477278..5d4fc6f745 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ContentRootProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ContentRootProperties.java @@ -5,23 +5,23 @@ @ConfigurationProperties(prefix = "consumer.content.root") public class ContentRootProperties { - private String message = "message"; + private String message = "message"; - private String metadata = "metadata"; + private String metadata = "metadata"; - public String getMessage() { - return message; - } + public String getMessage() { + return message; + } - public void setMessage(String message) { - this.message = message; - } + public void setMessage(String message) { + this.message = message; + } - public String getMetadata() { - return metadata; - } + public String getMetadata() { + return metadata; + } - public void setMetadata(String metadata) { - this.metadata = metadata; - } + public void setMetadata(String metadata) { + this.metadata = metadata; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/DatacenterNameProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/DatacenterNameProperties.java index e3260cd59b..f34c8a5700 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/DatacenterNameProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/DatacenterNameProperties.java @@ -6,23 +6,23 @@ @ConfigurationProperties(prefix = "consumer.datacenter.name") public class DatacenterNameProperties { - private DcNameSource source; + private DcNameSource source; - private String env = "DC"; + private String env = "DC"; - public DcNameSource getSource() { - return source; - } + public DcNameSource getSource() { + return source; + } - public void setSource(DcNameSource source) { - this.source = source; - } + public void setSource(DcNameSource source) { + this.source = source; + } - public String getEnv() { - return env; - } + public String getEnv() { + return env; + } - public void setEnv(String env) { - this.env = env; - } + public void setEnv(String env) { + this.env = env; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/GooglePubSubCompressorProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/GooglePubSubCompressorProperties.java index 75ca9919ab..44f5545286 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/GooglePubSubCompressorProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/GooglePubSubCompressorProperties.java @@ -5,33 +5,33 @@ @ConfigurationProperties(prefix = "consumer.google.pubsub.compressor") public class GooglePubSubCompressorProperties { - private Boolean enabled = false; + private Boolean enabled = false; - private String compressionLevel = "high"; + private String compressionLevel = "high"; - private Long compressionThresholdBytes = 400L; + private Long compressionThresholdBytes = 400L; - public Boolean isEnabled() { - return enabled; - } + public Boolean isEnabled() { + return enabled; + } - public void setEnabled(Boolean enabled) { - this.enabled = enabled; - } + public void setEnabled(Boolean enabled) { + this.enabled = enabled; + } - public String getCompressionLevel() { - return compressionLevel; - } + public String getCompressionLevel() { + return compressionLevel; + } - public void setCompressionLevel(String compressionLevel) { - this.compressionLevel = compressionLevel; - } + public void setCompressionLevel(String compressionLevel) { + this.compressionLevel = compressionLevel; + } - public Long getCompressionThresholdBytes() { - return compressionThresholdBytes; - } + public Long getCompressionThresholdBytes() { + return compressionThresholdBytes; + } - public void setCompressionThresholdBytes(Long compressionThresholdBytes) { - this.compressionThresholdBytes = compressionThresholdBytes; - } + public void setCompressionThresholdBytes(Long compressionThresholdBytes) { + this.compressionThresholdBytes = compressionThresholdBytes; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/GooglePubSubConfiguration.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/GooglePubSubConfiguration.java index 038687d981..b533f14969 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/GooglePubSubConfiguration.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/GooglePubSubConfiguration.java @@ -8,6 +8,8 @@ import com.google.cloud.pubsub.v1.TopicAdminSettings; import com.google.common.util.concurrent.ThreadFactoryBuilder; import jakarta.inject.Named; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -15,68 +17,74 @@ import pl.allegro.tech.hermes.consumers.consumer.sender.googlepubsub.GooglePubSubMessageTransformerCreator; import pl.allegro.tech.hermes.consumers.consumer.sender.googlepubsub.GooglePubSubSenderTargetResolver; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; - @Configuration @EnableConfigurationProperties({ - GooglePubSubSenderProperties.class, GooglePubSubCompressorProperties.class + GooglePubSubSenderProperties.class, + GooglePubSubCompressorProperties.class }) public class GooglePubSubConfiguration { - @Bean - public TransportChannelProvider transportChannelProvider() { - return TopicAdminSettings.defaultGrpcTransportProviderBuilder().setChannelsPerCpu(1).build(); - } + @Bean + public TransportChannelProvider transportChannelProvider() { + return TopicAdminSettings.defaultGrpcTransportProviderBuilder().setChannelsPerCpu(1).build(); + } - @Bean - public GooglePubSubMessageTransformerCreator messageTransformerCreator(GooglePubSubCompressorProperties compressorProperties) { - return GooglePubSubMessageTransformerCreator.creator() - .withCompressionEnabled(compressorProperties.isEnabled()) - .withCompressionLevel(compressorProperties.getCompressionLevel()) - .withCompressionThresholdBytes(compressorProperties.getCompressionThresholdBytes()); - } + @Bean + public GooglePubSubMessageTransformerCreator messageTransformerCreator( + GooglePubSubCompressorProperties compressorProperties) { + return GooglePubSubMessageTransformerCreator.creator() + .withCompressionEnabled(compressorProperties.isEnabled()) + .withCompressionLevel(compressorProperties.getCompressionLevel()) + .withCompressionThresholdBytes(compressorProperties.getCompressionThresholdBytes()); + } - @Bean - public GooglePubSubSenderTargetResolver pubSubSenderTargetResolver() { - return new GooglePubSubSenderTargetResolver(); - } + @Bean + public GooglePubSubSenderTargetResolver pubSubSenderTargetResolver() { + return new GooglePubSubSenderTargetResolver(); + } - @Bean(name = "googlePubSubPublishingExecutor", destroyMethod = "shutdown") - public ScheduledExecutorService googlePubSubPublishingExecutor(GooglePubSubSenderProperties googlePubSubSenderProperties) { - return Executors.newScheduledThreadPool(googlePubSubSenderProperties.getCorePoolSize(), - new ThreadFactoryBuilder().setNameFormat("pubsub-publisher-%d").build()); - } + @Bean(name = "googlePubSubPublishingExecutor", destroyMethod = "shutdown") + public ScheduledExecutorService googlePubSubPublishingExecutor( + GooglePubSubSenderProperties googlePubSubSenderProperties) { + return Executors.newScheduledThreadPool( + googlePubSubSenderProperties.getCorePoolSize(), + new ThreadFactoryBuilder().setNameFormat("pubsub-publisher-%d").build()); + } - @Bean - public ExecutorProvider googlePubSubPublishingExecutorProvider( - @Named("googlePubSubPublishingExecutor") ScheduledExecutorService googlePubSubPublishingExecutor) { + @Bean + public ExecutorProvider googlePubSubPublishingExecutorProvider( + @Named("googlePubSubPublishingExecutor") + ScheduledExecutorService googlePubSubPublishingExecutor) { - return FixedExecutorProvider.create(googlePubSubPublishingExecutor); - } + return FixedExecutorProvider.create(googlePubSubPublishingExecutor); + } - @Bean - public BatchingSettings batchingSettings(GooglePubSubSenderProperties googlePubSubSenderProperties) { - long requestBytesThreshold = googlePubSubSenderProperties.getBatchingRequestBytesThreshold(); - long messageCountBatchSize = googlePubSubSenderProperties.getBatchingMessageCountBytesSize(); - Duration publishDelayThreshold = Duration.ofMillis(googlePubSubSenderProperties.getBatchingPublishDelayThreshold().toMillis()); + @Bean + public BatchingSettings batchingSettings( + GooglePubSubSenderProperties googlePubSubSenderProperties) { + long requestBytesThreshold = googlePubSubSenderProperties.getBatchingRequestBytesThreshold(); + long messageCountBatchSize = googlePubSubSenderProperties.getBatchingMessageCountBytesSize(); + Duration publishDelayThreshold = + Duration.ofMillis( + googlePubSubSenderProperties.getBatchingPublishDelayThreshold().toMillis()); - return BatchingSettings.newBuilder() - .setElementCountThreshold(messageCountBatchSize) - .setRequestByteThreshold(requestBytesThreshold) - .setDelayThreshold(publishDelayThreshold) - .build(); - } + return BatchingSettings.newBuilder() + .setElementCountThreshold(messageCountBatchSize) + .setRequestByteThreshold(requestBytesThreshold) + .setDelayThreshold(publishDelayThreshold) + .build(); + } - @Bean - public RetrySettings retrySettings(GooglePubSubSenderProperties googlePubSubSenderProperties) { - Duration totalTimeout = Duration.ofMillis(googlePubSubSenderProperties.getTotalTimeout().toMillis()); + @Bean + public RetrySettings retrySettings(GooglePubSubSenderProperties googlePubSubSenderProperties) { + Duration totalTimeout = + Duration.ofMillis(googlePubSubSenderProperties.getTotalTimeout().toMillis()); - return RetrySettings.newBuilder() - .setInitialRpcTimeout(totalTimeout) - .setMaxRpcTimeout(totalTimeout) - .setTotalTimeout(totalTimeout) - .setMaxAttempts(1) - .build(); - } + return RetrySettings.newBuilder() + .setInitialRpcTimeout(totalTimeout) + .setMaxRpcTimeout(totalTimeout) + .setTotalTimeout(totalTimeout) + .setMaxAttempts(1) + .build(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/GooglePubSubSenderProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/GooglePubSubSenderProperties.java index ecf6785e4a..35f4c5a76d 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/GooglePubSubSenderProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/GooglePubSubSenderProperties.java @@ -1,69 +1,68 @@ package pl.allegro.tech.hermes.consumers.config; -import org.springframework.boot.context.properties.ConfigurationProperties; - import java.time.Duration; +import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties(prefix = "consumer.google.pubsub.sender") public class GooglePubSubSenderProperties { - private int corePoolSize = 4; + private int corePoolSize = 4; - private Duration totalTimeout = Duration.ofMillis(600_000); + private Duration totalTimeout = Duration.ofMillis(600_000); - private long batchingRequestBytesThreshold = 1024L; + private long batchingRequestBytesThreshold = 1024L; - private long batchingMessageCountBytesSize = 1L; + private long batchingMessageCountBytesSize = 1L; - private Duration batchingPublishDelayThreshold = Duration.ofMillis(1); + private Duration batchingPublishDelayThreshold = Duration.ofMillis(1); - private String transportChannelProviderAddress = "integration"; + private String transportChannelProviderAddress = "integration"; - public int getCorePoolSize() { - return corePoolSize; - } + public int getCorePoolSize() { + return corePoolSize; + } - public void setCorePoolSize(int corePoolSize) { - this.corePoolSize = corePoolSize; - } + public void setCorePoolSize(int corePoolSize) { + this.corePoolSize = corePoolSize; + } - public Duration getTotalTimeout() { - return totalTimeout; - } + public Duration getTotalTimeout() { + return totalTimeout; + } - public void setTotalTimeout(Duration totalTimeout) { - this.totalTimeout = totalTimeout; - } + public void setTotalTimeout(Duration totalTimeout) { + this.totalTimeout = totalTimeout; + } - public long getBatchingRequestBytesThreshold() { - return batchingRequestBytesThreshold; - } + public long getBatchingRequestBytesThreshold() { + return batchingRequestBytesThreshold; + } - public void setBatchingRequestBytesThreshold(long batchingRequestBytesThreshold) { - this.batchingRequestBytesThreshold = batchingRequestBytesThreshold; - } + public void setBatchingRequestBytesThreshold(long batchingRequestBytesThreshold) { + this.batchingRequestBytesThreshold = batchingRequestBytesThreshold; + } - public long getBatchingMessageCountBytesSize() { - return batchingMessageCountBytesSize; - } + public long getBatchingMessageCountBytesSize() { + return batchingMessageCountBytesSize; + } - public void setBatchingMessageCountBytesSize(long batchingMessageCountBytesSize) { - this.batchingMessageCountBytesSize = batchingMessageCountBytesSize; - } + public void setBatchingMessageCountBytesSize(long batchingMessageCountBytesSize) { + this.batchingMessageCountBytesSize = batchingMessageCountBytesSize; + } - public Duration getBatchingPublishDelayThreshold() { - return batchingPublishDelayThreshold; - } + public Duration getBatchingPublishDelayThreshold() { + return batchingPublishDelayThreshold; + } - public void setBatchingPublishDelayThreshold(Duration batchingPublishDelayThreshold) { - this.batchingPublishDelayThreshold = batchingPublishDelayThreshold; - } + public void setBatchingPublishDelayThreshold(Duration batchingPublishDelayThreshold) { + this.batchingPublishDelayThreshold = batchingPublishDelayThreshold; + } - public String getTransportChannelProviderAddress() { - return transportChannelProviderAddress; - } + public String getTransportChannelProviderAddress() { + return transportChannelProviderAddress; + } - public void setTransportChannelProviderAddress(String transportChannelProviderAddress) { - this.transportChannelProviderAddress = transportChannelProviderAddress; - } + public void setTransportChannelProviderAddress(String transportChannelProviderAddress) { + this.transportChannelProviderAddress = transportChannelProviderAddress; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/Http1ClientProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/Http1ClientProperties.java index 21f2119552..f6948a9ba5 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/Http1ClientProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/Http1ClientProperties.java @@ -1,86 +1,84 @@ package pl.allegro.tech.hermes.consumers.config; -import pl.allegro.tech.hermes.consumers.consumer.sender.http.Http1ClientParameters; - import java.time.Duration; +import pl.allegro.tech.hermes.consumers.consumer.sender.http.Http1ClientParameters; public class Http1ClientProperties implements Http1ClientParameters { - private int threadPoolSize = 30; - - private boolean threadPoolMonitoringEnabled = false; + private int threadPoolSize = 30; - private boolean followRedirectsEnabled = false; + private boolean threadPoolMonitoringEnabled = false; - private int maxConnectionsPerDestination = 100; + private boolean followRedirectsEnabled = false; - private Duration idleTimeout = Duration.ofMillis(0); + private int maxConnectionsPerDestination = 100; - private int maxRequestsQueuedPerDestination = 100; + private Duration idleTimeout = Duration.ofMillis(0); - private Duration connectionTimeout = Duration.ofSeconds(15); + private int maxRequestsQueuedPerDestination = 100; + private Duration connectionTimeout = Duration.ofSeconds(15); - @Override - public int getThreadPoolSize() { - return threadPoolSize; - } + @Override + public int getThreadPoolSize() { + return threadPoolSize; + } - public void setThreadPoolSize(int threadPoolSize) { - this.threadPoolSize = threadPoolSize; - } + public void setThreadPoolSize(int threadPoolSize) { + this.threadPoolSize = threadPoolSize; + } - @Override - public boolean isThreadPoolMonitoringEnabled() { - return threadPoolMonitoringEnabled; - } + @Override + public boolean isThreadPoolMonitoringEnabled() { + return threadPoolMonitoringEnabled; + } - public void setThreadPoolMonitoringEnabled(boolean threadPoolMonitoringEnabled) { - this.threadPoolMonitoringEnabled = threadPoolMonitoringEnabled; - } + public void setThreadPoolMonitoringEnabled(boolean threadPoolMonitoringEnabled) { + this.threadPoolMonitoringEnabled = threadPoolMonitoringEnabled; + } - @Override - public boolean isFollowRedirectsEnabled() { - return followRedirectsEnabled; - } + @Override + public boolean isFollowRedirectsEnabled() { + return followRedirectsEnabled; + } - public void setFollowRedirectsEnabled(boolean followRedirectsEnabled) { - this.followRedirectsEnabled = followRedirectsEnabled; - } + public void setFollowRedirectsEnabled(boolean followRedirectsEnabled) { + this.followRedirectsEnabled = followRedirectsEnabled; + } - @Override - public int getMaxConnectionsPerDestination() { - return maxConnectionsPerDestination; - } + @Override + public int getMaxConnectionsPerDestination() { + return maxConnectionsPerDestination; + } - public void setMaxConnectionsPerDestination(int maxConnectionsPerDestination) { - this.maxConnectionsPerDestination = maxConnectionsPerDestination; - } + public void setMaxConnectionsPerDestination(int maxConnectionsPerDestination) { + this.maxConnectionsPerDestination = maxConnectionsPerDestination; + } - @Override - public Duration getIdleTimeout() { - return idleTimeout; - } + @Override + public Duration getIdleTimeout() { + return idleTimeout; + } - public void setIdleTimeout(Duration idleTimeout) { - this.idleTimeout = idleTimeout; - } + public void setIdleTimeout(Duration idleTimeout) { + this.idleTimeout = idleTimeout; + } - @Override - public int getMaxRequestsQueuedPerDestination() { - return maxRequestsQueuedPerDestination; - } + @Override + public int getMaxRequestsQueuedPerDestination() { + return maxRequestsQueuedPerDestination; + } - public void setMaxRequestsQueuedPerDestination(int maxRequestsQueuedPerDestination) { - this.maxRequestsQueuedPerDestination = maxRequestsQueuedPerDestination; - } + public void setMaxRequestsQueuedPerDestination(int maxRequestsQueuedPerDestination) { + this.maxRequestsQueuedPerDestination = maxRequestsQueuedPerDestination; + } - @Override - public Duration getConnectionTimeout() { - return connectionTimeout; - } + @Override + public Duration getConnectionTimeout() { + return connectionTimeout; + } - public void setConnectionTimeout(Duration connectionTimeout) { - this.connectionTimeout = connectionTimeout; - } + public void setConnectionTimeout(Duration connectionTimeout) { + this.connectionTimeout = connectionTimeout; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/Http2ClientProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/Http2ClientProperties.java index b4ab115178..14e44e0791 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/Http2ClientProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/Http2ClientProperties.java @@ -1,84 +1,83 @@ package pl.allegro.tech.hermes.consumers.config; -import pl.allegro.tech.hermes.consumers.consumer.sender.http.Http2ClientParameters; - import java.time.Duration; +import pl.allegro.tech.hermes.consumers.consumer.sender.http.Http2ClientParameters; public class Http2ClientProperties implements Http2ClientParameters { - private boolean enabled = true; + private boolean enabled = true; - private int threadPoolSize = 10; + private int threadPoolSize = 10; - private boolean threadPoolMonitoringEnabled = false; + private boolean threadPoolMonitoringEnabled = false; - private Duration idleTimeout = Duration.ofMillis(0); + private Duration idleTimeout = Duration.ofMillis(0); - private int maxRequestsQueuedPerDestination = 100; + private int maxRequestsQueuedPerDestination = 100; - private boolean followRedirectsEnabled = false; + private boolean followRedirectsEnabled = false; - private Duration connectionTimeout = Duration.ofSeconds(15); + private Duration connectionTimeout = Duration.ofSeconds(15); - public boolean isEnabled() { - return enabled; - } + public boolean isEnabled() { + return enabled; + } - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } - @Override - public int getThreadPoolSize() { - return threadPoolSize; - } + @Override + public int getThreadPoolSize() { + return threadPoolSize; + } - public void setThreadPoolSize(int threadPoolSize) { - this.threadPoolSize = threadPoolSize; - } + public void setThreadPoolSize(int threadPoolSize) { + this.threadPoolSize = threadPoolSize; + } - @Override - public boolean isThreadPoolMonitoringEnabled() { - return threadPoolMonitoringEnabled; - } + @Override + public boolean isThreadPoolMonitoringEnabled() { + return threadPoolMonitoringEnabled; + } - public void setThreadPoolMonitoringEnabled(boolean threadPoolMonitoringEnabled) { - this.threadPoolMonitoringEnabled = threadPoolMonitoringEnabled; - } + public void setThreadPoolMonitoringEnabled(boolean threadPoolMonitoringEnabled) { + this.threadPoolMonitoringEnabled = threadPoolMonitoringEnabled; + } - @Override - public Duration getIdleTimeout() { - return idleTimeout; - } + @Override + public Duration getIdleTimeout() { + return idleTimeout; + } - public void setIdleTimeout(Duration idleTimeout) { - this.idleTimeout = idleTimeout; - } + public void setIdleTimeout(Duration idleTimeout) { + this.idleTimeout = idleTimeout; + } - @Override - public int getMaxRequestsQueuedPerDestination() { - return maxRequestsQueuedPerDestination; - } + @Override + public int getMaxRequestsQueuedPerDestination() { + return maxRequestsQueuedPerDestination; + } - public void setMaxRequestsQueuedPerDestination(int maxRequestsQueuedPerDestination) { - this.maxRequestsQueuedPerDestination = maxRequestsQueuedPerDestination; - } + public void setMaxRequestsQueuedPerDestination(int maxRequestsQueuedPerDestination) { + this.maxRequestsQueuedPerDestination = maxRequestsQueuedPerDestination; + } - @Override - public boolean isFollowRedirectsEnabled() { - return followRedirectsEnabled; - } + @Override + public boolean isFollowRedirectsEnabled() { + return followRedirectsEnabled; + } - public void setFollowRedirectsEnabled(boolean followRedirectsEnabled) { - this.followRedirectsEnabled = followRedirectsEnabled; - } + public void setFollowRedirectsEnabled(boolean followRedirectsEnabled) { + this.followRedirectsEnabled = followRedirectsEnabled; + } - @Override - public Duration getConnectionTimeout() { - return connectionTimeout; - } + @Override + public Duration getConnectionTimeout() { + return connectionTimeout; + } - public void setConnectionTimeout(Duration connectionTimeout) { - this.connectionTimeout = connectionTimeout; - } + public void setConnectionTimeout(Duration connectionTimeout) { + this.connectionTimeout = connectionTimeout; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/HttpClientsMonitoringProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/HttpClientsMonitoringProperties.java index b0bb50d2a9..57db022c41 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/HttpClientsMonitoringProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/HttpClientsMonitoringProperties.java @@ -5,23 +5,23 @@ @ConfigurationProperties(prefix = "consumer.http-client.monitoring") public class HttpClientsMonitoringProperties { - private boolean connectionPoolMonitoringEnabled = false; + private boolean connectionPoolMonitoringEnabled = false; - private boolean requestQueueMonitoringEnabled = true; + private boolean requestQueueMonitoringEnabled = true; - public boolean isConnectionPoolMonitoringEnabled() { - return connectionPoolMonitoringEnabled; - } + public boolean isConnectionPoolMonitoringEnabled() { + return connectionPoolMonitoringEnabled; + } - public void setConnectionPoolMonitoringEnabled(boolean connectionPoolMonitoringEnabled) { - this.connectionPoolMonitoringEnabled = connectionPoolMonitoringEnabled; - } + public void setConnectionPoolMonitoringEnabled(boolean connectionPoolMonitoringEnabled) { + this.connectionPoolMonitoringEnabled = connectionPoolMonitoringEnabled; + } - public boolean isRequestQueueMonitoringEnabled() { - return requestQueueMonitoringEnabled; - } + public boolean isRequestQueueMonitoringEnabled() { + return requestQueueMonitoringEnabled; + } - public void setRequestQueueMonitoringEnabled(boolean requestQueueMonitoringEnabled) { - this.requestQueueMonitoringEnabled = requestQueueMonitoringEnabled; - } + public void setRequestQueueMonitoringEnabled(boolean requestQueueMonitoringEnabled) { + this.requestQueueMonitoringEnabled = requestQueueMonitoringEnabled; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/KafkaClustersProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/KafkaClustersProperties.java index 2475f95102..cf4a71dec0 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/KafkaClustersProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/KafkaClustersProperties.java @@ -1,50 +1,53 @@ package pl.allegro.tech.hermes.consumers.config; -import org.springframework.boot.context.properties.ConfigurationProperties; -import pl.allegro.tech.hermes.infrastructure.dc.DatacenterNameProvider; - import java.util.ArrayList; import java.util.List; +import org.springframework.boot.context.properties.ConfigurationProperties; +import pl.allegro.tech.hermes.infrastructure.dc.DatacenterNameProvider; @ConfigurationProperties(prefix = "consumer.kafka") public class KafkaClustersProperties { - private List clusters = new ArrayList<>(); + private List clusters = new ArrayList<>(); - private String namespace = ""; + private String namespace = ""; - private String namespaceSeparator = "_"; + private String namespaceSeparator = "_"; - public List getClusters() { - return clusters; - } + public List getClusters() { + return clusters; + } - public void setClusters(List clusters) { - this.clusters = clusters; - } + public void setClusters(List clusters) { + this.clusters = clusters; + } - public String getNamespace() { - return namespace; - } + public String getNamespace() { + return namespace; + } - public void setNamespace(String namespace) { - this.namespace = namespace; - } + public void setNamespace(String namespace) { + this.namespace = namespace; + } - public String getNamespaceSeparator() { - return namespaceSeparator; - } + public String getNamespaceSeparator() { + return namespaceSeparator; + } - public void setNamespaceSeparator(String namespaceSeparator) { - this.namespaceSeparator = namespaceSeparator; - } + public void setNamespaceSeparator(String namespaceSeparator) { + this.namespaceSeparator = namespaceSeparator; + } - public KafkaProperties toKafkaProperties(DatacenterNameProvider datacenterNameProvider) { - return this.clusters - .stream() - .filter(cluster -> cluster.getDatacenter().equals(datacenterNameProvider.getDatacenterName())) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException( - "No properties for datacenter: " + datacenterNameProvider.getDatacenterName() + " defined.")); - } + public KafkaProperties toKafkaProperties(DatacenterNameProvider datacenterNameProvider) { + return this.clusters.stream() + .filter( + cluster -> cluster.getDatacenter().equals(datacenterNameProvider.getDatacenterName())) + .findFirst() + .orElseThrow( + () -> + new IllegalArgumentException( + "No properties for datacenter: " + + datacenterNameProvider.getDatacenterName() + + " defined.")); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/KafkaConsumerProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/KafkaConsumerProperties.java index 0c2340ec67..a8d1e51e69 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/KafkaConsumerProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/KafkaConsumerProperties.java @@ -1,235 +1,235 @@ package pl.allegro.tech.hermes.consumers.config; +import static pl.allegro.tech.hermes.consumers.consumer.receiver.kafka.PartitionAssignmentStrategy.RANGE; + +import java.time.Duration; +import java.util.List; import org.springframework.boot.context.properties.ConfigurationProperties; import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.consumers.consumer.receiver.kafka.KafkaConsumerParameters; import pl.allegro.tech.hermes.consumers.consumer.receiver.kafka.PartitionAssignmentStrategy; -import java.time.Duration; -import java.util.List; - -import static pl.allegro.tech.hermes.consumers.consumer.receiver.kafka.PartitionAssignmentStrategy.RANGE; - @ConfigurationProperties(prefix = "consumer.kafka.consumer") public class KafkaConsumerProperties implements KafkaConsumerParameters { - private int sendBufferBytes = 256 * 1024; - - private int receiveBufferBytes = 256 * 1024; + private int sendBufferBytes = 256 * 1024; - private int fetchMinBytes = 1; + private int receiveBufferBytes = 256 * 1024; - private Duration fetchMaxWait = Duration.ofMillis(500); + private int fetchMinBytes = 1; - private Duration reconnectBackoff = Duration.ofMillis(500); + private Duration fetchMaxWait = Duration.ofMillis(500); - private Duration retryBackoff = Duration.ofMillis(500); + private Duration reconnectBackoff = Duration.ofMillis(500); - private boolean checkCrcs = true; + private Duration retryBackoff = Duration.ofMillis(500); - private Duration metricsSampleWindow = Duration.ofSeconds(30); + private boolean checkCrcs = true; - private int metricsNumSamples = 2; + private Duration metricsSampleWindow = Duration.ofSeconds(30); - private Duration requestTimeout = Duration.ofSeconds(250); + private int metricsNumSamples = 2; - private Duration connectionsMaxIdle = Duration.ofMinutes(9); + private Duration requestTimeout = Duration.ofSeconds(250); - private int maxPollRecords = 1; + private Duration connectionsMaxIdle = Duration.ofMinutes(9); - private Duration maxPollInterval = Duration.ofMillis(Integer.MAX_VALUE); + private int maxPollRecords = 1; - private String autoOffsetReset = "earliest"; + private Duration maxPollInterval = Duration.ofMillis(Integer.MAX_VALUE); - private Duration sessionTimeout = Duration.ofSeconds(200); + private String autoOffsetReset = "earliest"; - private Duration heartbeatInterval = Duration.ofSeconds(3); + private Duration sessionTimeout = Duration.ofSeconds(200); - private Duration metadataMaxAge = Duration.ofMinutes(5); + private Duration heartbeatInterval = Duration.ofSeconds(3); - private int maxPartitionFetchMin = Topic.MIN_MESSAGE_SIZE; + private Duration metadataMaxAge = Duration.ofMinutes(5); - private int maxPartitionFetchMax = Topic.MAX_MESSAGE_SIZE; + private int maxPartitionFetchMin = Topic.MIN_MESSAGE_SIZE; - private List partitionAssignmentStrategies = List.of(RANGE); + private int maxPartitionFetchMax = Topic.MAX_MESSAGE_SIZE; - @Override - public int getSendBufferBytes() { - return sendBufferBytes; - } + private List partitionAssignmentStrategies = List.of(RANGE); - public void setSendBufferBytes(int sendBufferBytes) { - this.sendBufferBytes = sendBufferBytes; - } + @Override + public int getSendBufferBytes() { + return sendBufferBytes; + } - @Override - public int getReceiveBufferBytes() { - return receiveBufferBytes; - } + public void setSendBufferBytes(int sendBufferBytes) { + this.sendBufferBytes = sendBufferBytes; + } - public void setReceiveBufferBytes(int receiveBufferBytes) { - this.receiveBufferBytes = receiveBufferBytes; - } + @Override + public int getReceiveBufferBytes() { + return receiveBufferBytes; + } - @Override - public int getFetchMinBytes() { - return fetchMinBytes; - } + public void setReceiveBufferBytes(int receiveBufferBytes) { + this.receiveBufferBytes = receiveBufferBytes; + } - public void setFetchMinBytes(int fetchMinBytes) { - this.fetchMinBytes = fetchMinBytes; - } + @Override + public int getFetchMinBytes() { + return fetchMinBytes; + } - @Override - public Duration getFetchMaxWait() { - return fetchMaxWait; - } + public void setFetchMinBytes(int fetchMinBytes) { + this.fetchMinBytes = fetchMinBytes; + } - public void setFetchMaxWait(Duration fetchMaxWait) { - this.fetchMaxWait = fetchMaxWait; - } + @Override + public Duration getFetchMaxWait() { + return fetchMaxWait; + } - @Override - public Duration getReconnectBackoff() { - return reconnectBackoff; - } + public void setFetchMaxWait(Duration fetchMaxWait) { + this.fetchMaxWait = fetchMaxWait; + } - public void setReconnectBackoff(Duration reconnectBackoff) { - this.reconnectBackoff = reconnectBackoff; - } + @Override + public Duration getReconnectBackoff() { + return reconnectBackoff; + } - @Override - public Duration getRetryBackoff() { - return retryBackoff; - } + public void setReconnectBackoff(Duration reconnectBackoff) { + this.reconnectBackoff = reconnectBackoff; + } - public void setRetryBackoff(Duration retryBackoff) { - this.retryBackoff = retryBackoff; - } + @Override + public Duration getRetryBackoff() { + return retryBackoff; + } - @Override - public boolean isCheckCrcs() { - return checkCrcs; - } + public void setRetryBackoff(Duration retryBackoff) { + this.retryBackoff = retryBackoff; + } - public void setCheckCrcs(boolean checkCrcs) { - this.checkCrcs = checkCrcs; - } + @Override + public boolean isCheckCrcs() { + return checkCrcs; + } - @Override - public Duration getMetricsSampleWindow() { - return metricsSampleWindow; - } + public void setCheckCrcs(boolean checkCrcs) { + this.checkCrcs = checkCrcs; + } - public void setMetricsSampleWindow(Duration metricsSampleWindow) { - this.metricsSampleWindow = metricsSampleWindow; - } + @Override + public Duration getMetricsSampleWindow() { + return metricsSampleWindow; + } - @Override - public int getMetricsNumSamples() { - return metricsNumSamples; - } + public void setMetricsSampleWindow(Duration metricsSampleWindow) { + this.metricsSampleWindow = metricsSampleWindow; + } - public void setMetricsNumSamples(int metricsNumSamples) { - this.metricsNumSamples = metricsNumSamples; - } + @Override + public int getMetricsNumSamples() { + return metricsNumSamples; + } - @Override - public Duration getRequestTimeout() { - return requestTimeout; - } - - public void setRequestTimeout(Duration requestTimeout) { - this.requestTimeout = requestTimeout; - } - - @Override - public Duration getConnectionsMaxIdle() { - return connectionsMaxIdle; - } - - public void setConnectionsMaxIdle(Duration connectionsMaxIdle) { - this.connectionsMaxIdle = connectionsMaxIdle; - } - - @Override - public int getMaxPollRecords() { - return maxPollRecords; - } - - public void setMaxPollRecords(int maxPollRecords) { - this.maxPollRecords = maxPollRecords; - } - - @Override - public Duration getMaxPollInterval() { - return maxPollInterval; - } - - public void setMaxPollInterval(Duration maxPollInterval) { - this.maxPollInterval = maxPollInterval; - } - - @Override - public String getAutoOffsetReset() { - return autoOffsetReset; - } - - public void setAutoOffsetReset(String autoOffsetReset) { - this.autoOffsetReset = autoOffsetReset; - } - - @Override - public Duration getSessionTimeout() { - return sessionTimeout; - } - - public void setSessionTimeout(Duration sessionTimeout) { - this.sessionTimeout = sessionTimeout; - } - - @Override - public Duration getHeartbeatInterval() { - return heartbeatInterval; - } - - public void setHeartbeatInterval(Duration heartbeatInterval) { - this.heartbeatInterval = heartbeatInterval; - } - - @Override - public Duration getMetadataMaxAge() { - return metadataMaxAge; - } - - public void setMetadataMaxAge(Duration metadataMaxAge) { - this.metadataMaxAge = metadataMaxAge; - } - - @Override - public int getMaxPartitionFetchMin() { - return maxPartitionFetchMin; - } - - public void setMaxPartitionFetchMin(int maxPartitionFetchMin) { - this.maxPartitionFetchMin = maxPartitionFetchMin; - } - - @Override - public int getMaxPartitionFetchMax() { - return maxPartitionFetchMax; - } - - public void setMaxPartitionFetchMax(int maxPartitionFetchMax) { - this.maxPartitionFetchMax = maxPartitionFetchMax; - } - - @Override - public List getPartitionAssignmentStrategies() { - return partitionAssignmentStrategies; - } - - public void setPartitionAssignmentStrategies(List partitionAssignmentStrategies) { - this.partitionAssignmentStrategies = partitionAssignmentStrategies; - } + public void setMetricsNumSamples(int metricsNumSamples) { + this.metricsNumSamples = metricsNumSamples; + } + + @Override + public Duration getRequestTimeout() { + return requestTimeout; + } + + public void setRequestTimeout(Duration requestTimeout) { + this.requestTimeout = requestTimeout; + } + + @Override + public Duration getConnectionsMaxIdle() { + return connectionsMaxIdle; + } + + public void setConnectionsMaxIdle(Duration connectionsMaxIdle) { + this.connectionsMaxIdle = connectionsMaxIdle; + } + + @Override + public int getMaxPollRecords() { + return maxPollRecords; + } + + public void setMaxPollRecords(int maxPollRecords) { + this.maxPollRecords = maxPollRecords; + } + + @Override + public Duration getMaxPollInterval() { + return maxPollInterval; + } + + public void setMaxPollInterval(Duration maxPollInterval) { + this.maxPollInterval = maxPollInterval; + } + + @Override + public String getAutoOffsetReset() { + return autoOffsetReset; + } + + public void setAutoOffsetReset(String autoOffsetReset) { + this.autoOffsetReset = autoOffsetReset; + } + + @Override + public Duration getSessionTimeout() { + return sessionTimeout; + } + + public void setSessionTimeout(Duration sessionTimeout) { + this.sessionTimeout = sessionTimeout; + } + + @Override + public Duration getHeartbeatInterval() { + return heartbeatInterval; + } + + public void setHeartbeatInterval(Duration heartbeatInterval) { + this.heartbeatInterval = heartbeatInterval; + } + + @Override + public Duration getMetadataMaxAge() { + return metadataMaxAge; + } + + public void setMetadataMaxAge(Duration metadataMaxAge) { + this.metadataMaxAge = metadataMaxAge; + } + + @Override + public int getMaxPartitionFetchMin() { + return maxPartitionFetchMin; + } + + public void setMaxPartitionFetchMin(int maxPartitionFetchMin) { + this.maxPartitionFetchMin = maxPartitionFetchMin; + } + + @Override + public int getMaxPartitionFetchMax() { + return maxPartitionFetchMax; + } + + public void setMaxPartitionFetchMax(int maxPartitionFetchMax) { + this.maxPartitionFetchMax = maxPartitionFetchMax; + } + + @Override + public List getPartitionAssignmentStrategies() { + return partitionAssignmentStrategies; + } + + public void setPartitionAssignmentStrategies( + List partitionAssignmentStrategies) { + this.partitionAssignmentStrategies = partitionAssignmentStrategies; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/KafkaHeaderNameProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/KafkaHeaderNameProperties.java index 4d90038b9e..c0ca77718b 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/KafkaHeaderNameProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/KafkaHeaderNameProperties.java @@ -6,37 +6,36 @@ @ConfigurationProperties(prefix = "consumer.kafka.header.name") public class KafkaHeaderNameProperties implements KafkaHeaderNameParameters { - private String schemaVersion = "sv"; + private String schemaVersion = "sv"; - private String schemaId = "sid"; + private String schemaId = "sid"; - private String messageId = "id"; + private String messageId = "id"; - @Override - public String getSchemaVersion() { - return schemaVersion; - } + @Override + public String getSchemaVersion() { + return schemaVersion; + } - public void setSchemaVersion(String schemaVersion) { - this.schemaVersion = schemaVersion; - } + public void setSchemaVersion(String schemaVersion) { + this.schemaVersion = schemaVersion; + } - @Override - public String getSchemaId() { - return schemaId; - } + @Override + public String getSchemaId() { + return schemaId; + } - public void setSchemaId(String schemaId) { - this.schemaId = schemaId; - } + public void setSchemaId(String schemaId) { + this.schemaId = schemaId; + } - @Override - public String getMessageId() { - return messageId; - } - - public void setMessageId(String messageId) { - this.messageId = messageId; - } + @Override + public String getMessageId() { + return messageId; + } + public void setMessageId(String messageId) { + this.messageId = messageId; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/KafkaProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/KafkaProperties.java index 0909d7f349..b8efcc00d4 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/KafkaProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/KafkaProperties.java @@ -5,69 +5,69 @@ public class KafkaProperties implements KafkaParameters { - private KafkaAuthenticationProperties authentication = new KafkaAuthenticationProperties(); + private KafkaAuthenticationProperties authentication = new KafkaAuthenticationProperties(); - private String datacenter = "dc"; + private String datacenter = "dc"; - private String clusterName = "primary"; + private String clusterName = "primary"; - private String brokerList = "localhost:9092"; + private String brokerList = "localhost:9092"; - public KafkaAuthenticationProperties getAuthentication() { - return authentication; - } + public KafkaAuthenticationProperties getAuthentication() { + return authentication; + } - @Deprecated - public void setAuthorization(KafkaAuthenticationProperties authorization) { - this.authentication = authorization; - } + @Deprecated + public void setAuthorization(KafkaAuthenticationProperties authorization) { + this.authentication = authorization; + } - public void setAuthentication(KafkaAuthenticationProperties authentication) { - this.authentication = authentication; - } + public void setAuthentication(KafkaAuthenticationProperties authentication) { + this.authentication = authentication; + } - public String getDatacenter() { - return datacenter; - } + public String getDatacenter() { + return datacenter; + } - public void setDatacenter(String datacenter) { - this.datacenter = datacenter; - } + public void setDatacenter(String datacenter) { + this.datacenter = datacenter; + } - public String getClusterName() { - return clusterName; - } + public String getClusterName() { + return clusterName; + } - public void setClusterName(String clusterName) { - this.clusterName = clusterName; - } + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } - @Override - public boolean isAuthenticationEnabled() { - return authentication.isEnabled(); - } + @Override + public boolean isAuthenticationEnabled() { + return authentication.isEnabled(); + } - @Override - public String getAuthenticationMechanism() { - return authentication.getMechanism(); - } + @Override + public String getAuthenticationMechanism() { + return authentication.getMechanism(); + } - @Override - public String getAuthenticationProtocol() { - return authentication.getProtocol(); - } + @Override + public String getAuthenticationProtocol() { + return authentication.getProtocol(); + } - @Override - public String getBrokerList() { - return brokerList; - } + @Override + public String getBrokerList() { + return brokerList; + } - public void setBrokerList(String brokerList) { - this.brokerList = brokerList; - } + public void setBrokerList(String brokerList) { + this.brokerList = brokerList; + } - @Override - public String getJaasConfig() { - return authentication.getJaasConfig(); - } + @Override + public String getJaasConfig() { + return authentication.getJaasConfig(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/MaxRateProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/MaxRateProperties.java index 227739f7a7..3817342acc 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/MaxRateProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/MaxRateProperties.java @@ -1,97 +1,97 @@ package pl.allegro.tech.hermes.consumers.config; +import java.time.Duration; import org.springframework.boot.context.properties.ConfigurationProperties; import pl.allegro.tech.hermes.consumers.consumer.rate.maxrate.MaxRateParameters; -import java.time.Duration; - @ConfigurationProperties(prefix = "consumer.maxrate") public class MaxRateProperties implements MaxRateParameters { - private RegistryBinaryEncoderProperties registryBinaryEncoder = new RegistryBinaryEncoderProperties(); + private RegistryBinaryEncoderProperties registryBinaryEncoder = + new RegistryBinaryEncoderProperties(); - private Duration balanceInterval = Duration.ofSeconds(30); + private Duration balanceInterval = Duration.ofSeconds(30); - private Duration updateInterval = Duration.ofSeconds(15); + private Duration updateInterval = Duration.ofSeconds(15); - private int historySize = 1; + private int historySize = 1; - private double busyTolerance = 0.1; + private double busyTolerance = 0.1; - private double minMaxRate = 1.0; + private double minMaxRate = 1.0; - private double minAllowedChangePercent = 1.0; + private double minAllowedChangePercent = 1.0; - private double minSignificantUpdatePercent = 9.0; + private double minSignificantUpdatePercent = 9.0; - public RegistryBinaryEncoderProperties getRegistryBinaryEncoder() { - return registryBinaryEncoder; - } + public RegistryBinaryEncoderProperties getRegistryBinaryEncoder() { + return registryBinaryEncoder; + } - public void setRegistryBinaryEncoder(RegistryBinaryEncoderProperties registryBinaryEncoder) { - this.registryBinaryEncoder = registryBinaryEncoder; - } + public void setRegistryBinaryEncoder(RegistryBinaryEncoderProperties registryBinaryEncoder) { + this.registryBinaryEncoder = registryBinaryEncoder; + } - @Override - public Duration getBalanceInterval() { - return balanceInterval; - } + @Override + public Duration getBalanceInterval() { + return balanceInterval; + } - public void setBalanceInterval(Duration balanceInterval) { - this.balanceInterval = balanceInterval; - } + public void setBalanceInterval(Duration balanceInterval) { + this.balanceInterval = balanceInterval; + } - @Override - public Duration getUpdateInterval() { - return updateInterval; - } + @Override + public Duration getUpdateInterval() { + return updateInterval; + } - public void setUpdateInterval(Duration updateInterval) { - this.updateInterval = updateInterval; - } + public void setUpdateInterval(Duration updateInterval) { + this.updateInterval = updateInterval; + } - @Override - public int getHistorySize() { - return historySize; - } + @Override + public int getHistorySize() { + return historySize; + } - public void setHistorySize(int historySize) { - this.historySize = historySize; - } + public void setHistorySize(int historySize) { + this.historySize = historySize; + } - @Override - public double getBusyTolerance() { - return busyTolerance; - } + @Override + public double getBusyTolerance() { + return busyTolerance; + } - public void setBusyTolerance(double busyTolerance) { - this.busyTolerance = busyTolerance; - } + public void setBusyTolerance(double busyTolerance) { + this.busyTolerance = busyTolerance; + } - @Override - public double getMinMaxRate() { - return minMaxRate; - } + @Override + public double getMinMaxRate() { + return minMaxRate; + } - public void setMinMaxRate(double minMaxRate) { - this.minMaxRate = minMaxRate; - } + public void setMinMaxRate(double minMaxRate) { + this.minMaxRate = minMaxRate; + } - @Override - public double getMinAllowedChangePercent() { - return minAllowedChangePercent; - } + @Override + public double getMinAllowedChangePercent() { + return minAllowedChangePercent; + } - public void setMinAllowedChangePercent(double minAllowedChangePercent) { - this.minAllowedChangePercent = minAllowedChangePercent; - } + public void setMinAllowedChangePercent(double minAllowedChangePercent) { + this.minAllowedChangePercent = minAllowedChangePercent; + } - @Override - public double getMinSignificantUpdatePercent() { - return minSignificantUpdatePercent; - } + @Override + public double getMinSignificantUpdatePercent() { + return minSignificantUpdatePercent; + } - public void setMinSignificantUpdatePercent(double minSignificantUpdatePercent) { - this.minSignificantUpdatePercent = minSignificantUpdatePercent; - } + public void setMinSignificantUpdatePercent(double minSignificantUpdatePercent) { + this.minSignificantUpdatePercent = minSignificantUpdatePercent; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/MessageConfiguration.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/MessageConfiguration.java index acd9db4186..253d89f1b8 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/MessageConfiguration.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/MessageConfiguration.java @@ -10,9 +10,11 @@ @EnableConfigurationProperties(CommonConsumerProperties.class) public class MessageConfiguration { - @Bean - public UndeliveredMessageLogPersister undeliveredMessageLogPersister(UndeliveredMessageLog undeliveredMessageLog, - CommonConsumerProperties commonConsumerProperties) { - return new UndeliveredMessageLogPersister(undeliveredMessageLog, commonConsumerProperties.getUndeliveredMessageLogPersistPeriod()); - } + @Bean + public UndeliveredMessageLogPersister undeliveredMessageLogPersister( + UndeliveredMessageLog undeliveredMessageLog, + CommonConsumerProperties commonConsumerProperties) { + return new UndeliveredMessageLogPersister( + undeliveredMessageLog, commonConsumerProperties.getUndeliveredMessageLogPersistPeriod()); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/MetricsProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/MetricsProperties.java index 835ccb19bd..46977a3bd8 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/MetricsProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/MetricsProperties.java @@ -1,19 +1,18 @@ package pl.allegro.tech.hermes.consumers.config; -import org.springframework.boot.context.properties.ConfigurationProperties; - import java.time.Duration; +import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties(prefix = "consumer.metrics.metric-registry") public class MetricsProperties { - private Duration counterExpireAfterAccess = Duration.ofHours(72); + private Duration counterExpireAfterAccess = Duration.ofHours(72); - public Duration getCounterExpireAfterAccess() { - return counterExpireAfterAccess; - } + public Duration getCounterExpireAfterAccess() { + return counterExpireAfterAccess; + } - public void setCounterExpireAfterAccess(Duration counterExpireAfterAccess) { - this.counterExpireAfterAccess = counterExpireAfterAccess; - } + public void setCounterExpireAfterAccess(Duration counterExpireAfterAccess) { + this.counterExpireAfterAccess = counterExpireAfterAccess; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/MicrometerRegistryProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/MicrometerRegistryProperties.java index d5431b7de1..833084e2d7 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/MicrometerRegistryProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/MicrometerRegistryProperties.java @@ -1,50 +1,49 @@ package pl.allegro.tech.hermes.consumers.config; -import org.springframework.boot.context.properties.ConfigurationProperties; -import pl.allegro.tech.hermes.common.di.factories.MicrometerRegistryParameters; - import java.time.Duration; import java.util.List; +import org.springframework.boot.context.properties.ConfigurationProperties; +import pl.allegro.tech.hermes.common.di.factories.MicrometerRegistryParameters; @ConfigurationProperties(prefix = "consumer.metrics.micrometer") public class MicrometerRegistryProperties implements MicrometerRegistryParameters { - private List percentiles = List.of(0.5, 0.99, 0.999); - private boolean zookeeperReporterEnabled = true; - private Duration reportPeriod = Duration.ofSeconds(20); + private List percentiles = List.of(0.5, 0.99, 0.999); + private boolean zookeeperReporterEnabled = true; + private Duration reportPeriod = Duration.ofSeconds(20); - @Override - public List getPercentiles() { - return percentiles; - } + @Override + public List getPercentiles() { + return percentiles; + } - @Override - public boolean zookeeperReporterEnabled() { - return zookeeperReporterEnabled; - } + @Override + public boolean zookeeperReporterEnabled() { + return zookeeperReporterEnabled; + } - @Override - public Duration zookeeperReportPeriod() { - return reportPeriod; - } + @Override + public Duration zookeeperReportPeriod() { + return reportPeriod; + } - public void setPercentiles(List percentiles) { - this.percentiles = percentiles; - } + public void setPercentiles(List percentiles) { + this.percentiles = percentiles; + } - public boolean isZookeeperReporterEnabled() { - return zookeeperReporterEnabled; - } + public boolean isZookeeperReporterEnabled() { + return zookeeperReporterEnabled; + } - public void setZookeeperReporterEnabled(boolean zookeeperReporterEnabled) { - this.zookeeperReporterEnabled = zookeeperReporterEnabled; - } + public void setZookeeperReporterEnabled(boolean zookeeperReporterEnabled) { + this.zookeeperReporterEnabled = zookeeperReporterEnabled; + } - public Duration getReportPeriod() { - return reportPeriod; - } + public Duration getReportPeriod() { + return reportPeriod; + } - public void setReportPeriod(Duration reportPeriod) { - this.reportPeriod = reportPeriod; - } + public void setReportPeriod(Duration reportPeriod) { + this.reportPeriod = reportPeriod; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/OAuthConfiguration.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/OAuthConfiguration.java index 82c55cea79..1ddfab771c 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/OAuthConfiguration.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/OAuthConfiguration.java @@ -3,6 +3,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.util.concurrent.ThreadFactoryBuilder; import jakarta.inject.Named; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; import org.apache.curator.framework.CuratorFramework; import org.eclipse.jetty.client.HttpClient; import org.springframework.boot.context.properties.EnableConfigurationProperties; @@ -23,69 +26,75 @@ import pl.allegro.tech.hermes.domain.subscription.SubscriptionRepository; import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadFactory; - @Configuration @EnableConfigurationProperties(OAuthProperties.class) public class OAuthConfiguration { - @Bean - public OAuthTokenRequestRateLimiterFactory oAuthTokenRequestRateLimiterFactory(OAuthProviderRepository oAuthProviderRepository, - OAuthProperties oAuthProperties) { - return new OAuthTokenRequestRateLimiterFactory(oAuthProviderRepository, - oAuthProperties.getProvidersTokenRequestRateLimiterRateReductionFactor()); - } + @Bean + public OAuthTokenRequestRateLimiterFactory oAuthTokenRequestRateLimiterFactory( + OAuthProviderRepository oAuthProviderRepository, OAuthProperties oAuthProperties) { + return new OAuthTokenRequestRateLimiterFactory( + oAuthProviderRepository, + oAuthProperties.getProvidersTokenRequestRateLimiterRateReductionFactor()); + } - @Bean - public OAuthAccessTokens oAuthSubscriptionAccessTokens(OAuthAccessTokensLoader tokenLoader, - OAuthProperties oAuthProperties) { - return new OAuthSubscriptionAccessTokens(tokenLoader, oAuthProperties.getSubscriptionTokensCacheMaxSize()); - } + @Bean + public OAuthAccessTokens oAuthSubscriptionAccessTokens( + OAuthAccessTokensLoader tokenLoader, OAuthProperties oAuthProperties) { + return new OAuthSubscriptionAccessTokens( + tokenLoader, oAuthProperties.getSubscriptionTokensCacheMaxSize()); + } - @Bean(initMethod = "start", destroyMethod = "stop") - public OAuthClient oAuthHttpClient(@Named("oauth-http-client") HttpClient httpClient, - ObjectMapper objectMapper) { - return new OAuthHttpClient(httpClient, objectMapper); - } + @Bean(initMethod = "start", destroyMethod = "stop") + public OAuthClient oAuthHttpClient( + @Named("oauth-http-client") HttpClient httpClient, ObjectMapper objectMapper) { + return new OAuthHttpClient(httpClient, objectMapper); + } - @Bean - public OAuthSubscriptionHandlerFactory oAuthSubscriptionHandlerFactory(SubscriptionRepository subscriptionRepository, - OAuthAccessTokens accessTokens, - OAuthTokenRequestRateLimiterFactory rateLimiterLoader) { - return new OAuthSubscriptionHandlerFactory(subscriptionRepository, accessTokens, rateLimiterLoader); - } + @Bean + public OAuthSubscriptionHandlerFactory oAuthSubscriptionHandlerFactory( + SubscriptionRepository subscriptionRepository, + OAuthAccessTokens accessTokens, + OAuthTokenRequestRateLimiterFactory rateLimiterLoader) { + return new OAuthSubscriptionHandlerFactory( + subscriptionRepository, accessTokens, rateLimiterLoader); + } - @Bean - public OAuthAccessTokensLoader oAuthAccessTokensLoader(SubscriptionRepository subscriptionRepository, - OAuthProviderRepository oAuthProviderRepository, - OAuthClient oAuthClient, - MetricsFacade metrics) { - return new OAuthAccessTokensLoader(subscriptionRepository, oAuthProviderRepository, oAuthClient, metrics); - } + @Bean + public OAuthAccessTokensLoader oAuthAccessTokensLoader( + SubscriptionRepository subscriptionRepository, + OAuthProviderRepository oAuthProviderRepository, + OAuthClient oAuthClient, + MetricsFacade metrics) { + return new OAuthAccessTokensLoader( + subscriptionRepository, oAuthProviderRepository, oAuthClient, metrics); + } - @Bean - public OAuthProvidersNotifyingCache oAuthProvidersNotifyingCache(CuratorFramework curator, - ZookeeperPaths paths, - ObjectMapper objectMapper) { - String path = paths.oAuthProvidersPath(); - ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat("oauth-providers-notifying-cache-%d").build(); - ExecutorService executorService = Executors.newSingleThreadExecutor(threadFactory); - OAuthProvidersNotifyingCache cache = new OAuthProvidersNotifyingCache(curator, path, executorService, objectMapper); - try { - cache.start(); - } catch (Exception e) { - throw new IllegalStateException("Unable to start Zookeeper cache for path " + path, e); - } - return cache; + @Bean + public OAuthProvidersNotifyingCache oAuthProvidersNotifyingCache( + CuratorFramework curator, ZookeeperPaths paths, ObjectMapper objectMapper) { + String path = paths.oAuthProvidersPath(); + ThreadFactory threadFactory = + new ThreadFactoryBuilder().setNameFormat("oauth-providers-notifying-cache-%d").build(); + ExecutorService executorService = Executors.newSingleThreadExecutor(threadFactory); + OAuthProvidersNotifyingCache cache = + new OAuthProvidersNotifyingCache(curator, path, executorService, objectMapper); + try { + cache.start(); + } catch (Exception e) { + throw new IllegalStateException("Unable to start Zookeeper cache for path " + path, e); } + return cache; + } - @Bean - public ConsumerAuthorizationHandler oAuthConsumerAuthorizationHandler(OAuthSubscriptionHandlerFactory handlerFactory, - OAuthProperties oAuthProperties, - OAuthProvidersNotifyingCache oAuthProvidersCache) { - return new OAuthConsumerAuthorizationHandler(handlerFactory, oAuthProperties.getMissingSubscriptionHandlersCreationDelay(), - oAuthProvidersCache); - } + @Bean + public ConsumerAuthorizationHandler oAuthConsumerAuthorizationHandler( + OAuthSubscriptionHandlerFactory handlerFactory, + OAuthProperties oAuthProperties, + OAuthProvidersNotifyingCache oAuthProvidersCache) { + return new OAuthConsumerAuthorizationHandler( + handlerFactory, + oAuthProperties.getMissingSubscriptionHandlersCreationDelay(), + oAuthProvidersCache); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/OAuthProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/OAuthProperties.java index bd895c0a22..49244692eb 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/OAuthProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/OAuthProperties.java @@ -1,39 +1,41 @@ package pl.allegro.tech.hermes.consumers.config; -import org.springframework.boot.context.properties.ConfigurationProperties; - import java.time.Duration; +import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties(prefix = "consumer.oauth") public class OAuthProperties { - private Duration missingSubscriptionHandlersCreationDelay = Duration.ofSeconds(10); + private Duration missingSubscriptionHandlersCreationDelay = Duration.ofSeconds(10); - private long subscriptionTokensCacheMaxSize = 1000L; + private long subscriptionTokensCacheMaxSize = 1000L; - private double providersTokenRequestRateLimiterRateReductionFactor = 2.0; + private double providersTokenRequestRateLimiterRateReductionFactor = 2.0; - public Duration getMissingSubscriptionHandlersCreationDelay() { - return missingSubscriptionHandlersCreationDelay; - } + public Duration getMissingSubscriptionHandlersCreationDelay() { + return missingSubscriptionHandlersCreationDelay; + } - public void setMissingSubscriptionHandlersCreationDelay(Duration missingSubscriptionHandlersCreationDelay) { - this.missingSubscriptionHandlersCreationDelay = missingSubscriptionHandlersCreationDelay; - } + public void setMissingSubscriptionHandlersCreationDelay( + Duration missingSubscriptionHandlersCreationDelay) { + this.missingSubscriptionHandlersCreationDelay = missingSubscriptionHandlersCreationDelay; + } - public long getSubscriptionTokensCacheMaxSize() { - return subscriptionTokensCacheMaxSize; - } + public long getSubscriptionTokensCacheMaxSize() { + return subscriptionTokensCacheMaxSize; + } - public void setSubscriptionTokensCacheMaxSize(long subscriptionTokensCacheMaxSize) { - this.subscriptionTokensCacheMaxSize = subscriptionTokensCacheMaxSize; - } + public void setSubscriptionTokensCacheMaxSize(long subscriptionTokensCacheMaxSize) { + this.subscriptionTokensCacheMaxSize = subscriptionTokensCacheMaxSize; + } - public double getProvidersTokenRequestRateLimiterRateReductionFactor() { - return providersTokenRequestRateLimiterRateReductionFactor; - } + public double getProvidersTokenRequestRateLimiterRateReductionFactor() { + return providersTokenRequestRateLimiterRateReductionFactor; + } - public void setProvidersTokenRequestRateLimiterRateReductionFactor(double providersTokenRequestRateLimiterRateReductionFactor) { - this.providersTokenRequestRateLimiterRateReductionFactor = providersTokenRequestRateLimiterRateReductionFactor; - } + public void setProvidersTokenRequestRateLimiterRateReductionFactor( + double providersTokenRequestRateLimiterRateReductionFactor) { + this.providersTokenRequestRateLimiterRateReductionFactor = + providersTokenRequestRateLimiterRateReductionFactor; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/OnGoogleDefaultCredentials.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/OnGoogleDefaultCredentials.java index 95b98ce6f0..1735d8d108 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/OnGoogleDefaultCredentials.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/OnGoogleDefaultCredentials.java @@ -1,20 +1,19 @@ package pl.allegro.tech.hermes.consumers.config; import com.google.auth.oauth2.GoogleCredentials; +import java.io.IOException; import org.springframework.context.annotation.Condition; import org.springframework.context.annotation.ConditionContext; import org.springframework.core.type.AnnotatedTypeMetadata; -import java.io.IOException; - public class OnGoogleDefaultCredentials implements Condition { - @Override - public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { - try { - GoogleCredentials.getApplicationDefault(); - return true; - } catch (IOException e) { - return false; - } + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + try { + GoogleCredentials.getApplicationDefault(); + return true; + } catch (IOException e) { + return false; } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/PrometheusConfigAdapter.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/PrometheusConfigAdapter.java index eb71f796c2..43c1e2d9e8 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/PrometheusConfigAdapter.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/PrometheusConfigAdapter.java @@ -1,32 +1,29 @@ package pl.allegro.tech.hermes.consumers.config; import io.micrometer.prometheus.PrometheusConfig; - import java.time.Duration; public class PrometheusConfigAdapter implements PrometheusConfig { - private final PrometheusProperties prometheusReporterProperties; + private final PrometheusProperties prometheusReporterProperties; - public PrometheusConfigAdapter(PrometheusProperties prometheusReporterProperties) { - this.prometheusReporterProperties = prometheusReporterProperties; - } + public PrometheusConfigAdapter(PrometheusProperties prometheusReporterProperties) { + this.prometheusReporterProperties = prometheusReporterProperties; + } - @Override - public boolean descriptions() { - return prometheusReporterProperties.isDescriptions(); - } + @Override + public boolean descriptions() { + return prometheusReporterProperties.isDescriptions(); + } - @Override - public Duration step() { - return prometheusReporterProperties.getStep(); - } + @Override + public Duration step() { + return prometheusReporterProperties.getStep(); + } - /** - * Returning null is fine since we override all the methods from PrometheusConfig. - */ - @Override - public String get(String key) { - return null; // Nothing to see here, move along. - } + /** Returning null is fine since we override all the methods from PrometheusConfig. */ + @Override + public String get(String key) { + return null; // Nothing to see here, move along. + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/PrometheusProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/PrometheusProperties.java index 6455a38fd1..dca0531075 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/PrometheusProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/PrometheusProperties.java @@ -1,28 +1,27 @@ package pl.allegro.tech.hermes.consumers.config; -import org.springframework.boot.context.properties.ConfigurationProperties; - import java.time.Duration; +import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties(prefix = "consumer.prometheus") public class PrometheusProperties { - private Duration step = Duration.ofMinutes(1); - private boolean descriptions = true; + private Duration step = Duration.ofMinutes(1); + private boolean descriptions = true; - public Duration getStep() { - return this.step; - } + public Duration getStep() { + return this.step; + } - public void setStep(Duration step) { - this.step = step; - } + public void setStep(Duration step) { + this.step = step; + } - public boolean isDescriptions() { - return this.descriptions; - } + public boolean isDescriptions() { + return this.descriptions; + } - public void setDescriptions(boolean descriptions) { - this.descriptions = descriptions; - } + public void setDescriptions(boolean descriptions) { + this.descriptions = descriptions; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/RateProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/RateProperties.java index b27a94bbdd..33ccce3056 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/RateProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/RateProperties.java @@ -1,95 +1,95 @@ package pl.allegro.tech.hermes.consumers.config; +import java.time.Duration; import org.springframework.boot.context.properties.ConfigurationProperties; import pl.allegro.tech.hermes.consumers.consumer.rate.calculator.RateCalculatorParameters; -import java.time.Duration; - @ConfigurationProperties(prefix = "consumer.rate") public class RateProperties implements RateCalculatorParameters { - private Duration limiterSupervisorPeriod = Duration.ofSeconds(30); + private Duration limiterSupervisorPeriod = Duration.ofSeconds(30); - private int limiterReportingThreadPoolSize = 30; + private int limiterReportingThreadPoolSize = 30; - private boolean limiterReportingThreadMonitoringEnabled = false; + private boolean limiterReportingThreadMonitoringEnabled = false; - private Duration limiterHeartbeatModeDelay = Duration.ofMinutes(1); + private Duration limiterHeartbeatModeDelay = Duration.ofMinutes(1); - private Duration limiterSlowModeDelay = Duration.ofSeconds(1); + private Duration limiterSlowModeDelay = Duration.ofSeconds(1); - private double convergenceFactor = 0.2; + private double convergenceFactor = 0.2; - private double failuresNoChangeToleranceRatio = 0.05; + private double failuresNoChangeToleranceRatio = 0.05; - private double failuresSpeedUpToleranceRatio = 0.01; + private double failuresSpeedUpToleranceRatio = 0.01; - public Duration getLimiterSupervisorPeriod() { - return limiterSupervisorPeriod; - } + public Duration getLimiterSupervisorPeriod() { + return limiterSupervisorPeriod; + } - public void setLimiterSupervisorPeriod(Duration limiterSupervisorPeriod) { - this.limiterSupervisorPeriod = limiterSupervisorPeriod; - } + public void setLimiterSupervisorPeriod(Duration limiterSupervisorPeriod) { + this.limiterSupervisorPeriod = limiterSupervisorPeriod; + } - public int getLimiterReportingThreadPoolSize() { - return limiterReportingThreadPoolSize; - } + public int getLimiterReportingThreadPoolSize() { + return limiterReportingThreadPoolSize; + } - public void setLimiterReportingThreadPoolSize(int limiterReportingThreadPoolSize) { - this.limiterReportingThreadPoolSize = limiterReportingThreadPoolSize; - } + public void setLimiterReportingThreadPoolSize(int limiterReportingThreadPoolSize) { + this.limiterReportingThreadPoolSize = limiterReportingThreadPoolSize; + } - public boolean isLimiterReportingThreadMonitoringEnabled() { - return limiterReportingThreadMonitoringEnabled; - } + public boolean isLimiterReportingThreadMonitoringEnabled() { + return limiterReportingThreadMonitoringEnabled; + } - public void setLimiterReportingThreadMonitoringEnabled(boolean limiterReportingThreadMonitoringEnabled) { - this.limiterReportingThreadMonitoringEnabled = limiterReportingThreadMonitoringEnabled; - } + public void setLimiterReportingThreadMonitoringEnabled( + boolean limiterReportingThreadMonitoringEnabled) { + this.limiterReportingThreadMonitoringEnabled = limiterReportingThreadMonitoringEnabled; + } - @Override - public Duration getLimiterHeartbeatModeDelay() { - return limiterHeartbeatModeDelay; - } + @Override + public Duration getLimiterHeartbeatModeDelay() { + return limiterHeartbeatModeDelay; + } - public void setLimiterHeartbeatModeDelay(Duration limiterHeartbeatModeDelay) { - this.limiterHeartbeatModeDelay = limiterHeartbeatModeDelay; - } + public void setLimiterHeartbeatModeDelay(Duration limiterHeartbeatModeDelay) { + this.limiterHeartbeatModeDelay = limiterHeartbeatModeDelay; + } - @Override - public Duration getLimiterSlowModeDelay() { - return limiterSlowModeDelay; - } + @Override + public Duration getLimiterSlowModeDelay() { + return limiterSlowModeDelay; + } - public void setLimiterSlowModeDelay(Duration limiterSlowModeDelay) { - this.limiterSlowModeDelay = limiterSlowModeDelay; - } + public void setLimiterSlowModeDelay(Duration limiterSlowModeDelay) { + this.limiterSlowModeDelay = limiterSlowModeDelay; + } - @Override - public double getConvergenceFactor() { - return convergenceFactor; - } + @Override + public double getConvergenceFactor() { + return convergenceFactor; + } - public void setConvergenceFactor(double convergenceFactor) { - this.convergenceFactor = convergenceFactor; - } + public void setConvergenceFactor(double convergenceFactor) { + this.convergenceFactor = convergenceFactor; + } - @Override - public double getFailuresNoChangeToleranceRatio() { - return failuresNoChangeToleranceRatio; - } + @Override + public double getFailuresNoChangeToleranceRatio() { + return failuresNoChangeToleranceRatio; + } - public void setFailuresNoChangeToleranceRatio(double failuresNoChangeToleranceRatio) { - this.failuresNoChangeToleranceRatio = failuresNoChangeToleranceRatio; - } + public void setFailuresNoChangeToleranceRatio(double failuresNoChangeToleranceRatio) { + this.failuresNoChangeToleranceRatio = failuresNoChangeToleranceRatio; + } - @Override - public double getFailuresSpeedUpToleranceRatio() { - return failuresSpeedUpToleranceRatio; - } + @Override + public double getFailuresSpeedUpToleranceRatio() { + return failuresSpeedUpToleranceRatio; + } - public void setFailuresSpeedUpToleranceRatio(double failuresSpeedUpToleranceRatio) { - this.failuresSpeedUpToleranceRatio = failuresSpeedUpToleranceRatio; - } + public void setFailuresSpeedUpToleranceRatio(double failuresSpeedUpToleranceRatio) { + this.failuresSpeedUpToleranceRatio = failuresSpeedUpToleranceRatio; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/RegistryBinaryEncoderProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/RegistryBinaryEncoderProperties.java index cbb65ac065..7e9ca21627 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/RegistryBinaryEncoderProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/RegistryBinaryEncoderProperties.java @@ -2,23 +2,23 @@ public class RegistryBinaryEncoderProperties { - private int maxRateBufferSizeBytes = 100_000; + private int maxRateBufferSizeBytes = 100_000; - private int historyBufferSizeBytes = 100_000; + private int historyBufferSizeBytes = 100_000; - public int getMaxRateBufferSizeBytes() { - return maxRateBufferSizeBytes; - } + public int getMaxRateBufferSizeBytes() { + return maxRateBufferSizeBytes; + } - public void setMaxRateBufferSizeBytes(int maxRateBufferSizeBytes) { - this.maxRateBufferSizeBytes = maxRateBufferSizeBytes; - } + public void setMaxRateBufferSizeBytes(int maxRateBufferSizeBytes) { + this.maxRateBufferSizeBytes = maxRateBufferSizeBytes; + } - public int getHistoryBufferSizeBytes() { - return historyBufferSizeBytes; - } + public int getHistoryBufferSizeBytes() { + return historyBufferSizeBytes; + } - public void setHistoryBufferSizeBytes(int historyBufferSizeBytes) { - this.historyBufferSizeBytes = historyBufferSizeBytes; - } + public void setHistoryBufferSizeBytes(int historyBufferSizeBytes) { + this.historyBufferSizeBytes = historyBufferSizeBytes; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/RegistryConfiguration.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/RegistryConfiguration.java index b4e1d6a55d..9b74cf5fe6 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/RegistryConfiguration.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/RegistryConfiguration.java @@ -1,6 +1,10 @@ package pl.allegro.tech.hermes.consumers.config; +import static java.util.concurrent.Executors.newSingleThreadExecutor; + import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.time.Clock; +import java.util.concurrent.ThreadFactory; import org.apache.curator.framework.CuratorFramework; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; @@ -10,34 +14,34 @@ import pl.allegro.tech.hermes.infrastructure.dc.DatacenterNameProvider; import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; -import java.time.Clock; -import java.util.concurrent.ThreadFactory; - -import static java.util.concurrent.Executors.newSingleThreadExecutor; - @Configuration -@EnableConfigurationProperties({ - KafkaClustersProperties.class, - WorkloadProperties.class -}) +@EnableConfigurationProperties({KafkaClustersProperties.class, WorkloadProperties.class}) public class RegistryConfiguration { - @Bean(initMethod = "start", destroyMethod = "stop") - public ConsumerNodesRegistry consumerNodesRegistry(CuratorFramework curatorFramework, - KafkaClustersProperties kafkaClustersProperties, - WorkloadProperties workloadProperties, - ZookeeperPaths zookeeperPaths, - Clock clock, - DatacenterNameProvider datacenterNameProvider) { - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setNameFormat("ConsumerRegistryExecutor-%d").build(); + @Bean(initMethod = "start", destroyMethod = "stop") + public ConsumerNodesRegistry consumerNodesRegistry( + CuratorFramework curatorFramework, + KafkaClustersProperties kafkaClustersProperties, + WorkloadProperties workloadProperties, + ZookeeperPaths zookeeperPaths, + Clock clock, + DatacenterNameProvider datacenterNameProvider) { + ThreadFactory threadFactory = + new ThreadFactoryBuilder().setNameFormat("ConsumerRegistryExecutor-%d").build(); - String consumerNodeId = workloadProperties.getNodeId(); - KafkaProperties kafkaProperties = kafkaClustersProperties.toKafkaProperties(datacenterNameProvider); - long deadAfterSeconds = workloadProperties.getDeadAfter().toSeconds(); - ConsumerNodesRegistryPaths registryPaths = new ConsumerNodesRegistryPaths(zookeeperPaths, kafkaProperties.getClusterName()); + String consumerNodeId = workloadProperties.getNodeId(); + KafkaProperties kafkaProperties = + kafkaClustersProperties.toKafkaProperties(datacenterNameProvider); + long deadAfterSeconds = workloadProperties.getDeadAfter().toSeconds(); + ConsumerNodesRegistryPaths registryPaths = + new ConsumerNodesRegistryPaths(zookeeperPaths, kafkaProperties.getClusterName()); - return new ConsumerNodesRegistry(curatorFramework, newSingleThreadExecutor(threadFactory), - registryPaths, consumerNodeId, deadAfterSeconds, clock); - } + return new ConsumerNodesRegistry( + curatorFramework, + newSingleThreadExecutor(threadFactory), + registryPaths, + consumerNodeId, + deadAfterSeconds, + clock); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/SchemaConfiguration.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/SchemaConfiguration.java index fb3bcc9c82..5bd89dd14f 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/SchemaConfiguration.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/SchemaConfiguration.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.jakarta.rs.json.JacksonJsonProvider; import jakarta.ws.rs.client.Client; import jakarta.ws.rs.client.ClientBuilder; +import java.net.URI; import org.apache.avro.Schema; import org.glassfish.jersey.client.ClientConfig; import org.glassfish.jersey.client.ClientProperties; @@ -23,64 +24,80 @@ import pl.allegro.tech.hermes.schema.resolver.DefaultSchemaRepositoryInstanceResolver; import pl.allegro.tech.hermes.schema.resolver.SchemaRepositoryInstanceResolver; -import java.net.URI; - @Configuration -@EnableConfigurationProperties({ - SchemaProperties.class, - KafkaClustersProperties.class -}) +@EnableConfigurationProperties({SchemaProperties.class, KafkaClustersProperties.class}) public class SchemaConfiguration { - @Bean - public SchemaRepository schemaRepository(SchemaVersionsRepository schemaVersionsRepository, - CompiledSchemaRepository compiledAvroSchemaRepository) { - return new SchemaRepositoryFactory(schemaVersionsRepository, compiledAvroSchemaRepository).provide(); - } + @Bean + public SchemaRepository schemaRepository( + SchemaVersionsRepository schemaVersionsRepository, + CompiledSchemaRepository compiledAvroSchemaRepository) { + return new SchemaRepositoryFactory(schemaVersionsRepository, compiledAvroSchemaRepository) + .provide(); + } - @Bean - public CompiledSchemaRepository avroCompiledSchemaRepository(RawSchemaClient rawSchemaClient, - SchemaProperties schemaProperties) { - return new AvroCompiledSchemaRepositoryFactory( - rawSchemaClient, schemaProperties.getCache().getCompiledMaximumSize(), - schemaProperties.getCache().getCompiledExpireAfterAccess(), schemaProperties.getCache().isEnabled() - ).provide(); - } + @Bean + public CompiledSchemaRepository avroCompiledSchemaRepository( + RawSchemaClient rawSchemaClient, SchemaProperties schemaProperties) { + return new AvroCompiledSchemaRepositoryFactory( + rawSchemaClient, + schemaProperties.getCache().getCompiledMaximumSize(), + schemaProperties.getCache().getCompiledExpireAfterAccess(), + schemaProperties.getCache().isEnabled()) + .provide(); + } - @Bean - public RawSchemaClient rawSchemaClient(MetricsFacade metricsFacade, - ObjectMapper objectMapper, - SchemaRepositoryInstanceResolver resolver, - SchemaProperties schemaProperties, - KafkaClustersProperties kafkaProperties) { - return new RawSchemaClientFactory(kafkaProperties.getNamespace(), kafkaProperties.getNamespaceSeparator(), metricsFacade, - objectMapper, resolver, - schemaProperties.getRepository().isSubjectSuffixEnabled(), - schemaProperties.getRepository().isSubjectNamespaceEnabled()).provide(); - } + @Bean + public RawSchemaClient rawSchemaClient( + MetricsFacade metricsFacade, + ObjectMapper objectMapper, + SchemaRepositoryInstanceResolver resolver, + SchemaProperties schemaProperties, + KafkaClustersProperties kafkaProperties) { + return new RawSchemaClientFactory( + kafkaProperties.getNamespace(), + kafkaProperties.getNamespaceSeparator(), + metricsFacade, + objectMapper, + resolver, + schemaProperties.getRepository().isSubjectSuffixEnabled(), + schemaProperties.getRepository().isSubjectNamespaceEnabled()) + .provide(); + } - @Bean - public SchemaRepositoryInstanceResolver schemaRepositoryInstanceResolver(SchemaProperties schemaProperties, Client client) { - URI schemaRepositoryServerUri = URI.create(schemaProperties.getRepository().getServerUrl()); - return new DefaultSchemaRepositoryInstanceResolver(client, schemaRepositoryServerUri); - } + @Bean + public SchemaRepositoryInstanceResolver schemaRepositoryInstanceResolver( + SchemaProperties schemaProperties, Client client) { + URI schemaRepositoryServerUri = URI.create(schemaProperties.getRepository().getServerUrl()); + return new DefaultSchemaRepositoryInstanceResolver(client, schemaRepositoryServerUri); + } - @Bean - public Client schemaRepositoryClient(ObjectMapper mapper, SchemaProperties schemaProperties) { - ClientConfig config = new ClientConfig() - .property(ClientProperties.READ_TIMEOUT, (int) schemaProperties.getRepository().getHttpReadTimeout().toMillis()) - .property(ClientProperties.CONNECT_TIMEOUT, (int) schemaProperties.getRepository().getHttpConnectTimeout().toMillis()) - .register(new JacksonJsonProvider(mapper)); + @Bean + public Client schemaRepositoryClient(ObjectMapper mapper, SchemaProperties schemaProperties) { + ClientConfig config = + new ClientConfig() + .property( + ClientProperties.READ_TIMEOUT, + (int) schemaProperties.getRepository().getHttpReadTimeout().toMillis()) + .property( + ClientProperties.CONNECT_TIMEOUT, + (int) schemaProperties.getRepository().getHttpConnectTimeout().toMillis()) + .register(new JacksonJsonProvider(mapper)); - return ClientBuilder.newClient(config); - } + return ClientBuilder.newClient(config); + } - @Bean - public SchemaVersionsRepository schemaVersionsRepositoryFactory(RawSchemaClient rawSchemaClient, - SchemaProperties schemaProperties, - InternalNotificationsBus notificationsBus, - CompiledSchemaRepository compiledSchemaRepository) { - return new SchemaVersionsRepositoryFactory(rawSchemaClient, schemaProperties.getCache(), notificationsBus, compiledSchemaRepository) - .provide(); - } + @Bean + public SchemaVersionsRepository schemaVersionsRepositoryFactory( + RawSchemaClient rawSchemaClient, + SchemaProperties schemaProperties, + InternalNotificationsBus notificationsBus, + CompiledSchemaRepository compiledSchemaRepository) { + return new SchemaVersionsRepositoryFactory( + rawSchemaClient, + schemaProperties.getCache(), + notificationsBus, + compiledSchemaRepository) + .provide(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/SchemaProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/SchemaProperties.java index 9f7986736d..8113776008 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/SchemaProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/SchemaProperties.java @@ -7,53 +7,53 @@ @ConfigurationProperties(prefix = "consumer.schema") public class SchemaProperties { - private SchemaCacheProperties cache = new SchemaCacheProperties(); + private SchemaCacheProperties cache = new SchemaCacheProperties(); - private SchemaRepositoryProperties repository = new SchemaRepositoryProperties(); + private SchemaRepositoryProperties repository = new SchemaRepositoryProperties(); - private boolean idHeaderEnabled = false; + private boolean idHeaderEnabled = false; - private boolean idSerializationEnabled = false; + private boolean idSerializationEnabled = false; - private boolean versionTruncationEnabled = false; + private boolean versionTruncationEnabled = false; - public SchemaCacheProperties getCache() { - return cache; - } + public SchemaCacheProperties getCache() { + return cache; + } - public void setCache(SchemaCacheProperties cache) { - this.cache = cache; - } + public void setCache(SchemaCacheProperties cache) { + this.cache = cache; + } - public SchemaRepositoryProperties getRepository() { - return repository; - } + public SchemaRepositoryProperties getRepository() { + return repository; + } - public void setRepository(SchemaRepositoryProperties repository) { - this.repository = repository; - } + public void setRepository(SchemaRepositoryProperties repository) { + this.repository = repository; + } - public boolean isIdHeaderEnabled() { - return idHeaderEnabled; - } + public boolean isIdHeaderEnabled() { + return idHeaderEnabled; + } - public void setIdHeaderEnabled(boolean idHeaderEnabled) { - this.idHeaderEnabled = idHeaderEnabled; - } + public void setIdHeaderEnabled(boolean idHeaderEnabled) { + this.idHeaderEnabled = idHeaderEnabled; + } - public boolean isIdSerializationEnabled() { - return idSerializationEnabled; - } + public boolean isIdSerializationEnabled() { + return idSerializationEnabled; + } - public void setIdSerializationEnabled(boolean idSerializationEnabled) { - this.idSerializationEnabled = idSerializationEnabled; - } + public void setIdSerializationEnabled(boolean idSerializationEnabled) { + this.idSerializationEnabled = idSerializationEnabled; + } - public boolean isVersionTruncationEnabled() { - return versionTruncationEnabled; - } + public boolean isVersionTruncationEnabled() { + return versionTruncationEnabled; + } - public void setVersionTruncationEnabled(boolean versionTruncationEnabled) { - this.versionTruncationEnabled = versionTruncationEnabled; - } + public void setVersionTruncationEnabled(boolean versionTruncationEnabled) { + this.versionTruncationEnabled = versionTruncationEnabled; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/SenderAsyncTimeoutProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/SenderAsyncTimeoutProperties.java index a038232262..2ec89879cc 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/SenderAsyncTimeoutProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/SenderAsyncTimeoutProperties.java @@ -5,33 +5,33 @@ @ConfigurationProperties(prefix = "consumer.sender.async.timeout") public class SenderAsyncTimeoutProperties { - private int milliseconds = 5_000; + private int milliseconds = 5_000; - private int threadPoolSize = 32; + private int threadPoolSize = 32; - private boolean threadPoolMonitoringEnabled = false; + private boolean threadPoolMonitoringEnabled = false; - public int getMilliseconds() { - return milliseconds; - } + public int getMilliseconds() { + return milliseconds; + } - public void setMilliseconds(int milliseconds) { - this.milliseconds = milliseconds; - } + public void setMilliseconds(int milliseconds) { + this.milliseconds = milliseconds; + } - public int getThreadPoolSize() { - return threadPoolSize; - } + public int getThreadPoolSize() { + return threadPoolSize; + } - public void setThreadPoolSize(int threadPoolSize) { - this.threadPoolSize = threadPoolSize; - } + public void setThreadPoolSize(int threadPoolSize) { + this.threadPoolSize = threadPoolSize; + } - public boolean isThreadPoolMonitoringEnabled() { - return threadPoolMonitoringEnabled; - } + public boolean isThreadPoolMonitoringEnabled() { + return threadPoolMonitoringEnabled; + } - public void setThreadPoolMonitoringEnabled(boolean threadPoolMonitoringEnabled) { - this.threadPoolMonitoringEnabled = threadPoolMonitoringEnabled; - } + public void setThreadPoolMonitoringEnabled(boolean threadPoolMonitoringEnabled) { + this.threadPoolMonitoringEnabled = threadPoolMonitoringEnabled; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ServerConfiguration.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ServerConfiguration.java index e78237a450..8e2bfbfc0c 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ServerConfiguration.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ServerConfiguration.java @@ -2,28 +2,30 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.micrometer.prometheus.PrometheusMeterRegistry; +import java.io.IOException; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import pl.allegro.tech.hermes.consumers.health.ConsumerMonitor; import pl.allegro.tech.hermes.consumers.server.ConsumerHttpServer; -import java.io.IOException; - @Configuration @EnableConfigurationProperties(CommonConsumerProperties.class) public class ServerConfiguration { - @Bean(initMethod = "start", destroyMethod = "stop") - public ConsumerHttpServer consumerHttpServer(CommonConsumerProperties commonConsumerProperties, - ConsumerMonitor monitor, - ObjectMapper mapper, - PrometheusMeterRegistry meterRegistry) throws IOException { - return new ConsumerHttpServer(commonConsumerProperties.getHealthCheckPort(), monitor, mapper, meterRegistry); - } + @Bean(initMethod = "start", destroyMethod = "stop") + public ConsumerHttpServer consumerHttpServer( + CommonConsumerProperties commonConsumerProperties, + ConsumerMonitor monitor, + ObjectMapper mapper, + PrometheusMeterRegistry meterRegistry) + throws IOException { + return new ConsumerHttpServer( + commonConsumerProperties.getHealthCheckPort(), monitor, mapper, meterRegistry); + } - @Bean - public ConsumerMonitor consumerMonitor() { - return new ConsumerMonitor(); - } + @Bean + public ConsumerMonitor consumerMonitor() { + return new ConsumerMonitor(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/SslContextProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/SslContextProperties.java index 3efa6cffea..7413d8056e 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/SslContextProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/SslContextProperties.java @@ -6,113 +6,113 @@ @ConfigurationProperties(prefix = "consumer.ssl") public class SslContextProperties implements SslContextParameters { - private boolean enabled = true; + private boolean enabled = true; - private String protocol = "TLS"; + private String protocol = "TLS"; - private String keystoreSource = "jre"; + private String keystoreSource = "jre"; - private String keystoreLocation = "classpath:client.keystore"; + private String keystoreLocation = "classpath:client.keystore"; - private String keystorePassword = "password"; + private String keystorePassword = "password"; - private String keystoreFormat = "JKS"; + private String keystoreFormat = "JKS"; - private String truststoreSource = "jre"; + private String truststoreSource = "jre"; - private String truststoreLocation = "classpath:client.truststore"; + private String truststoreLocation = "classpath:client.truststore"; - private String truststorePassword = "password"; + private String truststorePassword = "password"; - private String truststoreFormat = "JKS"; + private String truststoreFormat = "JKS"; - @Override - public boolean isEnabled() { - return enabled; - } + @Override + public boolean isEnabled() { + return enabled; + } - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } - @Override - public String getProtocol() { - return protocol; - } + @Override + public String getProtocol() { + return protocol; + } - public void setProtocol(String protocol) { - this.protocol = protocol; - } + public void setProtocol(String protocol) { + this.protocol = protocol; + } - @Override - public String getKeystoreSource() { - return keystoreSource; - } + @Override + public String getKeystoreSource() { + return keystoreSource; + } - public void setKeystoreSource(String keystoreSource) { - this.keystoreSource = keystoreSource; - } + public void setKeystoreSource(String keystoreSource) { + this.keystoreSource = keystoreSource; + } - @Override - public String getKeystoreLocation() { - return keystoreLocation; - } + @Override + public String getKeystoreLocation() { + return keystoreLocation; + } - public void setKeystoreLocation(String keystoreLocation) { - this.keystoreLocation = keystoreLocation; - } + public void setKeystoreLocation(String keystoreLocation) { + this.keystoreLocation = keystoreLocation; + } - @Override - public String getKeystorePassword() { - return keystorePassword; - } + @Override + public String getKeystorePassword() { + return keystorePassword; + } - public void setKeystorePassword(String keystorePassword) { - this.keystorePassword = keystorePassword; - } + public void setKeystorePassword(String keystorePassword) { + this.keystorePassword = keystorePassword; + } - @Override - public String getKeystoreFormat() { - return keystoreFormat; - } + @Override + public String getKeystoreFormat() { + return keystoreFormat; + } - public void setKeystoreFormat(String keystoreFormat) { - this.keystoreFormat = keystoreFormat; - } + public void setKeystoreFormat(String keystoreFormat) { + this.keystoreFormat = keystoreFormat; + } - @Override - public String getTruststoreSource() { - return truststoreSource; - } + @Override + public String getTruststoreSource() { + return truststoreSource; + } - public void setTruststoreSource(String truststoreSource) { - this.truststoreSource = truststoreSource; - } + public void setTruststoreSource(String truststoreSource) { + this.truststoreSource = truststoreSource; + } - @Override - public String getTruststoreLocation() { - return truststoreLocation; - } + @Override + public String getTruststoreLocation() { + return truststoreLocation; + } - public void setTruststoreLocation(String truststoreLocation) { - this.truststoreLocation = truststoreLocation; - } + public void setTruststoreLocation(String truststoreLocation) { + this.truststoreLocation = truststoreLocation; + } - @Override - public String getTruststorePassword() { - return truststorePassword; - } + @Override + public String getTruststorePassword() { + return truststorePassword; + } - public void setTruststorePassword(String truststorePassword) { - this.truststorePassword = truststorePassword; - } + public void setTruststorePassword(String truststorePassword) { + this.truststorePassword = truststorePassword; + } - @Override - public String getTruststoreFormat() { - return truststoreFormat; - } + @Override + public String getTruststoreFormat() { + return truststoreFormat; + } - public void setTruststoreFormat(String truststoreFormat) { - this.truststoreFormat = truststoreFormat; - } + public void setTruststoreFormat(String truststoreFormat) { + this.truststoreFormat = truststoreFormat; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/SubscriptionConfiguration.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/SubscriptionConfiguration.java index 5641600fd9..b712138cfe 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/SubscriptionConfiguration.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/SubscriptionConfiguration.java @@ -21,39 +21,39 @@ @EnableConfigurationProperties(CommonConsumerProperties.class) public class SubscriptionConfiguration { - @Bean - public SubscriptionIdProvider subscriptionIdProvider(CuratorFramework curatorFramework, ZookeeperPaths zookeeperPaths) { - return new ZookeeperSubscriptionIdProvider(curatorFramework, zookeeperPaths); - } + @Bean + public SubscriptionIdProvider subscriptionIdProvider( + CuratorFramework curatorFramework, ZookeeperPaths zookeeperPaths) { + return new ZookeeperSubscriptionIdProvider(curatorFramework, zookeeperPaths); + } - @Bean - public SubscriptionIds subscriptionIds(InternalNotificationsBus internalNotificationsBus, - SubscriptionsCache subscriptionsCache, - SubscriptionIdProvider subscriptionIdProvider, - CommonConsumerProperties commonConsumerProperties) { - NotificationAwareSubscriptionIdsCache cache = new NotificationAwareSubscriptionIdsCache( - internalNotificationsBus, - subscriptionsCache, - subscriptionIdProvider, - commonConsumerProperties.getSubscriptionIdsCacheRemovedExpireAfterAccess().toSeconds(), - Ticker.systemTicker() - ); - cache.start(); - return cache; - } + @Bean + public SubscriptionIds subscriptionIds( + InternalNotificationsBus internalNotificationsBus, + SubscriptionsCache subscriptionsCache, + SubscriptionIdProvider subscriptionIdProvider, + CommonConsumerProperties commonConsumerProperties) { + NotificationAwareSubscriptionIdsCache cache = + new NotificationAwareSubscriptionIdsCache( + internalNotificationsBus, + subscriptionsCache, + subscriptionIdProvider, + commonConsumerProperties.getSubscriptionIdsCacheRemovedExpireAfterAccess().toSeconds(), + Ticker.systemTicker()); + cache.start(); + return cache; + } - @Bean - public SubscriptionsCache subscriptionsCache(InternalNotificationsBus notificationsBus, - GroupRepository groupRepository, - TopicRepository topicRepository, - SubscriptionRepository subscriptionRepository) { - SubscriptionsCache cache = new NotificationsBasedSubscriptionCache( - notificationsBus, - groupRepository, - topicRepository, - subscriptionRepository - ); - cache.start(); - return cache; - } + @Bean + public SubscriptionsCache subscriptionsCache( + InternalNotificationsBus notificationsBus, + GroupRepository groupRepository, + TopicRepository topicRepository, + SubscriptionRepository subscriptionRepository) { + SubscriptionsCache cache = + new NotificationsBasedSubscriptionCache( + notificationsBus, groupRepository, topicRepository, subscriptionRepository); + cache.start(); + return cache; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/SupervisorConfiguration.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/SupervisorConfiguration.java index 0be54f5b08..964ca439f7 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/SupervisorConfiguration.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/SupervisorConfiguration.java @@ -1,6 +1,12 @@ package pl.allegro.tech.hermes.consumers.config; +import static java.util.concurrent.Executors.newFixedThreadPool; +import static org.slf4j.LoggerFactory.getLogger; + import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.time.Clock; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadFactory; import org.apache.curator.framework.CuratorFramework; import org.slf4j.Logger; import org.springframework.boot.context.properties.EnableConfigurationProperties; @@ -63,321 +69,343 @@ import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; import pl.allegro.tech.hermes.tracker.consumers.Trackers; -import java.time.Clock; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.ThreadFactory; - -import static java.util.concurrent.Executors.newFixedThreadPool; -import static org.slf4j.LoggerFactory.getLogger; - @Configuration @EnableConfigurationProperties({ - CommitOffsetProperties.class, - KafkaClustersProperties.class, - WorkloadProperties.class, - CommonConsumerProperties.class + CommitOffsetProperties.class, + KafkaClustersProperties.class, + WorkloadProperties.class, + CommonConsumerProperties.class }) public class SupervisorConfiguration { - private static final Logger logger = getLogger(SupervisorConfiguration.class); + private static final Logger logger = getLogger(SupervisorConfiguration.class); - @Bean(initMethod = "start", destroyMethod = "shutdown") - public WorkloadSupervisor workloadSupervisor(InternalNotificationsBus notificationsBus, - ConsumerNodesRegistry consumerNodesRegistry, - ConsumerAssignmentRegistry assignmentRegistry, - ConsumerAssignmentCache consumerAssignmentCache, - ClusterAssignmentCache clusterAssignmentCache, - SubscriptionsCache subscriptionsCache, - ConsumersSupervisor supervisor, - ZookeeperAdminCache adminCache, - MetricsFacade metrics, - WorkloadProperties workloadProperties, - KafkaClustersProperties kafkaClustersProperties, - WorkloadConstraintsRepository workloadConstraintsRepository, - DatacenterNameProvider datacenterNameProvider, - WorkBalancer workBalancer, - BalancingListener balancingListener) { - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setNameFormat("AssignmentExecutor-%d") - .setUncaughtExceptionHandler((t, e) -> logger.error("AssignmentExecutor failed {}", t.getName(), e)).build(); - ExecutorService assignmentExecutor = newFixedThreadPool(workloadProperties.getAssignmentProcessingThreadPoolSize(), threadFactory); - KafkaProperties kafkaProperties = kafkaClustersProperties.toKafkaProperties(datacenterNameProvider); - return new WorkloadSupervisor( - supervisor, - notificationsBus, - subscriptionsCache, - consumerAssignmentCache, - assignmentRegistry, - clusterAssignmentCache, - consumerNodesRegistry, - adminCache, - assignmentExecutor, - workloadProperties, - kafkaProperties.getClusterName(), - metrics, - workloadConstraintsRepository, - workBalancer, - balancingListener - ); - } + @Bean(initMethod = "start", destroyMethod = "shutdown") + public WorkloadSupervisor workloadSupervisor( + InternalNotificationsBus notificationsBus, + ConsumerNodesRegistry consumerNodesRegistry, + ConsumerAssignmentRegistry assignmentRegistry, + ConsumerAssignmentCache consumerAssignmentCache, + ClusterAssignmentCache clusterAssignmentCache, + SubscriptionsCache subscriptionsCache, + ConsumersSupervisor supervisor, + ZookeeperAdminCache adminCache, + MetricsFacade metrics, + WorkloadProperties workloadProperties, + KafkaClustersProperties kafkaClustersProperties, + WorkloadConstraintsRepository workloadConstraintsRepository, + DatacenterNameProvider datacenterNameProvider, + WorkBalancer workBalancer, + BalancingListener balancingListener) { + ThreadFactory threadFactory = + new ThreadFactoryBuilder() + .setNameFormat("AssignmentExecutor-%d") + .setUncaughtExceptionHandler( + (t, e) -> logger.error("AssignmentExecutor failed {}", t.getName(), e)) + .build(); + ExecutorService assignmentExecutor = + newFixedThreadPool( + workloadProperties.getAssignmentProcessingThreadPoolSize(), threadFactory); + KafkaProperties kafkaProperties = + kafkaClustersProperties.toKafkaProperties(datacenterNameProvider); + return new WorkloadSupervisor( + supervisor, + notificationsBus, + subscriptionsCache, + consumerAssignmentCache, + assignmentRegistry, + clusterAssignmentCache, + consumerNodesRegistry, + adminCache, + assignmentExecutor, + workloadProperties, + kafkaProperties.getClusterName(), + metrics, + workloadConstraintsRepository, + workBalancer, + balancingListener); + } - @Bean - public WorkBalancer workBalancer(WorkloadProperties workloadProperties, - Clock clock, - CurrentLoadProvider currentLoadProvider, - TargetWeightCalculator targetWeightCalculator) { - switch (workloadProperties.getWorkBalancingStrategy()) { - case SELECTIVE: - return new SelectiveWorkBalancer(); - case WEIGHTED: - WeightedWorkBalancingProperties weightedWorkBalancingProperties = workloadProperties.getWeightedWorkBalancing(); - return new WeightedWorkBalancer( - clock, - weightedWorkBalancingProperties.getStabilizationWindowSize(), - weightedWorkBalancingProperties.getMinSignificantChangePercent(), - currentLoadProvider, - targetWeightCalculator - ); - default: - throw new UnknownWorkBalancingStrategyException(); - } + @Bean + public WorkBalancer workBalancer( + WorkloadProperties workloadProperties, + Clock clock, + CurrentLoadProvider currentLoadProvider, + TargetWeightCalculator targetWeightCalculator) { + switch (workloadProperties.getWorkBalancingStrategy()) { + case SELECTIVE: + return new SelectiveWorkBalancer(); + case WEIGHTED: + WeightedWorkBalancingProperties weightedWorkBalancingProperties = + workloadProperties.getWeightedWorkBalancing(); + return new WeightedWorkBalancer( + clock, + weightedWorkBalancingProperties.getStabilizationWindowSize(), + weightedWorkBalancingProperties.getMinSignificantChangePercent(), + currentLoadProvider, + targetWeightCalculator); + default: + throw new UnknownWorkBalancingStrategyException(); } + } - @Bean(initMethod = "start", destroyMethod = "stop") - public ConsumerNodeLoadRegistry consumerNodeLoadRegistry(CuratorFramework curator, - SubscriptionIds subscriptionIds, - ZookeeperPaths zookeeperPaths, - WorkloadProperties workloadProperties, - KafkaClustersProperties kafkaClustersProperties, - DatacenterNameProvider datacenterNameProvider, - ExecutorServiceFactory executorServiceFactory, - Clock clock, - MetricsFacade metrics) { - switch (workloadProperties.getWorkBalancingStrategy()) { - case SELECTIVE: - return new NoOpConsumerNodeLoadRegistry(); - case WEIGHTED: - KafkaProperties kafkaProperties = kafkaClustersProperties.toKafkaProperties(datacenterNameProvider); - WeightedWorkBalancingProperties weightedWorkBalancing = workloadProperties.getWeightedWorkBalancing(); - return new ZookeeperConsumerNodeLoadRegistry( - curator, - subscriptionIds, - zookeeperPaths, - workloadProperties.getNodeId(), - kafkaProperties.getClusterName(), - weightedWorkBalancing.getLoadReportingInterval(), - executorServiceFactory, - clock, - metrics, - weightedWorkBalancing.getConsumerLoadEncoderBufferSizeBytes() - ); - default: - throw new UnknownWorkBalancingStrategyException(); - } + @Bean(initMethod = "start", destroyMethod = "stop") + public ConsumerNodeLoadRegistry consumerNodeLoadRegistry( + CuratorFramework curator, + SubscriptionIds subscriptionIds, + ZookeeperPaths zookeeperPaths, + WorkloadProperties workloadProperties, + KafkaClustersProperties kafkaClustersProperties, + DatacenterNameProvider datacenterNameProvider, + ExecutorServiceFactory executorServiceFactory, + Clock clock, + MetricsFacade metrics) { + switch (workloadProperties.getWorkBalancingStrategy()) { + case SELECTIVE: + return new NoOpConsumerNodeLoadRegistry(); + case WEIGHTED: + KafkaProperties kafkaProperties = + kafkaClustersProperties.toKafkaProperties(datacenterNameProvider); + WeightedWorkBalancingProperties weightedWorkBalancing = + workloadProperties.getWeightedWorkBalancing(); + return new ZookeeperConsumerNodeLoadRegistry( + curator, + subscriptionIds, + zookeeperPaths, + workloadProperties.getNodeId(), + kafkaProperties.getClusterName(), + weightedWorkBalancing.getLoadReportingInterval(), + executorServiceFactory, + clock, + metrics, + weightedWorkBalancing.getConsumerLoadEncoderBufferSizeBytes()); + default: + throw new UnknownWorkBalancingStrategyException(); } + } - @Bean - public TargetWeightCalculator targetWeightCalculator(WorkloadProperties workloadProperties, - WeightedWorkloadMetricsReporter metricsReporter, - Clock clock) { - WeightedWorkBalancingProperties weightedWorkBalancing = workloadProperties.getWeightedWorkBalancing(); - switch (weightedWorkBalancing.getTargetWeightCalculationStrategy()) { - case AVG: - return new AvgTargetWeightCalculator(metricsReporter); - case SCORING: - return new ScoringTargetWeightCalculator( - metricsReporter, - clock, - weightedWorkBalancing.getWeightWindowSize(), - weightedWorkBalancing.getScoringGain() - ); - default: - throw new UnknownTargetWeightCalculationStrategyException(); - } + @Bean + public TargetWeightCalculator targetWeightCalculator( + WorkloadProperties workloadProperties, + WeightedWorkloadMetricsReporter metricsReporter, + Clock clock) { + WeightedWorkBalancingProperties weightedWorkBalancing = + workloadProperties.getWeightedWorkBalancing(); + switch (weightedWorkBalancing.getTargetWeightCalculationStrategy()) { + case AVG: + return new AvgTargetWeightCalculator(metricsReporter); + case SCORING: + return new ScoringTargetWeightCalculator( + metricsReporter, + clock, + weightedWorkBalancing.getWeightWindowSize(), + weightedWorkBalancing.getScoringGain()); + default: + throw new UnknownTargetWeightCalculationStrategyException(); } + } - @Bean - public BalancingListener balancingListener(ConsumerNodeLoadRegistry consumerNodeLoadRegistry, - SubscriptionProfileRegistry subscriptionProfileRegistry, - WorkloadProperties workloadProperties, - CurrentLoadProvider currentLoadProvider, - WeightedWorkloadMetricsReporter weightedWorkloadMetrics, - Clock clock) { - switch (workloadProperties.getWorkBalancingStrategy()) { - case SELECTIVE: - return new NoOpBalancingListener(); - case WEIGHTED: - return new WeightedWorkBalancingListener( - consumerNodeLoadRegistry, - subscriptionProfileRegistry, - currentLoadProvider, - weightedWorkloadMetrics, - clock, - workloadProperties.getWeightedWorkBalancing().getWeightWindowSize() - ); - default: - throw new UnknownWorkBalancingStrategyException(); - } + @Bean + public BalancingListener balancingListener( + ConsumerNodeLoadRegistry consumerNodeLoadRegistry, + SubscriptionProfileRegistry subscriptionProfileRegistry, + WorkloadProperties workloadProperties, + CurrentLoadProvider currentLoadProvider, + WeightedWorkloadMetricsReporter weightedWorkloadMetrics, + Clock clock) { + switch (workloadProperties.getWorkBalancingStrategy()) { + case SELECTIVE: + return new NoOpBalancingListener(); + case WEIGHTED: + return new WeightedWorkBalancingListener( + consumerNodeLoadRegistry, + subscriptionProfileRegistry, + currentLoadProvider, + weightedWorkloadMetrics, + clock, + workloadProperties.getWeightedWorkBalancing().getWeightWindowSize()); + default: + throw new UnknownWorkBalancingStrategyException(); } + } - @Bean - public CurrentLoadProvider currentLoadProvider() { - return new CurrentLoadProvider(); - } + @Bean + public CurrentLoadProvider currentLoadProvider() { + return new CurrentLoadProvider(); + } - @Bean - public WeightedWorkloadMetricsReporter weightedWorkloadMetrics(MetricsFacade metrics) { - return new WeightedWorkloadMetricsReporter(metrics); - } + @Bean + public WeightedWorkloadMetricsReporter weightedWorkloadMetrics(MetricsFacade metrics) { + return new WeightedWorkloadMetricsReporter(metrics); + } - @Bean - public SubscriptionProfileRegistry subscriptionProfileRegistry(CuratorFramework curator, - SubscriptionIds subscriptionIds, - ZookeeperPaths zookeeperPaths, - WorkloadProperties workloadProperties, - KafkaClustersProperties kafkaClustersProperties, - DatacenterNameProvider datacenterNameProvider) { - KafkaProperties kafkaProperties = kafkaClustersProperties.toKafkaProperties(datacenterNameProvider); - WeightedWorkBalancingProperties weightedWorkBalancing = workloadProperties.getWeightedWorkBalancing(); - return new ZookeeperSubscriptionProfileRegistry( - curator, - subscriptionIds, - zookeeperPaths, - kafkaProperties.getClusterName(), - weightedWorkBalancing.getSubscriptionProfilesEncoderBufferSizeBytes() - ); - } + @Bean + public SubscriptionProfileRegistry subscriptionProfileRegistry( + CuratorFramework curator, + SubscriptionIds subscriptionIds, + ZookeeperPaths zookeeperPaths, + WorkloadProperties workloadProperties, + KafkaClustersProperties kafkaClustersProperties, + DatacenterNameProvider datacenterNameProvider) { + KafkaProperties kafkaProperties = + kafkaClustersProperties.toKafkaProperties(datacenterNameProvider); + WeightedWorkBalancingProperties weightedWorkBalancing = + workloadProperties.getWeightedWorkBalancing(); + return new ZookeeperSubscriptionProfileRegistry( + curator, + subscriptionIds, + zookeeperPaths, + kafkaProperties.getClusterName(), + weightedWorkBalancing.getSubscriptionProfilesEncoderBufferSizeBytes()); + } - @Bean - public Retransmitter retransmitter(SubscriptionOffsetChangeIndicator subscriptionOffsetChangeIndicator, - KafkaClustersProperties kafkaClustersProperties, - DatacenterNameProvider datacenterNameProvider) { - KafkaProperties kafkaProperties = kafkaClustersProperties.toKafkaProperties(datacenterNameProvider); + @Bean + public Retransmitter retransmitter( + SubscriptionOffsetChangeIndicator subscriptionOffsetChangeIndicator, + KafkaClustersProperties kafkaClustersProperties, + DatacenterNameProvider datacenterNameProvider) { + KafkaProperties kafkaProperties = + kafkaClustersProperties.toKafkaProperties(datacenterNameProvider); - return new Retransmitter(subscriptionOffsetChangeIndicator, kafkaProperties.getClusterName()); - } + return new Retransmitter(subscriptionOffsetChangeIndicator, kafkaProperties.getClusterName()); + } - @Bean - public ConsumerFactory consumerFactory(ReceiverFactory messageReceiverFactory, - MetricsFacade metrics, - CommonConsumerProperties commonConsumerProperties, - ConsumerRateLimitSupervisor consumerRateLimitSupervisor, - OutputRateCalculatorFactory outputRateCalculatorFactory, - Trackers trackers, - ConsumerMessageSenderFactory consumerMessageSenderFactory, - TopicRepository topicRepository, - MessageConverterResolver messageConverterResolver, - MessageBatchFactory byteBufferMessageBatchFactory, - CompositeMessageContentWrapper compositeMessageContentWrapper, - MessageBatchSenderFactory batchSenderFactory, - ConsumerAuthorizationHandler consumerAuthorizationHandler, - Clock clock, - SubscriptionLoadRecordersRegistry subscriptionLoadRecordersRegistry, - ConsumerPartitionAssignmentState consumerPartitionAssignmentState, - CommitOffsetProperties commitOffsetProperties) { - return new ConsumerFactory( - messageReceiverFactory, - metrics, - commonConsumerProperties, - consumerRateLimitSupervisor, - outputRateCalculatorFactory, - trackers, - consumerMessageSenderFactory, - topicRepository, - messageConverterResolver, - byteBufferMessageBatchFactory, - compositeMessageContentWrapper, - batchSenderFactory, - consumerAuthorizationHandler, - clock, - subscriptionLoadRecordersRegistry, - consumerPartitionAssignmentState, - commitOffsetProperties.getPeriod(), - commitOffsetProperties.getQueuesSize() - ); - } + @Bean + public ConsumerFactory consumerFactory( + ReceiverFactory messageReceiverFactory, + MetricsFacade metrics, + CommonConsumerProperties commonConsumerProperties, + ConsumerRateLimitSupervisor consumerRateLimitSupervisor, + OutputRateCalculatorFactory outputRateCalculatorFactory, + Trackers trackers, + ConsumerMessageSenderFactory consumerMessageSenderFactory, + TopicRepository topicRepository, + MessageConverterResolver messageConverterResolver, + MessageBatchFactory byteBufferMessageBatchFactory, + CompositeMessageContentWrapper compositeMessageContentWrapper, + MessageBatchSenderFactory batchSenderFactory, + ConsumerAuthorizationHandler consumerAuthorizationHandler, + Clock clock, + SubscriptionLoadRecordersRegistry subscriptionLoadRecordersRegistry, + ConsumerPartitionAssignmentState consumerPartitionAssignmentState, + CommitOffsetProperties commitOffsetProperties) { + return new ConsumerFactory( + messageReceiverFactory, + metrics, + commonConsumerProperties, + consumerRateLimitSupervisor, + outputRateCalculatorFactory, + trackers, + consumerMessageSenderFactory, + topicRepository, + messageConverterResolver, + byteBufferMessageBatchFactory, + compositeMessageContentWrapper, + batchSenderFactory, + consumerAuthorizationHandler, + clock, + subscriptionLoadRecordersRegistry, + consumerPartitionAssignmentState, + commitOffsetProperties.getPeriod(), + commitOffsetProperties.getQueuesSize()); + } - @Bean - public ConsumersExecutorService consumersExecutorService(CommonConsumerProperties commonConsumerProperties, - MetricsFacade metrics) { - return new ConsumersExecutorService(commonConsumerProperties.getThreadPoolSize(), metrics); - } + @Bean + public ConsumersExecutorService consumersExecutorService( + CommonConsumerProperties commonConsumerProperties, MetricsFacade metrics) { + return new ConsumersExecutorService(commonConsumerProperties.getThreadPoolSize(), metrics); + } - @Bean - public ConsumersSupervisor nonblockingConsumersSupervisor(CommonConsumerProperties commonConsumerProperties, - ConsumersExecutorService executor, - ConsumerFactory consumerFactory, - ConsumerPartitionAssignmentState consumerPartitionAssignmentState, - Retransmitter retransmitter, - UndeliveredMessageLogPersister undeliveredMessageLogPersister, - SubscriptionRepository subscriptionRepository, - MetricsFacade metrics, - ConsumerMonitor monitor, - Clock clock) { - return new NonblockingConsumersSupervisor(commonConsumerProperties, executor, consumerFactory, - consumerPartitionAssignmentState, retransmitter, undeliveredMessageLogPersister, - subscriptionRepository, metrics, monitor, clock); - } + @Bean + public ConsumersSupervisor nonblockingConsumersSupervisor( + CommonConsumerProperties commonConsumerProperties, + ConsumersExecutorService executor, + ConsumerFactory consumerFactory, + ConsumerPartitionAssignmentState consumerPartitionAssignmentState, + Retransmitter retransmitter, + UndeliveredMessageLogPersister undeliveredMessageLogPersister, + SubscriptionRepository subscriptionRepository, + MetricsFacade metrics, + ConsumerMonitor monitor, + Clock clock) { + return new NonblockingConsumersSupervisor( + commonConsumerProperties, + executor, + consumerFactory, + consumerPartitionAssignmentState, + retransmitter, + undeliveredMessageLogPersister, + subscriptionRepository, + metrics, + monitor, + clock); + } - @Bean(initMethod = "start", destroyMethod = "shutdown") - public ConsumersRuntimeMonitor consumersRuntimeMonitor(ConsumersSupervisor consumerSupervisor, - WorkloadSupervisor workloadSupervisor, - MetricsFacade metrics, - SubscriptionsCache subscriptionsCache, - WorkloadProperties workloadProperties) { - return new ConsumersRuntimeMonitor( - consumerSupervisor, - workloadSupervisor, - metrics, - subscriptionsCache, - workloadProperties.getMonitorScanInterval() - ); - } + @Bean(initMethod = "start", destroyMethod = "shutdown") + public ConsumersRuntimeMonitor consumersRuntimeMonitor( + ConsumersSupervisor consumerSupervisor, + WorkloadSupervisor workloadSupervisor, + MetricsFacade metrics, + SubscriptionsCache subscriptionsCache, + WorkloadProperties workloadProperties) { + return new ConsumersRuntimeMonitor( + consumerSupervisor, + workloadSupervisor, + metrics, + subscriptionsCache, + workloadProperties.getMonitorScanInterval()); + } - @Bean - public ConsumerAssignmentRegistry consumerAssignmentRegistry(CuratorFramework curator, - WorkloadProperties workloadProperties, - KafkaClustersProperties kafkaClustersProperties, - ZookeeperPaths zookeeperPaths, - SubscriptionIds subscriptionIds, - DatacenterNameProvider datacenterNameProvider) { - KafkaProperties kafkaProperties = kafkaClustersProperties.toKafkaProperties(datacenterNameProvider); - return new ConsumerAssignmentRegistry( - curator, - workloadProperties.getRegistryBinaryEncoderAssignmentsBufferSizeBytes(), - kafkaProperties.getClusterName(), - zookeeperPaths, - subscriptionIds); - } + @Bean + public ConsumerAssignmentRegistry consumerAssignmentRegistry( + CuratorFramework curator, + WorkloadProperties workloadProperties, + KafkaClustersProperties kafkaClustersProperties, + ZookeeperPaths zookeeperPaths, + SubscriptionIds subscriptionIds, + DatacenterNameProvider datacenterNameProvider) { + KafkaProperties kafkaProperties = + kafkaClustersProperties.toKafkaProperties(datacenterNameProvider); + return new ConsumerAssignmentRegistry( + curator, + workloadProperties.getRegistryBinaryEncoderAssignmentsBufferSizeBytes(), + kafkaProperties.getClusterName(), + zookeeperPaths, + subscriptionIds); + } - @Bean - public ClusterAssignmentCache clusterAssignmentCache(CuratorFramework curator, - KafkaClustersProperties kafkaClustersProperties, - ZookeeperPaths zookeeperPaths, - SubscriptionIds subscriptionIds, - ConsumerNodesRegistry consumerNodesRegistry, - DatacenterNameProvider datacenterNameProvider) { - KafkaProperties kafkaProperties = kafkaClustersProperties.toKafkaProperties(datacenterNameProvider); - return new ClusterAssignmentCache( - curator, - kafkaProperties.getClusterName(), - zookeeperPaths, - subscriptionIds, - consumerNodesRegistry); - } + @Bean + public ClusterAssignmentCache clusterAssignmentCache( + CuratorFramework curator, + KafkaClustersProperties kafkaClustersProperties, + ZookeeperPaths zookeeperPaths, + SubscriptionIds subscriptionIds, + ConsumerNodesRegistry consumerNodesRegistry, + DatacenterNameProvider datacenterNameProvider) { + KafkaProperties kafkaProperties = + kafkaClustersProperties.toKafkaProperties(datacenterNameProvider); + return new ClusterAssignmentCache( + curator, + kafkaProperties.getClusterName(), + zookeeperPaths, + subscriptionIds, + consumerNodesRegistry); + } - @Bean(initMethod = "start", destroyMethod = "stop") - public ConsumerAssignmentCache consumerAssignmentCache(CuratorFramework curator, - WorkloadProperties workloadProperties, - KafkaClustersProperties kafkaClustersProperties, - ZookeeperPaths zookeeperPaths, - SubscriptionIds subscriptionIds, - DatacenterNameProvider datacenterNameProvider) { - KafkaProperties kafkaProperties = kafkaClustersProperties.toKafkaProperties(datacenterNameProvider); - return new ConsumerAssignmentCache( - curator, - workloadProperties.getNodeId(), - kafkaProperties.getClusterName(), - zookeeperPaths, - subscriptionIds); - } + @Bean(initMethod = "start", destroyMethod = "stop") + public ConsumerAssignmentCache consumerAssignmentCache( + CuratorFramework curator, + WorkloadProperties workloadProperties, + KafkaClustersProperties kafkaClustersProperties, + ZookeeperPaths zookeeperPaths, + SubscriptionIds subscriptionIds, + DatacenterNameProvider datacenterNameProvider) { + KafkaProperties kafkaProperties = + kafkaClustersProperties.toKafkaProperties(datacenterNameProvider); + return new ConsumerAssignmentCache( + curator, + workloadProperties.getNodeId(), + kafkaProperties.getClusterName(), + zookeeperPaths, + subscriptionIds); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/WorkloadProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/WorkloadProperties.java index 948bd932cb..559dbbfd6c 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/WorkloadProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/WorkloadProperties.java @@ -1,240 +1,249 @@ package pl.allegro.tech.hermes.consumers.config; +import static java.lang.Math.abs; +import static java.util.UUID.randomUUID; + +import java.time.Duration; +import java.util.Arrays; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.NestedConfigurationProperty; import pl.allegro.tech.hermes.common.exception.InternalProcessingException; import pl.allegro.tech.hermes.common.util.InetAddressInstanceIdResolver; import pl.allegro.tech.hermes.consumers.supervisor.workload.WorkBalancingParameters; -import java.time.Duration; -import java.util.Arrays; - -import static java.lang.Math.abs; -import static java.util.UUID.randomUUID; - @ConfigurationProperties(prefix = "consumer.workload") public class WorkloadProperties implements WorkBalancingParameters { - private int registryBinaryEncoderAssignmentsBufferSizeBytes = 100_000; + private int registryBinaryEncoderAssignmentsBufferSizeBytes = 100_000; - private Duration rebalanceInterval = Duration.ofSeconds(30); + private Duration rebalanceInterval = Duration.ofSeconds(30); - private int consumersPerSubscription = 2; + private int consumersPerSubscription = 2; - private int maxSubscriptionsPerConsumer = 200; + private int maxSubscriptionsPerConsumer = 200; - private int assignmentProcessingThreadPoolSize = 5; + private int assignmentProcessingThreadPoolSize = 5; - private String nodeId = - new InetAddressInstanceIdResolver().resolve().replaceAll("\\.", "_") + "$" + abs(randomUUID().getMostSignificantBits()); + private String nodeId = + new InetAddressInstanceIdResolver().resolve().replaceAll("\\.", "_") + + "$" + + abs(randomUUID().getMostSignificantBits()); - private Duration monitorScanInterval = Duration.ofSeconds(120); + private Duration monitorScanInterval = Duration.ofSeconds(120); - private boolean autoRebalance = true; + private boolean autoRebalance = true; - private Duration deadAfter = Duration.ofSeconds(120); + private Duration deadAfter = Duration.ofSeconds(120); - private WorkBalancingStrategy workBalancingStrategy = WorkBalancingStrategy.SELECTIVE; + private WorkBalancingStrategy workBalancingStrategy = WorkBalancingStrategy.SELECTIVE; - @NestedConfigurationProperty - private WeightedWorkBalancingProperties weightedWorkBalancing = new WeightedWorkBalancingProperties(); + @NestedConfigurationProperty + private WeightedWorkBalancingProperties weightedWorkBalancing = + new WeightedWorkBalancingProperties(); - public int getRegistryBinaryEncoderAssignmentsBufferSizeBytes() { - return registryBinaryEncoderAssignmentsBufferSizeBytes; - } + public int getRegistryBinaryEncoderAssignmentsBufferSizeBytes() { + return registryBinaryEncoderAssignmentsBufferSizeBytes; + } - public void setRegistryBinaryEncoderAssignmentsBufferSizeBytes(int registryBinaryEncoderAssignmentsBufferSizeBytes) { - this.registryBinaryEncoderAssignmentsBufferSizeBytes = registryBinaryEncoderAssignmentsBufferSizeBytes; - } + public void setRegistryBinaryEncoderAssignmentsBufferSizeBytes( + int registryBinaryEncoderAssignmentsBufferSizeBytes) { + this.registryBinaryEncoderAssignmentsBufferSizeBytes = + registryBinaryEncoderAssignmentsBufferSizeBytes; + } - @Override - public Duration getRebalanceInterval() { - return rebalanceInterval; - } + @Override + public Duration getRebalanceInterval() { + return rebalanceInterval; + } - public void setRebalanceInterval(Duration rebalanceInterval) { - this.rebalanceInterval = rebalanceInterval; - } + public void setRebalanceInterval(Duration rebalanceInterval) { + this.rebalanceInterval = rebalanceInterval; + } - @Override - public int getConsumersPerSubscription() { - return consumersPerSubscription; - } + @Override + public int getConsumersPerSubscription() { + return consumersPerSubscription; + } - public void setConsumersPerSubscription(int consumersPerSubscription) { - this.consumersPerSubscription = consumersPerSubscription; - } + public void setConsumersPerSubscription(int consumersPerSubscription) { + this.consumersPerSubscription = consumersPerSubscription; + } - @Override - public int getMaxSubscriptionsPerConsumer() { - return maxSubscriptionsPerConsumer; - } + @Override + public int getMaxSubscriptionsPerConsumer() { + return maxSubscriptionsPerConsumer; + } - public void setMaxSubscriptionsPerConsumer(int maxSubscriptionsPerConsumer) { - this.maxSubscriptionsPerConsumer = maxSubscriptionsPerConsumer; - } + public void setMaxSubscriptionsPerConsumer(int maxSubscriptionsPerConsumer) { + this.maxSubscriptionsPerConsumer = maxSubscriptionsPerConsumer; + } - public int getAssignmentProcessingThreadPoolSize() { - return assignmentProcessingThreadPoolSize; - } + public int getAssignmentProcessingThreadPoolSize() { + return assignmentProcessingThreadPoolSize; + } - public void setAssignmentProcessingThreadPoolSize(int assignmentProcessingThreadPoolSize) { - this.assignmentProcessingThreadPoolSize = assignmentProcessingThreadPoolSize; - } + public void setAssignmentProcessingThreadPoolSize(int assignmentProcessingThreadPoolSize) { + this.assignmentProcessingThreadPoolSize = assignmentProcessingThreadPoolSize; + } - public String getNodeId() { - return nodeId; - } + public String getNodeId() { + return nodeId; + } - public void setNodeId(String nodeId) { - this.nodeId = nodeId; - } + public void setNodeId(String nodeId) { + this.nodeId = nodeId; + } - public Duration getMonitorScanInterval() { - return monitorScanInterval; - } + public Duration getMonitorScanInterval() { + return monitorScanInterval; + } - public void setMonitorScanInterval(Duration monitorScanInterval) { - this.monitorScanInterval = monitorScanInterval; - } + public void setMonitorScanInterval(Duration monitorScanInterval) { + this.monitorScanInterval = monitorScanInterval; + } - @Override - public boolean isAutoRebalance() { - return autoRebalance; - } + @Override + public boolean isAutoRebalance() { + return autoRebalance; + } - public void setAutoRebalance(boolean autoRebalance) { - this.autoRebalance = autoRebalance; - } + public void setAutoRebalance(boolean autoRebalance) { + this.autoRebalance = autoRebalance; + } - public Duration getDeadAfter() { - return deadAfter; - } + public Duration getDeadAfter() { + return deadAfter; + } - public void setDeadAfter(Duration deadAfter) { - this.deadAfter = deadAfter; - } + public void setDeadAfter(Duration deadAfter) { + this.deadAfter = deadAfter; + } - public WorkBalancingStrategy getWorkBalancingStrategy() { - return workBalancingStrategy; - } + public WorkBalancingStrategy getWorkBalancingStrategy() { + return workBalancingStrategy; + } - public void setWorkBalancingStrategy(WorkBalancingStrategy workBalancingStrategy) { - this.workBalancingStrategy = workBalancingStrategy; - } + public void setWorkBalancingStrategy(WorkBalancingStrategy workBalancingStrategy) { + this.workBalancingStrategy = workBalancingStrategy; + } - public WeightedWorkBalancingProperties getWeightedWorkBalancing() { - return weightedWorkBalancing; - } + public WeightedWorkBalancingProperties getWeightedWorkBalancing() { + return weightedWorkBalancing; + } - public void setWeightedWorkBalancing(WeightedWorkBalancingProperties weightedWorkBalancing) { - this.weightedWorkBalancing = weightedWorkBalancing; - } + public void setWeightedWorkBalancing(WeightedWorkBalancingProperties weightedWorkBalancing) { + this.weightedWorkBalancing = weightedWorkBalancing; + } - public static class WeightedWorkBalancingProperties { + public static class WeightedWorkBalancingProperties { - private int consumerLoadEncoderBufferSizeBytes = 100_000; + private int consumerLoadEncoderBufferSizeBytes = 100_000; - private int subscriptionProfilesEncoderBufferSizeBytes = 100_000; + private int subscriptionProfilesEncoderBufferSizeBytes = 100_000; - private Duration loadReportingInterval = Duration.ofMinutes(1); + private Duration loadReportingInterval = Duration.ofMinutes(1); - private Duration stabilizationWindowSize = Duration.ofMinutes(30); + private Duration stabilizationWindowSize = Duration.ofMinutes(30); - private double minSignificantChangePercent = 5; + private double minSignificantChangePercent = 5; - private Duration weightWindowSize = Duration.ofMinutes(15); + private Duration weightWindowSize = Duration.ofMinutes(15); - private TargetWeightCalculationStrategy targetWeightCalculationStrategy = TargetWeightCalculationStrategy.AVG; + private TargetWeightCalculationStrategy targetWeightCalculationStrategy = + TargetWeightCalculationStrategy.AVG; - private double scoringGain = 1.0d; + private double scoringGain = 1.0d; - public int getConsumerLoadEncoderBufferSizeBytes() { - return consumerLoadEncoderBufferSizeBytes; - } + public int getConsumerLoadEncoderBufferSizeBytes() { + return consumerLoadEncoderBufferSizeBytes; + } - public void setConsumerLoadEncoderBufferSizeBytes(int consumerLoadEncoderBufferSizeBytes) { - this.consumerLoadEncoderBufferSizeBytes = consumerLoadEncoderBufferSizeBytes; - } + public void setConsumerLoadEncoderBufferSizeBytes(int consumerLoadEncoderBufferSizeBytes) { + this.consumerLoadEncoderBufferSizeBytes = consumerLoadEncoderBufferSizeBytes; + } - public int getSubscriptionProfilesEncoderBufferSizeBytes() { - return subscriptionProfilesEncoderBufferSizeBytes; - } + public int getSubscriptionProfilesEncoderBufferSizeBytes() { + return subscriptionProfilesEncoderBufferSizeBytes; + } - public void setSubscriptionProfilesEncoderBufferSizeBytes(int subscriptionProfilesEncoderBufferSizeBytes) { - this.subscriptionProfilesEncoderBufferSizeBytes = subscriptionProfilesEncoderBufferSizeBytes; - } + public void setSubscriptionProfilesEncoderBufferSizeBytes( + int subscriptionProfilesEncoderBufferSizeBytes) { + this.subscriptionProfilesEncoderBufferSizeBytes = subscriptionProfilesEncoderBufferSizeBytes; + } - public Duration getLoadReportingInterval() { - return loadReportingInterval; - } + public Duration getLoadReportingInterval() { + return loadReportingInterval; + } - public void setLoadReportingInterval(Duration loadReportingInterval) { - this.loadReportingInterval = loadReportingInterval; - } + public void setLoadReportingInterval(Duration loadReportingInterval) { + this.loadReportingInterval = loadReportingInterval; + } - public Duration getStabilizationWindowSize() { - return stabilizationWindowSize; - } + public Duration getStabilizationWindowSize() { + return stabilizationWindowSize; + } - public void setStabilizationWindowSize(Duration stabilizationWindowSize) { - this.stabilizationWindowSize = stabilizationWindowSize; - } + public void setStabilizationWindowSize(Duration stabilizationWindowSize) { + this.stabilizationWindowSize = stabilizationWindowSize; + } - public double getMinSignificantChangePercent() { - return minSignificantChangePercent; - } + public double getMinSignificantChangePercent() { + return minSignificantChangePercent; + } - public void setMinSignificantChangePercent(double minSignificantChangePercent) { - this.minSignificantChangePercent = minSignificantChangePercent; - } + public void setMinSignificantChangePercent(double minSignificantChangePercent) { + this.minSignificantChangePercent = minSignificantChangePercent; + } - public Duration getWeightWindowSize() { - return weightWindowSize; - } + public Duration getWeightWindowSize() { + return weightWindowSize; + } - public void setWeightWindowSize(Duration weightWindowSize) { - this.weightWindowSize = weightWindowSize; - } + public void setWeightWindowSize(Duration weightWindowSize) { + this.weightWindowSize = weightWindowSize; + } - public TargetWeightCalculationStrategy getTargetWeightCalculationStrategy() { - return targetWeightCalculationStrategy; - } + public TargetWeightCalculationStrategy getTargetWeightCalculationStrategy() { + return targetWeightCalculationStrategy; + } - public void setTargetWeightCalculationStrategy(TargetWeightCalculationStrategy targetWeightCalculationStrategy) { - this.targetWeightCalculationStrategy = targetWeightCalculationStrategy; - } + public void setTargetWeightCalculationStrategy( + TargetWeightCalculationStrategy targetWeightCalculationStrategy) { + this.targetWeightCalculationStrategy = targetWeightCalculationStrategy; + } - public double getScoringGain() { - return scoringGain; - } + public double getScoringGain() { + return scoringGain; + } - public void setScoringGain(double scoringGain) { - this.scoringGain = scoringGain; - } + public void setScoringGain(double scoringGain) { + this.scoringGain = scoringGain; } + } - public enum WorkBalancingStrategy { - SELECTIVE, - WEIGHTED; + public enum WorkBalancingStrategy { + SELECTIVE, + WEIGHTED; - public static class UnknownWorkBalancingStrategyException extends InternalProcessingException { + public static class UnknownWorkBalancingStrategyException extends InternalProcessingException { - public UnknownWorkBalancingStrategyException() { - super("Unknown work balancing strategy. Use one of: " + Arrays.toString(values())); - } - } + public UnknownWorkBalancingStrategyException() { + super("Unknown work balancing strategy. Use one of: " + Arrays.toString(values())); + } } + } - public enum TargetWeightCalculationStrategy { - AVG, - SCORING; + public enum TargetWeightCalculationStrategy { + AVG, + SCORING; - public static class UnknownTargetWeightCalculationStrategyException extends InternalProcessingException { + public static class UnknownTargetWeightCalculationStrategyException + extends InternalProcessingException { - public UnknownTargetWeightCalculationStrategyException() { - super("Unknown target weight calculation strategy. Use one of: " + Arrays.toString(values())); - } - } + public UnknownTargetWeightCalculationStrategyException() { + super( + "Unknown target weight calculation strategy. Use one of: " + Arrays.toString(values())); + } } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ZookeeperClustersProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ZookeeperClustersProperties.java index f79f48e8d0..7e0f2d8019 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ZookeeperClustersProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ZookeeperClustersProperties.java @@ -1,30 +1,33 @@ package pl.allegro.tech.hermes.consumers.config; -import org.springframework.boot.context.properties.ConfigurationProperties; -import pl.allegro.tech.hermes.infrastructure.dc.DatacenterNameProvider; - import java.util.ArrayList; import java.util.List; +import org.springframework.boot.context.properties.ConfigurationProperties; +import pl.allegro.tech.hermes.infrastructure.dc.DatacenterNameProvider; @ConfigurationProperties(prefix = "consumer.zookeeper") public class ZookeeperClustersProperties { - private List clusters = new ArrayList<>(); + private List clusters = new ArrayList<>(); - public List getClusters() { - return clusters; - } + public List getClusters() { + return clusters; + } - public void setClusters(List clusters) { - this.clusters = clusters; - } + public void setClusters(List clusters) { + this.clusters = clusters; + } - public ZookeeperProperties toZookeeperProperties(DatacenterNameProvider datacenterNameProvider) { - return this.clusters - .stream() - .filter(cluster -> cluster.getDatacenter().equals(datacenterNameProvider.getDatacenterName())) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException( - "No properties for datacenter: " + datacenterNameProvider.getDatacenterName() + " defined.")); - } + public ZookeeperProperties toZookeeperProperties(DatacenterNameProvider datacenterNameProvider) { + return this.clusters.stream() + .filter( + cluster -> cluster.getDatacenter().equals(datacenterNameProvider.getDatacenterName())) + .findFirst() + .orElseThrow( + () -> + new IllegalArgumentException( + "No properties for datacenter: " + + datacenterNameProvider.getDatacenterName() + + " defined.")); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ZookeeperProperties.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ZookeeperProperties.java index cbeabac595..5fb8b2f714 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ZookeeperProperties.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/config/ZookeeperProperties.java @@ -1,179 +1,178 @@ package pl.allegro.tech.hermes.consumers.config; -import pl.allegro.tech.hermes.common.di.factories.ZookeeperParameters; - import java.time.Duration; +import pl.allegro.tech.hermes.common.di.factories.ZookeeperParameters; public class ZookeeperProperties implements ZookeeperParameters { - private String connectionString = "localhost:2181"; + private String connectionString = "localhost:2181"; - private String datacenter = "dc"; + private String datacenter = "dc"; - private Duration baseSleepTime = Duration.ofSeconds(1000); + private Duration baseSleepTime = Duration.ofSeconds(1000); - private Duration maxSleepTime = Duration.ofSeconds(30); + private Duration maxSleepTime = Duration.ofSeconds(30); - private int maxRetries = 29; + private int maxRetries = 29; - private Duration connectionTimeout = Duration.ofSeconds(10); + private Duration connectionTimeout = Duration.ofSeconds(10); - private Duration sessionTimeout = Duration.ofSeconds(10); + private Duration sessionTimeout = Duration.ofSeconds(10); - private String root = "/hermes"; + private String root = "/hermes"; - private int processingThreadPoolSize = 5; + private int processingThreadPoolSize = 5; - private ZookeeperAuthorizationProperties authorization = new ZookeeperAuthorizationProperties(); + private ZookeeperAuthorizationProperties authorization = new ZookeeperAuthorizationProperties(); - @Override - public String getConnectionString() { - return connectionString; - } + @Override + public String getConnectionString() { + return connectionString; + } - public void setConnectionString(String connectionString) { - this.connectionString = connectionString; - } + public void setConnectionString(String connectionString) { + this.connectionString = connectionString; + } - public String getDatacenter() { - return datacenter; - } + public String getDatacenter() { + return datacenter; + } - public void setDatacenter(String datacenter) { - this.datacenter = datacenter; - } + public void setDatacenter(String datacenter) { + this.datacenter = datacenter; + } - @Override - public Duration getBaseSleepTime() { - return baseSleepTime; - } + @Override + public Duration getBaseSleepTime() { + return baseSleepTime; + } - public void setBaseSleepTime(Duration baseSleepTime) { - this.baseSleepTime = baseSleepTime; - } + public void setBaseSleepTime(Duration baseSleepTime) { + this.baseSleepTime = baseSleepTime; + } - @Override - public Duration getMaxSleepTime() { - return maxSleepTime; - } + @Override + public Duration getMaxSleepTime() { + return maxSleepTime; + } - public void setMaxSleepTime(Duration maxSleepTime) { - this.maxSleepTime = maxSleepTime; - } + public void setMaxSleepTime(Duration maxSleepTime) { + this.maxSleepTime = maxSleepTime; + } - @Override - public int getMaxRetries() { - return maxRetries; - } + @Override + public int getMaxRetries() { + return maxRetries; + } - public void setMaxRetries(int maxRetries) { - this.maxRetries = maxRetries; - } + public void setMaxRetries(int maxRetries) { + this.maxRetries = maxRetries; + } - @Override - public Duration getConnectionTimeout() { - return connectionTimeout; - } + @Override + public Duration getConnectionTimeout() { + return connectionTimeout; + } - public void setConnectionTimeout(Duration connectionTimeout) { - this.connectionTimeout = connectionTimeout; - } + public void setConnectionTimeout(Duration connectionTimeout) { + this.connectionTimeout = connectionTimeout; + } - @Override - public Duration getSessionTimeout() { - return sessionTimeout; - } + @Override + public Duration getSessionTimeout() { + return sessionTimeout; + } - public void setSessionTimeout(Duration sessionTimeout) { - this.sessionTimeout = sessionTimeout; - } + public void setSessionTimeout(Duration sessionTimeout) { + this.sessionTimeout = sessionTimeout; + } - @Override - public String getRoot() { - return root; - } + @Override + public String getRoot() { + return root; + } - public void setRoot(String root) { - this.root = root; - } + public void setRoot(String root) { + this.root = root; + } - @Override - public int getProcessingThreadPoolSize() { - return processingThreadPoolSize; - } + @Override + public int getProcessingThreadPoolSize() { + return processingThreadPoolSize; + } - @Override - public boolean isAuthorizationEnabled() { - return authorization.enabled; - } + @Override + public boolean isAuthorizationEnabled() { + return authorization.enabled; + } - @Override - public String getScheme() { - return authorization.scheme; - } + @Override + public String getScheme() { + return authorization.scheme; + } - @Override - public String getUser() { - return authorization.user; - } + @Override + public String getUser() { + return authorization.user; + } - @Override - public String getPassword() { - return authorization.password; - } + @Override + public String getPassword() { + return authorization.password; + } - public void setProcessingThreadPoolSize(int processingThreadPoolSize) { - this.processingThreadPoolSize = processingThreadPoolSize; - } + public void setProcessingThreadPoolSize(int processingThreadPoolSize) { + this.processingThreadPoolSize = processingThreadPoolSize; + } - public ZookeeperAuthorizationProperties getAuthorization() { - return authorization; - } + public ZookeeperAuthorizationProperties getAuthorization() { + return authorization; + } - public void setAuthorization(ZookeeperAuthorizationProperties authorization) { - this.authorization = authorization; - } + public void setAuthorization(ZookeeperAuthorizationProperties authorization) { + this.authorization = authorization; + } - public static class ZookeeperAuthorizationProperties { + public static class ZookeeperAuthorizationProperties { - private boolean enabled = false; + private boolean enabled = false; - private String scheme = "digest"; + private String scheme = "digest"; - private String user = "user"; + private String user = "user"; - private String password = "password"; + private String password = "password"; - public boolean isEnabled() { - return enabled; - } + public boolean isEnabled() { + return enabled; + } - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } - public String getScheme() { - return scheme; - } + public String getScheme() { + return scheme; + } - public void setScheme(String scheme) { - this.scheme = scheme; - } + public void setScheme(String scheme) { + this.scheme = scheme; + } - public String getUser() { - return user; - } + public String getUser() { + return user; + } - public void setUser(String user) { - this.user = user; - } + public void setUser(String user) { + this.user = user; + } - public String getPassword() { - return password; - } + public String getPassword() { + return password; + } - public void setPassword(String password) { - this.password = password; - } + public void setPassword(String password) { + this.password = password; } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/BatchConsumer.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/BatchConsumer.java index 03d67d1615..9f73598e31 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/BatchConsumer.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/BatchConsumer.java @@ -1,9 +1,22 @@ package pl.allegro.tech.hermes.consumers.consumer; +import static com.github.rholder.retry.WaitStrategies.fixedWait; +import static java.util.Optional.of; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; + import com.github.rholder.retry.Attempt; import com.github.rholder.retry.RetryListener; import com.github.rholder.retry.Retryer; import com.github.rholder.retry.RetryerBuilder; +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.BatchSubscriptionPolicy; @@ -28,297 +41,318 @@ import pl.allegro.tech.hermes.metrics.HermesTimerContext; import pl.allegro.tech.hermes.tracker.consumers.Trackers; -import java.io.IOException; -import java.time.Duration; -import java.time.Instant; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Optional; -import java.util.Set; - -import static com.github.rholder.retry.WaitStrategies.fixedWait; -import static java.util.Optional.of; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.SECONDS; - public class BatchConsumer implements Consumer { - private static final Logger logger = LoggerFactory.getLogger(BatchConsumer.class); - - private final ReceiverFactory messageReceiverFactory; - private final MessageBatchSender sender; - private final MessageBatchFactory batchFactory; - private final boolean useTopicMessageSize; - private final MessageConverterResolver messageConverterResolver; - private final CompositeMessageContentWrapper compositeMessageContentWrapper; - private final Trackers trackers; - private final SubscriptionLoadRecorder loadRecorder; - private final Duration commitPeriod; - - private Topic topic; - private Subscription subscription; - - private volatile boolean consuming = true; - - private final MetricsFacade metricsFacade; - private final BatchConsumerMetrics metrics; - private MessageBatchReceiver receiver; - - private final Map maxPendingOffsets = new HashMap<>(); - - private Instant lastCommitTime; - - public BatchConsumer(ReceiverFactory messageReceiverFactory, - MessageBatchSender sender, - MessageBatchFactory batchFactory, - MessageConverterResolver messageConverterResolver, - CompositeMessageContentWrapper compositeMessageContentWrapper, - MetricsFacade metricsFacade, - Trackers trackers, - Subscription subscription, - Topic topic, - boolean useTopicMessageSize, - SubscriptionLoadRecorder loadRecorder, - Duration commitPeriod) { - this.messageReceiverFactory = messageReceiverFactory; - this.sender = sender; - this.batchFactory = batchFactory; - this.subscription = subscription; - this.useTopicMessageSize = useTopicMessageSize; - this.loadRecorder = loadRecorder; - this.metricsFacade = metricsFacade; - this.metrics = new BatchConsumerMetrics(metricsFacade, subscription.getQualifiedName()); - this.messageConverterResolver = messageConverterResolver; - this.compositeMessageContentWrapper = compositeMessageContentWrapper; - this.topic = topic; - this.trackers = trackers; - this.commitPeriod = commitPeriod; - this.lastCommitTime = Instant.now(); - } - - @Override - public void consume(Runnable signalsInterrupt) { - Optional inflight = Optional.empty(); - try { - logger.debug("Trying to create new batch [subscription={}].", subscription.getQualifiedName()); - - signalsInterrupt.run(); - commitIfReady(); - - MessageBatchingResult result = receiver.next(subscription, signalsInterrupt); - inflight = of(result.getBatch()); - - inflight.ifPresent(batch -> { - logger.debug("Delivering batch [subscription={}].", subscription.getQualifiedName()); - - deliver(signalsInterrupt, batch, createRetryer(batch, subscription.getBatchSubscriptionPolicy())); - - offerProcessedOffsets(batch); - logger.debug("Finished delivering batch [subscription={}]", subscription.getQualifiedName()); - }); - - result.getDiscarded().forEach(m -> { + private static final Logger logger = LoggerFactory.getLogger(BatchConsumer.class); + + private final ReceiverFactory messageReceiverFactory; + private final MessageBatchSender sender; + private final MessageBatchFactory batchFactory; + private final boolean useTopicMessageSize; + private final MessageConverterResolver messageConverterResolver; + private final CompositeMessageContentWrapper compositeMessageContentWrapper; + private final Trackers trackers; + private final SubscriptionLoadRecorder loadRecorder; + private final Duration commitPeriod; + + private Topic topic; + private Subscription subscription; + + private volatile boolean consuming = true; + + private final MetricsFacade metricsFacade; + private final BatchConsumerMetrics metrics; + private MessageBatchReceiver receiver; + + private final Map maxPendingOffsets = new HashMap<>(); + + private Instant lastCommitTime; + + public BatchConsumer( + ReceiverFactory messageReceiverFactory, + MessageBatchSender sender, + MessageBatchFactory batchFactory, + MessageConverterResolver messageConverterResolver, + CompositeMessageContentWrapper compositeMessageContentWrapper, + MetricsFacade metricsFacade, + Trackers trackers, + Subscription subscription, + Topic topic, + boolean useTopicMessageSize, + SubscriptionLoadRecorder loadRecorder, + Duration commitPeriod) { + this.messageReceiverFactory = messageReceiverFactory; + this.sender = sender; + this.batchFactory = batchFactory; + this.subscription = subscription; + this.useTopicMessageSize = useTopicMessageSize; + this.loadRecorder = loadRecorder; + this.metricsFacade = metricsFacade; + this.metrics = new BatchConsumerMetrics(metricsFacade, subscription.getQualifiedName()); + this.messageConverterResolver = messageConverterResolver; + this.compositeMessageContentWrapper = compositeMessageContentWrapper; + this.topic = topic; + this.trackers = trackers; + this.commitPeriod = commitPeriod; + this.lastCommitTime = Instant.now(); + } + + @Override + public void consume(Runnable signalsInterrupt) { + Optional inflight = Optional.empty(); + try { + logger.debug( + "Trying to create new batch [subscription={}].", subscription.getQualifiedName()); + + signalsInterrupt.run(); + commitIfReady(); + + MessageBatchingResult result = receiver.next(subscription, signalsInterrupt); + inflight = of(result.getBatch()); + + inflight.ifPresent( + batch -> { + logger.debug("Delivering batch [subscription={}].", subscription.getQualifiedName()); + + deliver( + signalsInterrupt, + batch, + createRetryer(batch, subscription.getBatchSubscriptionPolicy())); + + offerProcessedOffsets(batch); + logger.debug( + "Finished delivering batch [subscription={}]", subscription.getQualifiedName()); + }); + + result + .getDiscarded() + .forEach( + m -> { metrics.markDiscarded(); trackers.get(subscription).logDiscarded(m, "too large"); - }); - } finally { - logger.debug("Cleaning batch [subscription={}]", subscription.getQualifiedName()); - inflight.ifPresent(this::clean); - } - } - - private void commitIfReady() { - if (isReadyToCommit()) { - Set offsetsToCommit = new HashSet<>(); - - for (Map.Entry entry : maxPendingOffsets.entrySet()) { - offsetsToCommit.add(new SubscriptionPartitionOffset(entry.getKey(), entry.getValue())); - } - - if (!offsetsToCommit.isEmpty()) { - commit(offsetsToCommit); - } - maxPendingOffsets.clear(); - lastCommitTime = Instant.now(); - } - } - - private boolean isReadyToCommit() { - return Duration.between(lastCommitTime, Instant.now()).toMillis() > commitPeriod.toMillis(); - } - - private void offerProcessedOffsets(MessageBatch batch) { - for (SubscriptionPartitionOffset offset : batch.getPartitionOffsets()) { - putOffset(offset); - } + }); + } finally { + logger.debug("Cleaning batch [subscription={}]", subscription.getQualifiedName()); + inflight.ifPresent(this::clean); } + } - private void putOffset(SubscriptionPartitionOffset offset) { - maxPendingOffsets.compute(offset.getSubscriptionPartition(), (subscriptionPartition, maxOffset) -> - maxOffset == null ? offset.getOffset() : Math.max(maxOffset, offset.getOffset()) - ); - } + private void commitIfReady() { + if (isReadyToCommit()) { + Set offsetsToCommit = new HashSet<>(); - @Override - public void initialize() { - loadRecorder.initialize(); - logger.debug("Consumer: preparing receiver for subscription {}", subscription.getQualifiedName()); - MessageReceiver receiver = messageReceiverFactory.createMessageReceiver( - topic, - subscription, - new BatchConsumerRateLimiter(), - loadRecorder, - metricsFacade, - this::putOffset - ); - - logger.debug("Consumer: preparing batch receiver for subscription {}", subscription.getQualifiedName()); - this.receiver = new MessageBatchReceiver( - receiver, - batchFactory, - messageConverterResolver, - compositeMessageContentWrapper, - topic, - trackers, - loadRecorder, - this::commitIfReady - ); - metrics.initialize(); - } + for (Map.Entry entry : maxPendingOffsets.entrySet()) { + offsetsToCommit.add(new SubscriptionPartitionOffset(entry.getKey(), entry.getValue())); + } - @Override - public void tearDown() { - consuming = false; - if (receiver != null) { - receiver.stop(); - } else { - logger.info("No batch receiver to stop [subscription={}].", subscription.getQualifiedName()); - } - loadRecorder.shutdown(); - metrics.shutdown(); + if (!offsetsToCommit.isEmpty()) { + commit(offsetsToCommit); + } + maxPendingOffsets.clear(); + lastCommitTime = Instant.now(); } + } - @Override - public void updateSubscription(Subscription subscription) { - this.subscription = subscription; - receiver.updateSubscription(subscription); - } + private boolean isReadyToCommit() { + return Duration.between(lastCommitTime, Instant.now()).toMillis() > commitPeriod.toMillis(); + } - @Override - public void updateTopic(Topic newTopic) { - if (this.topic.getContentType() != newTopic.getContentType() || messageSizeChanged(newTopic)) { - logger.info("Reinitializing message receiver, contentType or messageSize changed."); - this.topic = newTopic; - tearDown(); - initialize(); - } + private void offerProcessedOffsets(MessageBatch batch) { + for (SubscriptionPartitionOffset offset : batch.getPartitionOffsets()) { + putOffset(offset); } - - private boolean messageSizeChanged(Topic newTopic) { - return this.topic.getMaxMessageSize() != newTopic.getMaxMessageSize() - && useTopicMessageSize; + } + + private void putOffset(SubscriptionPartitionOffset offset) { + maxPendingOffsets.compute( + offset.getSubscriptionPartition(), + (subscriptionPartition, maxOffset) -> + maxOffset == null ? offset.getOffset() : Math.max(maxOffset, offset.getOffset())); + } + + @Override + public void initialize() { + loadRecorder.initialize(); + logger.debug( + "Consumer: preparing receiver for subscription {}", subscription.getQualifiedName()); + MessageReceiver receiver = + messageReceiverFactory.createMessageReceiver( + topic, + subscription, + new BatchConsumerRateLimiter(), + loadRecorder, + metricsFacade, + this::putOffset); + + logger.debug( + "Consumer: preparing batch receiver for subscription {}", subscription.getQualifiedName()); + this.receiver = + new MessageBatchReceiver( + receiver, + batchFactory, + messageConverterResolver, + compositeMessageContentWrapper, + topic, + trackers, + loadRecorder, + this::commitIfReady); + metrics.initialize(); + } + + @Override + public void tearDown() { + consuming = false; + if (receiver != null) { + receiver.stop(); + } else { + logger.info("No batch receiver to stop [subscription={}].", subscription.getQualifiedName()); } - - @Override - public void commit(Set offsetsToCommit) { - if (receiver != null) { - receiver.commit(offsetsToCommit); - } + loadRecorder.shutdown(); + metrics.shutdown(); + } + + @Override + public void updateSubscription(Subscription subscription) { + this.subscription = subscription; + receiver.updateSubscription(subscription); + } + + @Override + public void updateTopic(Topic newTopic) { + if (this.topic.getContentType() != newTopic.getContentType() || messageSizeChanged(newTopic)) { + logger.info("Reinitializing message receiver, contentType or messageSize changed."); + this.topic = newTopic; + tearDown(); + initialize(); } + } - @Override - public boolean moveOffset(PartitionOffset partitionOffset) { - if (receiver != null) { - return receiver.moveOffset(partitionOffset); - } - return false; - } + private boolean messageSizeChanged(Topic newTopic) { + return this.topic.getMaxMessageSize() != newTopic.getMaxMessageSize() && useTopicMessageSize; + } - @Override - public Subscription getSubscription() { - return subscription; + @Override + public void commit(Set offsetsToCommit) { + if (receiver != null) { + receiver.commit(offsetsToCommit); } + } - private Retryer createRetryer(MessageBatch batch, BatchSubscriptionPolicy policy) { - return createRetryer(batch, - policy.getMessageBackoff(), - SECONDS.toMillis(policy.getMessageTtl()), - policy.isRetryClientErrors()); + @Override + public boolean moveOffset(PartitionOffset partitionOffset) { + if (receiver != null) { + return receiver.moveOffset(partitionOffset); } - - private Retryer createRetryer(final MessageBatch batch, - int messageBackoff, - long messageTtlMillis, - boolean retryClientErrors) { - return RetryerBuilder.newBuilder() - .retryIfExceptionOfType(IOException.class) - .retryIfRuntimeException() - .retryIfResult(result -> consuming && !result.succeeded() && shouldRetryOnClientError(retryClientErrors, result)) - .withWaitStrategy(fixedWait(messageBackoff, MILLISECONDS)) - .withStopStrategy(attempt -> attempt.getDelaySinceFirstAttempt() > messageTtlMillis - || Thread.currentThread().isInterrupted()) - .withRetryListener(getRetryListener(result -> { - batch.incrementRetryCounter(); - markSendingResult(batch, result); + return false; + } + + @Override + public Subscription getSubscription() { + return subscription; + } + + private Retryer createRetryer( + MessageBatch batch, BatchSubscriptionPolicy policy) { + return createRetryer( + batch, + policy.getMessageBackoff(), + SECONDS.toMillis(policy.getMessageTtl()), + policy.isRetryClientErrors()); + } + + private Retryer createRetryer( + final MessageBatch batch, + int messageBackoff, + long messageTtlMillis, + boolean retryClientErrors) { + return RetryerBuilder.newBuilder() + .retryIfExceptionOfType(IOException.class) + .retryIfRuntimeException() + .retryIfResult( + result -> + consuming + && !result.succeeded() + && shouldRetryOnClientError(retryClientErrors, result)) + .withWaitStrategy(fixedWait(messageBackoff, MILLISECONDS)) + .withStopStrategy( + attempt -> + attempt.getDelaySinceFirstAttempt() > messageTtlMillis + || Thread.currentThread().isInterrupted()) + .withRetryListener( + getRetryListener( + result -> { + batch.incrementRetryCounter(); + markSendingResult(batch, result); })) - .build(); - } - - private void markSendingResult(MessageBatch batch, MessageSendingResult result) { - if (result.succeeded()) { - metrics.recordAttemptAsFinished(batch.getMessageCount()); - metrics.markSuccess(batch, result); - batch.getMessagesMetadata().forEach( - m -> trackers.get(subscription).logSent(m, result.getHostname()) - ); - } else { - metrics.markFailure(batch, result); - batch.getMessagesMetadata().forEach( - m -> trackers.get(subscription).logFailed(m, result.getRootCause(), result.getHostname()) - ); - } + .build(); + } + + private void markSendingResult(MessageBatch batch, MessageSendingResult result) { + if (result.succeeded()) { + metrics.recordAttemptAsFinished(batch.getMessageCount()); + metrics.markSuccess(batch, result); + batch + .getMessagesMetadata() + .forEach(m -> trackers.get(subscription).logSent(m, result.getHostname())); + } else { + metrics.markFailure(batch, result); + batch + .getMessagesMetadata() + .forEach( + m -> + trackers + .get(subscription) + .logFailed(m, result.getRootCause(), result.getHostname())); } - - private boolean shouldRetryOnClientError(boolean retryClientErrors, MessageSendingResult result) { - return !result.isClientError() || retryClientErrors; + } + + private boolean shouldRetryOnClientError(boolean retryClientErrors, MessageSendingResult result) { + return !result.isClientError() || retryClientErrors; + } + + private void deliver( + Runnable signalsInterrupt, MessageBatch batch, Retryer retryer) { + metrics.recordAttempt(batch.getMessageCount()); + try (HermesTimerContext ignored = metrics.latencyTimer().time()) { + retryer.call( + () -> { + loadRecorder.recordSingleOperation(); + signalsInterrupt.run(); + return sender.send( + batch, + subscription.getEndpoint(), + subscription.getEndpointAddressResolverMetadata(), + subscription.getBatchSubscriptionPolicy().getRequestTimeout()); + }); + } catch (Exception e) { + logger.error( + "Batch was rejected [batch_id={}, subscription={}].", + batch.getId(), + subscription.getQualifiedName(), + e); + metrics.recordAttemptAsFinished(batch.getMessageCount()); + metrics.markDiscarded(batch); + batch + .getMessagesMetadata() + .forEach(m -> trackers.get(subscription).logDiscarded(m, e.getMessage())); } - - private void deliver(Runnable signalsInterrupt, MessageBatch batch, Retryer retryer) { - metrics.recordAttempt(batch.getMessageCount()); - try (HermesTimerContext ignored = metrics.latencyTimer().time()) { - retryer.call(() -> { - loadRecorder.recordSingleOperation(); - signalsInterrupt.run(); - return sender.send( - batch, - subscription.getEndpoint(), - subscription.getEndpointAddressResolverMetadata(), - subscription.getBatchSubscriptionPolicy().getRequestTimeout() - ); - }); - } catch (Exception e) { - logger.error("Batch was rejected [batch_id={}, subscription={}].", batch.getId(), subscription.getQualifiedName(), e); - metrics.recordAttemptAsFinished(batch.getMessageCount()); - metrics.markDiscarded(batch); - batch.getMessagesMetadata().forEach(m -> trackers.get(subscription).logDiscarded(m, e.getMessage())); + } + + private void clean(MessageBatch batch) { + batchFactory.destroyBatch(batch); + } + + private RetryListener getRetryListener( + java.util.function.Consumer consumer) { + return new RetryListener() { + @Override + public void onRetry(Attempt attempt) { + if (attempt.hasException()) { + consumer.accept(MessageSendingResult.failedResult(attempt.getExceptionCause())); + } else { + consumer.accept((MessageSendingResult) attempt.getResult()); } - } - - private void clean(MessageBatch batch) { - batchFactory.destroyBatch(batch); - } - - private RetryListener getRetryListener(java.util.function.Consumer consumer) { - return new RetryListener() { - @Override - public void onRetry(Attempt attempt) { - if (attempt.hasException()) { - consumer.accept(MessageSendingResult.failedResult(attempt.getExceptionCause())); - } else { - consumer.accept((MessageSendingResult) attempt.getResult()); - } - } - }; - } + } + }; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/BatchConsumerMetrics.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/BatchConsumerMetrics.java index d75c9418bf..534623ba5a 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/BatchConsumerMetrics.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/BatchConsumerMetrics.java @@ -1,5 +1,8 @@ package pl.allegro.tech.hermes.consumers.consumer; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.LongAdder; import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.common.metric.MetricsFacade; import pl.allegro.tech.hermes.consumers.consumer.batch.MessageBatch; @@ -7,96 +10,94 @@ import pl.allegro.tech.hermes.metrics.HermesCounter; import pl.allegro.tech.hermes.metrics.HermesHistogram; import pl.allegro.tech.hermes.metrics.HermesTimer; -import pl.allegro.tech.hermes.tracker.consumers.MessageMetadata; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.LongAdder; class BatchConsumerMetrics { - private final MetricsFacade metrics; - private final SubscriptionName subscriptionName; - private final LongAdder inflightCount = new LongAdder(); - private final HermesCounter failures; - private final HermesCounter timeouts; - private final HermesCounter otherErrors; - private final HermesCounter discarded; - private final HermesHistogram inflightTime; - private final HermesCounter throughputInBytes; - private final HermesCounter successes; - private final HermesCounter batchSuccesses; - private final HermesTimer latency; - private final Map httpStatusCodes = new ConcurrentHashMap<>(); + private final MetricsFacade metrics; + private final SubscriptionName subscriptionName; + private final LongAdder inflightCount = new LongAdder(); + private final HermesCounter failures; + private final HermesCounter timeouts; + private final HermesCounter otherErrors; + private final HermesCounter discarded; + private final HermesHistogram inflightTime; + private final HermesCounter throughputInBytes; + private final HermesCounter successes; + private final HermesCounter batchSuccesses; + private final HermesTimer latency; + private final Map httpStatusCodes = new ConcurrentHashMap<>(); - BatchConsumerMetrics(MetricsFacade metrics, SubscriptionName subscriptionName) { - this.metrics = metrics; - this.subscriptionName = subscriptionName; - this.failures = metrics.subscriptions().failuresCounter(subscriptionName); - this.timeouts = metrics.subscriptions().timeoutsCounter(subscriptionName); - this.otherErrors = metrics.subscriptions().otherErrorsCounter(subscriptionName); - this.discarded = metrics.subscriptions().discarded(subscriptionName); - this.inflightTime = metrics.subscriptions().inflightTimeInMillisHistogram(subscriptionName); - this.throughputInBytes = metrics.subscriptions().throughputInBytes(subscriptionName); - this.successes = metrics.subscriptions().successes(subscriptionName); - this.batchSuccesses = metrics.subscriptions().batchSuccesses(subscriptionName); - this.latency = metrics.subscriptions().latency(subscriptionName); - } + BatchConsumerMetrics(MetricsFacade metrics, SubscriptionName subscriptionName) { + this.metrics = metrics; + this.subscriptionName = subscriptionName; + this.failures = metrics.subscriptions().failuresCounter(subscriptionName); + this.timeouts = metrics.subscriptions().timeoutsCounter(subscriptionName); + this.otherErrors = metrics.subscriptions().otherErrorsCounter(subscriptionName); + this.discarded = metrics.subscriptions().discarded(subscriptionName); + this.inflightTime = metrics.subscriptions().inflightTimeInMillisHistogram(subscriptionName); + this.throughputInBytes = metrics.subscriptions().throughputInBytes(subscriptionName); + this.successes = metrics.subscriptions().successes(subscriptionName); + this.batchSuccesses = metrics.subscriptions().batchSuccesses(subscriptionName); + this.latency = metrics.subscriptions().latency(subscriptionName); + } - void recordAttempt(int messageCount) { - inflightCount.add(messageCount); - } + void recordAttempt(int messageCount) { + inflightCount.add(messageCount); + } - void recordAttemptAsFinished(int messageCount) { - inflightCount.add(-1 * messageCount); - } + void recordAttemptAsFinished(int messageCount) { + inflightCount.add(-1 * messageCount); + } - void markFailure(MessageBatch batch, MessageSendingResult result) { - failures.increment(); - if (result.hasHttpAnswer()) { - markHttpStatusCode(result.getStatusCode()); - } else if (result.isTimeout()) { - timeouts.increment(); - } else { - otherErrors.increment(); - } - throughputInBytes.increment(batch.getSize()); + void markFailure(MessageBatch batch, MessageSendingResult result) { + failures.increment(); + if (result.hasHttpAnswer()) { + markHttpStatusCode(result.getStatusCode()); + } else if (result.isTimeout()) { + timeouts.increment(); + } else { + otherErrors.increment(); } + throughputInBytes.increment(batch.getSize()); + } - void markSuccess(MessageBatch batch, MessageSendingResult result) { - successes.increment(batch.getMessageCount()); - batchSuccesses.increment(); - throughputInBytes.increment(batch.getSize()); - markHttpStatusCode(result.getStatusCode()); - inflightTime.record(batch.getLifetime()); - } + void markSuccess(MessageBatch batch, MessageSendingResult result) { + successes.increment(batch.getMessageCount()); + batchSuccesses.increment(); + throughputInBytes.increment(batch.getSize()); + markHttpStatusCode(result.getStatusCode()); + inflightTime.record(batch.getLifetime()); + } - private void markHttpStatusCode(int statusCode) { - httpStatusCodes.computeIfAbsent( - statusCode, - integer -> metrics.subscriptions().httpAnswerCounter(subscriptionName, statusCode) - ).increment(); - } + private void markHttpStatusCode(int statusCode) { + httpStatusCodes + .computeIfAbsent( + statusCode, + integer -> metrics.subscriptions().httpAnswerCounter(subscriptionName, statusCode)) + .increment(); + } - void shutdown() { - metrics.unregisterAllMetricsRelatedTo(subscriptionName); - } + void shutdown() { + metrics.unregisterAllMetricsRelatedTo(subscriptionName); + } - void initialize() { - metrics.subscriptions() - .registerInflightGauge(subscriptionName, this, metrics -> metrics.inflightCount.doubleValue()); - } + void initialize() { + metrics + .subscriptions() + .registerInflightGauge( + subscriptionName, this, metrics -> metrics.inflightCount.doubleValue()); + } - void markDiscarded() { - discarded.increment(); - } + void markDiscarded() { + discarded.increment(); + } - void markDiscarded(MessageBatch batch) { - discarded.increment(batch.getMessageCount()); - inflightTime.record(batch.getLifetime()); - } + void markDiscarded(MessageBatch batch) { + discarded.increment(batch.getMessageCount()); + inflightTime.record(batch.getLifetime()); + } - HermesTimer latencyTimer() { - return latency; - } + HermesTimer latencyTimer() { + return latency; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/Consumer.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/Consumer.java index 996157ced2..5e88be3da4 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/Consumer.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/Consumer.java @@ -1,32 +1,32 @@ package pl.allegro.tech.hermes.consumers.consumer; +import java.util.Set; import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.common.kafka.offset.PartitionOffset; import pl.allegro.tech.hermes.consumers.consumer.offset.SubscriptionPartitionOffset; -import java.util.Set; - public interface Consumer { - /** - * Consume **must** make sure that interrupted status is restored as it is needed for stopping unhealthy consumers. - * Swallowing the interrupt by consume or any of its dependencies will result in consumer being marked - * as unhealthy and will prevent commits despite messages being sent to subscribers. - */ - void consume(Runnable signalsInterrupt); + /** + * Consume **must** make sure that interrupted status is restored as it is needed for stopping + * unhealthy consumers. Swallowing the interrupt by consume or any of its dependencies will result + * in consumer being marked as unhealthy and will prevent commits despite messages being sent to + * subscribers. + */ + void consume(Runnable signalsInterrupt); - void initialize(); + void initialize(); - void tearDown(); + void tearDown(); - void updateSubscription(Subscription subscription); + void updateSubscription(Subscription subscription); - void updateTopic(Topic topic); + void updateTopic(Topic topic); - void commit(Set offsets); + void commit(Set offsets); - boolean moveOffset(PartitionOffset subscriptionPartitionOffset); + boolean moveOffset(PartitionOffset subscriptionPartitionOffset); - Subscription getSubscription(); + Subscription getSubscription(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/ConsumerAuthorizationHandler.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/ConsumerAuthorizationHandler.java index e291fa64df..170049a52b 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/ConsumerAuthorizationHandler.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/ConsumerAuthorizationHandler.java @@ -6,9 +6,9 @@ public interface ConsumerAuthorizationHandler extends SuccessHandler, ErrorHandler { - void createSubscriptionHandler(SubscriptionName subscriptionName); + void createSubscriptionHandler(SubscriptionName subscriptionName); - void removeSubscriptionHandler(SubscriptionName subscriptionName); + void removeSubscriptionHandler(SubscriptionName subscriptionName); - void updateSubscription(SubscriptionName subscriptionName); + void updateSubscription(SubscriptionName subscriptionName); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/ConsumerMessageSender.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/ConsumerMessageSender.java index 14fc29a8bf..3d644776b1 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/ConsumerMessageSender.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/ConsumerMessageSender.java @@ -1,6 +1,21 @@ package pl.allegro.tech.hermes.consumers.consumer; +import static java.lang.String.format; +import static org.apache.commons.lang3.math.NumberUtils.INTEGER_ZERO; +import static pl.allegro.tech.hermes.consumers.consumer.offset.SubscriptionPartitionOffset.subscriptionPartitionOffset; + import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.net.URI; +import java.time.Clock; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.LongAdder; import org.eclipse.jetty.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,301 +40,329 @@ import pl.allegro.tech.hermes.metrics.HermesTimer; import pl.allegro.tech.hermes.metrics.HermesTimerContext; -import java.net.URI; -import java.time.Clock; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.LongAdder; - -import static java.lang.String.format; -import static org.apache.commons.lang3.math.NumberUtils.INTEGER_ZERO; -import static pl.allegro.tech.hermes.consumers.consumer.offset.SubscriptionPartitionOffset.subscriptionPartitionOffset; - public class ConsumerMessageSender { - private static final Logger logger = LoggerFactory.getLogger(ConsumerMessageSender.class); - private final ExecutorService deliveryReportingExecutor; - private final List successHandlers; - private final List errorHandlers; - private final MessageSenderFactory messageSenderFactory; - private final Clock clock; - private final PendingOffsets pendingOffsets; - private final SubscriptionLoadRecorder loadRecorder; - private final HermesTimer consumerLatencyTimer; - private final HermesCounter retries; - private final SerialConsumerRateLimiter rateLimiter; - private final HermesTimer rateLimiterAcquireTimer; - private final FutureAsyncTimeout async; - private final int asyncTimeoutMs; - private final LongAdder inflightCount = new LongAdder(); - - private MessageSender messageSender; - private Subscription subscription; - - private ScheduledExecutorService retrySingleThreadExecutor; - private volatile boolean running = true; - - public ConsumerMessageSender(Subscription subscription, - MessageSenderFactory messageSenderFactory, - List successHandlers, - List errorHandlers, - SerialConsumerRateLimiter rateLimiter, - ExecutorService deliveryReportingExecutor, - PendingOffsets pendingOffsets, - MetricsFacade metrics, - int asyncTimeoutMs, - FutureAsyncTimeout futureAsyncTimeout, - Clock clock, - SubscriptionLoadRecorder loadRecorder) { - this.deliveryReportingExecutor = deliveryReportingExecutor; - this.successHandlers = successHandlers; - this.errorHandlers = errorHandlers; - this.messageSenderFactory = messageSenderFactory; - this.clock = clock; - this.loadRecorder = loadRecorder; - this.async = futureAsyncTimeout; - this.rateLimiter = rateLimiter; - this.asyncTimeoutMs = asyncTimeoutMs; - this.messageSender = messageSender(subscription); - this.subscription = subscription; - this.pendingOffsets = pendingOffsets; - this.consumerLatencyTimer = metrics.subscriptions().latency(subscription.getQualifiedName()); - metrics.subscriptions().registerInflightGauge(subscription.getQualifiedName(), this, sender -> sender.inflightCount.doubleValue()); - this.retries = metrics.subscriptions().retries(subscription.getQualifiedName()); - this.rateLimiterAcquireTimer = metrics.subscriptions().rateLimiterAcquire(subscription.getQualifiedName()); - } - - public void initialize() { - running = true; - ThreadFactory threadFactory = - new ThreadFactoryBuilder().setNameFormat(subscription.getQualifiedName() + "-retry-executor-%d").build(); - this.retrySingleThreadExecutor = Executors.newScheduledThreadPool(1, threadFactory); - } - - public void shutdown() { - running = false; - messageSender.stop(); - retrySingleThreadExecutor.shutdownNow(); - try { - retrySingleThreadExecutor.awaitTermination(1, TimeUnit.MINUTES); - } catch (InterruptedException e) { - logger.warn("Failed to stop retry executor within one minute with following exception", e); - } - } - - public void sendAsync(Message message, ConsumerProfiler profiler) { - inflightCount.increment(); - sendAsync(message, calculateMessageDelay(message.getPublishingTimestamp()), profiler); - } - - private void sendAsync(Message message, int delayMillis, ConsumerProfiler profiler) { - profiler.measure(Measurement.SCHEDULE_MESSAGE_SENDING); - retrySingleThreadExecutor.schedule(() -> sendMessage(message, profiler), delayMillis, TimeUnit.MILLISECONDS); - } - - private int calculateMessageDelay(long publishingMessageTimestamp) { - Integer delay = subscription.getSerialSubscriptionPolicy().getSendingDelay(); - if (INTEGER_ZERO.equals(delay)) { - return delay; - } - - long messageAgeAtThisPoint = clock.millis() - publishingMessageTimestamp; - - delay = delay - (int) messageAgeAtThisPoint; - - return Math.max(delay, INTEGER_ZERO); + private static final Logger logger = LoggerFactory.getLogger(ConsumerMessageSender.class); + private final ExecutorService deliveryReportingExecutor; + private final List successHandlers; + private final List errorHandlers; + private final MessageSenderFactory messageSenderFactory; + private final Clock clock; + private final PendingOffsets pendingOffsets; + private final SubscriptionLoadRecorder loadRecorder; + private final HermesTimer consumerLatencyTimer; + private final HermesCounter retries; + private final SerialConsumerRateLimiter rateLimiter; + private final HermesTimer rateLimiterAcquireTimer; + private final FutureAsyncTimeout async; + private final int asyncTimeoutMs; + private final LongAdder inflightCount = new LongAdder(); + + private MessageSender messageSender; + private Subscription subscription; + + private ScheduledExecutorService retrySingleThreadExecutor; + private volatile boolean running = true; + + public ConsumerMessageSender( + Subscription subscription, + MessageSenderFactory messageSenderFactory, + List successHandlers, + List errorHandlers, + SerialConsumerRateLimiter rateLimiter, + ExecutorService deliveryReportingExecutor, + PendingOffsets pendingOffsets, + MetricsFacade metrics, + int asyncTimeoutMs, + FutureAsyncTimeout futureAsyncTimeout, + Clock clock, + SubscriptionLoadRecorder loadRecorder) { + this.deliveryReportingExecutor = deliveryReportingExecutor; + this.successHandlers = successHandlers; + this.errorHandlers = errorHandlers; + this.messageSenderFactory = messageSenderFactory; + this.clock = clock; + this.loadRecorder = loadRecorder; + this.async = futureAsyncTimeout; + this.rateLimiter = rateLimiter; + this.asyncTimeoutMs = asyncTimeoutMs; + this.messageSender = messageSender(subscription); + this.subscription = subscription; + this.pendingOffsets = pendingOffsets; + this.consumerLatencyTimer = metrics.subscriptions().latency(subscription.getQualifiedName()); + metrics + .subscriptions() + .registerInflightGauge( + subscription.getQualifiedName(), this, sender -> sender.inflightCount.doubleValue()); + this.retries = metrics.subscriptions().retries(subscription.getQualifiedName()); + this.rateLimiterAcquireTimer = + metrics.subscriptions().rateLimiterAcquire(subscription.getQualifiedName()); + } + + public void initialize() { + running = true; + ThreadFactory threadFactory = + new ThreadFactoryBuilder() + .setNameFormat(subscription.getQualifiedName() + "-retry-executor-%d") + .build(); + this.retrySingleThreadExecutor = Executors.newScheduledThreadPool(1, threadFactory); + } + + public void shutdown() { + running = false; + messageSender.stop(); + retrySingleThreadExecutor.shutdownNow(); + try { + retrySingleThreadExecutor.awaitTermination(1, TimeUnit.MINUTES); + } catch (InterruptedException e) { + logger.warn("Failed to stop retry executor within one minute with following exception", e); } - - - /** - * Method is calling MessageSender and is registering listeners to handle response. - * Main responsibility of this method is that no message will be fully processed or rejected without release on semaphore. - */ - private void sendMessage(final Message message, ConsumerProfiler profiler) { - loadRecorder.recordSingleOperation(); - profiler.measure(Measurement.ACQUIRE_RATE_LIMITER); - acquireRateLimiterWithTimer(); - HermesTimerContext timer = consumerLatencyTimer.time(); - profiler.measure(Measurement.MESSAGE_SENDER_SEND); - CompletableFuture response = messageSender.send(message); - - response.thenAcceptAsync(new ResponseHandlingListener(message, timer, profiler), deliveryReportingExecutor) - .exceptionally(e -> { - logger.error( - "An error occurred while handling message sending response of subscription {} [partition={}, offset={}, id={}]", - subscription.getQualifiedName(), message.getPartition(), message.getOffset(), message.getId(), e); - return null; - }); + } + + public void sendAsync(Message message, ConsumerProfiler profiler) { + inflightCount.increment(); + sendAsync(message, calculateMessageDelay(message.getPublishingTimestamp()), profiler); + } + + private void sendAsync(Message message, int delayMillis, ConsumerProfiler profiler) { + profiler.measure(Measurement.SCHEDULE_MESSAGE_SENDING); + retrySingleThreadExecutor.schedule( + () -> sendMessage(message, profiler), delayMillis, TimeUnit.MILLISECONDS); + } + + private int calculateMessageDelay(long publishingMessageTimestamp) { + Integer delay = subscription.getSerialSubscriptionPolicy().getSendingDelay(); + if (INTEGER_ZERO.equals(delay)) { + return delay; } - private void acquireRateLimiterWithTimer() { - HermesTimerContext acquireTimer = rateLimiterAcquireTimer.time(); - rateLimiter.acquire(); - acquireTimer.close(); + long messageAgeAtThisPoint = clock.millis() - publishingMessageTimestamp; + + delay = delay - (int) messageAgeAtThisPoint; + + return Math.max(delay, INTEGER_ZERO); + } + + /** + * Method is calling MessageSender and is registering listeners to handle response. Main + * responsibility of this method is that no message will be fully processed or rejected without + * release on semaphore. + */ + private void sendMessage(final Message message, ConsumerProfiler profiler) { + loadRecorder.recordSingleOperation(); + profiler.measure(Measurement.ACQUIRE_RATE_LIMITER); + acquireRateLimiterWithTimer(); + HermesTimerContext timer = consumerLatencyTimer.time(); + profiler.measure(Measurement.MESSAGE_SENDER_SEND); + CompletableFuture response = messageSender.send(message); + + response + .thenAcceptAsync( + new ResponseHandlingListener(message, timer, profiler), deliveryReportingExecutor) + .exceptionally( + e -> { + logger.error( + "An error occurred while handling message sending response of subscription {} [partition={}, offset={}, id={}]", + subscription.getQualifiedName(), + message.getPartition(), + message.getOffset(), + message.getId(), + e); + return null; + }); + } + + private void acquireRateLimiterWithTimer() { + HermesTimerContext acquireTimer = rateLimiterAcquireTimer.time(); + rateLimiter.acquire(); + acquireTimer.close(); + } + + private MessageSender messageSender(Subscription subscription) { + Integer requestTimeoutMs = subscription.getSerialSubscriptionPolicy().getRequestTimeout(); + ResilientMessageSender resilientMessageSender = + new ResilientMessageSender( + this.rateLimiter, subscription, this.async, requestTimeoutMs, this.asyncTimeoutMs); + + return this.messageSenderFactory.create(subscription, resilientMessageSender); + } + + public void updateSubscription(Subscription newSubscription) { + boolean endpointUpdated = + !this.subscription.getEndpoint().equals(newSubscription.getEndpoint()); + boolean subscriptionPolicyUpdated = + !Objects.equals( + this.subscription.getSerialSubscriptionPolicy(), + newSubscription.getSerialSubscriptionPolicy()); + boolean endpointAddressResolverMetadataChanged = + !Objects.equals( + this.subscription.getEndpointAddressResolverMetadata(), + newSubscription.getEndpointAddressResolverMetadata()); + boolean oAuthPolicyChanged = + !Objects.equals(this.subscription.getOAuthPolicy(), newSubscription.getOAuthPolicy()); + + this.subscription = newSubscription; + + boolean httpClientChanged = + this.subscription.isHttp2Enabled() != newSubscription.isHttp2Enabled(); + + if (endpointUpdated + || subscriptionPolicyUpdated + || endpointAddressResolverMetadataChanged + || oAuthPolicyChanged + || httpClientChanged) { + this.messageSender.stop(); + this.messageSender = messageSender(newSubscription); } - - private MessageSender messageSender(Subscription subscription) { - Integer requestTimeoutMs = subscription.getSerialSubscriptionPolicy().getRequestTimeout(); - ResilientMessageSender resilientMessageSender = new ResilientMessageSender( - this.rateLimiter, - subscription, - this.async, - requestTimeoutMs, - this.asyncTimeoutMs - ); - - return this.messageSenderFactory.create( - subscription, resilientMessageSender - ); + } + + private boolean willExceedTtl(Message message, long delay) { + long ttl = + TimeUnit.SECONDS.toMillis(subscription.getSerialSubscriptionPolicy().getMessageTtl()); + long remainingTtl = Math.max(ttl - delay, 0); + return message.isTtlExceeded(remainingTtl); + } + + private void handleFailedSending( + Message message, MessageSendingResult result, ConsumerProfiler profiler) { + errorHandlers.forEach(h -> h.handleFailed(message, subscription, result)); + retrySendingOrDiscard(message, result, profiler); + } + + private void retrySendingOrDiscard( + Message message, MessageSendingResult result, ConsumerProfiler profiler) { + List succeededUris = + result.getSucceededUris(ConsumerMessageSender.this::messageSentSucceeded); + message.incrementRetryCounter(succeededUris); + + long retryDelay = extractRetryDelay(message, result); + if (shouldAttemptResending(message, result, retryDelay)) { + retries.increment(); + profiler.flushMeasurements(ConsumerRun.RETRIED); + ConsumerProfiler resendProfiler = + subscription.isProfilingEnabled() + ? new DefaultConsumerProfiler( + subscription.getQualifiedName(), subscription.getProfilingThresholdMs()) + : new NoOpConsumerProfiler(); + resendProfiler.startMeasurements(Measurement.SCHEDULE_RESEND); + resendProfiler.saveRetryDelay(retryDelay); + retrySingleThreadExecutor.schedule( + () -> resend(message, result, resendProfiler), retryDelay, TimeUnit.MILLISECONDS); + } else { + handleMessageDiscarding(message, result, profiler); } - - public void updateSubscription(Subscription newSubscription) { - boolean endpointUpdated = !this.subscription.getEndpoint().equals(newSubscription.getEndpoint()); - boolean subscriptionPolicyUpdated = !Objects.equals( - this.subscription.getSerialSubscriptionPolicy(), - newSubscription.getSerialSubscriptionPolicy() - ); - boolean endpointAddressResolverMetadataChanged = !Objects.equals( - this.subscription.getEndpointAddressResolverMetadata(), - newSubscription.getEndpointAddressResolverMetadata() - ); - boolean oAuthPolicyChanged = !Objects.equals( - this.subscription.getOAuthPolicy(), newSubscription.getOAuthPolicy() - ); - - this.subscription = newSubscription; - - boolean httpClientChanged = this.subscription.isHttp2Enabled() != newSubscription.isHttp2Enabled(); - - if (endpointUpdated || subscriptionPolicyUpdated || endpointAddressResolverMetadataChanged - || oAuthPolicyChanged || httpClientChanged) { - this.messageSender.stop(); - this.messageSender = messageSender(newSubscription); - } + } + + private boolean shouldAttemptResending( + Message message, MessageSendingResult result, long retryDelay) { + return !willExceedTtl(message, retryDelay) && shouldResendMessage(result); + } + + private long extractRetryDelay(Message message, MessageSendingResult result) { + long defaultBackoff = + message.updateAndGetCurrentMessageBackoff(subscription.getSerialSubscriptionPolicy()); + long ttl = + TimeUnit.SECONDS.toMillis(subscription.getSerialSubscriptionPolicy().getMessageTtl()); + return result.getRetryAfterMillis().map(delay -> Math.min(delay, ttl)).orElse(defaultBackoff); + } + + private void resend(Message message, MessageSendingResult result, ConsumerProfiler profiler) { + if (result.isLoggable()) { + result.getLogInfo().forEach(logInfo -> logResultInfo(message, logInfo)); } - - private boolean willExceedTtl(Message message, long delay) { - long ttl = TimeUnit.SECONDS.toMillis(subscription.getSerialSubscriptionPolicy().getMessageTtl()); - long remainingTtl = Math.max(ttl - delay, 0); - return message.isTtlExceeded(remainingTtl); + sendMessage(message, profiler); + } + + private void logResultInfo(Message message, MessageSendingResultLogInfo logInfo) { + logger.debug( + format( + "Retrying message send to endpoint %s; messageId %s; offset: %s; partition: %s; sub id: %s; rootCause: %s", + logInfo.getUrlString(), + message.getId(), + message.getOffset(), + message.getPartition(), + subscription.getQualifiedName(), + logInfo.getRootCause()), + logInfo.getFailure()); + } + + private void handleMessageDiscarding( + Message message, MessageSendingResult result, ConsumerProfiler profiler) { + pendingOffsets.markAsProcessed( + subscriptionPartitionOffset( + subscription.getQualifiedName(), + message.getPartitionOffset(), + message.getPartitionAssignmentTerm())); + inflightCount.decrement(); + errorHandlers.forEach(h -> h.handleDiscarded(message, subscription, result)); + profiler.flushMeasurements(ConsumerRun.DISCARDED); + } + + private void handleMessageSendingSuccess( + Message message, MessageSendingResult result, ConsumerProfiler profiler) { + pendingOffsets.markAsProcessed( + subscriptionPartitionOffset( + subscription.getQualifiedName(), + message.getPartitionOffset(), + message.getPartitionAssignmentTerm())); + inflightCount.decrement(); + successHandlers.forEach(h -> h.handleSuccess(message, subscription, result)); + profiler.flushMeasurements(ConsumerRun.DELIVERED); + } + + private boolean messageSentSucceeded(MessageSendingResult result) { + return result.succeeded() || (result.isClientError() && !shouldRetryOnClientError()); + } + + private boolean shouldResendMessage(MessageSendingResult result) { + return !result.succeeded() + && (!result.isClientError() + || shouldRetryOnClientError() + || isUnauthorizedForOAuthSecuredSubscription(result)); + } + + private boolean shouldRetryOnClientError() { + return subscription.getSerialSubscriptionPolicy().isRetryClientErrors(); + } + + private boolean isUnauthorizedForOAuthSecuredSubscription(MessageSendingResult result) { + return subscription.hasOAuthPolicy() && result.getStatusCode() == HttpStatus.UNAUTHORIZED_401; + } + + class ResponseHandlingListener implements java.util.function.Consumer { + + private final Message message; + private final HermesTimerContext timer; + private final ConsumerProfiler profiler; + + public ResponseHandlingListener( + Message message, HermesTimerContext timer, ConsumerProfiler profiler) { + this.message = message; + this.timer = timer; + this.profiler = profiler; } - private void handleFailedSending(Message message, MessageSendingResult result, ConsumerProfiler profiler) { - errorHandlers.forEach(h -> h.handleFailed(message, subscription, result)); - retrySendingOrDiscard(message, result, profiler); - } - - private void retrySendingOrDiscard(Message message, MessageSendingResult result, ConsumerProfiler profiler) { - List succeededUris = result.getSucceededUris(ConsumerMessageSender.this::messageSentSucceeded); - message.incrementRetryCounter(succeededUris); - - long retryDelay = extractRetryDelay(message, result); - if (shouldAttemptResending(message, result, retryDelay)) { - retries.increment(); - profiler.flushMeasurements(ConsumerRun.RETRIED); - ConsumerProfiler resendProfiler = subscription.isProfilingEnabled() - ? new DefaultConsumerProfiler(subscription.getQualifiedName(), subscription.getProfilingThresholdMs()) : new NoOpConsumerProfiler(); - resendProfiler.startMeasurements(Measurement.SCHEDULE_RESEND); - resendProfiler.saveRetryDelay(retryDelay); - retrySingleThreadExecutor.schedule(() -> resend(message, result, resendProfiler), retryDelay, TimeUnit.MILLISECONDS); + @Override + public void accept(MessageSendingResult result) { + timer.close(); + loadRecorder.recordSingleOperation(); + profiler.measure(Measurement.HANDLERS); + if (running) { + if (result.succeeded()) { + handleMessageSendingSuccess(message, result, profiler); } else { - handleMessageDiscarding(message, result, profiler); - } - } - - private boolean shouldAttemptResending(Message message, MessageSendingResult result, long retryDelay) { - return !willExceedTtl(message, retryDelay) && shouldResendMessage(result); - } - - private long extractRetryDelay(Message message, MessageSendingResult result) { - long defaultBackoff = message.updateAndGetCurrentMessageBackoff(subscription.getSerialSubscriptionPolicy()); - long ttl = TimeUnit.SECONDS.toMillis(subscription.getSerialSubscriptionPolicy().getMessageTtl()); - return result.getRetryAfterMillis().map(delay -> Math.min(delay, ttl)).orElse(defaultBackoff); - } - - private void resend(Message message, MessageSendingResult result, ConsumerProfiler profiler) { - if (result.isLoggable()) { - result.getLogInfo().forEach(logInfo -> logResultInfo(message, logInfo)); - } - sendMessage(message, profiler); - } - - private void logResultInfo(Message message, MessageSendingResultLogInfo logInfo) { - logger.debug( - format("Retrying message send to endpoint %s; messageId %s; offset: %s; partition: %s; sub id: %s; rootCause: %s", - logInfo.getUrlString(), message.getId(), message.getOffset(), message.getPartition(), - subscription.getQualifiedName(), logInfo.getRootCause()), - logInfo.getFailure()); - } - - private void handleMessageDiscarding(Message message, MessageSendingResult result, ConsumerProfiler profiler) { - pendingOffsets.markAsProcessed(subscriptionPartitionOffset(subscription.getQualifiedName(), - message.getPartitionOffset(), message.getPartitionAssignmentTerm())); - inflightCount.decrement(); - errorHandlers.forEach(h -> h.handleDiscarded(message, subscription, result)); - profiler.flushMeasurements(ConsumerRun.DISCARDED); - } - - private void handleMessageSendingSuccess(Message message, MessageSendingResult result, ConsumerProfiler profiler) { - pendingOffsets.markAsProcessed(subscriptionPartitionOffset(subscription.getQualifiedName(), - message.getPartitionOffset(), message.getPartitionAssignmentTerm())); - inflightCount.decrement(); - successHandlers.forEach(h -> h.handleSuccess(message, subscription, result)); - profiler.flushMeasurements(ConsumerRun.DELIVERED); - } - - private boolean messageSentSucceeded(MessageSendingResult result) { - return result.succeeded() || (result.isClientError() && !shouldRetryOnClientError()); - } - - private boolean shouldResendMessage(MessageSendingResult result) { - return !result.succeeded() && (!result.isClientError() || shouldRetryOnClientError() - || isUnauthorizedForOAuthSecuredSubscription(result)); - } - - private boolean shouldRetryOnClientError() { - return subscription.getSerialSubscriptionPolicy().isRetryClientErrors(); - } - - private boolean isUnauthorizedForOAuthSecuredSubscription(MessageSendingResult result) { - return subscription.hasOAuthPolicy() && result.getStatusCode() == HttpStatus.UNAUTHORIZED_401; - } - - class ResponseHandlingListener implements java.util.function.Consumer { - - private final Message message; - private final HermesTimerContext timer; - private final ConsumerProfiler profiler; - - public ResponseHandlingListener(Message message, HermesTimerContext timer, ConsumerProfiler profiler) { - this.message = message; - this.timer = timer; - this.profiler = profiler; - } - - @Override - public void accept(MessageSendingResult result) { - timer.close(); - loadRecorder.recordSingleOperation(); - profiler.measure(Measurement.HANDLERS); - if (running) { - if (result.succeeded()) { - handleMessageSendingSuccess(message, result, profiler); - } else { - handleFailedSending(message, result, profiler); - } - } else { - logger.warn("Process of subscription {} is not running. " - + "Ignoring sending message result [successful={}, partition={}, offset={}, id={}]", - subscription.getQualifiedName(), result.succeeded(), message.getPartition(), - message.getOffset(), message.getId()); - } + handleFailedSending(message, result, profiler); } + } else { + logger.warn( + "Process of subscription {} is not running. " + + "Ignoring sending message result [successful={}, partition={}, offset={}, id={}]", + subscription.getQualifiedName(), + result.succeeded(), + message.getPartition(), + message.getOffset(), + message.getId()); + } } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/ConsumerMessageSenderFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/ConsumerMessageSenderFactory.java index 066670df9b..7e73192f47 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/ConsumerMessageSenderFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/ConsumerMessageSenderFactory.java @@ -1,5 +1,9 @@ package pl.allegro.tech.hermes.consumers.consumer; +import java.time.Clock; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutorService; import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.common.message.undelivered.UndeliveredMessageLog; import pl.allegro.tech.hermes.common.metric.MetricsFacade; @@ -15,80 +19,81 @@ import pl.allegro.tech.hermes.consumers.consumer.sender.timeout.FutureAsyncTimeout; import pl.allegro.tech.hermes.tracker.consumers.Trackers; -import java.time.Clock; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.ExecutorService; - public class ConsumerMessageSenderFactory { - private final String kafkaClusterName; - private final MessageSenderFactory messageSenderFactory; - private final Trackers trackers; - private final FutureAsyncTimeout futureAsyncTimeout; - private final UndeliveredMessageLog undeliveredMessageLog; - private final Clock clock; - private final ConsumerAuthorizationHandler consumerAuthorizationHandler; - private final ExecutorService rateLimiterReportingExecutor; - private final int senderAsyncTimeoutMs; - - public ConsumerMessageSenderFactory(String kafkaClusterName, MessageSenderFactory messageSenderFactory, - Trackers trackers, FutureAsyncTimeout futureAsyncTimeout, - UndeliveredMessageLog undeliveredMessageLog, Clock clock, - InstrumentedExecutorServiceFactory instrumentedExecutorServiceFactory, - ConsumerAuthorizationHandler consumerAuthorizationHandler, - int senderAsyncTimeoutMs, - int rateLimiterReportingThreadPoolSize, - boolean rateLimiterReportingThreadMonitoringEnabled) { + private final String kafkaClusterName; + private final MessageSenderFactory messageSenderFactory; + private final Trackers trackers; + private final FutureAsyncTimeout futureAsyncTimeout; + private final UndeliveredMessageLog undeliveredMessageLog; + private final Clock clock; + private final ConsumerAuthorizationHandler consumerAuthorizationHandler; + private final ExecutorService rateLimiterReportingExecutor; + private final int senderAsyncTimeoutMs; - this.kafkaClusterName = kafkaClusterName; - this.messageSenderFactory = messageSenderFactory; - this.trackers = trackers; - this.futureAsyncTimeout = futureAsyncTimeout; - this.undeliveredMessageLog = undeliveredMessageLog; - this.clock = clock; - this.consumerAuthorizationHandler = consumerAuthorizationHandler; - this.rateLimiterReportingExecutor = instrumentedExecutorServiceFactory.getExecutorService( - "rate-limiter-reporter", rateLimiterReportingThreadPoolSize, - rateLimiterReportingThreadMonitoringEnabled); - this.senderAsyncTimeoutMs = senderAsyncTimeoutMs; - } + public ConsumerMessageSenderFactory( + String kafkaClusterName, + MessageSenderFactory messageSenderFactory, + Trackers trackers, + FutureAsyncTimeout futureAsyncTimeout, + UndeliveredMessageLog undeliveredMessageLog, + Clock clock, + InstrumentedExecutorServiceFactory instrumentedExecutorServiceFactory, + ConsumerAuthorizationHandler consumerAuthorizationHandler, + int senderAsyncTimeoutMs, + int rateLimiterReportingThreadPoolSize, + boolean rateLimiterReportingThreadMonitoringEnabled) { - public ConsumerMessageSender create(Subscription subscription, - SerialConsumerRateLimiter consumerRateLimiter, - PendingOffsets pendingOffsets, - SubscriptionLoadRecorder subscriptionLoadRecorder, - MetricsFacade metrics) { + this.kafkaClusterName = kafkaClusterName; + this.messageSenderFactory = messageSenderFactory; + this.trackers = trackers; + this.futureAsyncTimeout = futureAsyncTimeout; + this.undeliveredMessageLog = undeliveredMessageLog; + this.clock = clock; + this.consumerAuthorizationHandler = consumerAuthorizationHandler; + this.rateLimiterReportingExecutor = + instrumentedExecutorServiceFactory.getExecutorService( + "rate-limiter-reporter", + rateLimiterReportingThreadPoolSize, + rateLimiterReportingThreadMonitoringEnabled); + this.senderAsyncTimeoutMs = senderAsyncTimeoutMs; + } - List successHandlers = Arrays.asList( - consumerAuthorizationHandler, - new DefaultSuccessHandler(metrics, trackers, subscription.getQualifiedName())); + public ConsumerMessageSender create( + Subscription subscription, + SerialConsumerRateLimiter consumerRateLimiter, + PendingOffsets pendingOffsets, + SubscriptionLoadRecorder subscriptionLoadRecorder, + MetricsFacade metrics) { - List errorHandlers = Arrays.asList( - consumerAuthorizationHandler, - new DefaultErrorHandler( - metrics, - undeliveredMessageLog, - clock, - trackers, - kafkaClusterName, - subscription.getQualifiedName() - ) - ); + List successHandlers = + Arrays.asList( + consumerAuthorizationHandler, + new DefaultSuccessHandler(metrics, trackers, subscription.getQualifiedName())); - return new ConsumerMessageSender(subscription, - messageSenderFactory, - successHandlers, - errorHandlers, - consumerRateLimiter, - rateLimiterReportingExecutor, - pendingOffsets, + List errorHandlers = + Arrays.asList( + consumerAuthorizationHandler, + new DefaultErrorHandler( metrics, - senderAsyncTimeoutMs, - futureAsyncTimeout, + undeliveredMessageLog, clock, - subscriptionLoadRecorder - ); - } + trackers, + kafkaClusterName, + subscription.getQualifiedName())); + return new ConsumerMessageSender( + subscription, + messageSenderFactory, + successHandlers, + errorHandlers, + consumerRateLimiter, + rateLimiterReportingExecutor, + pendingOffsets, + metrics, + senderAsyncTimeoutMs, + futureAsyncTimeout, + clock, + subscriptionLoadRecorder); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/Message.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/Message.java index 46fd852618..ebb2002ae0 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/Message.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/Message.java @@ -3,6 +3,14 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; +import java.net.URI; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; import org.apache.avro.Schema; import org.apache.commons.lang3.ArrayUtils; import pl.allegro.tech.hermes.api.ContentType; @@ -13,282 +21,287 @@ import pl.allegro.tech.hermes.domain.filtering.FilterableMessage; import pl.allegro.tech.hermes.schema.CompiledSchema; -import java.net.URI; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; - /** - * Implementation note: this class is partially mutable and may be accessed from multiple - * threads involved in message lifecycle, it must be thread safe. + * Implementation note: this class is partially mutable and may be accessed from multiple threads + * involved in message lifecycle, it must be thread safe. */ public class Message implements FilterableMessage { - private final String id; - private final PartitionOffset partitionOffset; - - private final String topic; - private final String subscription; - private final boolean hasSubscriptionIdentityHeaders; - private final ContentType contentType; - private final Optional> schema; - - private final long publishingTimestamp; - private final long readingTimestamp; - private final byte[] data; - - private int retryCounter = 0; - private final long partitionAssignmentTerm; - private final Map externalMetadata; - - private final List
additionalHeaders; - - private final Set succeededUris = Sets.newHashSet(); - - private long currentMessageBackoff = -1; - - private boolean isFiltered = false; - - public Message(String id, - String topic, - byte[] content, - ContentType contentType, - Optional> schema, - long publishingTimestamp, - long readingTimestamp, - PartitionOffset partitionOffset, - long partitionAssignmentTerm, - Map externalMetadata, - List
additionalHeaders, - String subscription, - boolean hasSubscriptionIdentityHeaders) { - this.id = id; - this.data = content; - this.topic = topic; - this.contentType = contentType; - this.schema = schema; - this.publishingTimestamp = publishingTimestamp; - this.readingTimestamp = readingTimestamp; - this.partitionOffset = partitionOffset; - this.partitionAssignmentTerm = partitionAssignmentTerm; - this.externalMetadata = ImmutableMap.copyOf(externalMetadata); - this.additionalHeaders = ImmutableList.copyOf(additionalHeaders); - this.subscription = subscription; - this.hasSubscriptionIdentityHeaders = hasSubscriptionIdentityHeaders; - } - - public long getPublishingTimestamp() { - return publishingTimestamp; - } - - public long getReadingTimestamp() { - return readingTimestamp; - } - - public long getOffset() { - return partitionOffset.getOffset(); + private final String id; + private final PartitionOffset partitionOffset; + + private final String topic; + private final String subscription; + private final boolean hasSubscriptionIdentityHeaders; + private final ContentType contentType; + private final Optional> schema; + + private final long publishingTimestamp; + private final long readingTimestamp; + private final byte[] data; + + private int retryCounter = 0; + private final long partitionAssignmentTerm; + private final Map externalMetadata; + + private final List
additionalHeaders; + + private final Set succeededUris = Sets.newHashSet(); + + private long currentMessageBackoff = -1; + + private boolean isFiltered = false; + + public Message( + String id, + String topic, + byte[] content, + ContentType contentType, + Optional> schema, + long publishingTimestamp, + long readingTimestamp, + PartitionOffset partitionOffset, + long partitionAssignmentTerm, + Map externalMetadata, + List
additionalHeaders, + String subscription, + boolean hasSubscriptionIdentityHeaders) { + this.id = id; + this.data = content; + this.topic = topic; + this.contentType = contentType; + this.schema = schema; + this.publishingTimestamp = publishingTimestamp; + this.readingTimestamp = readingTimestamp; + this.partitionOffset = partitionOffset; + this.partitionAssignmentTerm = partitionAssignmentTerm; + this.externalMetadata = ImmutableMap.copyOf(externalMetadata); + this.additionalHeaders = ImmutableList.copyOf(additionalHeaders); + this.subscription = subscription; + this.hasSubscriptionIdentityHeaders = hasSubscriptionIdentityHeaders; + } + + public long getPublishingTimestamp() { + return publishingTimestamp; + } + + public long getReadingTimestamp() { + return readingTimestamp; + } + + public long getOffset() { + return partitionOffset.getOffset(); + } + + public long getPartitionAssignmentTerm() { + return partitionAssignmentTerm; + } + + @Override + public byte[] getData() { + return data; + } + + @Override + public ContentType getContentType() { + return contentType; + } + + public int getPartition() { + return partitionOffset.getPartition(); + } + + public String getTopic() { + return topic; + } + + public boolean isTtlExceeded(long ttlMillis) { + long currentTimestamp = System.currentTimeMillis(); + return currentTimestamp > readingTimestamp + ttlMillis; + } + + public synchronized void incrementRetryCounter(Collection succeededUris) { + this.retryCounter++; + this.succeededUris.addAll(succeededUris.stream().map(URI::toString).toList()); + } + + public synchronized int getRetryCounter() { + return retryCounter; + } + + @Override + public Optional> getSchema() { + return schema; + } + + public String getId() { + return id; + } + + public synchronized Set getSucceededUris() { + return succeededUris; + } + + @Override + public Map getExternalMetadata() { + return externalMetadata; + } + + public List
getAdditionalHeaders() { + return additionalHeaders; + } + + public synchronized long updateAndGetCurrentMessageBackoff( + SubscriptionPolicy subscriptionPolicy) { + if (currentMessageBackoff == -1) { + currentMessageBackoff = subscriptionPolicy.getMessageBackoff(); + } else { + currentMessageBackoff = + Math.min( + subscriptionPolicy.getBackoffMaxIntervalMillis(), + (long) (currentMessageBackoff * subscriptionPolicy.getBackoffMultiplier())); } - - public long getPartitionAssignmentTerm() { - return partitionAssignmentTerm; + return currentMessageBackoff; + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; } - - @Override - public byte[] getData() { - return data; + if (obj == null || getClass() != obj.getClass()) { + return false; } - - @Override - public ContentType getContentType() { - return contentType; + final Message other = (Message) obj; + return Objects.equals(this.id, other.id); + } + + public static Builder message() { + return new Builder(); + } + + public KafkaTopicName getKafkaTopic() { + return partitionOffset.getTopic(); + } + + public PartitionOffset getPartitionOffset() { + return partitionOffset; + } + + public synchronized boolean hasNotBeenSentTo(String uri) { + return !succeededUris.contains(uri); + } + + public long getSize() { + return ArrayUtils.getLength(data); + } + + public boolean hasSubscriptionIdentityHeaders() { + return hasSubscriptionIdentityHeaders; + } + + public String getSubscription() { + return subscription; + } + + public synchronized boolean isFiltered() { + return isFiltered; + } + + public synchronized void setFiltered(boolean filtered) { + isFiltered = filtered; + } + + public static class Builder { + private String id; + private PartitionOffset partitionOffset; + + private String topic; + private String subscription; + private boolean hasSubscriptionIdentityHeaders; + private ContentType contentType; + private Optional> schema; + + private long publishingTimestamp; + private long readingTimestamp; + private byte[] data; + + private long partitionAssignmentTerm = -1; + private Map externalMetadata = Collections.emptyMap(); + + private List
additionalHeaders = Collections.emptyList(); + + public Builder() {} + + public Builder fromMessage(Message message) { + this.id = message.getId(); + this.data = message.getData(); + this.contentType = message.getContentType(); + this.topic = message.getTopic(); + this.subscription = message.getSubscription(); + this.hasSubscriptionIdentityHeaders = message.hasSubscriptionIdentityHeaders(); + this.publishingTimestamp = message.getPublishingTimestamp(); + this.readingTimestamp = message.getReadingTimestamp(); + this.partitionOffset = message.partitionOffset; + this.partitionAssignmentTerm = message.partitionAssignmentTerm; + this.externalMetadata = message.getExternalMetadata(); + this.additionalHeaders = message.getAdditionalHeaders(); + this.schema = message.getSchema(); + + return this; } - public int getPartition() { - return partitionOffset.getPartition(); + public Builder withData(byte[] data) { + this.data = data; + return this; } - public String getTopic() { - return topic; + public Builder withSchema(CompiledSchema schema) { + this.schema = Optional.of(schema); + return this; } - public boolean isTtlExceeded(long ttlMillis) { - long currentTimestamp = System.currentTimeMillis(); - return currentTimestamp > readingTimestamp + ttlMillis; + public Builder withExternalMetadata(Map externalMetadata) { + this.externalMetadata = ImmutableMap.copyOf(externalMetadata); + return this; } - public synchronized void incrementRetryCounter(Collection succeededUris) { - this.retryCounter++; - this.succeededUris.addAll(succeededUris.stream().map(URI::toString).toList()); + public Builder withAdditionalHeaders(List
additionalHeaders) { + this.additionalHeaders = ImmutableList.copyOf(additionalHeaders); + return this; } - public synchronized int getRetryCounter() { - return retryCounter; - } - - @Override - public Optional> getSchema() { - return schema; - } - - public String getId() { - return id; - } - - public synchronized Set getSucceededUris() { - return succeededUris; - } - - @Override - public Map getExternalMetadata() { - return externalMetadata; - } - - public List
getAdditionalHeaders() { - return additionalHeaders; - } - - public synchronized long updateAndGetCurrentMessageBackoff(SubscriptionPolicy subscriptionPolicy) { - if (currentMessageBackoff == -1) { - currentMessageBackoff = subscriptionPolicy.getMessageBackoff(); - } else { - currentMessageBackoff = Math.min(subscriptionPolicy.getBackoffMaxIntervalMillis(), - (long) (currentMessageBackoff * subscriptionPolicy.getBackoffMultiplier())); - } - return currentMessageBackoff; - } - - @Override - public int hashCode() { - return Objects.hash(id); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - final Message other = (Message) obj; - return Objects.equals(this.id, other.id); - } - - public static Builder message() { - return new Builder(); - } - - public KafkaTopicName getKafkaTopic() { - return partitionOffset.getTopic(); - } - - public PartitionOffset getPartitionOffset() { - return partitionOffset; - } - - public synchronized boolean hasNotBeenSentTo(String uri) { - return !succeededUris.contains(uri); - } - - public long getSize() { - return ArrayUtils.getLength(data); - } - - public boolean hasSubscriptionIdentityHeaders() { - return hasSubscriptionIdentityHeaders; - } - - public String getSubscription() { - return subscription; - } + public Builder withContentType(ContentType contentType) { + this.contentType = contentType; - public synchronized boolean isFiltered() { - return isFiltered; + return this; } - public synchronized void setFiltered(boolean filtered) { - isFiltered = filtered; + public Builder withNoSchema() { + this.schema = Optional.empty(); + return this; } - public static class Builder { - private String id; - private PartitionOffset partitionOffset; - - private String topic; - private String subscription; - private boolean hasSubscriptionIdentityHeaders; - private ContentType contentType; - private Optional> schema; - - private long publishingTimestamp; - private long readingTimestamp; - private byte[] data; - - private long partitionAssignmentTerm = -1; - private Map externalMetadata = Collections.emptyMap(); - - private List
additionalHeaders = Collections.emptyList(); - - public Builder() { - } - - public Builder fromMessage(Message message) { - this.id = message.getId(); - this.data = message.getData(); - this.contentType = message.getContentType(); - this.topic = message.getTopic(); - this.subscription = message.getSubscription(); - this.hasSubscriptionIdentityHeaders = message.hasSubscriptionIdentityHeaders(); - this.publishingTimestamp = message.getPublishingTimestamp(); - this.readingTimestamp = message.getReadingTimestamp(); - this.partitionOffset = message.partitionOffset; - this.partitionAssignmentTerm = message.partitionAssignmentTerm; - this.externalMetadata = message.getExternalMetadata(); - this.additionalHeaders = message.getAdditionalHeaders(); - this.schema = message.getSchema(); - - return this; - } - - public Builder withData(byte[] data) { - this.data = data; - return this; - } - - public Builder withSchema(CompiledSchema schema) { - this.schema = Optional.of(schema); - return this; - } - - public Builder withExternalMetadata(Map externalMetadata) { - this.externalMetadata = ImmutableMap.copyOf(externalMetadata); - return this; - } - - public Builder withAdditionalHeaders(List
additionalHeaders) { - this.additionalHeaders = ImmutableList.copyOf(additionalHeaders); - return this; - } - - public Builder withContentType(ContentType contentType) { - this.contentType = contentType; - - return this; - } - - public Builder withNoSchema() { - this.schema = Optional.empty(); - return this; - } - - public Message build() { - return new Message( - id, topic, data, contentType, schema, publishingTimestamp, readingTimestamp, partitionOffset, partitionAssignmentTerm, externalMetadata, additionalHeaders, subscription, hasSubscriptionIdentityHeaders - ); - } + public Message build() { + return new Message( + id, + topic, + data, + contentType, + schema, + publishingTimestamp, + readingTimestamp, + partitionOffset, + partitionAssignmentTerm, + externalMetadata, + additionalHeaders, + subscription, + hasSubscriptionIdentityHeaders); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/ResilientMessageSender.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/ResilientMessageSender.java index 7f09b6417d..f53149bd09 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/ResilientMessageSender.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/ResilientMessageSender.java @@ -1,10 +1,5 @@ package pl.allegro.tech.hermes.consumers.consumer; -import pl.allegro.tech.hermes.api.Subscription; -import pl.allegro.tech.hermes.consumers.consumer.rate.ConsumerRateLimiter; -import pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult; -import pl.allegro.tech.hermes.consumers.consumer.sender.timeout.FutureAsyncTimeout; - import java.time.Duration; import java.util.Collections; import java.util.List; @@ -12,78 +7,78 @@ import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; +import pl.allegro.tech.hermes.api.Subscription; +import pl.allegro.tech.hermes.consumers.consumer.rate.ConsumerRateLimiter; +import pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult; +import pl.allegro.tech.hermes.consumers.consumer.sender.timeout.FutureAsyncTimeout; public class ResilientMessageSender { - private final ConsumerRateLimiter rateLimiter; - private final List> ignore; - private final FutureAsyncTimeout async; - private final int requestTimeoutMs; - private final int asyncTimeoutMs; + private final ConsumerRateLimiter rateLimiter; + private final List> ignore; + private final FutureAsyncTimeout async; + private final int requestTimeoutMs; + private final int asyncTimeoutMs; - public ResilientMessageSender(ConsumerRateLimiter rateLimiter, - Subscription subscription, - FutureAsyncTimeout async, - int requestTimeoutMs, - int asyncTimeoutMs) { - this.rateLimiter = rateLimiter; - this.ignore = ignorableErrors(subscription); - this.async = async; - this.requestTimeoutMs = requestTimeoutMs; - this.asyncTimeoutMs = asyncTimeoutMs; - } + public ResilientMessageSender( + ConsumerRateLimiter rateLimiter, + Subscription subscription, + FutureAsyncTimeout async, + int requestTimeoutMs, + int asyncTimeoutMs) { + this.rateLimiter = rateLimiter; + this.ignore = ignorableErrors(subscription); + this.async = async; + this.requestTimeoutMs = requestTimeoutMs; + this.asyncTimeoutMs = asyncTimeoutMs; + } - private static List> ignorableErrors(Subscription subscription) { - Predicate ignore = - result -> result.ignoreInRateCalculation( - subscription.getSerialSubscriptionPolicy().isRetryClientErrors(), - subscription.hasOAuthPolicy() - ); - return Collections.singletonList(ignore); - } + private static List> ignorableErrors(Subscription subscription) { + Predicate ignore = + result -> + result.ignoreInRateCalculation( + subscription.getSerialSubscriptionPolicy().isRetryClientErrors(), + subscription.hasOAuthPolicy()); + return Collections.singletonList(ignore); + } - public CompletableFuture send( - Consumer> resultFutureConsumer, - Function exceptionMapper - ) { - try { - CompletableFuture resultFuture = new CompletableFuture<>(); - resultFutureConsumer.accept(resultFuture); - CompletableFuture timeoutGuardedResultFuture = async.within( - resultFuture, - Duration.ofMillis(asyncTimeoutMs + requestTimeoutMs), - exceptionMapper); - return withCompletionHandle(timeoutGuardedResultFuture, exceptionMapper); - } catch (Exception e) { - rateLimiter.registerFailedSending(); - return CompletableFuture.completedFuture(exceptionMapper.apply(e)); - } + public CompletableFuture send( + Consumer> resultFutureConsumer, Function exceptionMapper) { + try { + CompletableFuture resultFuture = new CompletableFuture<>(); + resultFutureConsumer.accept(resultFuture); + CompletableFuture timeoutGuardedResultFuture = + async.within( + resultFuture, Duration.ofMillis(asyncTimeoutMs + requestTimeoutMs), exceptionMapper); + return withCompletionHandle(timeoutGuardedResultFuture, exceptionMapper); + } catch (Exception e) { + rateLimiter.registerFailedSending(); + return CompletableFuture.completedFuture(exceptionMapper.apply(e)); } + } - private CompletableFuture withCompletionHandle( - CompletableFuture future, - Function exceptionMapper - ) { - return future.handle((result, throwable) -> { - if (throwable != null) { - rateLimiter.registerFailedSending(); - return exceptionMapper.apply(throwable); + private CompletableFuture withCompletionHandle( + CompletableFuture future, Function exceptionMapper) { + return future.handle( + (result, throwable) -> { + if (throwable != null) { + rateLimiter.registerFailedSending(); + return exceptionMapper.apply(throwable); + } else { + if (result.succeeded()) { + rateLimiter.registerSuccessfulSending(); } else { - if (result.succeeded()) { - rateLimiter.registerSuccessfulSending(); - } else { - registerResultInRateLimiter(result); - } - return result; + registerResultInRateLimiter(result); } + return result; + } }); - } + } - private void registerResultInRateLimiter(MessageSendingResult result) { - if (ignore.stream().anyMatch(p -> p.test(result))) { - rateLimiter.registerSuccessfulSending(); - } else { - rateLimiter.registerFailedSending(); - } + private void registerResultInRateLimiter(MessageSendingResult result) { + if (ignore.stream().anyMatch(p -> p.test(result))) { + rateLimiter.registerSuccessfulSending(); + } else { + rateLimiter.registerFailedSending(); } - + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/SerialConsumer.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/SerialConsumer.java index bf4f4f2025..0897a48e34 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/SerialConsumer.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/SerialConsumer.java @@ -1,5 +1,12 @@ package pl.allegro.tech.hermes.consumers.consumer; +import static pl.allegro.tech.hermes.consumers.consumer.message.MessageConverter.toMessageMetadata; +import static pl.allegro.tech.hermes.consumers.consumer.offset.SubscriptionPartitionOffset.subscriptionPartitionOffset; + +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.Subscription; @@ -24,234 +31,243 @@ import pl.allegro.tech.hermes.consumers.consumer.receiver.UninitializedMessageReceiver; import pl.allegro.tech.hermes.tracker.consumers.Trackers; -import java.time.Duration; -import java.time.Instant; -import java.util.Optional; -import java.util.Set; - -import static pl.allegro.tech.hermes.consumers.consumer.message.MessageConverter.toMessageMetadata; -import static pl.allegro.tech.hermes.consumers.consumer.offset.SubscriptionPartitionOffset.subscriptionPartitionOffset; - public class SerialConsumer implements Consumer { - private static final Logger logger = LoggerFactory.getLogger(SerialConsumer.class); - - private final ReceiverFactory messageReceiverFactory; - private final MetricsFacade metrics; - private final SerialConsumerRateLimiter rateLimiter; - private final Trackers trackers; - private final MessageConverterResolver messageConverterResolver; - private final ConsumerMessageSender sender; - private final boolean useTopicMessageSizeEnabled; - private final PendingOffsets pendingOffsets; - private final ConsumerAuthorizationHandler consumerAuthorizationHandler; - private final SubscriptionLoadRecorder loadRecorder; - private final OffsetCommitter offsetCommitter; - private final Duration commitPeriod; - - private final int defaultInflight; - private final Duration signalProcessingInterval; - - private Topic topic; - private Subscription subscription; - - private MessageReceiver messageReceiver; - - private Instant lastCommitTime; - - public SerialConsumer(ReceiverFactory messageReceiverFactory, - MetricsFacade metrics, - Subscription subscription, - SerialConsumerRateLimiter rateLimiter, - ConsumerMessageSenderFactory consumerMessageSenderFactory, - Trackers trackers, - MessageConverterResolver messageConverterResolver, - Topic topic, - CommonConsumerParameters commonConsumerParameters, - ConsumerAuthorizationHandler consumerAuthorizationHandler, - SubscriptionLoadRecorder loadRecorder, - ConsumerPartitionAssignmentState consumerPartitionAssignmentState, - Duration commitPeriod, - int offsetQueueSize) { - this.defaultInflight = commonConsumerParameters.getSerialConsumer().getInflightSize(); - this.signalProcessingInterval = commonConsumerParameters.getSerialConsumer().getSignalProcessingInterval(); - this.messageReceiverFactory = messageReceiverFactory; - this.metrics = metrics; - this.subscription = subscription; - this.rateLimiter = rateLimiter; - this.useTopicMessageSizeEnabled = commonConsumerParameters.isUseTopicMessageSizeEnabled(); - this.pendingOffsets = new PendingOffsets(subscription.getQualifiedName(), metrics, calculateInflightSize(subscription), offsetQueueSize); - this.consumerAuthorizationHandler = consumerAuthorizationHandler; - this.trackers = trackers; - this.messageConverterResolver = messageConverterResolver; - this.loadRecorder = loadRecorder; - this.messageReceiver = new UninitializedMessageReceiver(); - this.topic = topic; - this.offsetCommitter = new OffsetCommitter(consumerPartitionAssignmentState, metrics); - this.sender = consumerMessageSenderFactory.create( - subscription, - rateLimiter, - pendingOffsets, - loadRecorder, - metrics - ); - this.commitPeriod = commitPeriod; - this.lastCommitTime = Instant.now(); - } - - private int calculateInflightSize(Subscription subscription) { - Optional subscriptionInflight = Optional.ofNullable(subscription.getSerialSubscriptionPolicy().getInflightSize()); - return subscriptionInflight.orElse(defaultInflight); - } - - @Override - public void consume(Runnable signalsInterrupt) { - try { - ConsumerProfiler profiler = subscription.isProfilingEnabled() ? new DefaultConsumerProfiler(subscription.getQualifiedName(), subscription.getProfilingThresholdMs()) : new NoOpConsumerProfiler(); - profiler.startMeasurements(Measurement.SIGNALS_AND_SEMAPHORE_ACQUIRE); - do { - loadRecorder.recordSingleOperation(); - profiler.startPartialMeasurement(Measurement.SIGNALS_INTERRUPT_RUN); - signalsInterrupt.run(); - commitIfReady(); - profiler.stopPartialMeasurement(); - } while (!pendingOffsets.tryAcquireSlot(signalProcessingInterval)); - - profiler.measure(Measurement.MESSAGE_RECEIVER_NEXT); - Optional maybeMessage = messageReceiver.next(); - - profiler.measure(Measurement.MESSAGE_CONVERSION); - if (maybeMessage.isPresent()) { - Message message = maybeMessage.get(); - - if (message.isFiltered()) { - profiler.flushMeasurements(ConsumerRun.FILTERED); - } else { - if (logger.isDebugEnabled()) { - logger.debug( - "Read message {} partition {} offset {}", - message.getContentType(), message.getPartition(), message.getOffset() - ); - } - - Message convertedMessage = messageConverterResolver.converterFor(message, subscription).convert(message, topic); - sendMessage(convertedMessage, profiler); - } - } else { - pendingOffsets.releaseSlot(); - profiler.flushMeasurements(ConsumerRun.EMPTY); - } - } catch (InterruptedException e) { - logger.info("Restoring interrupted status {}", subscription.getQualifiedName(), e); - Thread.currentThread().interrupt(); - } catch (Exception e) { - logger.error("Consumer loop failed for {}", subscription.getQualifiedName(), e); - } - } - - private void commitIfReady() { - if (isReadyToCommit()) { - Set offsetsToCommit = offsetCommitter.calculateOffsetsToBeCommitted(pendingOffsets.getOffsetsSnapshotAndReleaseProcessedSlots()); - if (!offsetsToCommit.isEmpty()) { - commit(offsetsToCommit); - } - lastCommitTime = Instant.now(); - } - } - - private boolean isReadyToCommit() { - return Duration.between(lastCommitTime, Instant.now()).toMillis() > commitPeriod.toMillis(); - } - - private void sendMessage(Message message, ConsumerProfiler profiler) throws InterruptedException { - profiler.measure(Measurement.OFFER_INFLIGHT_OFFSET); - pendingOffsets.markAsInflight( - subscriptionPartitionOffset(subscription.getQualifiedName(), - message.getPartitionOffset(), - message.getPartitionAssignmentTerm() - )); - - profiler.measure(Measurement.TRACKERS_LOG_INFLIGHT); - trackers.get(subscription).logInflight(toMessageMetadata(message, subscription)); - - sender.sendAsync(message, profiler); - } - - @Override - public void initialize() { - logger.info("Consumer: preparing message receiver for subscription {}", subscription.getQualifiedName()); - initializeMessageReceiver(); - sender.initialize(); - rateLimiter.initialize(); - loadRecorder.initialize(); - consumerAuthorizationHandler.createSubscriptionHandler(subscription.getQualifiedName()); - } - - private void initializeMessageReceiver() { - this.messageReceiver = messageReceiverFactory.createMessageReceiver( - topic, - subscription, - rateLimiter, - loadRecorder, - metrics, - pendingOffsets::markAsProcessed - ); - } - - /** - * Try to keep shutdown order the same as initialization so nothing will left to clean up when error occurs during initialization. - */ - @Override - public void tearDown() { - messageReceiver.stop(); - sender.shutdown(); - rateLimiter.shutdown(); - loadRecorder.shutdown(); - consumerAuthorizationHandler.removeSubscriptionHandler(subscription.getQualifiedName()); - metrics.unregisterAllMetricsRelatedTo(subscription.getQualifiedName()); - } - - @Override - public void updateSubscription(Subscription newSubscription) { - logger.info("Updating consumer for subscription {}", subscription.getQualifiedName()); - pendingOffsets.setInflightSize(calculateInflightSize(newSubscription)); - rateLimiter.updateSubscription(newSubscription); - sender.updateSubscription(newSubscription); - messageReceiver.update(newSubscription); - consumerAuthorizationHandler.updateSubscription(newSubscription.getQualifiedName()); - this.subscription = newSubscription; - } - - @Override - public void updateTopic(Topic newTopic) { - if (this.topic.getContentType() != newTopic.getContentType() - || messageSizeChanged(newTopic) - || this.topic.isSchemaIdAwareSerializationEnabled() != newTopic.isSchemaIdAwareSerializationEnabled()) { - logger.info("Reinitializing message receiver, contentType, messageSize or schemaIdAwareSerialization changed."); - this.topic = newTopic; - - messageReceiver.stop(); - initializeMessageReceiver(); + private static final Logger logger = LoggerFactory.getLogger(SerialConsumer.class); + + private final ReceiverFactory messageReceiverFactory; + private final MetricsFacade metrics; + private final SerialConsumerRateLimiter rateLimiter; + private final Trackers trackers; + private final MessageConverterResolver messageConverterResolver; + private final ConsumerMessageSender sender; + private final boolean useTopicMessageSizeEnabled; + private final PendingOffsets pendingOffsets; + private final ConsumerAuthorizationHandler consumerAuthorizationHandler; + private final SubscriptionLoadRecorder loadRecorder; + private final OffsetCommitter offsetCommitter; + private final Duration commitPeriod; + + private final int defaultInflight; + private final Duration signalProcessingInterval; + + private Topic topic; + private Subscription subscription; + + private MessageReceiver messageReceiver; + + private Instant lastCommitTime; + + public SerialConsumer( + ReceiverFactory messageReceiverFactory, + MetricsFacade metrics, + Subscription subscription, + SerialConsumerRateLimiter rateLimiter, + ConsumerMessageSenderFactory consumerMessageSenderFactory, + Trackers trackers, + MessageConverterResolver messageConverterResolver, + Topic topic, + CommonConsumerParameters commonConsumerParameters, + ConsumerAuthorizationHandler consumerAuthorizationHandler, + SubscriptionLoadRecorder loadRecorder, + ConsumerPartitionAssignmentState consumerPartitionAssignmentState, + Duration commitPeriod, + int offsetQueueSize) { + this.defaultInflight = commonConsumerParameters.getSerialConsumer().getInflightSize(); + this.signalProcessingInterval = + commonConsumerParameters.getSerialConsumer().getSignalProcessingInterval(); + this.messageReceiverFactory = messageReceiverFactory; + this.metrics = metrics; + this.subscription = subscription; + this.rateLimiter = rateLimiter; + this.useTopicMessageSizeEnabled = commonConsumerParameters.isUseTopicMessageSizeEnabled(); + this.pendingOffsets = + new PendingOffsets( + subscription.getQualifiedName(), + metrics, + calculateInflightSize(subscription), + offsetQueueSize); + this.consumerAuthorizationHandler = consumerAuthorizationHandler; + this.trackers = trackers; + this.messageConverterResolver = messageConverterResolver; + this.loadRecorder = loadRecorder; + this.messageReceiver = new UninitializedMessageReceiver(); + this.topic = topic; + this.offsetCommitter = new OffsetCommitter(consumerPartitionAssignmentState, metrics); + this.sender = + consumerMessageSenderFactory.create( + subscription, rateLimiter, pendingOffsets, loadRecorder, metrics); + this.commitPeriod = commitPeriod; + this.lastCommitTime = Instant.now(); + } + + private int calculateInflightSize(Subscription subscription) { + Optional subscriptionInflight = + Optional.ofNullable(subscription.getSerialSubscriptionPolicy().getInflightSize()); + return subscriptionInflight.orElse(defaultInflight); + } + + @Override + public void consume(Runnable signalsInterrupt) { + try { + ConsumerProfiler profiler = + subscription.isProfilingEnabled() + ? new DefaultConsumerProfiler( + subscription.getQualifiedName(), subscription.getProfilingThresholdMs()) + : new NoOpConsumerProfiler(); + profiler.startMeasurements(Measurement.SIGNALS_AND_SEMAPHORE_ACQUIRE); + do { + loadRecorder.recordSingleOperation(); + profiler.startPartialMeasurement(Measurement.SIGNALS_INTERRUPT_RUN); + signalsInterrupt.run(); + commitIfReady(); + profiler.stopPartialMeasurement(); + } while (!pendingOffsets.tryAcquireSlot(signalProcessingInterval)); + + profiler.measure(Measurement.MESSAGE_RECEIVER_NEXT); + Optional maybeMessage = messageReceiver.next(); + + profiler.measure(Measurement.MESSAGE_CONVERSION); + if (maybeMessage.isPresent()) { + Message message = maybeMessage.get(); + + if (message.isFiltered()) { + profiler.flushMeasurements(ConsumerRun.FILTERED); + } else { + if (logger.isDebugEnabled()) { + logger.debug( + "Read message {} partition {} offset {}", + message.getContentType(), + message.getPartition(), + message.getOffset()); + } + + Message convertedMessage = + messageConverterResolver.converterFor(message, subscription).convert(message, topic); + sendMessage(convertedMessage, profiler); } + } else { + pendingOffsets.releaseSlot(); + profiler.flushMeasurements(ConsumerRun.EMPTY); + } + } catch (InterruptedException e) { + logger.info("Restoring interrupted status {}", subscription.getQualifiedName(), e); + Thread.currentThread().interrupt(); + } catch (Exception e) { + logger.error("Consumer loop failed for {}", subscription.getQualifiedName(), e); } - - private boolean messageSizeChanged(Topic newTopic) { - return this.topic.getMaxMessageSize() != newTopic.getMaxMessageSize() - && useTopicMessageSizeEnabled; - } - - @Override - public void commit(Set offsets) { - messageReceiver.commit(offsets); + } + + private void commitIfReady() { + if (isReadyToCommit()) { + Set offsetsToCommit = + offsetCommitter.calculateOffsetsToBeCommitted( + pendingOffsets.getOffsetsSnapshotAndReleaseProcessedSlots()); + if (!offsetsToCommit.isEmpty()) { + commit(offsetsToCommit); + } + lastCommitTime = Instant.now(); } - - @Override - public boolean moveOffset(PartitionOffset offset) { - return messageReceiver.moveOffset(offset); - } - - @Override - public Subscription getSubscription() { - return subscription; + } + + private boolean isReadyToCommit() { + return Duration.between(lastCommitTime, Instant.now()).toMillis() > commitPeriod.toMillis(); + } + + private void sendMessage(Message message, ConsumerProfiler profiler) throws InterruptedException { + profiler.measure(Measurement.OFFER_INFLIGHT_OFFSET); + pendingOffsets.markAsInflight( + subscriptionPartitionOffset( + subscription.getQualifiedName(), + message.getPartitionOffset(), + message.getPartitionAssignmentTerm())); + + profiler.measure(Measurement.TRACKERS_LOG_INFLIGHT); + trackers.get(subscription).logInflight(toMessageMetadata(message, subscription)); + + sender.sendAsync(message, profiler); + } + + @Override + public void initialize() { + logger.info( + "Consumer: preparing message receiver for subscription {}", + subscription.getQualifiedName()); + initializeMessageReceiver(); + sender.initialize(); + rateLimiter.initialize(); + loadRecorder.initialize(); + consumerAuthorizationHandler.createSubscriptionHandler(subscription.getQualifiedName()); + } + + private void initializeMessageReceiver() { + this.messageReceiver = + messageReceiverFactory.createMessageReceiver( + topic, + subscription, + rateLimiter, + loadRecorder, + metrics, + pendingOffsets::markAsProcessed); + } + + /** + * Try to keep shutdown order the same as initialization so nothing will left to clean up when + * error occurs during initialization. + */ + @Override + public void tearDown() { + messageReceiver.stop(); + sender.shutdown(); + rateLimiter.shutdown(); + loadRecorder.shutdown(); + consumerAuthorizationHandler.removeSubscriptionHandler(subscription.getQualifiedName()); + metrics.unregisterAllMetricsRelatedTo(subscription.getQualifiedName()); + } + + @Override + public void updateSubscription(Subscription newSubscription) { + logger.info("Updating consumer for subscription {}", subscription.getQualifiedName()); + pendingOffsets.setInflightSize(calculateInflightSize(newSubscription)); + rateLimiter.updateSubscription(newSubscription); + sender.updateSubscription(newSubscription); + messageReceiver.update(newSubscription); + consumerAuthorizationHandler.updateSubscription(newSubscription.getQualifiedName()); + this.subscription = newSubscription; + } + + @Override + public void updateTopic(Topic newTopic) { + if (this.topic.getContentType() != newTopic.getContentType() + || messageSizeChanged(newTopic) + || this.topic.isSchemaIdAwareSerializationEnabled() + != newTopic.isSchemaIdAwareSerializationEnabled()) { + logger.info( + "Reinitializing message receiver, contentType, messageSize or schemaIdAwareSerialization changed."); + this.topic = newTopic; + + messageReceiver.stop(); + initializeMessageReceiver(); } + } + + private boolean messageSizeChanged(Topic newTopic) { + return this.topic.getMaxMessageSize() != newTopic.getMaxMessageSize() + && useTopicMessageSizeEnabled; + } + + @Override + public void commit(Set offsets) { + messageReceiver.commit(offsets); + } + + @Override + public boolean moveOffset(PartitionOffset offset) { + return messageReceiver.moveOffset(offset); + } + + @Override + public Subscription getSubscription() { + return subscription; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/SerialConsumerParameters.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/SerialConsumerParameters.java index 4b2a3be9ac..6cb077ff17 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/SerialConsumerParameters.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/SerialConsumerParameters.java @@ -4,7 +4,7 @@ public interface SerialConsumerParameters { - Duration getSignalProcessingInterval(); + Duration getSignalProcessingInterval(); - int getInflightSize(); + int getInflightSize(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/ByteBufferMessageBatchFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/ByteBufferMessageBatchFactory.java index 09ecbfc037..ddee065f08 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/ByteBufferMessageBatchFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/ByteBufferMessageBatchFactory.java @@ -1,46 +1,51 @@ package pl.allegro.tech.hermes.consumers.consumer.batch; -import pl.allegro.tech.hermes.api.Subscription; -import pl.allegro.tech.hermes.common.exception.InternalProcessingException; -import pl.allegro.tech.hermes.common.metric.MetricsFacade; +import static com.google.common.base.Preconditions.checkNotNull; +import static java.util.UUID.randomUUID; import java.nio.ByteBuffer; import java.time.Clock; - -import static com.google.common.base.Preconditions.checkNotNull; -import static java.util.UUID.randomUUID; +import pl.allegro.tech.hermes.api.Subscription; +import pl.allegro.tech.hermes.common.exception.InternalProcessingException; +import pl.allegro.tech.hermes.common.metric.MetricsFacade; public class ByteBufferMessageBatchFactory implements MessageBatchFactory { - private final DirectBufferPool bufferPool; - private final Clock clock; + private final DirectBufferPool bufferPool; + private final Clock clock; - public ByteBufferMessageBatchFactory(int poolableSize, int maxPoolSize, Clock clock, MetricsFacade metrics) { - this.clock = clock; - this.bufferPool = new DirectBufferPool(maxPoolSize, poolableSize, true); - metrics.consumer().registerBatchBufferTotalBytesGauge(bufferPool, DirectBufferPool::totalMemory); - metrics.consumer().registerBatchBufferAvailableBytesGauge(bufferPool, DirectBufferPool::availableMemory); - } + public ByteBufferMessageBatchFactory( + int poolableSize, int maxPoolSize, Clock clock, MetricsFacade metrics) { + this.clock = clock; + this.bufferPool = new DirectBufferPool(maxPoolSize, poolableSize, true); + metrics + .consumer() + .registerBatchBufferTotalBytesGauge(bufferPool, DirectBufferPool::totalMemory); + metrics + .consumer() + .registerBatchBufferAvailableBytesGauge(bufferPool, DirectBufferPool::availableMemory); + } - @Override - public MessageBatch createBatch(Subscription subscription) { - try { - ByteBuffer buffer = bufferPool.allocate(subscription.getBatchSubscriptionPolicy().getBatchVolume()); - switch (subscription.getContentType()) { - case JSON: - return new JsonMessageBatch(randomUUID().toString(), buffer, subscription, clock); - case AVRO: - default: - throw new UnsupportedOperationException( - "Batching is not supported yet for contentType " + subscription.getContentType()); - } - } catch (InterruptedException e) { - throw new InternalProcessingException(e); - } + @Override + public MessageBatch createBatch(Subscription subscription) { + try { + ByteBuffer buffer = + bufferPool.allocate(subscription.getBatchSubscriptionPolicy().getBatchVolume()); + switch (subscription.getContentType()) { + case JSON: + return new JsonMessageBatch(randomUUID().toString(), buffer, subscription, clock); + case AVRO: + default: + throw new UnsupportedOperationException( + "Batching is not supported yet for contentType " + subscription.getContentType()); + } + } catch (InterruptedException e) { + throw new InternalProcessingException(e); } + } - @Override - public void destroyBatch(MessageBatch batch) { - checkNotNull(batch); - bufferPool.deallocate(batch.getContent()); - } + @Override + public void destroyBatch(MessageBatch batch) { + checkNotNull(batch); + bufferPool.deallocate(batch.getContent()); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/DirectBufferPool.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/DirectBufferPool.java index 4db98100bc..f0544db6cf 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/DirectBufferPool.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/DirectBufferPool.java @@ -1,17 +1,15 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package pl.allegro.tech.hermes.consumers.consumer.batch; @@ -35,221 +33,210 @@ * * */ - /** - * A pool of ByteBuffers kept under a given memory limit. This class is fairly specific to the needs of the producer. In - * particular it has the following properties: + * A pool of ByteBuffers kept under a given memory limit. This class is fairly specific to the needs + * of the producer. In particular it has the following properties: + * *

    - *
  1. There is a special "poolable size" and buffers of this size are kept in a free list and recycled - *
  2. It is fair. That is all memory is given to the longest waiting thread until it has sufficient memory. This - * prevents starvation or deadlock when a thread asks for a large chunk of memory and needs to block until multiple - * buffers are deallocated. + *
  3. There is a special "poolable size" and buffers of this size are kept in a free list and + * recycled + *
  4. It is fair. That is all memory is given to the longest waiting thread until it has + * sufficient memory. This prevents starvation or deadlock when a thread asks for a large + * chunk of memory and needs to block until multiple buffers are deallocated. *
*/ public final class DirectBufferPool { - private final long totalMemory; - private final int poolableSize; - private final boolean blockOnExhaustion; - private final ReentrantLock lock; - private final Deque free; - private final Deque waiters; - private long availableMemory; - - /** - * Create a new buffer pool - * - * @param memory The maximum amount of memory that this buffer pool can allocate - * @param poolableSize The buffer size to cache in the free list rather than deallocating - * @param blockOnExhaustion This controls the behavior when the buffer pool is out of memory. If true the - * {@link #allocate(int)} call will block and wait for memory to be returned to the pool. If false - * {@link #allocate(int)} will throw an exception if the buffer is out of memory. - */ - public DirectBufferPool(long memory, int poolableSize, boolean blockOnExhaustion) { - this.poolableSize = poolableSize; - this.blockOnExhaustion = blockOnExhaustion; - this.lock = new ReentrantLock(); - this.free = new ArrayDeque<>(); - this.waiters = new ArrayDeque<>(); - this.totalMemory = memory; - this.availableMemory = memory; - } - - /** - * Allocate a buffer of the given size. This method blocks if there is not enough memory and the buffer pool - * is configured with blocking mode. - * - * @param size The buffer size to allocate in bytes - * @return The buffer - * @throws InterruptedException If the thread is interrupted while blocked - * @throws IllegalArgumentException if size is larger than the total memory controlled by the pool (and hence we would block - * forever) - * @throws BufferOverflowException if the pool is in non-blocking mode and size exceeds the free memory in the pool - */ - public ByteBuffer allocate(int size) throws InterruptedException { - if (size > this.totalMemory) - throw new IllegalArgumentException("Attempt to allocate " + size - + " bytes, but there is a hard limit of " - + this.totalMemory - + " on memory allocations."); - - this.lock.lock(); - try { - // check if we have a free buffer of the right size pooled - if (size == poolableSize && !this.free.isEmpty()) - return this.free.pollFirst(); - - // now check if the request is immediately satisfiable with the - // memory on hand or if we need to block - int freeListSize = this.free.size() * this.poolableSize; - if (this.availableMemory + freeListSize >= size) { - // we have enough unallocated or pooled memory to immediately - // satisfy the request - freeUp(size); - this.availableMemory -= size; - lock.unlock(); - return ByteBuffer.allocateDirect(size); - } else if (!blockOnExhaustion) { - throw new BufferOverflowException(); - } else { - // we are out of memory and will have to block - int accumulated = 0; - ByteBuffer buffer = null; - Condition moreMemory = this.lock.newCondition(); - this.waiters.addLast(moreMemory); - // loop over and over until we have a buffer or have reserved - // enough memory to allocate one - while (accumulated < size) { - moreMemory.await(); - - // check if we can satisfy this request from the free list, - // otherwise allocate memory - if (accumulated == 0 && size == this.poolableSize && !this.free.isEmpty()) { - // just grab a buffer from the free list - buffer = this.free.pollFirst(); - accumulated = size; - } else { - // we'll need to allocate memory, but we may only get - // part of what we need on this iteration - freeUp(size - accumulated); - int got = (int) Math.min(size - accumulated, this.availableMemory); - this.availableMemory -= got; - accumulated += got; - } - } - - // remove the condition for this thread to let the next thread - // in line start getting memory - Condition removed = this.waiters.removeFirst(); - if (removed != moreMemory) - throw new IllegalStateException("Wrong condition: this shouldn't happen."); - - // signal any additional waiters if there is more memory left - // over for them - if (this.availableMemory > 0 || !this.free.isEmpty()) { - if (!this.waiters.isEmpty()) - this.waiters.peekFirst().signal(); - } - - // unlock and return the buffer - lock.unlock(); - if (buffer == null) - return ByteBuffer.allocateDirect(size); - else - return buffer; - } - } finally { - if (lock.isHeldByCurrentThread()) - lock.unlock(); + private final long totalMemory; + private final int poolableSize; + private final boolean blockOnExhaustion; + private final ReentrantLock lock; + private final Deque free; + private final Deque waiters; + private long availableMemory; + + /** + * Create a new buffer pool + * + * @param memory The maximum amount of memory that this buffer pool can allocate + * @param poolableSize The buffer size to cache in the free list rather than deallocating + * @param blockOnExhaustion This controls the behavior when the buffer pool is out of memory. If + * true the {@link #allocate(int)} call will block and wait for memory to be returned to the + * pool. If false {@link #allocate(int)} will throw an exception if the buffer is out of + * memory. + */ + public DirectBufferPool(long memory, int poolableSize, boolean blockOnExhaustion) { + this.poolableSize = poolableSize; + this.blockOnExhaustion = blockOnExhaustion; + this.lock = new ReentrantLock(); + this.free = new ArrayDeque<>(); + this.waiters = new ArrayDeque<>(); + this.totalMemory = memory; + this.availableMemory = memory; + } + + /** + * Allocate a buffer of the given size. This method blocks if there is not enough memory and the + * buffer pool is configured with blocking mode. + * + * @param size The buffer size to allocate in bytes + * @return The buffer + * @throws InterruptedException If the thread is interrupted while blocked + * @throws IllegalArgumentException if size is larger than the total memory controlled by the pool + * (and hence we would block forever) + * @throws BufferOverflowException if the pool is in non-blocking mode and size exceeds the free + * memory in the pool + */ + public ByteBuffer allocate(int size) throws InterruptedException { + if (size > this.totalMemory) + throw new IllegalArgumentException( + "Attempt to allocate " + + size + + " bytes, but there is a hard limit of " + + this.totalMemory + + " on memory allocations."); + + this.lock.lock(); + try { + // check if we have a free buffer of the right size pooled + if (size == poolableSize && !this.free.isEmpty()) return this.free.pollFirst(); + + // now check if the request is immediately satisfiable with the + // memory on hand or if we need to block + int freeListSize = this.free.size() * this.poolableSize; + if (this.availableMemory + freeListSize >= size) { + // we have enough unallocated or pooled memory to immediately + // satisfy the request + freeUp(size); + this.availableMemory -= size; + lock.unlock(); + return ByteBuffer.allocateDirect(size); + } else if (!blockOnExhaustion) { + throw new BufferOverflowException(); + } else { + // we are out of memory and will have to block + int accumulated = 0; + ByteBuffer buffer = null; + Condition moreMemory = this.lock.newCondition(); + this.waiters.addLast(moreMemory); + // loop over and over until we have a buffer or have reserved + // enough memory to allocate one + while (accumulated < size) { + moreMemory.await(); + + // check if we can satisfy this request from the free list, + // otherwise allocate memory + if (accumulated == 0 && size == this.poolableSize && !this.free.isEmpty()) { + // just grab a buffer from the free list + buffer = this.free.pollFirst(); + accumulated = size; + } else { + // we'll need to allocate memory, but we may only get + // part of what we need on this iteration + freeUp(size - accumulated); + int got = (int) Math.min(size - accumulated, this.availableMemory); + this.availableMemory -= got; + accumulated += got; + } } - } - /** - * Attempt to ensure we have at least the requested number of bytes of memory for allocation by deallocating pooled - * buffers (if needed) - */ - private void freeUp(int size) { - while (!this.free.isEmpty() && this.availableMemory < size) - this.availableMemory += this.free.pollLast().capacity(); - } + // remove the condition for this thread to let the next thread + // in line start getting memory + Condition removed = this.waiters.removeFirst(); + if (removed != moreMemory) + throw new IllegalStateException("Wrong condition: this shouldn't happen."); - /** - * Return buffers to the pool. If they are of the poolable size add them to the free list, otherwise just mark the - * memory as free. - * - * @param buffer The buffer to return - * @param size The size of the buffer to mark as deallocated, note that this maybe smaller than buffer.capacity - * since the buffer may re-allocate itself during in-place compression - */ - public void deallocate(ByteBuffer buffer, int size) { - lock.lock(); - try { - if (size == this.poolableSize && size == buffer.capacity()) { - buffer.clear(); - this.free.add(buffer); - } else { - DirectBufferUtils.release(buffer); - this.availableMemory += size; - } - Condition moreMem = this.waiters.peekFirst(); - if (moreMem != null) - moreMem.signal(); - } finally { - lock.unlock(); + // signal any additional waiters if there is more memory left + // over for them + if (this.availableMemory > 0 || !this.free.isEmpty()) { + if (!this.waiters.isEmpty()) this.waiters.peekFirst().signal(); } - } - public void deallocate(ByteBuffer buffer) { - deallocate(buffer, buffer.capacity()); + // unlock and return the buffer + lock.unlock(); + if (buffer == null) return ByteBuffer.allocateDirect(size); + else return buffer; + } + } finally { + if (lock.isHeldByCurrentThread()) lock.unlock(); } - - /** - * the total free memory both unallocated and in the free list - */ - public long availableMemory() { - lock.lock(); - try { - return this.availableMemory + this.free.size() * this.poolableSize; - } finally { - lock.unlock(); - } + } + + /** + * Attempt to ensure we have at least the requested number of bytes of memory for allocation by + * deallocating pooled buffers (if needed) + */ + private void freeUp(int size) { + while (!this.free.isEmpty() && this.availableMemory < size) + this.availableMemory += this.free.pollLast().capacity(); + } + + /** + * Return buffers to the pool. If they are of the poolable size add them to the free list, + * otherwise just mark the memory as free. + * + * @param buffer The buffer to return + * @param size The size of the buffer to mark as deallocated, note that this maybe smaller than + * buffer.capacity since the buffer may re-allocate itself during in-place compression + */ + public void deallocate(ByteBuffer buffer, int size) { + lock.lock(); + try { + if (size == this.poolableSize && size == buffer.capacity()) { + buffer.clear(); + this.free.add(buffer); + } else { + DirectBufferUtils.release(buffer); + this.availableMemory += size; + } + Condition moreMem = this.waiters.peekFirst(); + if (moreMem != null) moreMem.signal(); + } finally { + lock.unlock(); } - - /** - * Get the unallocated memory (not in the free list or in use) - */ - public long unallocatedMemory() { - lock.lock(); - try { - return this.availableMemory; - } finally { - lock.unlock(); - } + } + + public void deallocate(ByteBuffer buffer) { + deallocate(buffer, buffer.capacity()); + } + + /** the total free memory both unallocated and in the free list */ + public long availableMemory() { + lock.lock(); + try { + return this.availableMemory + this.free.size() * this.poolableSize; + } finally { + lock.unlock(); } - - /** - * The number of threads blocked waiting on memory - */ - public int queued() { - lock.lock(); - try { - return this.waiters.size(); - } finally { - lock.unlock(); - } + } + + /** Get the unallocated memory (not in the free list or in use) */ + public long unallocatedMemory() { + lock.lock(); + try { + return this.availableMemory; + } finally { + lock.unlock(); } - - /** - * The buffer size that will be retained in the free list after use - */ - public int poolableSize() { - return this.poolableSize; + } + + /** The number of threads blocked waiting on memory */ + public int queued() { + lock.lock(); + try { + return this.waiters.size(); + } finally { + lock.unlock(); } + } - /** - * The total memory managed by this pool - */ - public long totalMemory() { - return this.totalMemory; - } + /** The buffer size that will be retained in the free list after use */ + public int poolableSize() { + return this.poolableSize; + } + + /** The total memory managed by this pool */ + public long totalMemory() { + return this.totalMemory; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/DirectBufferUtils.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/DirectBufferUtils.java index 96dfa34ec2..81a15e92d0 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/DirectBufferUtils.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/DirectBufferUtils.java @@ -1,56 +1,54 @@ package pl.allegro.tech.hermes.consumers.consumer.batch; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.lang.reflect.Field; import java.lang.reflect.Method; import java.nio.ByteBuffer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; class DirectBufferUtils { - private static final Logger logger = LoggerFactory.getLogger(DirectBufferUtils.class); - - private static final DirectBufferCleaner CLEANER; - - static { - DirectBufferCleaner cleaner; - try { - cleaner = new DirectBufferCleaner(); - } catch (ReflectiveOperationException e) { - cleaner = null; - } - CLEANER = cleaner; - } - - static boolean supportsReleasing() { - return CLEANER != null; - } + private static final Logger logger = LoggerFactory.getLogger(DirectBufferUtils.class); + private static final DirectBufferCleaner CLEANER; - static void release(ByteBuffer buffer) { - try { - if (supportsReleasing() && buffer.isDirect()) { - CLEANER.clean(buffer); - } - } catch (ReflectiveOperationException e) { - logger.warn("Releasing ByteBuffer failed", e); - } + static { + DirectBufferCleaner cleaner; + try { + cleaner = new DirectBufferCleaner(); + } catch (ReflectiveOperationException e) { + cleaner = null; + } + CLEANER = cleaner; + } + + static boolean supportsReleasing() { + return CLEANER != null; + } + + static void release(ByteBuffer buffer) { + try { + if (supportsReleasing() && buffer.isDirect()) { + CLEANER.clean(buffer); + } + } catch (ReflectiveOperationException e) { + logger.warn("Releasing ByteBuffer failed", e); + } + } + + static class DirectBufferCleaner { + private final Object unsafe; + private final Method invokeCleaner; + + DirectBufferCleaner() throws ReflectiveOperationException { + Class clazz = Class.forName("sun.misc.Unsafe"); + Field field = clazz.getDeclaredField("theUnsafe"); + field.setAccessible(true); + unsafe = field.get(null); + invokeCleaner = clazz.getMethod("invokeCleaner", ByteBuffer.class); } - static class DirectBufferCleaner { - private final Object unsafe; - private final Method invokeCleaner; - - DirectBufferCleaner() throws ReflectiveOperationException { - Class clazz = Class.forName("sun.misc.Unsafe"); - Field field = clazz.getDeclaredField("theUnsafe"); - field.setAccessible(true); - unsafe = field.get(null); - invokeCleaner = clazz.getMethod("invokeCleaner", ByteBuffer.class); - } - - public void clean(ByteBuffer buffer) throws ReflectiveOperationException { - invokeCleaner.invoke(unsafe, buffer); - } + public void clean(ByteBuffer buffer) throws ReflectiveOperationException { + invokeCleaner.invoke(unsafe, buffer); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/JsonMessageBatch.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/JsonMessageBatch.java index 794f40a939..5e0ee0cebd 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/JsonMessageBatch.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/JsonMessageBatch.java @@ -1,13 +1,7 @@ package pl.allegro.tech.hermes.consumers.consumer.batch; -import pl.allegro.tech.hermes.api.ContentType; -import pl.allegro.tech.hermes.api.Header; -import pl.allegro.tech.hermes.api.Subscription; -import pl.allegro.tech.hermes.api.SubscriptionName; -import pl.allegro.tech.hermes.common.kafka.KafkaTopicName; -import pl.allegro.tech.hermes.common.kafka.offset.PartitionOffset; -import pl.allegro.tech.hermes.consumers.consumer.offset.SubscriptionPartitionOffset; -import pl.allegro.tech.hermes.tracker.consumers.MessageMetadata; +import static com.google.common.base.Preconditions.checkState; +import static pl.allegro.tech.hermes.consumers.consumer.offset.SubscriptionPartitionOffset.subscriptionPartitionOffset; import java.nio.BufferOverflowException; import java.nio.ByteBuffer; @@ -17,186 +11,195 @@ import java.util.List; import java.util.stream.Collectors; import javax.annotation.concurrent.NotThreadSafe; - -import static com.google.common.base.Preconditions.checkState; -import static pl.allegro.tech.hermes.consumers.consumer.offset.SubscriptionPartitionOffset.subscriptionPartitionOffset; +import pl.allegro.tech.hermes.api.ContentType; +import pl.allegro.tech.hermes.api.Header; +import pl.allegro.tech.hermes.api.Subscription; +import pl.allegro.tech.hermes.api.SubscriptionName; +import pl.allegro.tech.hermes.common.kafka.KafkaTopicName; +import pl.allegro.tech.hermes.common.kafka.offset.PartitionOffset; +import pl.allegro.tech.hermes.consumers.consumer.offset.SubscriptionPartitionOffset; +import pl.allegro.tech.hermes.tracker.consumers.MessageMetadata; @NotThreadSafe public class JsonMessageBatch implements MessageBatch { - private final Clock clock; - - private final int maxBatchTime; - private final int batchSize; - - private final String id; - private final String topic; - private final SubscriptionName subscription; - private final boolean hasSubscriptionIdentityHeaders; - private final ByteBuffer byteBuffer; - private final List metadata = new ArrayList<>(); - private final List
additionalHeaders; - - private int elements = 0; - private long batchStart; - private boolean closed = false; - private int retryCounter = 0; - - JsonMessageBatch(String id, ByteBuffer buffer, Subscription subscription, Clock clock) { - this.id = id; - this.clock = clock; - this.maxBatchTime = subscription.getBatchSubscriptionPolicy().getBatchTime(); - this.batchSize = subscription.getBatchSubscriptionPolicy().getBatchSize(); - this.byteBuffer = buffer; - this.additionalHeaders = subscription.getHeaders(); - this.topic = subscription.getQualifiedTopicName(); - this.subscription = subscription.getQualifiedName(); - this.hasSubscriptionIdentityHeaders = subscription.isSubscriptionIdentityHeadersEnabled(); - } - - @Override - public boolean isFull() { - return elements >= batchSize || byteBuffer.remaining() < 2; - } - - @Override - public void append(byte[] data, MessageMetadata metadata) { - checkState(!closed, "Batch already closed."); - if (!canFit(data)) { - throw new BufferOverflowException(); - } - if (isEmpty()) { - batchStart = clock.millis(); - } - - byteBuffer.put((byte) (isEmpty() ? '[' : ',')).put(data); - this.metadata.add(metadata); - elements++; - } - - @Override - public boolean canFit(byte[] data) { - return byteBuffer.remaining() >= requiredFreeSpace(data); - } - - private int requiredFreeSpace(byte[] data) { - return data.length + 2; - } - - @Override - public boolean isExpired() { - return !isEmpty() && getLifetime() > maxBatchTime; - } - - @Override - public String getId() { - return id; - } - - @Override - public ContentType getContentType() { - return ContentType.JSON; - } - - @Override - public MessageBatch close() { - if (!isEmpty()) { - byteBuffer.put((byte) ']'); - } - int position = byteBuffer.position(); - byteBuffer.position(0); - byteBuffer.limit(position); - this.closed = true; - return this; - } - - @Override - public ByteBuffer getContent() { - if (closed) { - byteBuffer.position(0); - } - return byteBuffer; - } - - @Override - public List getPartitionOffsets() { - return metadata.stream() - .map(m -> subscriptionPartitionOffset(this.subscription, - new PartitionOffset(KafkaTopicName.valueOf(m.getKafkaTopic()), m.getOffset(), m.getPartition()), - m.getPartitionAssignmentTerm())) - .collect(Collectors.toList()); - } - - @Override - public List getMessagesMetadata() { - return Collections.unmodifiableList(metadata); - } - - @Override - public List
getAdditionalHeaders() { - return Collections.unmodifiableList(additionalHeaders); - } - - @Override - public int getMessageCount() { - return elements; - } - - @Override - public long getLifetime() { - return clock.millis() - batchStart; - } - - @Override - public boolean isClosed() { - return closed; - } - - @Override - public boolean isEmpty() { - return elements == 0; - } - - @Override - public boolean isBiggerThanTotalCapacity(byte[] data) { - return requiredFreeSpace(data) > getCapacity(); - } - - @Override - public int getCapacity() { - return byteBuffer.capacity(); - } - - @Override - public int getSize() { - if (closed) { - return byteBuffer.limit(); - } - return byteBuffer.position(); - } - - @Override - public void incrementRetryCounter() { - this.retryCounter++; - } - - @Override - public int getRetryCounter() { - return retryCounter; - } - - @Override - public boolean hasSubscriptionIdentityHeaders() { - return hasSubscriptionIdentityHeaders; - } - - @Override - public String getTopic() { - return topic; - } - - @Override - public SubscriptionName getSubscription() { - return subscription; - } + private final Clock clock; + + private final int maxBatchTime; + private final int batchSize; + + private final String id; + private final String topic; + private final SubscriptionName subscription; + private final boolean hasSubscriptionIdentityHeaders; + private final ByteBuffer byteBuffer; + private final List metadata = new ArrayList<>(); + private final List
additionalHeaders; + + private int elements = 0; + private long batchStart; + private boolean closed = false; + private int retryCounter = 0; + + JsonMessageBatch(String id, ByteBuffer buffer, Subscription subscription, Clock clock) { + this.id = id; + this.clock = clock; + this.maxBatchTime = subscription.getBatchSubscriptionPolicy().getBatchTime(); + this.batchSize = subscription.getBatchSubscriptionPolicy().getBatchSize(); + this.byteBuffer = buffer; + this.additionalHeaders = subscription.getHeaders(); + this.topic = subscription.getQualifiedTopicName(); + this.subscription = subscription.getQualifiedName(); + this.hasSubscriptionIdentityHeaders = subscription.isSubscriptionIdentityHeadersEnabled(); + } + + @Override + public boolean isFull() { + return elements >= batchSize || byteBuffer.remaining() < 2; + } + + @Override + public void append(byte[] data, MessageMetadata metadata) { + checkState(!closed, "Batch already closed."); + if (!canFit(data)) { + throw new BufferOverflowException(); + } + if (isEmpty()) { + batchStart = clock.millis(); + } + + byteBuffer.put((byte) (isEmpty() ? '[' : ',')).put(data); + this.metadata.add(metadata); + elements++; + } + + @Override + public boolean canFit(byte[] data) { + return byteBuffer.remaining() >= requiredFreeSpace(data); + } + + private int requiredFreeSpace(byte[] data) { + return data.length + 2; + } + + @Override + public boolean isExpired() { + return !isEmpty() && getLifetime() > maxBatchTime; + } + + @Override + public String getId() { + return id; + } + + @Override + public ContentType getContentType() { + return ContentType.JSON; + } + + @Override + public MessageBatch close() { + if (!isEmpty()) { + byteBuffer.put((byte) ']'); + } + int position = byteBuffer.position(); + byteBuffer.position(0); + byteBuffer.limit(position); + this.closed = true; + return this; + } + + @Override + public ByteBuffer getContent() { + if (closed) { + byteBuffer.position(0); + } + return byteBuffer; + } + + @Override + public List getPartitionOffsets() { + return metadata.stream() + .map( + m -> + subscriptionPartitionOffset( + this.subscription, + new PartitionOffset( + KafkaTopicName.valueOf(m.getKafkaTopic()), m.getOffset(), m.getPartition()), + m.getPartitionAssignmentTerm())) + .collect(Collectors.toList()); + } + + @Override + public List getMessagesMetadata() { + return Collections.unmodifiableList(metadata); + } + + @Override + public List
getAdditionalHeaders() { + return Collections.unmodifiableList(additionalHeaders); + } + + @Override + public int getMessageCount() { + return elements; + } + + @Override + public long getLifetime() { + return clock.millis() - batchStart; + } + + @Override + public boolean isClosed() { + return closed; + } + + @Override + public boolean isEmpty() { + return elements == 0; + } + + @Override + public boolean isBiggerThanTotalCapacity(byte[] data) { + return requiredFreeSpace(data) > getCapacity(); + } + + @Override + public int getCapacity() { + return byteBuffer.capacity(); + } + + @Override + public int getSize() { + if (closed) { + return byteBuffer.limit(); + } + return byteBuffer.position(); + } + + @Override + public void incrementRetryCounter() { + this.retryCounter++; + } + + @Override + public int getRetryCounter() { + return retryCounter; + } + + @Override + public boolean hasSubscriptionIdentityHeaders() { + return hasSubscriptionIdentityHeaders; + } + + @Override + public String getTopic() { + return topic; + } + + @Override + public SubscriptionName getSubscription() { + return subscription; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/MessageBatch.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/MessageBatch.java index ca43d73998..bb33c9303c 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/MessageBatch.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/MessageBatch.java @@ -1,64 +1,63 @@ package pl.allegro.tech.hermes.consumers.consumer.batch; +import java.nio.BufferOverflowException; +import java.nio.ByteBuffer; +import java.util.List; import pl.allegro.tech.hermes.api.ContentType; import pl.allegro.tech.hermes.api.Header; import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.consumers.consumer.offset.SubscriptionPartitionOffset; import pl.allegro.tech.hermes.tracker.consumers.MessageMetadata; -import java.nio.BufferOverflowException; -import java.nio.ByteBuffer; -import java.util.List; - public interface MessageBatch { - default boolean isReadyForDelivery() { - return isClosed() || isFull() || isExpired(); - } + default boolean isReadyForDelivery() { + return isClosed() || isFull() || isExpired(); + } - void append(byte[] data, MessageMetadata batchMessageMetadata) throws BufferOverflowException; + void append(byte[] data, MessageMetadata batchMessageMetadata) throws BufferOverflowException; - boolean canFit(byte[] data); + boolean canFit(byte[] data); - boolean isExpired(); + boolean isExpired(); - boolean isClosed(); + boolean isClosed(); - boolean isFull(); + boolean isFull(); - String getId(); + String getId(); - ContentType getContentType(); + ContentType getContentType(); - ByteBuffer getContent(); + ByteBuffer getContent(); - List getPartitionOffsets(); + List getPartitionOffsets(); - List getMessagesMetadata(); + List getMessagesMetadata(); - List
getAdditionalHeaders(); + List
getAdditionalHeaders(); - long getLifetime(); + long getLifetime(); - int getMessageCount(); + int getMessageCount(); - MessageBatch close(); + MessageBatch close(); - boolean isEmpty(); + boolean isEmpty(); - boolean isBiggerThanTotalCapacity(byte[] data); + boolean isBiggerThanTotalCapacity(byte[] data); - int getCapacity(); + int getCapacity(); - int getSize(); + int getSize(); - void incrementRetryCounter(); + void incrementRetryCounter(); - int getRetryCounter(); + int getRetryCounter(); - boolean hasSubscriptionIdentityHeaders(); + boolean hasSubscriptionIdentityHeaders(); - String getTopic(); + String getTopic(); - SubscriptionName getSubscription(); + SubscriptionName getSubscription(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/MessageBatchFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/MessageBatchFactory.java index da39eedf45..d271cf61c6 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/MessageBatchFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/MessageBatchFactory.java @@ -4,7 +4,7 @@ public interface MessageBatchFactory { - MessageBatch createBatch(Subscription subscription); + MessageBatch createBatch(Subscription subscription); - void destroyBatch(MessageBatch batch); + void destroyBatch(MessageBatch batch); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/MessageBatchReceiver.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/MessageBatchReceiver.java index 66fccbacc7..1011bf9b14 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/MessageBatchReceiver.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/MessageBatchReceiver.java @@ -1,5 +1,16 @@ package pl.allegro.tech.hermes.consumers.consumer.batch; +import static com.google.common.base.Preconditions.checkArgument; +import static pl.allegro.tech.hermes.consumers.consumer.Message.message; +import static pl.allegro.tech.hermes.consumers.consumer.message.MessageConverter.toMessageMetadata; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Queue; +import java.util.Set; +import javax.annotation.concurrent.NotThreadSafe; import org.apache.avro.Schema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -16,148 +27,160 @@ import pl.allegro.tech.hermes.tracker.consumers.MessageMetadata; import pl.allegro.tech.hermes.tracker.consumers.Trackers; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.Queue; -import java.util.Set; -import javax.annotation.concurrent.NotThreadSafe; - -import static com.google.common.base.Preconditions.checkArgument; -import static pl.allegro.tech.hermes.consumers.consumer.Message.message; -import static pl.allegro.tech.hermes.consumers.consumer.message.MessageConverter.toMessageMetadata; - @NotThreadSafe public class MessageBatchReceiver { - private static final Logger logger = LoggerFactory.getLogger(MessageBatchReceiver.class); - - private final MessageReceiver receiver; - private final MessageBatchFactory batchFactory; - private final MessageConverterResolver messageConverterResolver; - private final CompositeMessageContentWrapper compositeMessageContentWrapper; - private final Trackers trackers; - private final Queue inflight; - private final Topic topic; - private final SubscriptionLoadRecorder loadRecorder; - private boolean receiving = true; - private final Runnable commitIfReady; - - public MessageBatchReceiver(MessageReceiver receiver, - MessageBatchFactory batchFactory, - MessageConverterResolver messageConverterResolver, - CompositeMessageContentWrapper compositeMessageContentWrapper, - Topic topic, - Trackers trackers, - SubscriptionLoadRecorder loadRecorder, - Runnable commitIfReady) { - this.receiver = receiver; - this.batchFactory = batchFactory; - this.messageConverterResolver = messageConverterResolver; - this.compositeMessageContentWrapper = compositeMessageContentWrapper; - this.topic = topic; - this.trackers = trackers; - this.loadRecorder = loadRecorder; - this.inflight = new ArrayDeque<>(1); - this.commitIfReady = commitIfReady; + private static final Logger logger = LoggerFactory.getLogger(MessageBatchReceiver.class); + + private final MessageReceiver receiver; + private final MessageBatchFactory batchFactory; + private final MessageConverterResolver messageConverterResolver; + private final CompositeMessageContentWrapper compositeMessageContentWrapper; + private final Trackers trackers; + private final Queue inflight; + private final Topic topic; + private final SubscriptionLoadRecorder loadRecorder; + private boolean receiving = true; + private final Runnable commitIfReady; + + public MessageBatchReceiver( + MessageReceiver receiver, + MessageBatchFactory batchFactory, + MessageConverterResolver messageConverterResolver, + CompositeMessageContentWrapper compositeMessageContentWrapper, + Topic topic, + Trackers trackers, + SubscriptionLoadRecorder loadRecorder, + Runnable commitIfReady) { + this.receiver = receiver; + this.batchFactory = batchFactory; + this.messageConverterResolver = messageConverterResolver; + this.compositeMessageContentWrapper = compositeMessageContentWrapper; + this.topic = topic; + this.trackers = trackers; + this.loadRecorder = loadRecorder; + this.inflight = new ArrayDeque<>(1); + this.commitIfReady = commitIfReady; + } + + public MessageBatchingResult next(Subscription subscription, Runnable signalsInterrupt) { + if (logger.isDebugEnabled()) { + logger.debug( + "Trying to allocate memory for new batch [subscription={}]", + subscription.getQualifiedName()); } - public MessageBatchingResult next(Subscription subscription, Runnable signalsInterrupt) { - if (logger.isDebugEnabled()) { - logger.debug("Trying to allocate memory for new batch [subscription={}]", subscription.getQualifiedName()); - } - - MessageBatch batch = batchFactory.createBatch(subscription); - if (logger.isDebugEnabled()) { - logger.debug("New batch allocated [subscription={}]", subscription.getQualifiedName()); - } - List discarded = new ArrayList<>(); - - while (isReceiving() && !batch.isReadyForDelivery() && !Thread.currentThread().isInterrupted()) { - loadRecorder.recordSingleOperation(); - signalsInterrupt.run(); - // We need a commit function call here to prevent cases where it takes a long time to create a batch and messages are filtered at the same time. - // Otherwise, this would lead to ever-increasing lag despite the processing and filtering of current messages. - commitIfReady.run(); - Optional maybeMessage = inflight.isEmpty() - ? readAndTransform(subscription, batch.getId()) - : Optional.ofNullable(inflight.poll()); - - if (maybeMessage.isPresent()) { - Message message = maybeMessage.get(); - - if (batch.canFit(message.getData())) { - batch.append(message.getData(), toMessageMetadata(message, subscription, batch.getId())); - } else if (batch.isBiggerThanTotalCapacity(message.getData())) { - logger.error("Message size exceeds buffer total capacity [size={}, capacity={}, subscription={}]", - message.getData().length, batch.getCapacity(), subscription.getQualifiedName()); - discarded.add(toMessageMetadata(message, subscription)); - } else { - logger.debug( - "Message too large for current batch [message_size={}, subscription={}]", - message.getData().length, subscription.getQualifiedName() - ); - checkArgument(inflight.offer(message)); - break; - } - } - } - if (logger.isDebugEnabled()) { - logger.debug("Batch is ready for delivery [subscription={}]", subscription.getQualifiedName()); + MessageBatch batch = batchFactory.createBatch(subscription); + if (logger.isDebugEnabled()) { + logger.debug("New batch allocated [subscription={}]", subscription.getQualifiedName()); + } + List discarded = new ArrayList<>(); + + while (isReceiving() + && !batch.isReadyForDelivery() + && !Thread.currentThread().isInterrupted()) { + loadRecorder.recordSingleOperation(); + signalsInterrupt.run(); + // We need a commit function call here to prevent cases where it takes a long time to create a + // batch and messages are filtered at the same time. + // Otherwise, this would lead to ever-increasing lag despite the processing and filtering of + // current messages. + commitIfReady.run(); + Optional maybeMessage = + inflight.isEmpty() + ? readAndTransform(subscription, batch.getId()) + : Optional.ofNullable(inflight.poll()); + + if (maybeMessage.isPresent()) { + Message message = maybeMessage.get(); + + if (batch.canFit(message.getData())) { + batch.append(message.getData(), toMessageMetadata(message, subscription, batch.getId())); + } else if (batch.isBiggerThanTotalCapacity(message.getData())) { + logger.error( + "Message size exceeds buffer total capacity [size={}, capacity={}, subscription={}]", + message.getData().length, + batch.getCapacity(), + subscription.getQualifiedName()); + discarded.add(toMessageMetadata(message, subscription)); + } else { + logger.debug( + "Message too large for current batch [message_size={}, subscription={}]", + message.getData().length, + subscription.getQualifiedName()); + checkArgument(inflight.offer(message)); + break; } - return new MessageBatchingResult(batch.close(), discarded); + } } + if (logger.isDebugEnabled()) { + logger.debug( + "Batch is ready for delivery [subscription={}]", subscription.getQualifiedName()); + } + return new MessageBatchingResult(batch.close(), discarded); + } - private Optional readAndTransform(Subscription subscription, String batchId) { - Optional maybeMessage = receiver.next(); + private Optional readAndTransform(Subscription subscription, String batchId) { + Optional maybeMessage = receiver.next(); - if (maybeMessage.isPresent()) { - Message message = maybeMessage.get(); + if (maybeMessage.isPresent()) { + Message message = maybeMessage.get(); - if (!message.isFiltered()) { - Message transformed = messageConverterResolver.converterFor(message, subscription).convert(message, topic); - transformed = message().fromMessage(transformed).withData(wrap(subscription, transformed)).build(); + if (!message.isFiltered()) { + Message transformed = + messageConverterResolver.converterFor(message, subscription).convert(message, topic); + transformed = + message().fromMessage(transformed).withData(wrap(subscription, transformed)).build(); - trackers.get(subscription).logInflight(toMessageMetadata(transformed, subscription, batchId)); + trackers + .get(subscription) + .logInflight(toMessageMetadata(transformed, subscription, batchId)); - return Optional.of(transformed); - } - } - return Optional.empty(); + return Optional.of(transformed); + } } - - private byte[] wrap(Subscription subscription, Message next) { - switch (subscription.getContentType()) { - case AVRO: - return compositeMessageContentWrapper.wrapAvro(next.getData(), next.getId(), next.getPublishingTimestamp(), topic, - next.getSchema().get(), next.getExternalMetadata()); - case JSON: - return compositeMessageContentWrapper.wrapJson(next.getData(), next.getId(), next.getPublishingTimestamp(), - next.getExternalMetadata()); - default: - throw new UnsupportedContentTypeException(subscription); - } + return Optional.empty(); + } + + private byte[] wrap(Subscription subscription, Message next) { + switch (subscription.getContentType()) { + case AVRO: + return compositeMessageContentWrapper.wrapAvro( + next.getData(), + next.getId(), + next.getPublishingTimestamp(), + topic, + next.getSchema().get(), + next.getExternalMetadata()); + case JSON: + return compositeMessageContentWrapper.wrapJson( + next.getData(), + next.getId(), + next.getPublishingTimestamp(), + next.getExternalMetadata()); + default: + throw new UnsupportedContentTypeException(subscription); } + } - private boolean isReceiving() { - return receiving; - } + private boolean isReceiving() { + return receiving; + } - public void stop() { - receiving = false; - receiver.stop(); - } + public void stop() { + receiving = false; + receiver.stop(); + } - public void updateSubscription(Subscription modifiedSubscription) { - receiver.update(modifiedSubscription); - } + public void updateSubscription(Subscription modifiedSubscription) { + receiver.update(modifiedSubscription); + } - public void commit(Set offsets) { - receiver.commit(offsets); - } + public void commit(Set offsets) { + receiver.commit(offsets); + } - public boolean moveOffset(PartitionOffset offset) { - return receiver.moveOffset(offset); - } + public boolean moveOffset(PartitionOffset offset) { + return receiver.moveOffset(offset); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/MessageBatchingResult.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/MessageBatchingResult.java index 8a602604df..6292311260 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/MessageBatchingResult.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/batch/MessageBatchingResult.java @@ -1,23 +1,22 @@ package pl.allegro.tech.hermes.consumers.consumer.batch; -import pl.allegro.tech.hermes.tracker.consumers.MessageMetadata; - import java.util.List; +import pl.allegro.tech.hermes.tracker.consumers.MessageMetadata; public class MessageBatchingResult { - private final MessageBatch batch; - private final List discarded; + private final MessageBatch batch; + private final List discarded; - public MessageBatchingResult(MessageBatch batch, List discarded) { - this.batch = batch; - this.discarded = discarded; - } + public MessageBatchingResult(MessageBatch batch, List discarded) { + this.batch = batch; + this.discarded = discarded; + } - public MessageBatch getBatch() { - return batch; - } + public MessageBatch getBatch() { + return batch; + } - public List getDiscarded() { - return discarded; - } + public List getDiscarded() { + return discarded; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/converter/AvroToJsonMessageConverter.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/converter/AvroToJsonMessageConverter.java index 9f601733b9..fe9e4345c5 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/converter/AvroToJsonMessageConverter.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/converter/AvroToJsonMessageConverter.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.consumers.consumer.converter; +import static java.util.stream.Collectors.toList; +import static pl.allegro.tech.hermes.common.message.converter.AvroRecordToBytesConverter.bytesToRecord; +import static pl.allegro.tech.hermes.common.message.wrapper.AvroMetadataMarker.METADATA_MARKER; +import static pl.allegro.tech.hermes.consumers.consumer.Message.message; + import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.GenericRecordBuilder; @@ -8,42 +13,44 @@ import pl.allegro.tech.hermes.consumers.consumer.Message; import tech.allegro.schema.json2avro.converter.JsonAvroConverter; -import static java.util.stream.Collectors.toList; -import static pl.allegro.tech.hermes.common.message.converter.AvroRecordToBytesConverter.bytesToRecord; -import static pl.allegro.tech.hermes.common.message.wrapper.AvroMetadataMarker.METADATA_MARKER; -import static pl.allegro.tech.hermes.consumers.consumer.Message.message; - public class AvroToJsonMessageConverter implements MessageConverter { - private final JsonAvroConverter converter; - - public AvroToJsonMessageConverter() { - this.converter = new JsonAvroConverter(); - } - - @Override - public Message convert(Message message, Topic topic) { - return message() - .fromMessage(message) - .withContentType(ContentType.JSON) - .withData(converter.convertToJson(recordWithoutMetadata(message.getData(), message.getSchema().get().getSchema()))) - .withNoSchema() - .build(); - } - - private GenericRecord recordWithoutMetadata(byte [] data, Schema schema) { - GenericRecord original = bytesToRecord(data, schema); - Schema schemaWithoutMetadata = removeMetadataField(schema); - GenericRecordBuilder builder = new GenericRecordBuilder(schemaWithoutMetadata); - schemaWithoutMetadata.getFields().forEach(field -> builder.set(field, original.get(field.name()))); - return builder.build(); - } - - private Schema removeMetadataField(Schema schema) { - return Schema.createRecord( - schema.getFields().stream() - .filter(field -> !METADATA_MARKER.equals(field.name())) - .map(field -> new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultVal())) - .collect(toList())); - } + private final JsonAvroConverter converter; + + public AvroToJsonMessageConverter() { + this.converter = new JsonAvroConverter(); + } + + @Override + public Message convert(Message message, Topic topic) { + return message() + .fromMessage(message) + .withContentType(ContentType.JSON) + .withData( + converter.convertToJson( + recordWithoutMetadata( + message.getData(), message.getSchema().get().getSchema()))) + .withNoSchema() + .build(); + } + + private GenericRecord recordWithoutMetadata(byte[] data, Schema schema) { + GenericRecord original = bytesToRecord(data, schema); + Schema schemaWithoutMetadata = removeMetadataField(schema); + GenericRecordBuilder builder = new GenericRecordBuilder(schemaWithoutMetadata); + schemaWithoutMetadata + .getFields() + .forEach(field -> builder.set(field, original.get(field.name()))); + return builder.build(); + } + + private Schema removeMetadataField(Schema schema) { + return Schema.createRecord( + schema.getFields().stream() + .filter(field -> !METADATA_MARKER.equals(field.name())) + .map( + field -> + new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultVal())) + .collect(toList())); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/converter/DefaultMessageConverterResolver.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/converter/DefaultMessageConverterResolver.java index 6fe140fd11..c66ead004b 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/converter/DefaultMessageConverterResolver.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/converter/DefaultMessageConverterResolver.java @@ -6,21 +6,23 @@ public class DefaultMessageConverterResolver implements MessageConverterResolver { - private final AvroToJsonMessageConverter avroToJsonMessageConverter; - private final NoOperationMessageConverter noOperationMessageConverter; + private final AvroToJsonMessageConverter avroToJsonMessageConverter; + private final NoOperationMessageConverter noOperationMessageConverter; - public DefaultMessageConverterResolver(AvroToJsonMessageConverter avroToJsonMessageConverter, - NoOperationMessageConverter noOperationMessageConverter) { - this.avroToJsonMessageConverter = avroToJsonMessageConverter; - this.noOperationMessageConverter = noOperationMessageConverter; - } - - @Override - public MessageConverter converterFor(Message message, Subscription subscription) { - if (message.getContentType() == ContentType.AVRO && subscription.getContentType() == ContentType.JSON) { - return avroToJsonMessageConverter; - } + public DefaultMessageConverterResolver( + AvroToJsonMessageConverter avroToJsonMessageConverter, + NoOperationMessageConverter noOperationMessageConverter) { + this.avroToJsonMessageConverter = avroToJsonMessageConverter; + this.noOperationMessageConverter = noOperationMessageConverter; + } - return noOperationMessageConverter; + @Override + public MessageConverter converterFor(Message message, Subscription subscription) { + if (message.getContentType() == ContentType.AVRO + && subscription.getContentType() == ContentType.JSON) { + return avroToJsonMessageConverter; } + + return noOperationMessageConverter; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/converter/MessageConverter.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/converter/MessageConverter.java index 4df96cfef8..769e99a9c2 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/converter/MessageConverter.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/converter/MessageConverter.java @@ -5,6 +5,5 @@ public interface MessageConverter { - Message convert(Message message, Topic topic); - + Message convert(Message message, Topic topic); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/converter/MessageConverterResolver.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/converter/MessageConverterResolver.java index 4d8e2b20b5..e48674a233 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/converter/MessageConverterResolver.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/converter/MessageConverterResolver.java @@ -4,5 +4,5 @@ import pl.allegro.tech.hermes.consumers.consumer.Message; public interface MessageConverterResolver { - MessageConverter converterFor(Message message, Subscription subscription); + MessageConverter converterFor(Message message, Subscription subscription); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/converter/NoOperationMessageConverter.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/converter/NoOperationMessageConverter.java index 925d8746cb..a5a93e3c97 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/converter/NoOperationMessageConverter.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/converter/NoOperationMessageConverter.java @@ -4,8 +4,8 @@ import pl.allegro.tech.hermes.consumers.consumer.Message; public class NoOperationMessageConverter implements MessageConverter { - @Override - public Message convert(Message message, Topic topic) { - return message; - } + @Override + public Message convert(Message message, Topic topic) { + return message; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/filtering/FilteredMessageHandler.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/filtering/FilteredMessageHandler.java index 58af3a135e..0d7eac9a00 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/filtering/FilteredMessageHandler.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/filtering/FilteredMessageHandler.java @@ -1,5 +1,9 @@ package pl.allegro.tech.hermes.consumers.consumer.filtering; +import static pl.allegro.tech.hermes.consumers.consumer.message.MessageConverter.toMessageMetadata; +import static pl.allegro.tech.hermes.consumers.consumer.offset.SubscriptionPartitionOffset.subscriptionPartitionOffset; + +import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.Subscription; @@ -12,47 +16,49 @@ import pl.allegro.tech.hermes.metrics.HermesCounter; import pl.allegro.tech.hermes.tracker.consumers.Trackers; -import java.util.Optional; +public class FilteredMessageHandler { -import static pl.allegro.tech.hermes.consumers.consumer.message.MessageConverter.toMessageMetadata; -import static pl.allegro.tech.hermes.consumers.consumer.offset.SubscriptionPartitionOffset.subscriptionPartitionOffset; + private final PendingOffsetsAppender pendingOffsets; + private final Optional consumerRateLimiter; + private final Trackers trackers; + private final HermesCounter filteredOutCounter; -public class FilteredMessageHandler { + private static final Logger logger = LoggerFactory.getLogger(FilteredMessageHandler.class); - private final PendingOffsetsAppender pendingOffsets; - private final Optional consumerRateLimiter; - private final Trackers trackers; - private final HermesCounter filteredOutCounter; - - private static final Logger logger = LoggerFactory.getLogger(FilteredMessageHandler.class); - - public FilteredMessageHandler(ConsumerRateLimiter consumerRateLimiter, - PendingOffsetsAppender pendingOffsetsAppender, - Trackers trackers, - MetricsFacade metrics, - SubscriptionName subscriptionName) { - this.consumerRateLimiter = Optional.ofNullable(consumerRateLimiter); - this.pendingOffsets = pendingOffsetsAppender; - this.trackers = trackers; - this.filteredOutCounter = metrics.subscriptions().filteredOutCounter(subscriptionName); - } + public FilteredMessageHandler( + ConsumerRateLimiter consumerRateLimiter, + PendingOffsetsAppender pendingOffsetsAppender, + Trackers trackers, + MetricsFacade metrics, + SubscriptionName subscriptionName) { + this.consumerRateLimiter = Optional.ofNullable(consumerRateLimiter); + this.pendingOffsets = pendingOffsetsAppender; + this.trackers = trackers; + this.filteredOutCounter = metrics.subscriptions().filteredOutCounter(subscriptionName); + } - public void handle(FilterResult result, Message message, Subscription subscription) { - if (result.isFiltered()) { - if (logger.isDebugEnabled()) { - logger.debug("Message filtered for subscription {} {}", subscription.getQualifiedName(), result); - } + public void handle(FilterResult result, Message message, Subscription subscription) { + if (result.isFiltered()) { + if (logger.isDebugEnabled()) { + logger.debug( + "Message filtered for subscription {} {}", subscription.getQualifiedName(), result); + } - pendingOffsets.markAsProcessed(subscriptionPartitionOffset(subscription.getQualifiedName(), - message.getPartitionOffset(), message.getPartitionAssignmentTerm())); + pendingOffsets.markAsProcessed( + subscriptionPartitionOffset( + subscription.getQualifiedName(), + message.getPartitionOffset(), + message.getPartitionAssignmentTerm())); - filteredOutCounter.increment(); + filteredOutCounter.increment(); - if (subscription.isTrackingEnabled()) { - trackers.get(subscription).logFiltered(toMessageMetadata(message, subscription), result.getFilterType().get()); - } + if (subscription.isTrackingEnabled()) { + trackers + .get(subscription) + .logFiltered(toMessageMetadata(message, subscription), result.getFilterType().get()); + } - consumerRateLimiter.ifPresent(ConsumerRateLimiter::acquireFiltered); - } + consumerRateLimiter.ifPresent(ConsumerRateLimiter::acquireFiltered); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/idletime/ExponentiallyGrowingIdleTimeCalculator.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/idletime/ExponentiallyGrowingIdleTimeCalculator.java index 5bb2b543db..3b11b5b6a8 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/idletime/ExponentiallyGrowingIdleTimeCalculator.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/idletime/ExponentiallyGrowingIdleTimeCalculator.java @@ -5,41 +5,44 @@ public class ExponentiallyGrowingIdleTimeCalculator implements IdleTimeCalculator { - private final long base; - private final long initialIdleTimeMs; - private final long maxIdleTimeMs; - private long currentIdleTimeMs; - - public ExponentiallyGrowingIdleTimeCalculator(long initialIdleTimeMs, long maxIdleTimeMs) { - this(2, initialIdleTimeMs, maxIdleTimeMs); - } - - public ExponentiallyGrowingIdleTimeCalculator(long base, long initialIdleTime, long maxIdleTimeMs) { - checkArgument(base > 0, "base should be greater than zero"); - checkArgument(initialIdleTime > 0, "initialIdleTimeMs should be greater than zero"); - checkArgument(maxIdleTimeMs > 0, "maxIdleTimeMs should be greater than zero"); - checkArgument(initialIdleTime <= maxIdleTimeMs, "maxIdleTimeMs should be grater or equal initialIdleTimeMs"); - - this.base = base; - this.initialIdleTimeMs = initialIdleTime; - this.maxIdleTimeMs = maxIdleTimeMs; - this.currentIdleTimeMs = initialIdleTime; - } - - @Override - public long increaseIdleTime() { - long previousIdleTime = this.currentIdleTimeMs; - this.currentIdleTimeMs = min(this.currentIdleTimeMs * base, maxIdleTimeMs); - return previousIdleTime; - } - - @Override - public long getIdleTime() { - return currentIdleTimeMs; - } - - @Override - public void reset() { - this.currentIdleTimeMs = initialIdleTimeMs; - } + private final long base; + private final long initialIdleTimeMs; + private final long maxIdleTimeMs; + private long currentIdleTimeMs; + + public ExponentiallyGrowingIdleTimeCalculator(long initialIdleTimeMs, long maxIdleTimeMs) { + this(2, initialIdleTimeMs, maxIdleTimeMs); + } + + public ExponentiallyGrowingIdleTimeCalculator( + long base, long initialIdleTime, long maxIdleTimeMs) { + checkArgument(base > 0, "base should be greater than zero"); + checkArgument(initialIdleTime > 0, "initialIdleTimeMs should be greater than zero"); + checkArgument(maxIdleTimeMs > 0, "maxIdleTimeMs should be greater than zero"); + checkArgument( + initialIdleTime <= maxIdleTimeMs, + "maxIdleTimeMs should be grater or equal initialIdleTimeMs"); + + this.base = base; + this.initialIdleTimeMs = initialIdleTime; + this.maxIdleTimeMs = maxIdleTimeMs; + this.currentIdleTimeMs = initialIdleTime; + } + + @Override + public long increaseIdleTime() { + long previousIdleTime = this.currentIdleTimeMs; + this.currentIdleTimeMs = min(this.currentIdleTimeMs * base, maxIdleTimeMs); + return previousIdleTime; + } + + @Override + public long getIdleTime() { + return currentIdleTimeMs; + } + + @Override + public void reset() { + this.currentIdleTimeMs = initialIdleTimeMs; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/idletime/IdleTimeCalculator.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/idletime/IdleTimeCalculator.java index aedc53f69a..c057dfa06b 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/idletime/IdleTimeCalculator.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/idletime/IdleTimeCalculator.java @@ -1,9 +1,9 @@ package pl.allegro.tech.hermes.consumers.consumer.idletime; public interface IdleTimeCalculator { - long increaseIdleTime(); + long increaseIdleTime(); - long getIdleTime(); + long getIdleTime(); - void reset(); + void reset(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/interpolation/InterpolationException.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/interpolation/InterpolationException.java index 016de70fe5..670a281e4a 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/interpolation/InterpolationException.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/interpolation/InterpolationException.java @@ -1,7 +1,7 @@ package pl.allegro.tech.hermes.consumers.consumer.interpolation; public class InterpolationException extends Exception { - public InterpolationException(String format, Throwable t) { - super(format, t); - } + public InterpolationException(String format, Throwable t) { + super(format, t); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/interpolation/MessageBodyInterpolator.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/interpolation/MessageBodyInterpolator.java index 64cc0fb6df..cc0dc4c822 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/interpolation/MessageBodyInterpolator.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/interpolation/MessageBodyInterpolator.java @@ -9,72 +9,66 @@ import com.google.common.collect.Maps; import com.jayway.jsonpath.InvalidPathException; import com.jayway.jsonpath.JsonPath; -import pl.allegro.tech.hermes.api.EndpointAddress; -import pl.allegro.tech.hermes.consumers.consumer.Message; - import java.net.URI; import java.util.Map; +import pl.allegro.tech.hermes.api.EndpointAddress; +import pl.allegro.tech.hermes.consumers.consumer.Message; public class MessageBodyInterpolator implements UriInterpolator { - private static final int MAX_CACHE_SIZE = 1000; - - private final LoadingCache templateCache = CacheBuilder - .newBuilder() - .maximumSize(MAX_CACHE_SIZE) - .build(new TemplateLoader()); + private static final int MAX_CACHE_SIZE = 1000; - private final LoadingCache variableCompiler = CacheBuilder - .newBuilder() - .maximumSize(MAX_CACHE_SIZE) - .build(new JsonPathLoader()); + private final LoadingCache templateCache = + CacheBuilder.newBuilder().maximumSize(MAX_CACHE_SIZE).build(new TemplateLoader()); - public URI interpolate(EndpointAddress endpoint, Message message) throws InterpolationException { - UriTemplate template = templateCache.getUnchecked(endpoint.getEndpoint()); - String[] variables = template.getVariables(); + private final LoadingCache variableCompiler = + CacheBuilder.newBuilder().maximumSize(MAX_CACHE_SIZE).build(new JsonPathLoader()); - if (variables.length > 0) { - Map values = Maps.newHashMap(); - String payload = new String(message.getData(), Charsets.UTF_8); + public URI interpolate(EndpointAddress endpoint, Message message) throws InterpolationException { + UriTemplate template = templateCache.getUnchecked(endpoint.getEndpoint()); + String[] variables = template.getVariables(); - for (String variable : variables) { + if (variables.length > 0) { + Map values = Maps.newHashMap(); + String payload = new String(message.getData(), Charsets.UTF_8); - JsonPath path = variableCompiler.getUnchecked(variable); + for (String variable : variables) { - try { - values.put(variable, path.read(payload)); - } catch (InvalidPathException e) { - throw new InterpolationException(String.format("Missing variable on path %s", path.getPath()), e); - } - } + JsonPath path = variableCompiler.getUnchecked(variable); - try { - return URI.create(template.expand(values)); - } catch (VariableExpansionException e) { - throw new InterpolationException("Cannot expand template", e); - } + try { + values.put(variable, path.read(payload)); + } catch (InvalidPathException e) { + throw new InterpolationException( + String.format("Missing variable on path %s", path.getPath()), e); } + } - return endpoint.getUri(); - + try { + return URI.create(template.expand(values)); + } catch (VariableExpansionException e) { + throw new InterpolationException("Cannot expand template", e); + } } - private static class TemplateLoader extends CacheLoader { + return endpoint.getUri(); + } - @Override - public UriTemplate load(String url) throws Exception { - return UriTemplate.fromTemplate(url); - } + private static class TemplateLoader extends CacheLoader { + + @Override + public UriTemplate load(String url) throws Exception { + return UriTemplate.fromTemplate(url); } + } - private static class JsonPathLoader extends CacheLoader { + private static class JsonPathLoader extends CacheLoader { - private static final String ROOT_PREFIX = "$."; + private static final String ROOT_PREFIX = "$."; - @Override - public JsonPath load(String key) throws Exception { - return JsonPath.compile(ROOT_PREFIX + key); - } + @Override + public JsonPath load(String key) throws Exception { + return JsonPath.compile(ROOT_PREFIX + key); } - + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/interpolation/UriInterpolator.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/interpolation/UriInterpolator.java index f284e924e3..57a2e354cd 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/interpolation/UriInterpolator.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/interpolation/UriInterpolator.java @@ -1,10 +1,9 @@ package pl.allegro.tech.hermes.consumers.consumer.interpolation; +import java.net.URI; import pl.allegro.tech.hermes.api.EndpointAddress; import pl.allegro.tech.hermes.consumers.consumer.Message; -import java.net.URI; - public interface UriInterpolator { - URI interpolate(EndpointAddress endpoint, Message message) throws InterpolationException; + URI interpolate(EndpointAddress endpoint, Message message) throws InterpolationException; } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/load/SubscriptionLoadRecorder.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/load/SubscriptionLoadRecorder.java index 95cbb53c33..4d05e7d213 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/load/SubscriptionLoadRecorder.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/load/SubscriptionLoadRecorder.java @@ -2,9 +2,9 @@ public interface SubscriptionLoadRecorder { - void initialize(); + void initialize(); - void recordSingleOperation(); + void recordSingleOperation(); - void shutdown(); + void shutdown(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/load/SubscriptionLoadRecordersRegistry.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/load/SubscriptionLoadRecordersRegistry.java index 62361821c9..05d29b6560 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/load/SubscriptionLoadRecordersRegistry.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/load/SubscriptionLoadRecordersRegistry.java @@ -4,5 +4,5 @@ public interface SubscriptionLoadRecordersRegistry { - SubscriptionLoadRecorder register(SubscriptionName subscriptionName); + SubscriptionLoadRecorder register(SubscriptionName subscriptionName); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/message/MessageConverter.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/message/MessageConverter.java index 92c94dd2ce..fd723f8917 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/message/MessageConverter.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/message/MessageConverter.java @@ -6,28 +6,31 @@ public class MessageConverter { - public static MessageMetadata toMessageMetadata(Message message, Subscription subscription) { - return new MessageMetadata(message.getId(), - message.getOffset(), - message.getPartition(), - message.getPartitionAssignmentTerm(), - message.getTopic(), - subscription.getName(), - message.getKafkaTopic().asString(), - message.getPublishingTimestamp(), - message.getReadingTimestamp()); - } + public static MessageMetadata toMessageMetadata(Message message, Subscription subscription) { + return new MessageMetadata( + message.getId(), + message.getOffset(), + message.getPartition(), + message.getPartitionAssignmentTerm(), + message.getTopic(), + subscription.getName(), + message.getKafkaTopic().asString(), + message.getPublishingTimestamp(), + message.getReadingTimestamp()); + } - public static MessageMetadata toMessageMetadata(Message message, Subscription subscription, String batchId) { - return new MessageMetadata(message.getId(), - batchId, - message.getOffset(), - message.getPartition(), - message.getPartitionAssignmentTerm(), - subscription.getQualifiedTopicName(), - subscription.getName(), - message.getKafkaTopic().asString(), - message.getPublishingTimestamp(), - message.getReadingTimestamp()); - } + public static MessageMetadata toMessageMetadata( + Message message, Subscription subscription, String batchId) { + return new MessageMetadata( + message.getId(), + batchId, + message.getOffset(), + message.getPartition(), + message.getPartitionAssignmentTerm(), + subscription.getQualifiedTopicName(), + subscription.getName(), + message.getKafkaTopic().asString(), + message.getPublishingTimestamp(), + message.getReadingTimestamp()); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthAccessToken.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthAccessToken.java index 481bf49b8a..f4d082f517 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthAccessToken.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthAccessToken.java @@ -4,38 +4,37 @@ public class OAuthAccessToken { - private final String tokenValue; + private final String tokenValue; - private final Integer expiresIn; + private final Integer expiresIn; - public OAuthAccessToken(String tokenValue, Integer expiresIn) { - this.tokenValue = tokenValue; - this.expiresIn = expiresIn; - } + public OAuthAccessToken(String tokenValue, Integer expiresIn) { + this.tokenValue = tokenValue; + this.expiresIn = expiresIn; + } - public String getTokenValue() { - return tokenValue; - } + public String getTokenValue() { + return tokenValue; + } - public Integer getExpiresIn() { - return expiresIn; - } + public Integer getExpiresIn() { + return expiresIn; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - OAuthAccessToken that = (OAuthAccessToken) o; - return Objects.equals(tokenValue, that.tokenValue) - && Objects.equals(expiresIn, that.expiresIn); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(tokenValue, expiresIn); + if (o == null || getClass() != o.getClass()) { + return false; } + OAuthAccessToken that = (OAuthAccessToken) o; + return Objects.equals(tokenValue, that.tokenValue) && Objects.equals(expiresIn, that.expiresIn); + } + + @Override + public int hashCode() { + return Objects.hash(tokenValue, expiresIn); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthAccessTokens.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthAccessTokens.java index 444a937f45..bd3f067615 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthAccessTokens.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthAccessTokens.java @@ -1,16 +1,15 @@ package pl.allegro.tech.hermes.consumers.consumer.oauth; -import pl.allegro.tech.hermes.api.SubscriptionName; - import java.util.Optional; +import pl.allegro.tech.hermes.api.SubscriptionName; public interface OAuthAccessTokens { - Optional loadToken(SubscriptionName subscription); + Optional loadToken(SubscriptionName subscription); - Optional getTokenIfPresent(SubscriptionName subscription); + Optional getTokenIfPresent(SubscriptionName subscription); - void refreshToken(SubscriptionName subscription); + void refreshToken(SubscriptionName subscription); - boolean tokenExists(SubscriptionName subscription); + boolean tokenExists(SubscriptionName subscription); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthAccessTokensLoader.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthAccessTokensLoader.java index 40fd06cd0a..649220ec62 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthAccessTokensLoader.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthAccessTokensLoader.java @@ -1,5 +1,8 @@ package pl.allegro.tech.hermes.consumers.consumer.oauth; +import static pl.allegro.tech.hermes.api.SubscriptionOAuthPolicy.GrantType.USERNAME_PASSWORD; +import static pl.allegro.tech.hermes.consumers.consumer.oauth.client.OAuthTokenRequest.oAuthTokenRequest; + import com.google.common.cache.CacheLoader; import pl.allegro.tech.hermes.api.OAuthProvider; import pl.allegro.tech.hermes.api.Subscription; @@ -12,71 +15,72 @@ import pl.allegro.tech.hermes.domain.subscription.SubscriptionRepository; import pl.allegro.tech.hermes.metrics.HermesTimerContext; -import static pl.allegro.tech.hermes.api.SubscriptionOAuthPolicy.GrantType.USERNAME_PASSWORD; -import static pl.allegro.tech.hermes.consumers.consumer.oauth.client.OAuthTokenRequest.oAuthTokenRequest; - public class OAuthAccessTokensLoader extends CacheLoader { - private final SubscriptionRepository subscriptionRepository; - - private final OAuthProviderRepository oAuthProviderRepository; + private final SubscriptionRepository subscriptionRepository; - private final OAuthClient oAuthClient; + private final OAuthProviderRepository oAuthProviderRepository; - private final MetricsFacade metrics; + private final OAuthClient oAuthClient; - public OAuthAccessTokensLoader(SubscriptionRepository subscriptionRepository, - OAuthProviderRepository oAuthProviderRepository, - OAuthClient oAuthClient, - MetricsFacade metrics) { - this.subscriptionRepository = subscriptionRepository; - this.oAuthProviderRepository = oAuthProviderRepository; - this.oAuthClient = oAuthClient; - this.metrics = metrics; - } + private final MetricsFacade metrics; - @Override - public OAuthAccessToken load(SubscriptionName subscriptionName) throws Exception { - Subscription subscription = subscriptionRepository.getSubscriptionDetails(subscriptionName); - SubscriptionOAuthPolicy oAuthPolicy = subscription.getOAuthPolicy(); - String providerName = oAuthPolicy.getProviderName(); - OAuthProvider oAuthProvider = oAuthProviderRepository.getOAuthProviderDetails(providerName); - OAuthTokenRequest request; - if (USERNAME_PASSWORD.equals(oAuthPolicy.getGrantType())) { - request = getOAuthUsernamePasswordGrantTokenRequest(oAuthPolicy, oAuthProvider); - } else { - request = getOAuthClientCredentialsGrantTokenRequest(oAuthPolicy, oAuthProvider); - } + public OAuthAccessTokensLoader( + SubscriptionRepository subscriptionRepository, + OAuthProviderRepository oAuthProviderRepository, + OAuthClient oAuthClient, + MetricsFacade metrics) { + this.subscriptionRepository = subscriptionRepository; + this.oAuthProviderRepository = oAuthProviderRepository; + this.oAuthClient = oAuthClient; + this.metrics = metrics; + } - metrics.consumer().oAuthSubscriptionTokenRequestCounter(subscription, providerName).increment(); - try (HermesTimerContext ignored = metrics.consumer().oAuthProviderLatencyTimer(providerName).time()) { - return oAuthClient.getToken(request); - } + @Override + public OAuthAccessToken load(SubscriptionName subscriptionName) throws Exception { + Subscription subscription = subscriptionRepository.getSubscriptionDetails(subscriptionName); + SubscriptionOAuthPolicy oAuthPolicy = subscription.getOAuthPolicy(); + String providerName = oAuthPolicy.getProviderName(); + OAuthProvider oAuthProvider = oAuthProviderRepository.getOAuthProviderDetails(providerName); + OAuthTokenRequest request; + if (USERNAME_PASSWORD.equals(oAuthPolicy.getGrantType())) { + request = getOAuthUsernamePasswordGrantTokenRequest(oAuthPolicy, oAuthProvider); + } else { + request = getOAuthClientCredentialsGrantTokenRequest(oAuthPolicy, oAuthProvider); } - private OAuthTokenRequest getOAuthUsernamePasswordGrantTokenRequest(SubscriptionOAuthPolicy policy, OAuthProvider provider) { - return oAuthTokenRequest() - .withUrl(provider.getTokenEndpoint()) - .withGrantType(OAuthTokenRequest.GrantTypeValue.RESOURCE_OWNER_USERNAME_PASSWORD) - .withScope(policy.getScope()) - .withClientId(provider.getClientId()) - .withClientSecret(provider.getClientSecret()) - .withUsername(policy.getUsername()) - .withPassword(policy.getPassword()) - .withRequestTimeout(provider.getRequestTimeout()) - .withSocketTimeout(provider.getSocketTimeout()) - .build(); + metrics.consumer().oAuthSubscriptionTokenRequestCounter(subscription, providerName).increment(); + try (HermesTimerContext ignored = + metrics.consumer().oAuthProviderLatencyTimer(providerName).time()) { + return oAuthClient.getToken(request); } + } - private OAuthTokenRequest getOAuthClientCredentialsGrantTokenRequest(SubscriptionOAuthPolicy policy, OAuthProvider provider) { - return oAuthTokenRequest() - .withUrl(provider.getTokenEndpoint()) - .withGrantType(OAuthTokenRequest.GrantTypeValue.CLIENT_CREDENTIALS) - .withScope(policy.getScope()) - .withClientId(provider.getClientId()) - .withClientSecret(provider.getClientSecret()) - .withRequestTimeout(provider.getRequestTimeout()) - .withSocketTimeout(provider.getSocketTimeout()) - .build(); - } + private OAuthTokenRequest getOAuthUsernamePasswordGrantTokenRequest( + SubscriptionOAuthPolicy policy, OAuthProvider provider) { + return oAuthTokenRequest() + .withUrl(provider.getTokenEndpoint()) + .withGrantType(OAuthTokenRequest.GrantTypeValue.RESOURCE_OWNER_USERNAME_PASSWORD) + .withScope(policy.getScope()) + .withClientId(provider.getClientId()) + .withClientSecret(provider.getClientSecret()) + .withUsername(policy.getUsername()) + .withPassword(policy.getPassword()) + .withRequestTimeout(provider.getRequestTimeout()) + .withSocketTimeout(provider.getSocketTimeout()) + .build(); + } + + private OAuthTokenRequest getOAuthClientCredentialsGrantTokenRequest( + SubscriptionOAuthPolicy policy, OAuthProvider provider) { + return oAuthTokenRequest() + .withUrl(provider.getTokenEndpoint()) + .withGrantType(OAuthTokenRequest.GrantTypeValue.CLIENT_CREDENTIALS) + .withScope(policy.getScope()) + .withClientId(provider.getClientId()) + .withClientSecret(provider.getClientSecret()) + .withRequestTimeout(provider.getRequestTimeout()) + .withSocketTimeout(provider.getSocketTimeout()) + .build(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthConsumerAuthorizationHandler.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthConsumerAuthorizationHandler.java index 4319e181af..988437450d 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthConsumerAuthorizationHandler.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthConsumerAuthorizationHandler.java @@ -1,6 +1,12 @@ package pl.allegro.tech.hermes.consumers.consumer.oauth; import com.google.common.util.concurrent.RateLimiter; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.OAuthProvider; @@ -10,98 +16,106 @@ import pl.allegro.tech.hermes.consumers.consumer.Message; import pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult; -import java.time.Duration; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; - -public class OAuthConsumerAuthorizationHandler implements ConsumerAuthorizationHandler, OAuthProviderCacheListener { - - private static final Logger logger = LoggerFactory.getLogger(OAuthConsumerAuthorizationHandler.class); - - private final OAuthSubscriptionHandlerFactory handlerFactory; - - private final Map handlers = new ConcurrentHashMap<>(); - - private final RateLimiter missingHandlersCreationRateLimiter; - - public OAuthConsumerAuthorizationHandler(OAuthSubscriptionHandlerFactory handlerFactory, - Duration missingSubscriptionHandlersCreationDelay, - OAuthProvidersNotifyingCache oAuthProvidersCache) { - this.handlerFactory = handlerFactory; - this.missingHandlersCreationRateLimiter = RateLimiter.create(missingSubscriptionHandlersCreationDelay.toSeconds()); - oAuthProvidersCache.setListener(this); - } - - @Override - public void createSubscriptionHandler(SubscriptionName subscriptionName) { - handlerFactory.create(subscriptionName).ifPresent(handler -> { - logger.info("OAuth handler for subscription {} created", subscriptionName); - handlers.put(subscriptionName, handler); - handler.initialize(); - }); - } - - @Override - public void removeSubscriptionHandler(SubscriptionName subscriptionName) { - if (handlers.remove(subscriptionName) != null) { - logger.info("OAuth handler for subscription {} removed", subscriptionName); - } - } - - @Override - public void updateSubscription(SubscriptionName subscriptionName) { - removeSubscriptionHandler(subscriptionName); - createSubscriptionHandler(subscriptionName); +public class OAuthConsumerAuthorizationHandler + implements ConsumerAuthorizationHandler, OAuthProviderCacheListener { + + private static final Logger logger = + LoggerFactory.getLogger(OAuthConsumerAuthorizationHandler.class); + + private final OAuthSubscriptionHandlerFactory handlerFactory; + + private final Map handlers = + new ConcurrentHashMap<>(); + + private final RateLimiter missingHandlersCreationRateLimiter; + + public OAuthConsumerAuthorizationHandler( + OAuthSubscriptionHandlerFactory handlerFactory, + Duration missingSubscriptionHandlersCreationDelay, + OAuthProvidersNotifyingCache oAuthProvidersCache) { + this.handlerFactory = handlerFactory; + this.missingHandlersCreationRateLimiter = + RateLimiter.create(missingSubscriptionHandlersCreationDelay.toSeconds()); + oAuthProvidersCache.setListener(this); + } + + @Override + public void createSubscriptionHandler(SubscriptionName subscriptionName) { + handlerFactory + .create(subscriptionName) + .ifPresent( + handler -> { + logger.info("OAuth handler for subscription {} created", subscriptionName); + handlers.put(subscriptionName, handler); + handler.initialize(); + }); + } + + @Override + public void removeSubscriptionHandler(SubscriptionName subscriptionName) { + if (handlers.remove(subscriptionName) != null) { + logger.info("OAuth handler for subscription {} removed", subscriptionName); } - - @Override - public void oAuthProviderUpdate(OAuthProvider oAuthProvider) { - logger.info("Updated OAuth provider {}", oAuthProvider.getName()); - List subscriptions = handlers.entrySet().stream() - .filter(entry -> entry.getValue().getProviderName().equals(oAuthProvider.getName())) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); - - subscriptions.forEach(this::updateSubscription); - } - - @Override - public void handleSuccess(Message message, Subscription subscription, MessageSendingResult result) { - if (shouldHandle(subscription)) { - getSubscriptionHandler(subscription.getQualifiedName()).ifPresent(OAuthSubscriptionHandler::handleSuccess); - } + } + + @Override + public void updateSubscription(SubscriptionName subscriptionName) { + removeSubscriptionHandler(subscriptionName); + createSubscriptionHandler(subscriptionName); + } + + @Override + public void oAuthProviderUpdate(OAuthProvider oAuthProvider) { + logger.info("Updated OAuth provider {}", oAuthProvider.getName()); + List subscriptions = + handlers.entrySet().stream() + .filter(entry -> entry.getValue().getProviderName().equals(oAuthProvider.getName())) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + + subscriptions.forEach(this::updateSubscription); + } + + @Override + public void handleSuccess( + Message message, Subscription subscription, MessageSendingResult result) { + if (shouldHandle(subscription)) { + getSubscriptionHandler(subscription.getQualifiedName()) + .ifPresent(OAuthSubscriptionHandler::handleSuccess); } - - @Override - public void handleFailed(Message message, Subscription subscription, MessageSendingResult result) { - if (shouldHandle(subscription)) { - getSubscriptionHandler(subscription.getQualifiedName()).ifPresent(h -> h.handleFailed(subscription, result)); - } + } + + @Override + public void handleFailed( + Message message, Subscription subscription, MessageSendingResult result) { + if (shouldHandle(subscription)) { + getSubscriptionHandler(subscription.getQualifiedName()) + .ifPresent(h -> h.handleFailed(subscription, result)); } + } - @Override - public void handleDiscarded(Message message, Subscription subscription, MessageSendingResult result) { - } + @Override + public void handleDiscarded( + Message message, Subscription subscription, MessageSendingResult result) {} - private boolean shouldHandle(Subscription subscription) { - return subscription.hasOAuthPolicy(); - } + private boolean shouldHandle(Subscription subscription) { + return subscription.hasOAuthPolicy(); + } - private Optional getSubscriptionHandler(SubscriptionName subscriptionName) { - if (handlers.containsKey(subscriptionName)) { - return Optional.of(handlers.get(subscriptionName)); - } - return tryCreatingMissingHandler(subscriptionName); + private Optional getSubscriptionHandler( + SubscriptionName subscriptionName) { + if (handlers.containsKey(subscriptionName)) { + return Optional.of(handlers.get(subscriptionName)); } - - private Optional tryCreatingMissingHandler(SubscriptionName subscriptionName) { - if (missingHandlersCreationRateLimiter.tryAcquire()) { - createSubscriptionHandler(subscriptionName); - return Optional.ofNullable(handlers.get(subscriptionName)); - } - return Optional.empty(); + return tryCreatingMissingHandler(subscriptionName); + } + + private Optional tryCreatingMissingHandler( + SubscriptionName subscriptionName) { + if (missingHandlersCreationRateLimiter.tryAcquire()) { + createSubscriptionHandler(subscriptionName); + return Optional.ofNullable(handlers.get(subscriptionName)); } + return Optional.empty(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthProviderCacheListener.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthProviderCacheListener.java index 53de439970..872de43813 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthProviderCacheListener.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthProviderCacheListener.java @@ -4,5 +4,5 @@ public interface OAuthProviderCacheListener { - void oAuthProviderUpdate(OAuthProvider oAuthProvider); + void oAuthProviderUpdate(OAuthProvider oAuthProvider); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthProvidersNotifyingCache.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthProvidersNotifyingCache.java index bcbf22958c..d8019e4b82 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthProvidersNotifyingCache.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthProvidersNotifyingCache.java @@ -1,6 +1,9 @@ package pl.allegro.tech.hermes.consumers.consumer.oauth; import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.util.Optional; +import java.util.concurrent.ExecutorService; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCache; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; @@ -9,50 +12,51 @@ import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.OAuthProvider; -import java.io.IOException; -import java.util.Optional; -import java.util.concurrent.ExecutorService; +public class OAuthProvidersNotifyingCache extends PathChildrenCache + implements PathChildrenCacheListener { -public class OAuthProvidersNotifyingCache extends PathChildrenCache implements PathChildrenCacheListener { + private static final Logger logger = LoggerFactory.getLogger(OAuthProvidersNotifyingCache.class); - private static final Logger logger = LoggerFactory.getLogger(OAuthProvidersNotifyingCache.class); + private final ObjectMapper objectMapper; - private final ObjectMapper objectMapper; + private OAuthProviderCacheListener listener; - private OAuthProviderCacheListener listener; + public OAuthProvidersNotifyingCache( + CuratorFramework curator, + String path, + ExecutorService executorService, + ObjectMapper objectMapper) { + super(curator, path, true, false, executorService); + this.objectMapper = objectMapper; + getListenable().addListener(this); + } - public OAuthProvidersNotifyingCache(CuratorFramework curator, String path, ExecutorService executorService, - ObjectMapper objectMapper) { - super(curator, path, true, false, executorService); - this.objectMapper = objectMapper; - getListenable().addListener(this); - } + @Override + public void start() throws Exception { + super.start(); + } - @Override - public void start() throws Exception { - super.start(); - } + public void setListener(OAuthProviderCacheListener listener) { + this.listener = listener; + } - public void setListener(OAuthProviderCacheListener listener) { - this.listener = listener; + @Override + public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception { + if (event.getData() == null || event.getData().getData() == null) { + return; } - - @Override - public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception { - if (event.getData() == null || event.getData().getData() == null) { - return; - } - if (event.getType() == PathChildrenCacheEvent.Type.CHILD_UPDATED) { - parseEvent(event.getData().getPath(), event.getData().getData()).ifPresent(listener::oAuthProviderUpdate); - } + if (event.getType() == PathChildrenCacheEvent.Type.CHILD_UPDATED) { + parseEvent(event.getData().getPath(), event.getData().getData()) + .ifPresent(listener::oAuthProviderUpdate); } - - private Optional parseEvent(String path, byte[] data) { - try { - return Optional.of(objectMapper.readValue(data, OAuthProvider.class)); - } catch (IOException e) { - logger.error("Failed to parse object at path {}", path); - return Optional.empty(); - } + } + + private Optional parseEvent(String path, byte[] data) { + try { + return Optional.of(objectMapper.readValue(data, OAuthProvider.class)); + } catch (IOException e) { + logger.error("Failed to parse object at path {}", path); + return Optional.empty(); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthSubscriptionAccessTokens.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthSubscriptionAccessTokens.java index 827a6be454..d4dc152b12 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthSubscriptionAccessTokens.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthSubscriptionAccessTokens.java @@ -2,56 +2,56 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.LoadingCache; +import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.SubscriptionName; -import java.util.Optional; - public class OAuthSubscriptionAccessTokens implements OAuthAccessTokens { - private static final Logger logger = LoggerFactory.getLogger(OAuthSubscriptionAccessTokens.class); + private static final Logger logger = LoggerFactory.getLogger(OAuthSubscriptionAccessTokens.class); - private final LoadingCache subscriptionTokens; + private final LoadingCache subscriptionTokens; - private final OAuthAccessTokensLoader tokenLoader; + private final OAuthAccessTokensLoader tokenLoader; - public OAuthSubscriptionAccessTokens(OAuthAccessTokensLoader tokenLoader, - long subscriptionTokensCacheMaxSize) { - this.tokenLoader = tokenLoader; - this.subscriptionTokens = CacheBuilder.newBuilder() - .maximumSize(subscriptionTokensCacheMaxSize) - .build(tokenLoader); - } + public OAuthSubscriptionAccessTokens( + OAuthAccessTokensLoader tokenLoader, long subscriptionTokensCacheMaxSize) { + this.tokenLoader = tokenLoader; + this.subscriptionTokens = + CacheBuilder.newBuilder().maximumSize(subscriptionTokensCacheMaxSize).build(tokenLoader); + } - @Override - public Optional getTokenIfPresent(SubscriptionName subscriptionName) { - return Optional.ofNullable(subscriptionTokens.getIfPresent(subscriptionName)); - } + @Override + public Optional getTokenIfPresent(SubscriptionName subscriptionName) { + return Optional.ofNullable(subscriptionTokens.getIfPresent(subscriptionName)); + } - @Override - public Optional loadToken(SubscriptionName subscriptionName) { - try { - return Optional.ofNullable(subscriptionTokens.get(subscriptionName)); - } catch (Exception e) { - logger.error("Could not get access token for subscription {}", subscriptionName, e); - return Optional.empty(); - } + @Override + public Optional loadToken(SubscriptionName subscriptionName) { + try { + return Optional.ofNullable(subscriptionTokens.get(subscriptionName)); + } catch (Exception e) { + logger.error("Could not get access token for subscription {}", subscriptionName, e); + return Optional.empty(); } - - @Override - public void refreshToken(SubscriptionName subscriptionName) { - try { - OAuthAccessToken token = tokenLoader.load(subscriptionName); - subscriptionTokens.put(subscriptionName, token); - } catch (Exception e) { - logger.error("An error occurred while refreshing access token for subscription {}", - subscriptionName, e); - } + } + + @Override + public void refreshToken(SubscriptionName subscriptionName) { + try { + OAuthAccessToken token = tokenLoader.load(subscriptionName); + subscriptionTokens.put(subscriptionName, token); + } catch (Exception e) { + logger.error( + "An error occurred while refreshing access token for subscription {}", + subscriptionName, + e); } + } - @Override - public boolean tokenExists(SubscriptionName subscriptionName) { - return subscriptionTokens.getIfPresent(subscriptionName) != null; - } + @Override + public boolean tokenExists(SubscriptionName subscriptionName) { + return subscriptionTokens.getIfPresent(subscriptionName) != null; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthSubscriptionHandler.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthSubscriptionHandler.java index f13399a69c..66eb215c5f 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthSubscriptionHandler.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthSubscriptionHandler.java @@ -1,6 +1,10 @@ package pl.allegro.tech.hermes.consumers.consumer.oauth; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; import org.eclipse.jetty.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -8,61 +12,64 @@ import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; - public class OAuthSubscriptionHandler { - private static final Logger logger = LoggerFactory.getLogger(OAuthSubscriptionHandler.class); + private static final Logger logger = LoggerFactory.getLogger(OAuthSubscriptionHandler.class); - private final SubscriptionName subscriptionName; + private final SubscriptionName subscriptionName; - private final String providerName; + private final String providerName; - private final OAuthAccessTokens accessTokens; + private final OAuthAccessTokens accessTokens; - private final OAuthTokenRequestRateLimiter rateLimiter; + private final OAuthTokenRequestRateLimiter rateLimiter; - private final ScheduledExecutorService executorService; + private final ScheduledExecutorService executorService; - public OAuthSubscriptionHandler(SubscriptionName subscriptionName, String providerName, OAuthAccessTokens accessTokens, - OAuthTokenRequestRateLimiter rateLimiter) { - this.subscriptionName = subscriptionName; - this.providerName = providerName; - this.accessTokens = accessTokens; - this.rateLimiter = rateLimiter; - ThreadFactory threadFactory = - new ThreadFactoryBuilder().setNameFormat(subscriptionName.getQualifiedName() + "-oauth-handler-%d").build(); - this.executorService = Executors.newScheduledThreadPool(1, threadFactory); - } + public OAuthSubscriptionHandler( + SubscriptionName subscriptionName, + String providerName, + OAuthAccessTokens accessTokens, + OAuthTokenRequestRateLimiter rateLimiter) { + this.subscriptionName = subscriptionName; + this.providerName = providerName; + this.accessTokens = accessTokens; + this.rateLimiter = rateLimiter; + ThreadFactory threadFactory = + new ThreadFactoryBuilder() + .setNameFormat(subscriptionName.getQualifiedName() + "-oauth-handler-%d") + .build(); + this.executorService = Executors.newScheduledThreadPool(1, threadFactory); + } - public void initialize() { - rateLimiter.tryAcquire(); - accessTokens.loadToken(subscriptionName); - } + public void initialize() { + rateLimiter.tryAcquire(); + accessTokens.loadToken(subscriptionName); + } - public String getProviderName() { - return providerName; - } + public String getProviderName() { + return providerName; + } - public void handleSuccess() { - rateLimiter.resetRate(); - } + public void handleSuccess() { + rateLimiter.resetRate(); + } - public void handleFailed(Subscription subscription, MessageSendingResult result) { - SubscriptionName subscriptionName = subscription.getQualifiedName(); - if (shouldTryRefreshingToken(subscriptionName, result)) { - if (rateLimiter.tryAcquire()) { - logger.info("Refreshing token for subscription {}", subscriptionName); - rateLimiter.reduceRate(); - executorService.schedule(() -> accessTokens.refreshToken(subscriptionName), 0, TimeUnit.MILLISECONDS); - } - } + public void handleFailed(Subscription subscription, MessageSendingResult result) { + SubscriptionName subscriptionName = subscription.getQualifiedName(); + if (shouldTryRefreshingToken(subscriptionName, result)) { + if (rateLimiter.tryAcquire()) { + logger.info("Refreshing token for subscription {}", subscriptionName); + rateLimiter.reduceRate(); + executorService.schedule( + () -> accessTokens.refreshToken(subscriptionName), 0, TimeUnit.MILLISECONDS); + } } + } - private boolean shouldTryRefreshingToken(SubscriptionName subscriptionName, MessageSendingResult result) { - return result.getStatusCode() == HttpStatus.UNAUTHORIZED_401 || !accessTokens.tokenExists(subscriptionName); - } + private boolean shouldTryRefreshingToken( + SubscriptionName subscriptionName, MessageSendingResult result) { + return result.getStatusCode() == HttpStatus.UNAUTHORIZED_401 + || !accessTokens.tokenExists(subscriptionName); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthSubscriptionHandlerFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthSubscriptionHandlerFactory.java index 993dfb48dd..f223747b03 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthSubscriptionHandlerFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthSubscriptionHandlerFactory.java @@ -1,45 +1,52 @@ package pl.allegro.tech.hermes.consumers.consumer.oauth; +import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.domain.subscription.SubscriptionRepository; -import java.util.Optional; - public class OAuthSubscriptionHandlerFactory { - private static final Logger logger = LoggerFactory.getLogger(OAuthSubscriptionHandlerFactory.class); - - private final SubscriptionRepository subscriptionRepository; - - private final OAuthAccessTokens accessTokens; - - private final OAuthTokenRequestRateLimiterFactory rateLimiterLoader; - - public OAuthSubscriptionHandlerFactory(SubscriptionRepository subscriptionRepository, - OAuthAccessTokens accessTokens, - OAuthTokenRequestRateLimiterFactory rateLimiterLoader) { - this.subscriptionRepository = subscriptionRepository; - this.accessTokens = accessTokens; - this.rateLimiterLoader = rateLimiterLoader; - } - - public Optional create(SubscriptionName subscriptionName) { - Subscription subscription = subscriptionRepository.getSubscriptionDetails(subscriptionName); - if (subscription.hasOAuthPolicy()) { - try { - String providerName = subscription.getOAuthPolicy().getProviderName(); - logger.info("Creating OAuth handler subscription {} using {} OAuth provider", - subscriptionName, providerName); - OAuthTokenRequestRateLimiter rateLimiter = rateLimiterLoader.create(subscription); - return Optional.of(new OAuthSubscriptionHandler(subscriptionName, providerName, accessTokens, rateLimiter)); - } catch (Exception e) { - logger.error("Failed to create OAuth handler for subscription {}, {}", - subscriptionName.getQualifiedName(), e.getMessage()); - } - } - return Optional.empty(); + private static final Logger logger = + LoggerFactory.getLogger(OAuthSubscriptionHandlerFactory.class); + + private final SubscriptionRepository subscriptionRepository; + + private final OAuthAccessTokens accessTokens; + + private final OAuthTokenRequestRateLimiterFactory rateLimiterLoader; + + public OAuthSubscriptionHandlerFactory( + SubscriptionRepository subscriptionRepository, + OAuthAccessTokens accessTokens, + OAuthTokenRequestRateLimiterFactory rateLimiterLoader) { + this.subscriptionRepository = subscriptionRepository; + this.accessTokens = accessTokens; + this.rateLimiterLoader = rateLimiterLoader; + } + + public Optional create(SubscriptionName subscriptionName) { + Subscription subscription = subscriptionRepository.getSubscriptionDetails(subscriptionName); + if (subscription.hasOAuthPolicy()) { + try { + String providerName = subscription.getOAuthPolicy().getProviderName(); + logger.info( + "Creating OAuth handler subscription {} using {} OAuth provider", + subscriptionName, + providerName); + OAuthTokenRequestRateLimiter rateLimiter = rateLimiterLoader.create(subscription); + return Optional.of( + new OAuthSubscriptionHandler( + subscriptionName, providerName, accessTokens, rateLimiter)); + } catch (Exception e) { + logger.error( + "Failed to create OAuth handler for subscription {}, {}", + subscriptionName.getQualifiedName(), + e.getMessage()); + } } + return Optional.empty(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthTokenRequestRateLimiter.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthTokenRequestRateLimiter.java index d6cf977c91..55f56396da 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthTokenRequestRateLimiter.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthTokenRequestRateLimiter.java @@ -1,47 +1,46 @@ package pl.allegro.tech.hermes.consumers.consumer.oauth; import com.google.common.util.concurrent.RateLimiter; - import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; public class OAuthTokenRequestRateLimiter { - private final double initialRate; + private final double initialRate; - private final double minimalRate; + private final double minimalRate; - private final double rateReductionFactor; + private final double rateReductionFactor; - private final RateLimiter rateLimiter; + private final RateLimiter rateLimiter; - private final AtomicInteger rateReductionsCount = new AtomicInteger(); + private final AtomicInteger rateReductionsCount = new AtomicInteger(); - public OAuthTokenRequestRateLimiter(double initialRate, double minimalRate, double rateReductionFactor, - long warmUpPeriod) { - this.initialRate = initialRate; - this.rateLimiter = RateLimiter.create(initialRate, warmUpPeriod, TimeUnit.MILLISECONDS); - this.minimalRate = minimalRate; - this.rateReductionFactor = rateReductionFactor; - } + public OAuthTokenRequestRateLimiter( + double initialRate, double minimalRate, double rateReductionFactor, long warmUpPeriod) { + this.initialRate = initialRate; + this.rateLimiter = RateLimiter.create(initialRate, warmUpPeriod, TimeUnit.MILLISECONDS); + this.minimalRate = minimalRate; + this.rateReductionFactor = rateReductionFactor; + } - public boolean tryAcquire() { - return rateLimiter.tryAcquire(); - } + public boolean tryAcquire() { + return rateLimiter.tryAcquire(); + } - public void reduceRate() { - rateReductionsCount.incrementAndGet(); - rateLimiter.setRate(Math.max(rateLimiter.getRate() / rateReductionFactor, minimalRate)); - } + public void reduceRate() { + rateReductionsCount.incrementAndGet(); + rateLimiter.setRate(Math.max(rateLimiter.getRate() / rateReductionFactor, minimalRate)); + } - public void resetRate() { - if (rateReductionsCount.get() > 0) { - rateLimiter.setRate(initialRate); - rateReductionsCount.set(0); - } + public void resetRate() { + if (rateReductionsCount.get() > 0) { + rateLimiter.setRate(initialRate); + rateReductionsCount.set(0); } + } - public double getCurrentRate() { - return rateLimiter.getRate(); - } + public double getCurrentRate() { + return rateLimiter.getRate(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthTokenRequestRateLimiterFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthTokenRequestRateLimiterFactory.java index df6868a87c..3d5b560415 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthTokenRequestRateLimiterFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/OAuthTokenRequestRateLimiterFactory.java @@ -7,29 +7,30 @@ public class OAuthTokenRequestRateLimiterFactory { - private final OAuthProviderRepository oAuthProviderRepository; - - private final double rateReductionFactor; - - public OAuthTokenRequestRateLimiterFactory(OAuthProviderRepository oAuthProviderRepository, - double rateReductionFactor) { - this.oAuthProviderRepository = oAuthProviderRepository; - this.rateReductionFactor = rateReductionFactor; - - Preconditions.checkArgument(rateReductionFactor >= 1, - "Token request rate limiter rate reduction factor must be greater or equal to 1"); - } - - public OAuthTokenRequestRateLimiter create(Subscription subscription) { - String providerName = subscription.getOAuthPolicy().getProviderName(); - OAuthProvider oAuthProvider = oAuthProviderRepository.getOAuthProviderDetails(providerName); - double initialRate = delayToRate(oAuthProvider.getTokenRequestInitialDelay()); - double minimalRate = delayToRate(oAuthProvider.getTokenRequestMaxDelay()); - return new OAuthTokenRequestRateLimiter(initialRate, minimalRate, rateReductionFactor, - oAuthProvider.getTokenRequestInitialDelay()); - } - - private double delayToRate(Integer delay) { - return 1000.0 / delay; - } + private final OAuthProviderRepository oAuthProviderRepository; + + private final double rateReductionFactor; + + public OAuthTokenRequestRateLimiterFactory( + OAuthProviderRepository oAuthProviderRepository, double rateReductionFactor) { + this.oAuthProviderRepository = oAuthProviderRepository; + this.rateReductionFactor = rateReductionFactor; + + Preconditions.checkArgument( + rateReductionFactor >= 1, + "Token request rate limiter rate reduction factor must be greater or equal to 1"); + } + + public OAuthTokenRequestRateLimiter create(Subscription subscription) { + String providerName = subscription.getOAuthPolicy().getProviderName(); + OAuthProvider oAuthProvider = oAuthProviderRepository.getOAuthProviderDetails(providerName); + double initialRate = delayToRate(oAuthProvider.getTokenRequestInitialDelay()); + double minimalRate = delayToRate(oAuthProvider.getTokenRequestMaxDelay()); + return new OAuthTokenRequestRateLimiter( + initialRate, minimalRate, rateReductionFactor, oAuthProvider.getTokenRequestInitialDelay()); + } + + private double delayToRate(Integer delay) { + return 1000.0 / delay; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/client/OAuthClient.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/client/OAuthClient.java index 6a460a4c2f..9bc3e510a0 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/client/OAuthClient.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/client/OAuthClient.java @@ -4,9 +4,9 @@ public interface OAuthClient { - OAuthAccessToken getToken(OAuthTokenRequest request); + OAuthAccessToken getToken(OAuthTokenRequest request); - void start(); + void start(); - void stop(); + void stop(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/client/OAuthHttpClient.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/client/OAuthHttpClient.java index e8a82866a7..8cd95f7473 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/client/OAuthHttpClient.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/client/OAuthHttpClient.java @@ -1,6 +1,12 @@ package pl.allegro.tech.hermes.consumers.consumer.oauth.client; +import static jakarta.ws.rs.core.MediaType.APPLICATION_FORM_URLENCODED; + import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.eclipse.jetty.client.ContentResponse; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.Request; @@ -9,98 +15,99 @@ import org.eclipse.jetty.http.HttpStatus; import pl.allegro.tech.hermes.consumers.consumer.oauth.OAuthAccessToken; -import java.io.IOException; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import static jakarta.ws.rs.core.MediaType.APPLICATION_FORM_URLENCODED; - public class OAuthHttpClient implements OAuthClient { - private final HttpClient httpClient; - - private final ObjectMapper objectMapper; - - public OAuthHttpClient(HttpClient httpClient, ObjectMapper objectMapper) { - this.httpClient = httpClient; - this.objectMapper = objectMapper; - } - - @Override - public OAuthAccessToken getToken(OAuthTokenRequest request) { - ContentResponse response = performHttpRequest(request); - validateHttpResponse(response); - OAuthTokenResponse accessTokenResponse = httpResponseToOAuthAccessTokenResponse(response); - - return accessTokenResponse.toAccessToken(); - } - - private Request createHttpRequest(OAuthTokenRequest request) { - Request httpRequest = httpClient.newRequest(request.getUrl()) - .timeout(request.getRequestTimeout(), TimeUnit.MILLISECONDS) - .idleTimeout(request.getSocketTimeout(), TimeUnit.MILLISECONDS) - .method(HttpMethod.POST) - .headers(headers -> { - headers.add(HttpHeader.KEEP_ALIVE, "true"); - headers.add(HttpHeader.CONTENT_TYPE, APPLICATION_FORM_URLENCODED); + private final HttpClient httpClient; + + private final ObjectMapper objectMapper; + + public OAuthHttpClient(HttpClient httpClient, ObjectMapper objectMapper) { + this.httpClient = httpClient; + this.objectMapper = objectMapper; + } + + @Override + public OAuthAccessToken getToken(OAuthTokenRequest request) { + ContentResponse response = performHttpRequest(request); + validateHttpResponse(response); + OAuthTokenResponse accessTokenResponse = httpResponseToOAuthAccessTokenResponse(response); + + return accessTokenResponse.toAccessToken(); + } + + private Request createHttpRequest(OAuthTokenRequest request) { + Request httpRequest = + httpClient + .newRequest(request.getUrl()) + .timeout(request.getRequestTimeout(), TimeUnit.MILLISECONDS) + .idleTimeout(request.getSocketTimeout(), TimeUnit.MILLISECONDS) + .method(HttpMethod.POST) + .headers( + headers -> { + headers.add(HttpHeader.KEEP_ALIVE, "true"); + headers.add(HttpHeader.CONTENT_TYPE, APPLICATION_FORM_URLENCODED); }); - addParamIfNotNull(httpRequest, OAuthTokenRequest.Param.GRANT_TYPE, request.getGrantType()); - addParamIfNotNull(httpRequest, OAuthTokenRequest.Param.SCOPE, request.getScope()); - addParamIfNotNull(httpRequest, OAuthTokenRequest.Param.CLIENT_ID, request.getClientId()); - addParamIfNotNull(httpRequest, OAuthTokenRequest.Param.CLIENT_SECRET, request.getClientSecret()); - - if (OAuthTokenRequest.GrantTypeValue.RESOURCE_OWNER_USERNAME_PASSWORD.equals(request.getGrantType())) { - addParamIfNotNull(httpRequest, OAuthTokenRequest.Param.USERNAME, request.getUsername()); - addParamIfNotNull(httpRequest, OAuthTokenRequest.Param.PASSWORD, request.getPassword()); - } - return httpRequest; + addParamIfNotNull(httpRequest, OAuthTokenRequest.Param.GRANT_TYPE, request.getGrantType()); + addParamIfNotNull(httpRequest, OAuthTokenRequest.Param.SCOPE, request.getScope()); + addParamIfNotNull(httpRequest, OAuthTokenRequest.Param.CLIENT_ID, request.getClientId()); + addParamIfNotNull( + httpRequest, OAuthTokenRequest.Param.CLIENT_SECRET, request.getClientSecret()); + + if (OAuthTokenRequest.GrantTypeValue.RESOURCE_OWNER_USERNAME_PASSWORD.equals( + request.getGrantType())) { + addParamIfNotNull(httpRequest, OAuthTokenRequest.Param.USERNAME, request.getUsername()); + addParamIfNotNull(httpRequest, OAuthTokenRequest.Param.PASSWORD, request.getPassword()); } + return httpRequest; + } - private void addParamIfNotNull(Request request, String name, String value) { - if (value != null) { - request.param(name, value); - } + private void addParamIfNotNull(Request request, String name, String value) { + if (value != null) { + request.param(name, value); } - - private ContentResponse performHttpRequest(OAuthTokenRequest request) { - try { - return createHttpRequest(request).send(); - } catch (InterruptedException | TimeoutException | ExecutionException e) { - throw new OAuthTokenRequestException("An exception occurred while performing token request", e); - } + } + + private ContentResponse performHttpRequest(OAuthTokenRequest request) { + try { + return createHttpRequest(request).send(); + } catch (InterruptedException | TimeoutException | ExecutionException e) { + throw new OAuthTokenRequestException( + "An exception occurred while performing token request", e); } - - private void validateHttpResponse(ContentResponse response) { - if (response.getStatus() != HttpStatus.OK_200) { - throw new OAuthTokenRequestException(String.format("%d %s response when performing token request", - response.getStatus(), response.getContentAsString())); - } + } + + private void validateHttpResponse(ContentResponse response) { + if (response.getStatus() != HttpStatus.OK_200) { + throw new OAuthTokenRequestException( + String.format( + "%d %s response when performing token request", + response.getStatus(), response.getContentAsString())); } + } - private OAuthTokenResponse httpResponseToOAuthAccessTokenResponse(ContentResponse response) { - try { - return objectMapper.readValue(response.getContentAsString(), OAuthTokenResponse.class); - } catch (IOException e) { - throw new OAuthTokenRequestException("An exception occurred while reading token response", e); - } + private OAuthTokenResponse httpResponseToOAuthAccessTokenResponse(ContentResponse response) { + try { + return objectMapper.readValue(response.getContentAsString(), OAuthTokenResponse.class); + } catch (IOException e) { + throw new OAuthTokenRequestException("An exception occurred while reading token response", e); } - - @Override - public void start() { - try { - this.httpClient.start(); - } catch (Exception ex) { - throw new RuntimeException(ex); - } + } + + @Override + public void start() { + try { + this.httpClient.start(); + } catch (Exception ex) { + throw new RuntimeException(ex); } - - @Override - public void stop() { - try { - this.httpClient.stop(); - } catch (Exception ex) { - throw new RuntimeException(ex); - } + } + + @Override + public void stop() { + try { + this.httpClient.stop(); + } catch (Exception ex) { + throw new RuntimeException(ex); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/client/OAuthTokenRequest.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/client/OAuthTokenRequest.java index 70879b3418..0b3308fee7 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/client/OAuthTokenRequest.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/client/OAuthTokenRequest.java @@ -2,161 +2,177 @@ public class OAuthTokenRequest { - public static class Param { - - public static final String GRANT_TYPE = "grant_type"; - public static final String SCOPE = "scope"; - public static final String CLIENT_ID = "client_id"; - public static final String CLIENT_SECRET = "client_secret"; - public static final String USERNAME = "username"; - public static final String PASSWORD = "password"; - } + public static class Param { - public static class GrantTypeValue { + public static final String GRANT_TYPE = "grant_type"; + public static final String SCOPE = "scope"; + public static final String CLIENT_ID = "client_id"; + public static final String CLIENT_SECRET = "client_secret"; + public static final String USERNAME = "username"; + public static final String PASSWORD = "password"; + } - public static final String RESOURCE_OWNER_USERNAME_PASSWORD = "password"; - public static final String CLIENT_CREDENTIALS = "client_credentials"; - } + public static class GrantTypeValue { - private final String url; + public static final String RESOURCE_OWNER_USERNAME_PASSWORD = "password"; + public static final String CLIENT_CREDENTIALS = "client_credentials"; + } - private final String grantType; + private final String url; - private final String scope; + private final String grantType; - private final String clientId; + private final String scope; - private final String clientSecret; + private final String clientId; - private final String username; + private final String clientSecret; - private final String password; + private final String username; - private final Integer requestTimeout; + private final String password; - private final Integer socketTimeout; + private final Integer requestTimeout; - private OAuthTokenRequest(String url, String grantType, String scope, String clientId, String clientSecret, - String username, String password, Integer requestTimeout, Integer socketTimeout) { - this.url = url; - this.grantType = grantType; - this.scope = scope; - this.clientId = clientId; - this.clientSecret = clientSecret; - this.username = username; - this.password = password; - this.requestTimeout = requestTimeout; - this.socketTimeout = socketTimeout; - } + private final Integer socketTimeout; - public String getUrl() { - return url; - } + private OAuthTokenRequest( + String url, + String grantType, + String scope, + String clientId, + String clientSecret, + String username, + String password, + Integer requestTimeout, + Integer socketTimeout) { + this.url = url; + this.grantType = grantType; + this.scope = scope; + this.clientId = clientId; + this.clientSecret = clientSecret; + this.username = username; + this.password = password; + this.requestTimeout = requestTimeout; + this.socketTimeout = socketTimeout; + } - public String getGrantType() { - return grantType; - } + public String getUrl() { + return url; + } - public String getScope() { - return scope; - } + public String getGrantType() { + return grantType; + } - public String getClientId() { - return clientId; - } + public String getScope() { + return scope; + } - public String getClientSecret() { - return clientSecret; - } + public String getClientId() { + return clientId; + } - public String getUsername() { - return username; - } + public String getClientSecret() { + return clientSecret; + } - public String getPassword() { - return password; - } + public String getUsername() { + return username; + } - public Integer getRequestTimeout() { - return requestTimeout; - } + public String getPassword() { + return password; + } - public Integer getSocketTimeout() { - return socketTimeout; - } + public Integer getRequestTimeout() { + return requestTimeout; + } - public static OAuthTokenRequestBuilder oAuthTokenRequest() { - return new OAuthTokenRequestBuilder(); - } + public Integer getSocketTimeout() { + return socketTimeout; + } - public static class OAuthTokenRequestBuilder { + public static OAuthTokenRequestBuilder oAuthTokenRequest() { + return new OAuthTokenRequestBuilder(); + } - private String url; + public static class OAuthTokenRequestBuilder { - private String grantType; + private String url; - private String scope; + private String grantType; - private String clientId; + private String scope; - private String clientSecret; + private String clientId; - private String username; + private String clientSecret; - private String password; + private String username; - private Integer requestTimeout; + private String password; - private Integer socketTimeout; + private Integer requestTimeout; - public OAuthTokenRequestBuilder withUrl(String url) { - this.url = url; - return this; - } + private Integer socketTimeout; - public OAuthTokenRequestBuilder withGrantType(String grantType) { - this.grantType = grantType; - return this; - } + public OAuthTokenRequestBuilder withUrl(String url) { + this.url = url; + return this; + } - public OAuthTokenRequestBuilder withScope(String scope) { - this.scope = scope; - return this; - } + public OAuthTokenRequestBuilder withGrantType(String grantType) { + this.grantType = grantType; + return this; + } - public OAuthTokenRequestBuilder withClientId(String clientId) { - this.clientId = clientId; - return this; - } + public OAuthTokenRequestBuilder withScope(String scope) { + this.scope = scope; + return this; + } - public OAuthTokenRequestBuilder withClientSecret(String clientSecret) { - this.clientSecret = clientSecret; - return this; - } + public OAuthTokenRequestBuilder withClientId(String clientId) { + this.clientId = clientId; + return this; + } - public OAuthTokenRequestBuilder withUsername(String username) { - this.username = username; - return this; - } + public OAuthTokenRequestBuilder withClientSecret(String clientSecret) { + this.clientSecret = clientSecret; + return this; + } - public OAuthTokenRequestBuilder withPassword(String password) { - this.password = password; - return this; - } + public OAuthTokenRequestBuilder withUsername(String username) { + this.username = username; + return this; + } - public OAuthTokenRequestBuilder withRequestTimeout(Integer requestTimeout) { - this.requestTimeout = requestTimeout; - return this; - } + public OAuthTokenRequestBuilder withPassword(String password) { + this.password = password; + return this; + } - public OAuthTokenRequestBuilder withSocketTimeout(Integer socketTimeout) { - this.socketTimeout = socketTimeout; - return this; - } + public OAuthTokenRequestBuilder withRequestTimeout(Integer requestTimeout) { + this.requestTimeout = requestTimeout; + return this; + } + + public OAuthTokenRequestBuilder withSocketTimeout(Integer socketTimeout) { + this.socketTimeout = socketTimeout; + return this; + } - public OAuthTokenRequest build() { - return new OAuthTokenRequest(url, grantType, scope, clientId, clientSecret, username, - password, requestTimeout, socketTimeout); - } + public OAuthTokenRequest build() { + return new OAuthTokenRequest( + url, + grantType, + scope, + clientId, + clientSecret, + username, + password, + requestTimeout, + socketTimeout); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/client/OAuthTokenRequestException.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/client/OAuthTokenRequestException.java index b14c17417a..74d6c9a939 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/client/OAuthTokenRequestException.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/client/OAuthTokenRequestException.java @@ -2,11 +2,11 @@ public class OAuthTokenRequestException extends RuntimeException { - public OAuthTokenRequestException(String message, Throwable cause) { - super(message, cause); - } + public OAuthTokenRequestException(String message, Throwable cause) { + super(message, cause); + } - public OAuthTokenRequestException(String message) { - super(message); - } + public OAuthTokenRequestException(String message) { + super(message); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/client/OAuthTokenResponse.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/client/OAuthTokenResponse.java index 7ab095095e..739ce368d1 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/client/OAuthTokenResponse.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/oauth/client/OAuthTokenResponse.java @@ -7,19 +7,19 @@ public class OAuthTokenResponse { - @NotEmpty - private final String accessToken; + @NotEmpty private final String accessToken; - private final Integer expiresIn; + private final Integer expiresIn; - @JsonCreator - public OAuthTokenResponse(@JsonProperty("access_token") String accessToken, - @JsonProperty("expires_in") Integer expiresIn) { - this.accessToken = accessToken; - this.expiresIn = expiresIn; - } + @JsonCreator + public OAuthTokenResponse( + @JsonProperty("access_token") String accessToken, + @JsonProperty("expires_in") Integer expiresIn) { + this.accessToken = accessToken; + this.expiresIn = expiresIn; + } - public OAuthAccessToken toAccessToken() { - return new OAuthAccessToken(accessToken, expiresIn); - } + public OAuthAccessToken toAccessToken() { + return new OAuthAccessToken(accessToken, expiresIn); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/ConsumerPartitionAssignmentState.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/ConsumerPartitionAssignmentState.java index d5ac81bdc1..e6c318f369 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/ConsumerPartitionAssignmentState.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/ConsumerPartitionAssignmentState.java @@ -1,66 +1,72 @@ package pl.allegro.tech.hermes.consumers.consumer.offset; -import org.slf4j.Logger; -import pl.allegro.tech.hermes.api.SubscriptionName; +import static java.util.stream.Collectors.toSet; +import static org.slf4j.LoggerFactory.getLogger; import java.util.Collection; import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; - -import static java.util.stream.Collectors.toSet; -import static org.slf4j.LoggerFactory.getLogger; +import org.slf4j.Logger; +import pl.allegro.tech.hermes.api.SubscriptionName; public class ConsumerPartitionAssignmentState { - private static final Logger logger = getLogger(ConsumerPartitionAssignmentState.class); + private static final Logger logger = getLogger(ConsumerPartitionAssignmentState.class); - private final Map> assigned = new ConcurrentHashMap<>(); + private final Map> assigned = new ConcurrentHashMap<>(); - private final Map terms = new ConcurrentHashMap<>(); + private final Map terms = new ConcurrentHashMap<>(); - public void assign(SubscriptionName name, Collection partitions) { - incrementTerm(name); - logger.info("Assigning partitions {} of {}, term={}", partitions, name, currentTerm(name)); - assigned.compute(name, ((subscriptionName, assigned) -> { - HashSet extended = new HashSet<>(partitions); - if (assigned == null) { - return extended; - } else { - extended.addAll(assigned); - return extended; - } + public void assign(SubscriptionName name, Collection partitions) { + incrementTerm(name); + logger.info("Assigning partitions {} of {}, term={}", partitions, name, currentTerm(name)); + assigned.compute( + name, + ((subscriptionName, assigned) -> { + HashSet extended = new HashSet<>(partitions); + if (assigned == null) { + return extended; + } else { + extended.addAll(assigned); + return extended; + } })); - } + } - private void incrementTerm(SubscriptionName name) { - terms.compute(name, ((subscriptionName, term) -> term == null ? 0L : term + 1L)); - } + private void incrementTerm(SubscriptionName name) { + terms.compute(name, ((subscriptionName, term) -> term == null ? 0L : term + 1L)); + } - public void revoke(SubscriptionName name, Collection partitions) { - logger.info("Revoking partitions {} of {}", partitions, name); - assigned.computeIfPresent(name, (subscriptionName, assigned) -> { - Set filtered = assigned.stream().filter(p -> !partitions.contains(p)).collect(toSet()); - return filtered.isEmpty() ? null : filtered; + public void revoke(SubscriptionName name, Collection partitions) { + logger.info("Revoking partitions {} of {}", partitions, name); + assigned.computeIfPresent( + name, + (subscriptionName, assigned) -> { + Set filtered = + assigned.stream().filter(p -> !partitions.contains(p)).collect(toSet()); + return filtered.isEmpty() ? null : filtered; }); - } + } - public void revokeAll(SubscriptionName name) { - logger.info("Revoking all partitions of {}", name); - assigned.remove(name); - } + public void revokeAll(SubscriptionName name) { + logger.info("Revoking all partitions of {}", name); + assigned.remove(name); + } - public long currentTerm(SubscriptionName name) { - return terms.getOrDefault(name, -1L); - } + public long currentTerm(SubscriptionName name) { + return terms.getOrDefault(name, -1L); + } - public boolean isAssignedPartitionAtCurrentTerm(SubscriptionPartition subscriptionPartition) { - return currentTerm(subscriptionPartition.getSubscriptionName()) == subscriptionPartition.getPartitionAssignmentTerm() - && isAssigned(subscriptionPartition.getSubscriptionName(), subscriptionPartition.getPartition()); - } + public boolean isAssignedPartitionAtCurrentTerm(SubscriptionPartition subscriptionPartition) { + return currentTerm(subscriptionPartition.getSubscriptionName()) + == subscriptionPartition.getPartitionAssignmentTerm() + && isAssigned( + subscriptionPartition.getSubscriptionName(), subscriptionPartition.getPartition()); + } - private boolean isAssigned(SubscriptionName name, int partition) { - return assigned.containsKey(name) && assigned.get(name).contains(partition); - } + private boolean isAssigned(SubscriptionName name, int partition) { + return assigned.containsKey(name) && assigned.get(name).contains(partition); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/MessageState.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/MessageState.java index db120576cc..b78a9e39b8 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/MessageState.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/MessageState.java @@ -1,6 +1,6 @@ package pl.allegro.tech.hermes.consumers.consumer.offset; public enum MessageState { - INFLIGHT, - PROCESSED + INFLIGHT, + PROCESSED } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/OffsetCommitter.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/OffsetCommitter.java index 4f6e539a93..a3fdc65409 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/OffsetCommitter.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/OffsetCommitter.java @@ -1,13 +1,6 @@ package pl.allegro.tech.hermes.consumers.consumer.offset; import com.google.common.collect.Sets; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import pl.allegro.tech.hermes.common.metric.MetricsFacade; -import pl.allegro.tech.hermes.metrics.HermesCounter; -import pl.allegro.tech.hermes.metrics.HermesTimer; -import pl.allegro.tech.hermes.metrics.HermesTimerContext; - import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -17,231 +10,269 @@ import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import pl.allegro.tech.hermes.common.metric.MetricsFacade; +import pl.allegro.tech.hermes.metrics.HermesCounter; +import pl.allegro.tech.hermes.metrics.HermesTimer; +import pl.allegro.tech.hermes.metrics.HermesTimerContext; /** - *

Note on algorithm used to calculate offsets to actually commit. - * The idea behind this algorithm is that we would like to commit:

+ * Note on algorithm used to calculate offsets to actually commit. The idea behind this algorithm is + * that we would like to commit: * *
    - *
  • maximal offset marked as committed,
  • - *
  • but not larger than smallest inflight offset (smallest inflight - 1).
  • + *
  • maximal offset marked as committed, + *
  • but not larger than smallest inflight offset (smallest inflight - 1). *
* - *

Important note! This class is Kafka OffsetCommiter, and so it perceives offsets in Kafka way. Most importantly - * committed offset marks message that is read as first on Consumer restart (offset is inclusive for reading and - * exclusive for writing).

+ *

Important note! This class is Kafka OffsetCommiter, and so it perceives offsets + * in Kafka way. Most importantly committed offset marks message that is read as first on Consumer + * restart (offset is inclusive for reading and exclusive for writing). + * + *

There are two queues which are used by Consumers to report message state: * - *

There are two queues which are used by Consumers to report message state:

*
    - *
  • inflightOffsets: message offsets that are currently being sent (inflight),
  • - *
  • commitedOffsets: message offsets that are ready to get committed.
  • + *
  • inflightOffsets: message offsets that are currently being sent (inflight), + *
  • commitedOffsets: message offsets that are ready to get committed. *
* - *

This committer class holds internal state in form of inflightOffsets and maxCommittedOffsets collections.

+ *

This committer class holds internal state in form of inflightOffsets and maxCommittedOffsets + * collections. + * *

    - *
  • inflightOffsets are all offsets that are currently in inflight state,
  • - *
  • maxCommittedOffsets are offsets (maximum per partition) of already committed messages that could not yet be committed - * to kafka due to an existing inflight offset on the same partition.
  • + *
  • inflightOffsets are all offsets that are currently in inflight state, + *
  • maxCommittedOffsets are offsets (maximum per partition) of already committed + * messages that could not yet be committed to kafka due to an existing inflight offset on the + * same partition. *
* - *

In scheduled periods, commit algorithm is run. It has three phases. First one is draining the queues and performing - * reductions:

+ *

In scheduled periods, commit algorithm is run. It has three phases. First one is draining the + * queues and performing reductions: + * *

    - *
  • drain committedOffsets queue to collection - it needs to be done before draining inflights, so this collection - * will not grow anymore, resulting in having inflights unmatched by commits; commits are incremented by 1 to match - * Kafka commit definition,
  • - *
  • update the maxCommittedOffsets map with largest committed offsets,
  • - *
  • drain inflightOffsets.
  • + *
  • drain committedOffsets queue to collection - it needs to be done before + * draining inflights, so this collection will not grow anymore, resulting in having inflights + * unmatched by commits; commits are incremented by 1 to match Kafka commit definition, + *
  • update the maxCommittedOffsets map with largest committed offsets, + *
  • drain inflightOffsets. *
* - *

Second phase is calculating the offsets:

+ *

Second phase is calculating the offsets: + * *

    - *
  • calculate maximal committed offset for each subscription and partition,
  • - *
  • calculate minimal inflight offset for each subscription and partition.
  • + *
  • calculate maximal committed offset for each subscription and partition, + *
  • calculate minimal inflight offset for each subscription and partition. *
* - *

Third phase is choosing which offset to commit for each subscription/partition. This is the minimal value of:

+ *

Third phase is choosing which offset to commit for each subscription/partition. This is the + * minimal value of: + * *

    - *
  • maximum committed offset,
  • - *
  • minimum inflight offset.
  • + *
  • maximum committed offset, + *
  • minimum inflight offset. *
- *

This algorithm is very simple, memory efficient, can be performed in single thread and introduces no locks.

+ * + *

This algorithm is very simple, memory efficient, can be performed in single thread and + * introduces no locks. */ public class OffsetCommitter { + private static final Logger logger = LoggerFactory.getLogger(OffsetCommitter.class); - private static final Logger logger = LoggerFactory.getLogger(OffsetCommitter.class); + private final ConsumerPartitionAssignmentState partitionAssignmentState; - private final ConsumerPartitionAssignmentState partitionAssignmentState; + private final HermesCounter obsoleteCounter; + private final HermesCounter committedCounter; + private final HermesTimer timer; - private final HermesCounter obsoleteCounter; - private final HermesCounter committedCounter; - private final HermesTimer timer; + private final Set inflightOffsets = new HashSet<>(); + private final Map maxCommittedOffsets = new HashMap<>(); - private final Set inflightOffsets = new HashSet<>(); - private final Map maxCommittedOffsets = new HashMap<>(); + public OffsetCommitter( + ConsumerPartitionAssignmentState partitionAssignmentState, MetricsFacade metrics) { + this.partitionAssignmentState = partitionAssignmentState; + this.obsoleteCounter = metrics.offsetCommits().obsoleteCounter(); + this.committedCounter = metrics.offsetCommits().committedCounter(); + this.timer = metrics.offsetCommits().duration(); + } - public OffsetCommitter( - ConsumerPartitionAssignmentState partitionAssignmentState, - MetricsFacade metrics - ) { - this.partitionAssignmentState = partitionAssignmentState; - this.obsoleteCounter = metrics.offsetCommits().obsoleteCounter(); - this.committedCounter = metrics.offsetCommits().committedCounter(); - this.timer = metrics.offsetCommits().duration(); - } + public Set calculateOffsetsToBeCommitted( + Map offsets) { + try (HermesTimerContext ignored = timer.time()) { + List processedOffsets = new ArrayList<>(); + for (Map.Entry entry : offsets.entrySet()) { + if (entry.getValue() == MessageState.PROCESSED) { + processedOffsets.add(entry.getKey()); + } + } - public Set calculateOffsetsToBeCommitted(Map offsets) { - try (HermesTimerContext ignored = timer.time()) { - List processedOffsets = new ArrayList<>(); - for (Map.Entry entry : offsets.entrySet()) { - if (entry.getValue() == MessageState.PROCESSED) { - processedOffsets.add(entry.getKey()); - } - } + List allOffsets = new ArrayList<>(); + for (Map.Entry entry : offsets.entrySet()) { + allOffsets.add(entry.getKey()); + } - List allOffsets = new ArrayList<>(); - for (Map.Entry entry : offsets.entrySet()) { - allOffsets.add(entry.getKey()); - } + ReducingConsumer processedOffsetsReducer = prepareProcessedOffsets(processedOffsets); - ReducingConsumer processedOffsetsReducer = prepareProcessedOffsets(processedOffsets); - - // update stored max committed offsets with offsets drained from queue - Map maxDrainedProcessedOffsets = processedOffsetsReducer.reduced; - updateMaxProcessedOffsets(maxDrainedProcessedOffsets); - - ReducingConsumer inflightOffsetReducer = prepareInflightOffsets(processedOffsetsReducer.all, allOffsets); - Map minInflightOffsets = inflightOffsetReducer.reduced; - - int scheduledToCommitCount = 0; - int obsoleteCount = 0; - - Set processedOffsetToBeRemoved = new HashSet<>(); - - Set offsetsToCommit = new HashSet<>(); - for (SubscriptionPartition partition : Sets.union(minInflightOffsets.keySet(), maxCommittedOffsets.keySet())) { - if (partitionAssignmentState.isAssignedPartitionAtCurrentTerm(partition)) { - long minInflight = minInflightOffsets.getOrDefault(partition, Long.MAX_VALUE); - long maxCommitted = maxCommittedOffsets.getOrDefault(partition, Long.MAX_VALUE); - - long offsetToBeCommitted = Math.min(minInflight, maxCommitted); - if (offsetToBeCommitted >= 0 && offsetToBeCommitted < Long.MAX_VALUE) { - scheduledToCommitCount++; - offsetsToCommit.add(new SubscriptionPartitionOffset(partition, offsetToBeCommitted)); - - // if we just committed the maximum possible offset for partition, we can safely forget about it - if (maxCommitted == offsetToBeCommitted) { - processedOffsetToBeRemoved.add(partition); - } - } else { - logger.warn("Skipping offset out of bounds for subscription {}: partition={}, offset={}", - partition.getSubscriptionName(), partition.getPartition(), offsetToBeCommitted); - } - } else { - obsoleteCount++; - } - } - processedOffsetToBeRemoved.forEach(maxCommittedOffsets::remove); - - obsoleteCounter.increment(obsoleteCount); - committedCounter.increment(scheduledToCommitCount); + // update stored max committed offsets with offsets drained from queue + Map maxDrainedProcessedOffsets = processedOffsetsReducer.reduced; + updateMaxProcessedOffsets(maxDrainedProcessedOffsets); - cleanupStoredOffsetsWithObsoleteTerms(); + ReducingConsumer inflightOffsetReducer = + prepareInflightOffsets(processedOffsetsReducer.all, allOffsets); + Map minInflightOffsets = inflightOffsetReducer.reduced; - return offsetsToCommit; - } catch (Exception exception) { - logger.error("Failed to run offset committer: {}", exception.getMessage(), exception); - } - return Set.of(); - } + int scheduledToCommitCount = 0; + int obsoleteCount = 0; - private ReducingConsumer prepareProcessedOffsets(List processedOffsets) { - ReducingConsumer processedOffsetsReducer = new ReducingConsumer(Math::max, c -> c + 1); - drain(processedOffsets, processedOffsetsReducer); - processedOffsetsReducer.resetModifierFunction(); - return processedOffsetsReducer; - } + Set processedOffsetToBeRemoved = new HashSet<>(); - private void updateMaxProcessedOffsets(Map maxDrainedCommittedOffsets) { - maxDrainedCommittedOffsets.forEach((partition, drainedOffset) -> - maxCommittedOffsets.compute(partition, (p, storedOffset) -> - storedOffset == null || storedOffset < drainedOffset ? drainedOffset : storedOffset) - ); - } + Set offsetsToCommit = new HashSet<>(); + for (SubscriptionPartition partition : + Sets.union(minInflightOffsets.keySet(), maxCommittedOffsets.keySet())) { + if (partitionAssignmentState.isAssignedPartitionAtCurrentTerm(partition)) { + long minInflight = minInflightOffsets.getOrDefault(partition, Long.MAX_VALUE); + long maxCommitted = maxCommittedOffsets.getOrDefault(partition, Long.MAX_VALUE); - private ReducingConsumer prepareInflightOffsets(Set processedOffsets, - List inflightOffsetsQueue) { - // smallest undelivered message - ReducingConsumer inflightOffsetReducer = new ReducingConsumer(Math::min); + long offsetToBeCommitted = Math.min(minInflight, maxCommitted); + if (offsetToBeCommitted >= 0 && offsetToBeCommitted < Long.MAX_VALUE) { + scheduledToCommitCount++; + offsetsToCommit.add(new SubscriptionPartitionOffset(partition, offsetToBeCommitted)); - // process inflights from the current iteration - drain(inflightOffsetsQueue, o -> reduceIfNotDelivered(o, inflightOffsetReducer, processedOffsets)); + // if we just committed the maximum possible offset for partition, we can safely forget + // about it + if (maxCommitted == offsetToBeCommitted) { + processedOffsetToBeRemoved.add(partition); + } + } else { + logger.warn( + "Skipping offset out of bounds for subscription {}: partition={}, offset={}", + partition.getSubscriptionName(), + partition.getPartition(), + offsetToBeCommitted); + } + } else { + obsoleteCount++; + } + } + processedOffsetToBeRemoved.forEach(maxCommittedOffsets::remove); - // process inflights from the previous iteration - inflightOffsets.forEach(o -> reduceIfNotDelivered(o, inflightOffsetReducer, processedOffsets)); + obsoleteCounter.increment(obsoleteCount); + committedCounter.increment(scheduledToCommitCount); - inflightOffsets.clear(); - inflightOffsets.addAll(inflightOffsetReducer.all); + cleanupStoredOffsetsWithObsoleteTerms(); - return inflightOffsetReducer; + return offsetsToCommit; + } catch (Exception exception) { + logger.error("Failed to run offset committer: {}", exception.getMessage(), exception); } - - private void reduceIfNotDelivered(SubscriptionPartitionOffset offset, - ReducingConsumer inflightOffsetReducer, - Set committedOffsets) { - if (!committedOffsets.contains(offset)) { - inflightOffsetReducer.accept(offset); - } + return Set.of(); + } + + private ReducingConsumer prepareProcessedOffsets( + List processedOffsets) { + ReducingConsumer processedOffsetsReducer = new ReducingConsumer(Math::max, c -> c + 1); + drain(processedOffsets, processedOffsetsReducer); + processedOffsetsReducer.resetModifierFunction(); + return processedOffsetsReducer; + } + + private void updateMaxProcessedOffsets( + Map maxDrainedCommittedOffsets) { + maxDrainedCommittedOffsets.forEach( + (partition, drainedOffset) -> + maxCommittedOffsets.compute( + partition, + (p, storedOffset) -> + storedOffset == null || storedOffset < drainedOffset + ? drainedOffset + : storedOffset)); + } + + private ReducingConsumer prepareInflightOffsets( + Set processedOffsets, + List inflightOffsetsQueue) { + // smallest undelivered message + ReducingConsumer inflightOffsetReducer = new ReducingConsumer(Math::min); + + // process inflights from the current iteration + drain( + inflightOffsetsQueue, + o -> reduceIfNotDelivered(o, inflightOffsetReducer, processedOffsets)); + + // process inflights from the previous iteration + inflightOffsets.forEach(o -> reduceIfNotDelivered(o, inflightOffsetReducer, processedOffsets)); + + inflightOffsets.clear(); + inflightOffsets.addAll(inflightOffsetReducer.all); + + return inflightOffsetReducer; + } + + private void reduceIfNotDelivered( + SubscriptionPartitionOffset offset, + ReducingConsumer inflightOffsetReducer, + Set committedOffsets) { + if (!committedOffsets.contains(offset)) { + inflightOffsetReducer.accept(offset); } - - private void cleanupStoredOffsetsWithObsoleteTerms() { - inflightOffsets.removeIf(o -> !partitionAssignmentState.isAssignedPartitionAtCurrentTerm(o.getSubscriptionPartition())); - maxCommittedOffsets.entrySet().removeIf(entry -> !partitionAssignmentState.isAssignedPartitionAtCurrentTerm(entry.getKey())); + } + + private void cleanupStoredOffsetsWithObsoleteTerms() { + inflightOffsets.removeIf( + o -> + !partitionAssignmentState.isAssignedPartitionAtCurrentTerm( + o.getSubscriptionPartition())); + maxCommittedOffsets + .entrySet() + .removeIf( + entry -> !partitionAssignmentState.isAssignedPartitionAtCurrentTerm(entry.getKey())); + } + + private void drain( + List subscriptionPartitionOffsets, + Consumer consumer) { + int size = subscriptionPartitionOffsets.size(); + for (int i = 0; i < size; i++) { + SubscriptionPartitionOffset element = subscriptionPartitionOffsets.get(i); + if (element != null) { + consumer.accept(element); + } else { + logger.warn("Unexpected null value while draining queue [idx={}, size={}]", i, size); + break; + } } - - private void drain(List subscriptionPartitionOffsets, Consumer consumer) { - int size = subscriptionPartitionOffsets.size(); - for (int i = 0; i < size; i++) { - SubscriptionPartitionOffset element = subscriptionPartitionOffsets.get(i); - if (element != null) { - consumer.accept(element); - } else { - logger.warn("Unexpected null value while draining queue [idx={}, size={}]", i, size); - break; - } - } + } + + private static final class ReducingConsumer implements Consumer { + private final BiFunction reductor; + private Function modifier; + private final Map reduced = new HashMap<>(); + private final Set all = new HashSet<>(); + + private ReducingConsumer( + BiFunction reductor, Function offsetModifier) { + this.reductor = reductor; + this.modifier = offsetModifier; } - private static final class ReducingConsumer implements Consumer { - private final BiFunction reductor; - private Function modifier; - private final Map reduced = new HashMap<>(); - private final Set all = new HashSet<>(); - - private ReducingConsumer(BiFunction reductor, Function offsetModifier) { - this.reductor = reductor; - this.modifier = offsetModifier; - } - - private ReducingConsumer(BiFunction reductor) { - this(reductor, Function.identity()); - } + private ReducingConsumer(BiFunction reductor) { + this(reductor, Function.identity()); + } - private void resetModifierFunction() { - this.modifier = Function.identity(); - } + private void resetModifierFunction() { + this.modifier = Function.identity(); + } - @Override - public void accept(SubscriptionPartitionOffset p) { - all.add(p); - reduced.compute( - p.getSubscriptionPartition(), - (k, v) -> { - long offset = modifier.apply(p.getOffset()); - return v == null ? offset : reductor.apply(v, offset); - } - ); - } + @Override + public void accept(SubscriptionPartitionOffset p) { + all.add(p); + reduced.compute( + p.getSubscriptionPartition(), + (k, v) -> { + long offset = modifier.apply(p.getOffset()); + return v == null ? offset : reductor.apply(v, offset); + }); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/OffsetCommitterConsumerRebalanceListener.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/OffsetCommitterConsumerRebalanceListener.java index 5d9c35d12b..3b296a7d68 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/OffsetCommitterConsumerRebalanceListener.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/OffsetCommitterConsumerRebalanceListener.java @@ -1,35 +1,35 @@ package pl.allegro.tech.hermes.consumers.consumer.offset; -import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; -import org.apache.kafka.common.TopicPartition; -import pl.allegro.tech.hermes.api.SubscriptionName; +import static java.util.stream.Collectors.toSet; import java.util.Collection; import java.util.Set; - -import static java.util.stream.Collectors.toSet; +import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; +import org.apache.kafka.common.TopicPartition; +import pl.allegro.tech.hermes.api.SubscriptionName; public class OffsetCommitterConsumerRebalanceListener implements ConsumerRebalanceListener { - private final SubscriptionName name; - private final ConsumerPartitionAssignmentState state; + private final SubscriptionName name; + private final ConsumerPartitionAssignmentState state; - public OffsetCommitterConsumerRebalanceListener(SubscriptionName name, ConsumerPartitionAssignmentState state) { - this.name = name; - this.state = state; - } + public OffsetCommitterConsumerRebalanceListener( + SubscriptionName name, ConsumerPartitionAssignmentState state) { + this.name = name; + this.state = state; + } - @Override - public void onPartitionsRevoked(Collection partitions) { - state.revoke(name, integerPartitions(partitions)); - } + @Override + public void onPartitionsRevoked(Collection partitions) { + state.revoke(name, integerPartitions(partitions)); + } - @Override - public void onPartitionsAssigned(Collection partitions) { - state.assign(name, integerPartitions(partitions)); - } + @Override + public void onPartitionsAssigned(Collection partitions) { + state.assign(name, integerPartitions(partitions)); + } - private Set integerPartitions(Collection partitions) { - return partitions.stream().map(TopicPartition::partition).collect(toSet()); - } + private Set integerPartitions(Collection partitions) { + return partitions.stream().map(TopicPartition::partition).collect(toSet()); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/PendingOffsets.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/PendingOffsets.java index 1abfac203e..54bfcad40b 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/PendingOffsets.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/PendingOffsets.java @@ -1,85 +1,96 @@ package pl.allegro.tech.hermes.consumers.consumer.offset; -import pl.allegro.tech.hermes.api.SubscriptionName; -import pl.allegro.tech.hermes.common.metric.MetricsFacade; -import pl.allegro.tech.hermes.consumers.consumer.rate.AdjustableSemaphore; - import java.time.Duration; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import pl.allegro.tech.hermes.api.SubscriptionName; +import pl.allegro.tech.hermes.common.metric.MetricsFacade; +import pl.allegro.tech.hermes.consumers.consumer.rate.AdjustableSemaphore; /** - * This class manages pending offsets for message consumption in a thread-safe manner. - * It ensures that the number of pending offsets does not exceed a specified maximum limit. + * This class manages pending offsets for message consumption in a thread-safe manner. It ensures + * that the number of pending offsets does not exceed a specified maximum limit. * - *

The {@code slots} map is effectively a bounded map, guarded by the {@code maxPendingOffsetsSemaphore}. - * This semaphore ensures that the number of entries in the {@code slots} map does not exceed {@code maxPendingOffsets} and prevents running out of memory. - * The semaphore is used to acquire permits before adding entries to the map and to release permits when entries are removed. - *

+ *

The {@code slots} map is effectively a bounded map, guarded by the {@code + * maxPendingOffsetsSemaphore}. This semaphore ensures that the number of entries in the {@code + * slots} map does not exceed {@code maxPendingOffsets} and prevents running out of memory. The + * semaphore is used to acquire permits before adding entries to the map and to release permits when + * entries are removed. * - *

The {@code inflightSemaphore} is used to limit the number of messages that are currently being processed (inflight). - * It helps control the concurrency level of message processing. - *

+ *

The {@code inflightSemaphore} is used to limit the number of messages that are currently being + * processed (inflight). It helps control the concurrency level of message processing. * - *

Note: Methods that modify the state of the {@code slots} map, such as {@code markAsProcessed} and {@code markAsInflight}, - * must only be called after successfully acquiring a permit using the {@code tryAcquireSlot} method. - *

+ *

Note: Methods that modify the state of the {@code slots} map, such as {@code markAsProcessed} + * and {@code markAsInflight}, must only be called after successfully acquiring a permit using the + * {@code tryAcquireSlot} method. */ public class PendingOffsets { - private final ConcurrentHashMap slots = new ConcurrentHashMap<>(); - private final AdjustableSemaphore inflightSemaphore; - private final Semaphore maxPendingOffsetsSemaphore; + private final ConcurrentHashMap slots = + new ConcurrentHashMap<>(); + private final AdjustableSemaphore inflightSemaphore; + private final Semaphore maxPendingOffsetsSemaphore; - public PendingOffsets(SubscriptionName subscriptionName, MetricsFacade metrics, int inflightQueueSize, int maxPendingOffsets) { - this.maxPendingOffsetsSemaphore = new Semaphore(maxPendingOffsets); - this.inflightSemaphore = new AdjustableSemaphore(inflightQueueSize); - metrics.subscriptions().registerPendingOffsetsGauge(subscriptionName, maxPendingOffsetsSemaphore, slots -> (maxPendingOffsets - (double) slots.availablePermits()) / maxPendingOffsets); - } + public PendingOffsets( + SubscriptionName subscriptionName, + MetricsFacade metrics, + int inflightQueueSize, + int maxPendingOffsets) { + this.maxPendingOffsetsSemaphore = new Semaphore(maxPendingOffsets); + this.inflightSemaphore = new AdjustableSemaphore(inflightQueueSize); + metrics + .subscriptions() + .registerPendingOffsetsGauge( + subscriptionName, + maxPendingOffsetsSemaphore, + slots -> (maxPendingOffsets - (double) slots.availablePermits()) / maxPendingOffsets); + } - public void setInflightSize(int inflightQueueSize) { - this.inflightSemaphore.setMaxPermits(inflightQueueSize); - } + public void setInflightSize(int inflightQueueSize) { + this.inflightSemaphore.setMaxPermits(inflightQueueSize); + } - public void markAsProcessed(SubscriptionPartitionOffset subscriptionPartitionOffset) { - inflightSemaphore.release(); - slots.put(subscriptionPartitionOffset, MessageState.PROCESSED); - } + public void markAsProcessed(SubscriptionPartitionOffset subscriptionPartitionOffset) { + inflightSemaphore.release(); + slots.put(subscriptionPartitionOffset, MessageState.PROCESSED); + } - public boolean tryAcquireSlot(Duration processingInterval) throws InterruptedException { - if (inflightSemaphore.tryAcquire(processingInterval.toMillis(), TimeUnit.MILLISECONDS)) { - if (maxPendingOffsetsSemaphore.tryAcquire(processingInterval.toMillis(), TimeUnit.MILLISECONDS)) { - return true; - } - inflightSemaphore.release(); - } - return false; + public boolean tryAcquireSlot(Duration processingInterval) throws InterruptedException { + if (inflightSemaphore.tryAcquire(processingInterval.toMillis(), TimeUnit.MILLISECONDS)) { + if (maxPendingOffsetsSemaphore.tryAcquire( + processingInterval.toMillis(), TimeUnit.MILLISECONDS)) { + return true; + } + inflightSemaphore.release(); } + return false; + } - public void markAsInflight(SubscriptionPartitionOffset subscriptionPartitionOffset) { - slots.put(subscriptionPartitionOffset, MessageState.INFLIGHT); - } + public void markAsInflight(SubscriptionPartitionOffset subscriptionPartitionOffset) { + slots.put(subscriptionPartitionOffset, MessageState.INFLIGHT); + } - public Map getOffsetsSnapshotAndReleaseProcessedSlots() { - int permitsReleased = 0; - Map offsetSnapshot = new HashMap<>(); + public Map + getOffsetsSnapshotAndReleaseProcessedSlots() { + int permitsReleased = 0; + Map offsetSnapshot = new HashMap<>(); - for (Map.Entry entry : slots.entrySet()) { - offsetSnapshot.put(entry.getKey(), entry.getValue()); - if (entry.getValue() == MessageState.PROCESSED) { - slots.remove(entry.getKey()); - permitsReleased++; - } - } - maxPendingOffsetsSemaphore.release(permitsReleased); - return offsetSnapshot; + for (Map.Entry entry : slots.entrySet()) { + offsetSnapshot.put(entry.getKey(), entry.getValue()); + if (entry.getValue() == MessageState.PROCESSED) { + slots.remove(entry.getKey()); + permitsReleased++; + } } + maxPendingOffsetsSemaphore.release(permitsReleased); + return offsetSnapshot; + } - public void releaseSlot() { - inflightSemaphore.release(); - maxPendingOffsetsSemaphore.release(); - } + public void releaseSlot() { + inflightSemaphore.release(); + maxPendingOffsetsSemaphore.release(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/PendingOffsetsAppender.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/PendingOffsetsAppender.java index a2405cf46d..8cdf6c4aca 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/PendingOffsetsAppender.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/PendingOffsetsAppender.java @@ -2,5 +2,5 @@ public interface PendingOffsetsAppender { - void markAsProcessed(SubscriptionPartitionOffset subscriptionPartitionOffset); + void markAsProcessed(SubscriptionPartitionOffset subscriptionPartitionOffset); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/SubscriptionPartition.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/SubscriptionPartition.java index d2e988b951..b524694c10 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/SubscriptionPartition.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/SubscriptionPartition.java @@ -1,73 +1,77 @@ package pl.allegro.tech.hermes.consumers.consumer.offset; +import java.util.Objects; import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.common.kafka.KafkaTopicName; -import java.util.Objects; - public class SubscriptionPartition { - private final KafkaTopicName kafkaTopicName; + private final KafkaTopicName kafkaTopicName; - private final SubscriptionName subscriptionName; + private final SubscriptionName subscriptionName; - private final int partition; + private final int partition; - private final long partitionAssignmentTerm; + private final long partitionAssignmentTerm; - public SubscriptionPartition(KafkaTopicName kafkaTopicName, - SubscriptionName subscriptionName, - int partition, - long partitionAssignmentTerm) { - this.kafkaTopicName = kafkaTopicName; - this.subscriptionName = subscriptionName; - this.partition = partition; - this.partitionAssignmentTerm = partitionAssignmentTerm; - } + public SubscriptionPartition( + KafkaTopicName kafkaTopicName, + SubscriptionName subscriptionName, + int partition, + long partitionAssignmentTerm) { + this.kafkaTopicName = kafkaTopicName; + this.subscriptionName = subscriptionName; + this.partition = partition; + this.partitionAssignmentTerm = partitionAssignmentTerm; + } - public KafkaTopicName getKafkaTopicName() { - return kafkaTopicName; - } + public KafkaTopicName getKafkaTopicName() { + return kafkaTopicName; + } - public SubscriptionName getSubscriptionName() { - return subscriptionName; - } + public SubscriptionName getSubscriptionName() { + return subscriptionName; + } - public int getPartition() { - return partition; - } + public int getPartition() { + return partition; + } - public long getPartitionAssignmentTerm() { - return partitionAssignmentTerm; - } + public long getPartitionAssignmentTerm() { + return partitionAssignmentTerm; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - SubscriptionPartition that = (SubscriptionPartition) o; - return partition == that.partition - && partitionAssignmentTerm == that.partitionAssignmentTerm - && Objects.equals(kafkaTopicName, that.kafkaTopicName) - && Objects.equals(subscriptionName, that.subscriptionName); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(kafkaTopicName, subscriptionName, partition, partitionAssignmentTerm); + if (o == null || getClass() != o.getClass()) { + return false; } + SubscriptionPartition that = (SubscriptionPartition) o; + return partition == that.partition + && partitionAssignmentTerm == that.partitionAssignmentTerm + && Objects.equals(kafkaTopicName, that.kafkaTopicName) + && Objects.equals(subscriptionName, that.subscriptionName); + } - @Override - public String toString() { - return "SubscriptionPartition{" - + "kafkaTopicName=" + kafkaTopicName - + ", subscriptionName=" + subscriptionName - + ", partition=" + partition - + ", partitionAssignmentTerm=" + partitionAssignmentTerm - + '}'; - } + @Override + public int hashCode() { + return Objects.hash(kafkaTopicName, subscriptionName, partition, partitionAssignmentTerm); + } + + @Override + public String toString() { + return "SubscriptionPartition{" + + "kafkaTopicName=" + + kafkaTopicName + + ", subscriptionName=" + + subscriptionName + + ", partition=" + + partition + + ", partitionAssignmentTerm=" + + partitionAssignmentTerm + + '}'; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/SubscriptionPartitionOffset.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/SubscriptionPartitionOffset.java index 120b8ad766..32f0eb8858 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/SubscriptionPartitionOffset.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/SubscriptionPartitionOffset.java @@ -1,86 +1,88 @@ package pl.allegro.tech.hermes.consumers.consumer.offset; +import java.util.Objects; import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.common.kafka.KafkaTopicName; import pl.allegro.tech.hermes.common.kafka.offset.PartitionOffset; -import java.util.Objects; - public class SubscriptionPartitionOffset { - private final SubscriptionPartition subscriptionPartition; - - private final long offset; - - public SubscriptionPartitionOffset(SubscriptionPartition subscriptionPartition, long offset) { - this.subscriptionPartition = subscriptionPartition; - this.offset = offset; - } - - public static SubscriptionPartitionOffset subscriptionPartitionOffset(SubscriptionName subscriptionName, - PartitionOffset partitionOffset, - long partitionAssignmentTerm) { - return new SubscriptionPartitionOffset( - new SubscriptionPartition( - partitionOffset.getTopic(), - subscriptionName, - partitionOffset.getPartition(), - partitionAssignmentTerm - ), - partitionOffset.getOffset()); - } - - public SubscriptionName getSubscriptionName() { - return subscriptionPartition.getSubscriptionName(); - } - - public KafkaTopicName getKafkaTopicName() { - return subscriptionPartition.getKafkaTopicName(); - } - - public int getPartition() { - return subscriptionPartition.getPartition(); + private final SubscriptionPartition subscriptionPartition; + + private final long offset; + + public SubscriptionPartitionOffset(SubscriptionPartition subscriptionPartition, long offset) { + this.subscriptionPartition = subscriptionPartition; + this.offset = offset; + } + + public static SubscriptionPartitionOffset subscriptionPartitionOffset( + SubscriptionName subscriptionName, + PartitionOffset partitionOffset, + long partitionAssignmentTerm) { + return new SubscriptionPartitionOffset( + new SubscriptionPartition( + partitionOffset.getTopic(), + subscriptionName, + partitionOffset.getPartition(), + partitionAssignmentTerm), + partitionOffset.getOffset()); + } + + public SubscriptionName getSubscriptionName() { + return subscriptionPartition.getSubscriptionName(); + } + + public KafkaTopicName getKafkaTopicName() { + return subscriptionPartition.getKafkaTopicName(); + } + + public int getPartition() { + return subscriptionPartition.getPartition(); + } + + public long getOffset() { + return offset; + } + + public PartitionOffset getPartitionOffset() { + return new PartitionOffset( + subscriptionPartition.getKafkaTopicName(), offset, subscriptionPartition.getPartition()); + } + + public long getPartitionAssignmentTerm() { + return subscriptionPartition.getPartitionAssignmentTerm(); + } + + public SubscriptionPartition getSubscriptionPartition() { + return subscriptionPartition; + } + + @Override + public String toString() { + return "SubscriptionPartitionOffset{" + + "subscriptionPartition=" + + subscriptionPartition + + ", offset=" + + offset + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - public long getOffset() { - return offset; - } - - public PartitionOffset getPartitionOffset() { - return new PartitionOffset(subscriptionPartition.getKafkaTopicName(), offset, subscriptionPartition.getPartition()); - } - - public long getPartitionAssignmentTerm() { - return subscriptionPartition.getPartitionAssignmentTerm(); - } - - public SubscriptionPartition getSubscriptionPartition() { - return subscriptionPartition; - } - - @Override - public String toString() { - return "SubscriptionPartitionOffset{" - + "subscriptionPartition=" + subscriptionPartition - + ", offset=" + offset - + '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - SubscriptionPartitionOffset that = (SubscriptionPartitionOffset) o; - return offset == that.offset - && Objects.equals(subscriptionPartition, that.subscriptionPartition); - } - - @Override - public int hashCode() { - return Objects.hash(subscriptionPartition, offset); + if (o == null || getClass() != o.getClass()) { + return false; } + SubscriptionPartitionOffset that = (SubscriptionPartitionOffset) o; + return offset == that.offset + && Objects.equals(subscriptionPartition, that.subscriptionPartition); + } + + @Override + public int hashCode() { + return Objects.hash(subscriptionPartition, offset); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/kafka/broker/BrokerOffsetCommitErrors.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/kafka/broker/BrokerOffsetCommitErrors.java index 78da6af7a7..e2250302c1 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/kafka/broker/BrokerOffsetCommitErrors.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/kafka/broker/BrokerOffsetCommitErrors.java @@ -1,21 +1,20 @@ package pl.allegro.tech.hermes.consumers.consumer.offset.kafka.broker; import com.google.common.base.Joiner; -import org.apache.kafka.common.TopicPartition; - import java.util.Map; +import org.apache.kafka.common.TopicPartition; public class BrokerOffsetCommitErrors { - private final Joiner.MapJoiner mapJoiner = Joiner.on(',').withKeyValueSeparator("="); - private final Map errors; + private final Joiner.MapJoiner mapJoiner = Joiner.on(',').withKeyValueSeparator("="); + private final Map errors; - public BrokerOffsetCommitErrors(Map errors) { - this.errors = errors; - } + public BrokerOffsetCommitErrors(Map errors) { + this.errors = errors; + } - @Override - public String toString() { - return mapJoiner.join(errors); - } + @Override + public String toString() { + return mapJoiner.join(errors); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/kafka/broker/KafkaConsumerOffsetMover.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/kafka/broker/KafkaConsumerOffsetMover.java index b35480177b..7f26db83f2 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/kafka/broker/KafkaConsumerOffsetMover.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/kafka/broker/KafkaConsumerOffsetMover.java @@ -9,33 +9,41 @@ public class KafkaConsumerOffsetMover { - private static final Logger logger = LoggerFactory.getLogger(KafkaConsumerOffsetMover.class); + private static final Logger logger = LoggerFactory.getLogger(KafkaConsumerOffsetMover.class); - private final SubscriptionName subscriptionName; - private KafkaConsumer consumer; + private final SubscriptionName subscriptionName; + private KafkaConsumer consumer; - public KafkaConsumerOffsetMover(SubscriptionName subscriptionName, KafkaConsumer consumer) { - this.subscriptionName = subscriptionName; - this.consumer = consumer; - } + public KafkaConsumerOffsetMover(SubscriptionName subscriptionName, KafkaConsumer consumer) { + this.subscriptionName = subscriptionName; + this.consumer = consumer; + } - public boolean move(PartitionOffset offset) { - try { - TopicPartition tp = new TopicPartition(offset.getTopic().asString(), offset.getPartition()); - if (consumer.assignment().contains(tp)) { - logger.info("Moving offset for assigned partition {} on subscription {}", - offset.getPartition(), subscriptionName); - consumer.seek(tp, offset.getOffset()); - return true; - } else { - logger.info("Not assigned to partition {} on subscription {}", - offset.getPartition(), subscriptionName); - return false; - } - } catch (IllegalStateException ex) { - logger.error("Failed to move offset for subscription={}, partition={}, offset={}", - subscriptionName, offset.getPartition(), offset.getOffset(), ex); - return false; - } + public boolean move(PartitionOffset offset) { + try { + TopicPartition tp = new TopicPartition(offset.getTopic().asString(), offset.getPartition()); + if (consumer.assignment().contains(tp)) { + logger.info( + "Moving offset for assigned partition {} on subscription {}", + offset.getPartition(), + subscriptionName); + consumer.seek(tp, offset.getOffset()); + return true; + } else { + logger.info( + "Not assigned to partition {} on subscription {}", + offset.getPartition(), + subscriptionName); + return false; + } + } catch (IllegalStateException ex) { + logger.error( + "Failed to move offset for subscription={}, partition={}, offset={}", + subscriptionName, + offset.getPartition(), + offset.getOffset(), + ex); + return false; } -} \ No newline at end of file + } +} diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/kafka/broker/PartitionNotAssignedException.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/kafka/broker/PartitionNotAssignedException.java index eecfbfed1a..29d12cb599 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/kafka/broker/PartitionNotAssignedException.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/kafka/broker/PartitionNotAssignedException.java @@ -3,11 +3,11 @@ import pl.allegro.tech.hermes.common.exception.RetransmissionException; public class PartitionNotAssignedException extends RetransmissionException { - public PartitionNotAssignedException() { - super(""); - } + public PartitionNotAssignedException() { + super(""); + } - public PartitionNotAssignedException(Throwable cause) { - super(cause); - } -} \ No newline at end of file + public PartitionNotAssignedException(Throwable cause) { + super(cause); + } +} diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/kafka/broker/ReadingConsumerMetadataException.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/kafka/broker/ReadingConsumerMetadataException.java index eeeb57de21..ce09a626d5 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/kafka/broker/ReadingConsumerMetadataException.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/offset/kafka/broker/ReadingConsumerMetadataException.java @@ -5,12 +5,12 @@ public class ReadingConsumerMetadataException extends HermesException { - public ReadingConsumerMetadataException(int errorCode) { - super(String.format("Cannot read consumer metadata, response error code: %s.", errorCode)); - } + public ReadingConsumerMetadataException(int errorCode) { + super(String.format("Cannot read consumer metadata, response error code: %s.", errorCode)); + } - @Override - public ErrorCode getCode() { - return ErrorCode.INTERNAL_ERROR; - } + @Override + public ErrorCode getCode() { + return ErrorCode.INTERNAL_ERROR; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/profiling/ConsumerProfiler.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/profiling/ConsumerProfiler.java index 5978ab90aa..c83bc2ef44 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/profiling/ConsumerProfiler.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/profiling/ConsumerProfiler.java @@ -2,26 +2,25 @@ public interface ConsumerProfiler { - void startMeasurements(String measurement); + void startMeasurements(String measurement); - /** - * Measures the execution time of a specific piece of code. - * The measurement starts with a call to this method, - * and is terminated by another call to the same method with a different parameter (to keep the measurement continuity), - * or by calling the flushMeasurements method. - */ - void measure(String measurement); + /** + * Measures the execution time of a specific piece of code. The measurement starts with a call to + * this method, and is terminated by another call to the same method with a different parameter + * (to keep the measurement continuity), or by calling the flushMeasurements method. + */ + void measure(String measurement); - /** - * Measures the same piece of code several times, for example, a method call in the middle of a loop. - * Default implementation stores individual measurements, as well as their sum. - * stopPartialMeasurements should be called before measuring again. - */ - void startPartialMeasurement(String measurement); + /** + * Measures the same piece of code several times, for example, a method call in the middle of a + * loop. Default implementation stores individual measurements, as well as their sum. + * stopPartialMeasurements should be called before measuring again. + */ + void startPartialMeasurement(String measurement); - void stopPartialMeasurement(); + void stopPartialMeasurement(); - void saveRetryDelay(long retryDelay); + void saveRetryDelay(long retryDelay); - void flushMeasurements(ConsumerRun consumerRun); + void flushMeasurements(ConsumerRun consumerRun); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/profiling/ConsumerRun.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/profiling/ConsumerRun.java index 41cce38a1a..7911c803d1 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/profiling/ConsumerRun.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/profiling/ConsumerRun.java @@ -1,5 +1,9 @@ package pl.allegro.tech.hermes.consumers.consumer.profiling; public enum ConsumerRun { - EMPTY, DELIVERED, DISCARDED, RETRIED, FILTERED + EMPTY, + DELIVERED, + DISCARDED, + RETRIED, + FILTERED } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/profiling/DefaultConsumerProfiler.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/profiling/DefaultConsumerProfiler.java index 347c2cc379..783be2d435 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/profiling/DefaultConsumerProfiler.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/profiling/DefaultConsumerProfiler.java @@ -1,77 +1,82 @@ package pl.allegro.tech.hermes.consumers.consumer.profiling; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.util.StopWatch; import pl.allegro.tech.hermes.api.SubscriptionName; -import java.util.concurrent.TimeUnit; - -/** - * This class is designed to be fully thread safe. - */ +/** This class is designed to be fully thread safe. */ public class DefaultConsumerProfiler implements ConsumerProfiler { - private static final Logger logger = LoggerFactory.getLogger(DefaultConsumerProfiler.class); + private static final Logger logger = LoggerFactory.getLogger(DefaultConsumerProfiler.class); - private final SubscriptionName subscriptionName; - private final long profilingThresholdMs; - private StopWatch stopWatch; - private StopWatch partialMeasurements; - private long retryDelayMillis = 0; + private final SubscriptionName subscriptionName; + private final long profilingThresholdMs; + private StopWatch stopWatch; + private StopWatch partialMeasurements; + private long retryDelayMillis = 0; - public DefaultConsumerProfiler(SubscriptionName subscriptionName, long profilingThresholdMs) { - this.subscriptionName = subscriptionName; - this.profilingThresholdMs = profilingThresholdMs; - } + public DefaultConsumerProfiler(SubscriptionName subscriptionName, long profilingThresholdMs) { + this.subscriptionName = subscriptionName; + this.profilingThresholdMs = profilingThresholdMs; + } - @Override - public synchronized void startMeasurements(String measurement) { - this.stopWatch = new StopWatch(); - this.stopWatch.start(measurement); - } + @Override + public synchronized void startMeasurements(String measurement) { + this.stopWatch = new StopWatch(); + this.stopWatch.start(measurement); + } - @Override - public synchronized void measure(String measurement) { - this.stopWatch.stop(); - this.stopWatch.start(measurement); - } + @Override + public synchronized void measure(String measurement) { + this.stopWatch.stop(); + this.stopWatch.start(measurement); + } - @Override - public synchronized void startPartialMeasurement(String measurement) { - if (partialMeasurements == null) { - partialMeasurements = new StopWatch(); - } - partialMeasurements.start(measurement); + @Override + public synchronized void startPartialMeasurement(String measurement) { + if (partialMeasurements == null) { + partialMeasurements = new StopWatch(); } + partialMeasurements.start(measurement); + } - @Override - public synchronized void stopPartialMeasurement() { - partialMeasurements.stop(); - } + @Override + public synchronized void stopPartialMeasurement() { + partialMeasurements.stop(); + } - @Override - public synchronized void saveRetryDelay(long retryDelay) { - this.retryDelayMillis = retryDelay; - } + @Override + public synchronized void saveRetryDelay(long retryDelay) { + this.retryDelayMillis = retryDelay; + } - @Override - public synchronized void flushMeasurements(ConsumerRun consumerRun) { - this.stopWatch.stop(); - if (stopWatch.getTotalTimeMillis() > profilingThresholdMs) { - logMeasurements(consumerRun); - } + @Override + public synchronized void flushMeasurements(ConsumerRun consumerRun) { + this.stopWatch.stop(); + if (stopWatch.getTotalTimeMillis() > profilingThresholdMs) { + logMeasurements(consumerRun); } + } - private void logMeasurements(ConsumerRun consumerRun) { - if (partialMeasurements != null) { - logger.info("Consumer profiler measurements for subscription {} and {} run: \n {} partialMeasurements: {} retryDelayMillis {}", - subscriptionName, consumerRun, stopWatch.prettyPrint(TimeUnit.MILLISECONDS), - partialMeasurements.prettyPrint(TimeUnit.MILLISECONDS), retryDelayMillis); - } else { - logger.info("Consumer profiler measurements for subscription {} and {} run: \n {} partialMeasurements: {}, retryDelayMillis {}", - subscriptionName, consumerRun, stopWatch.prettyPrint(TimeUnit.MILLISECONDS), - null, retryDelayMillis); - } + private void logMeasurements(ConsumerRun consumerRun) { + if (partialMeasurements != null) { + logger.info( + "Consumer profiler measurements for subscription {} and {} run: \n {} partialMeasurements: {} retryDelayMillis {}", + subscriptionName, + consumerRun, + stopWatch.prettyPrint(TimeUnit.MILLISECONDS), + partialMeasurements.prettyPrint(TimeUnit.MILLISECONDS), + retryDelayMillis); + } else { + logger.info( + "Consumer profiler measurements for subscription {} and {} run: \n {} partialMeasurements: {}, retryDelayMillis {}", + subscriptionName, + consumerRun, + stopWatch.prettyPrint(TimeUnit.MILLISECONDS), + null, + retryDelayMillis); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/profiling/Measurement.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/profiling/Measurement.java index a601674687..6ac8fa8cf7 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/profiling/Measurement.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/profiling/Measurement.java @@ -1,15 +1,15 @@ package pl.allegro.tech.hermes.consumers.consumer.profiling; public class Measurement { - public static final String SIGNALS_AND_SEMAPHORE_ACQUIRE = "signalsAndSemaphoreAcquire"; - public static final String SIGNALS_INTERRUPT_RUN = "signalsInterrupt.run"; - public static final String SCHEDULE_RESEND = "schedule.resend"; - public static final String MESSAGE_RECEIVER_NEXT = "messageReceiver.next"; - public static final String MESSAGE_CONVERSION = "messageConverter.convert"; - public static final String OFFER_INFLIGHT_OFFSET = "offsetQueue.offerInflightOffset"; - public static final String TRACKERS_LOG_INFLIGHT = "trackers.logInflight"; - public static final String SCHEDULE_MESSAGE_SENDING = "retrySingleThreadExecutor.schedule"; - public static final String ACQUIRE_RATE_LIMITER = "acquireRateLimiter"; - public static final String MESSAGE_SENDER_SEND = "messageSender.send"; - public static final String HANDLERS = "handlers"; + public static final String SIGNALS_AND_SEMAPHORE_ACQUIRE = "signalsAndSemaphoreAcquire"; + public static final String SIGNALS_INTERRUPT_RUN = "signalsInterrupt.run"; + public static final String SCHEDULE_RESEND = "schedule.resend"; + public static final String MESSAGE_RECEIVER_NEXT = "messageReceiver.next"; + public static final String MESSAGE_CONVERSION = "messageConverter.convert"; + public static final String OFFER_INFLIGHT_OFFSET = "offsetQueue.offerInflightOffset"; + public static final String TRACKERS_LOG_INFLIGHT = "trackers.logInflight"; + public static final String SCHEDULE_MESSAGE_SENDING = "retrySingleThreadExecutor.schedule"; + public static final String ACQUIRE_RATE_LIMITER = "acquireRateLimiter"; + public static final String MESSAGE_SENDER_SEND = "messageSender.send"; + public static final String HANDLERS = "handlers"; } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/profiling/NoOpConsumerProfiler.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/profiling/NoOpConsumerProfiler.java index 9b3355f4a1..f478c04d27 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/profiling/NoOpConsumerProfiler.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/profiling/NoOpConsumerProfiler.java @@ -2,33 +2,21 @@ public class NoOpConsumerProfiler implements ConsumerProfiler { - @Override - public void startMeasurements(String measurement) { + @Override + public void startMeasurements(String measurement) {} - } + @Override + public void measure(String measurement) {} - @Override - public void measure(String measurement) { + @Override + public void startPartialMeasurement(String measurement) {} - } + @Override + public void stopPartialMeasurement() {} - @Override - public void startPartialMeasurement(String measurement) { + @Override + public void saveRetryDelay(long retryDelay) {} - } - - @Override - public void stopPartialMeasurement() { - - } - - @Override - public void saveRetryDelay(long retryDelay) { - - } - - @Override - public void flushMeasurements(ConsumerRun consumerRun) { - - } + @Override + public void flushMeasurements(ConsumerRun consumerRun) {} } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/AdjustableSemaphore.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/AdjustableSemaphore.java index 03ce96488a..3d9b08cdd0 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/AdjustableSemaphore.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/AdjustableSemaphore.java @@ -17,109 +17,97 @@ * * */ -/** - * A simple implementation of an adjustable semaphore. - */ +/** A simple implementation of an adjustable semaphore. */ @ThreadSafe public final class AdjustableSemaphore { - /** - * Semaphore starts at 0 capacity; must be set by setMaxPermits before use. - */ - private final ResizeableSemaphore semaphore; - - /** - * How many permits are allowed as governed by this semaphore. - * Access must be synchronized on this object. - */ - private int maxPermits = 0; - - /** - * New instances should be configured with setMaxPermits(). - */ - public AdjustableSemaphore(int maxPermits) { - this.maxPermits = maxPermits; - this.semaphore = new ResizeableSemaphore(maxPermits); + /** Semaphore starts at 0 capacity; must be set by setMaxPermits before use. */ + private final ResizeableSemaphore semaphore; + + /** + * How many permits are allowed as governed by this semaphore. Access must be synchronized on this + * object. + */ + private int maxPermits = 0; + + /** New instances should be configured with setMaxPermits(). */ + public AdjustableSemaphore(int maxPermits) { + this.maxPermits = maxPermits; + this.semaphore = new ResizeableSemaphore(maxPermits); + } + + /* + * Must be synchronized because the underlying int is not thread safe + */ + + /** + * Set the max number of permits. Must be greater than zero. + * + *

Note that if there are more than the new max number of permits currently outstanding, any + * currently blocking threads or any new threads that start to block after the call will wait + * until enough permits have been released to have the number of outstanding permits fall below + * the new maximum. In other words, it does what you probably think it should. + */ + public synchronized void setMaxPermits(int newMax) { + if (newMax < 1) { + throw new IllegalArgumentException("Semaphore size must be at least 1," + " was " + newMax); } - /* - * Must be synchronized because the underlying int is not thread safe - */ - - /** - *

Set the max number of permits. Must be greater than zero.

- * - *

Note that if there are more than the new max number of permits currently - * outstanding, any currently blocking threads or any new threads that start - * to block after the call will wait until enough permits have been released to - * have the number of outstanding permits fall below the new maximum. In - * other words, it does what you probably think it should.

- */ - public synchronized void setMaxPermits(int newMax) { - if (newMax < 1) { - throw new IllegalArgumentException("Semaphore size must be at least 1," - + " was " + newMax); - } - - int delta = newMax - this.maxPermits; - - if (delta == 0) { - return; - } else if (delta > 0) { - // new max is higher, so release that many permits - this.semaphore.release(delta); - } else { - delta *= -1; - // delta < 0. - // reducePermits needs a positive #, though. - this.semaphore.reducePermits(delta); - } - - this.maxPermits = newMax; + int delta = newMax - this.maxPermits; + + if (delta == 0) { + return; + } else if (delta > 0) { + // new max is higher, so release that many permits + this.semaphore.release(delta); + } else { + delta *= -1; + // delta < 0. + // reducePermits needs a positive #, though. + this.semaphore.reducePermits(delta); } - /** - * Release a permit back to the semaphore. Make sure not to double-release. - */ - public void release() { - this.semaphore.release(); + this.maxPermits = newMax; + } + + /** Release a permit back to the semaphore. Make sure not to double-release. */ + public void release() { + this.semaphore.release(); + } + + /** + * Get a permit, blocking if necessary. + * + * @throws InterruptedException if interrupted while waiting for a permit + */ + public void acquire() throws InterruptedException { + this.semaphore.acquire(); + } + + public boolean tryAcquire(long timeout, TimeUnit unit) throws InterruptedException { + return this.semaphore.tryAcquire(timeout, unit); + } + + public int availablePermits() { + return this.semaphore.availablePermits(); + } + + /** + * A trivial subclass of Semaphore that exposes the reducePermits call to the parent + * class. Doug Lea says it's ok... + * http://osdir.com/ml/java.jsr.166-concurrency/2003-10/msg00042.html + */ + private static final class ResizeableSemaphore extends Semaphore { + private static final long serialVersionUID = 1L; + + /** Create a new semaphore with 0 permits. */ + ResizeableSemaphore(int maxPermits) { + super(maxPermits); } - /** - * Get a permit, blocking if necessary. - * - * @throws InterruptedException if interrupted while waiting for a permit - */ - public void acquire() throws InterruptedException { - this.semaphore.acquire(); - } - - public boolean tryAcquire(long timeout, TimeUnit unit) throws InterruptedException { - return this.semaphore.tryAcquire(timeout, unit); - } - - public int availablePermits() { - return this.semaphore.availablePermits(); - } - - /** - * A trivial subclass of Semaphore that exposes the reducePermits - * call to the parent class. Doug Lea says it's ok... - * http://osdir.com/ml/java.jsr.166-concurrency/2003-10/msg00042.html - */ - private static final class ResizeableSemaphore extends Semaphore { - private static final long serialVersionUID = 1L; - - /** - * Create a new semaphore with 0 permits. - */ - ResizeableSemaphore(int maxPermits) { - super(maxPermits); - } - - @Override - protected void reducePermits(int reduction) { - super.reducePermits(reduction); - } + @Override + protected void reducePermits(int reduction) { + super.reducePermits(reduction); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/BatchConsumerRateLimiter.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/BatchConsumerRateLimiter.java index c90ac028f6..25c1f0653f 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/BatchConsumerRateLimiter.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/BatchConsumerRateLimiter.java @@ -4,35 +4,27 @@ public class BatchConsumerRateLimiter implements ConsumerRateLimiter { - @Override - public void initialize() { - } + @Override + public void initialize() {} - @Override - public void shutdown() { - } + @Override + public void shutdown() {} - @Override - public void acquire() { - } + @Override + public void acquire() {} - @Override - public void acquireFiltered() { - } + @Override + public void acquireFiltered() {} - @Override - public void adjustConsumerRate() { - } + @Override + public void adjustConsumerRate() {} - @Override - public void updateSubscription(Subscription newSubscription) { - } + @Override + public void updateSubscription(Subscription newSubscription) {} - @Override - public void registerSuccessfulSending() { - } + @Override + public void registerSuccessfulSending() {} - @Override - public void registerFailedSending() { - } + @Override + public void registerFailedSending() {} } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/ConsumerRateLimitSupervisor.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/ConsumerRateLimitSupervisor.java index 086107f0a0..9c923c5e05 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/ConsumerRateLimitSupervisor.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/ConsumerRateLimitSupervisor.java @@ -1,9 +1,6 @@ package pl.allegro.tech.hermes.consumers.consumer.rate; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.time.Duration; import java.util.Collections; import java.util.Set; @@ -11,36 +8,43 @@ import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ConsumerRateLimitSupervisor implements Runnable { - private static final Logger logger = LoggerFactory.getLogger(ConsumerRateLimitSupervisor.class); - - private final Set consumerRateLimiters = Collections.newSetFromMap(new ConcurrentHashMap<>()); - - public ConsumerRateLimitSupervisor(Duration rateLimiterSupervisorPeriod) { - ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat("rate-limit-supervisor-%d").build(); - Executors.newSingleThreadScheduledExecutor(threadFactory) - .scheduleAtFixedRate(this, rateLimiterSupervisorPeriod.toSeconds(), rateLimiterSupervisorPeriod.toSeconds(), - TimeUnit.SECONDS); + private static final Logger logger = LoggerFactory.getLogger(ConsumerRateLimitSupervisor.class); + + private final Set consumerRateLimiters = + Collections.newSetFromMap(new ConcurrentHashMap<>()); + + public ConsumerRateLimitSupervisor(Duration rateLimiterSupervisorPeriod) { + ThreadFactory threadFactory = + new ThreadFactoryBuilder().setNameFormat("rate-limit-supervisor-%d").build(); + Executors.newSingleThreadScheduledExecutor(threadFactory) + .scheduleAtFixedRate( + this, + rateLimiterSupervisorPeriod.toSeconds(), + rateLimiterSupervisorPeriod.toSeconds(), + TimeUnit.SECONDS); + } + + @Override + public void run() { + for (ConsumerRateLimiter limiter : consumerRateLimiters) { + try { + limiter.adjustConsumerRate(); + } catch (Exception e) { + logger.warn("Issue adjusting consumer rate", e); + } } + } - @Override - public void run() { - for (ConsumerRateLimiter limiter : consumerRateLimiters) { - try { - limiter.adjustConsumerRate(); - } catch (Exception e) { - logger.warn("Issue adjusting consumer rate", e); - } - } - } + public void register(ConsumerRateLimiter consumerRateLimiter) { + consumerRateLimiters.add(consumerRateLimiter); + } - public void register(ConsumerRateLimiter consumerRateLimiter) { - consumerRateLimiters.add(consumerRateLimiter); - } - - public void unregister(ConsumerRateLimiter consumerRateLimiter) { - consumerRateLimiters.remove(consumerRateLimiter); - } + public void unregister(ConsumerRateLimiter consumerRateLimiter) { + consumerRateLimiters.remove(consumerRateLimiter); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/ConsumerRateLimiter.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/ConsumerRateLimiter.java index 26cee215f6..ba2951a1ba 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/ConsumerRateLimiter.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/ConsumerRateLimiter.java @@ -4,19 +4,19 @@ public interface ConsumerRateLimiter { - void initialize(); + void initialize(); - void shutdown(); + void shutdown(); - void acquire(); + void acquire(); - void acquireFiltered(); + void acquireFiltered(); - void adjustConsumerRate(); + void adjustConsumerRate(); - void updateSubscription(Subscription newSubscription); + void updateSubscription(Subscription newSubscription); - void registerSuccessfulSending(); + void registerSuccessfulSending(); - void registerFailedSending(); + void registerFailedSending(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/InflightsPool.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/InflightsPool.java index 79d8c8190e..e57965bc96 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/InflightsPool.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/InflightsPool.java @@ -1,5 +1,5 @@ package pl.allegro.tech.hermes.consumers.consumer.rate; public interface InflightsPool { - void release(); + void release(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/SendCounters.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/SendCounters.java index af891d932d..28debbf3e1 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/SendCounters.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/SendCounters.java @@ -1,86 +1,84 @@ package pl.allegro.tech.hermes.consumers.consumer.rate; -import com.google.common.util.concurrent.AtomicDouble; -import org.apache.commons.lang3.math.Fraction; +import static org.apache.commons.lang3.math.Fraction.getFraction; +import com.google.common.util.concurrent.AtomicDouble; import java.time.Clock; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; - -import static org.apache.commons.lang3.math.Fraction.getFraction; - +import org.apache.commons.lang3.math.Fraction; public class SendCounters { - private final AtomicInteger successes = new AtomicInteger(0); + private final AtomicInteger successes = new AtomicInteger(0); - private final AtomicInteger failures = new AtomicInteger(0); + private final AtomicInteger failures = new AtomicInteger(0); - private final AtomicInteger attempted = new AtomicInteger(0); + private final AtomicInteger attempted = new AtomicInteger(0); - private final AtomicDouble rate = new AtomicDouble(0); + private final AtomicDouble rate = new AtomicDouble(0); - private final Clock clock; + private final Clock clock; - private volatile long lastReset; + private volatile long lastReset; - public SendCounters(Clock clock) { - this.clock = clock; - } + public SendCounters(Clock clock) { + this.clock = clock; + } - public SendCounters incrementFailures() { - failures.incrementAndGet(); - return this; - } + public SendCounters incrementFailures() { + failures.incrementAndGet(); + return this; + } - public SendCounters incrementSuccesses() { - successes.incrementAndGet(); - return this; - } + public SendCounters incrementSuccesses() { + successes.incrementAndGet(); + return this; + } - public SendCounters incrementAttempted() { - attempted.incrementAndGet(); - return this; - } + public SendCounters incrementAttempted() { + attempted.incrementAndGet(); + return this; + } - public void reset() { - failures.set(0); - successes.set(0); + public void reset() { + failures.set(0); + successes.set(0); - long now = clock.millis(); - long elapsedSeconds = TimeUnit.MILLISECONDS.toSeconds(now - lastReset); - rate.set(attempted.doubleValue() / Math.max(elapsedSeconds, 1)); + long now = clock.millis(); + long elapsedSeconds = TimeUnit.MILLISECONDS.toSeconds(now - lastReset); + rate.set(attempted.doubleValue() / Math.max(elapsedSeconds, 1)); - attempted.set(0); - lastReset = now; - } + attempted.set(0); + lastReset = now; + } - public boolean noFailures() { - return failures.intValue() == 0; - } + public boolean noFailures() { + return failures.intValue() == 0; + } - public boolean hasFailures() { - return failures.intValue() > 0; - } + public boolean hasFailures() { + return failures.intValue() > 0; + } - public boolean majorityOfFailures() { - return failures.intValue() > successes.intValue(); - } - - public boolean onlySuccessess() { - return successes.intValue() > 0 && failures.intValue() == 0; - } + public boolean majorityOfFailures() { + return failures.intValue() > successes.intValue(); + } - public boolean failuresRatioExceeds(double threshold) { - if (hasFailures()) { - Fraction failuresRatio = getFraction(failures.intValue(), failures.intValue() + successes.intValue()); - return failuresRatio.compareTo(getFraction(threshold)) > 0; - } - return false; - } + public boolean onlySuccessess() { + return successes.intValue() > 0 && failures.intValue() == 0; + } - public double getRate() { - return rate.get(); + public boolean failuresRatioExceeds(double threshold) { + if (hasFailures()) { + Fraction failuresRatio = + getFraction(failures.intValue(), failures.intValue() + successes.intValue()); + return failuresRatio.compareTo(getFraction(threshold)) > 0; } + return false; + } + public double getRate() { + return rate.get(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/SerialConsumerRateLimiter.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/SerialConsumerRateLimiter.java index c2bf40d93f..5e9bf40aa0 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/SerialConsumerRateLimiter.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/SerialConsumerRateLimiter.java @@ -1,123 +1,128 @@ package pl.allegro.tech.hermes.consumers.consumer.rate; import com.google.common.util.concurrent.RateLimiter; +import java.time.Clock; +import java.util.Objects; import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.common.metric.MetricsFacade; import pl.allegro.tech.hermes.consumers.consumer.rate.calculator.OutputRateCalculationResult; import pl.allegro.tech.hermes.consumers.consumer.rate.calculator.OutputRateCalculator; import pl.allegro.tech.hermes.consumers.consumer.rate.calculator.OutputRateCalculatorFactory; -import java.time.Clock; -import java.util.Objects; - public class SerialConsumerRateLimiter implements ConsumerRateLimiter { - private Subscription subscription; - - private final MetricsFacade metrics; - - private final ConsumerRateLimitSupervisor rateLimitSupervisor; - - private final RateLimiter rateLimiter; - - private final RateLimiter filterRateLimiter; - - private final OutputRateCalculator outputRateCalculator; - - private final SendCounters sendCounters; - - private OutputRateCalculator.Mode currentMode; - - public SerialConsumerRateLimiter(Subscription subscription, - OutputRateCalculatorFactory outputRateCalculatorFactory, - MetricsFacade metrics, - ConsumerRateLimitSupervisor rateLimitSupervisor, - Clock clock) { - this.subscription = subscription; - this.metrics = metrics; - this.rateLimitSupervisor = rateLimitSupervisor; - this.sendCounters = new SendCounters(clock); - this.outputRateCalculator = outputRateCalculatorFactory.createCalculator(subscription, sendCounters, metrics); - this.currentMode = OutputRateCalculator.Mode.NORMAL; - this.rateLimiter = RateLimiter.create(calculateInitialRate().rate()); - this.filterRateLimiter = RateLimiter.create(subscription.getSerialSubscriptionPolicy().getRate()); + private Subscription subscription; + + private final MetricsFacade metrics; + + private final ConsumerRateLimitSupervisor rateLimitSupervisor; + + private final RateLimiter rateLimiter; + + private final RateLimiter filterRateLimiter; + + private final OutputRateCalculator outputRateCalculator; + + private final SendCounters sendCounters; + + private OutputRateCalculator.Mode currentMode; + + public SerialConsumerRateLimiter( + Subscription subscription, + OutputRateCalculatorFactory outputRateCalculatorFactory, + MetricsFacade metrics, + ConsumerRateLimitSupervisor rateLimitSupervisor, + Clock clock) { + this.subscription = subscription; + this.metrics = metrics; + this.rateLimitSupervisor = rateLimitSupervisor; + this.sendCounters = new SendCounters(clock); + this.outputRateCalculator = + outputRateCalculatorFactory.createCalculator(subscription, sendCounters, metrics); + this.currentMode = OutputRateCalculator.Mode.NORMAL; + this.rateLimiter = RateLimiter.create(calculateInitialRate().rate()); + this.filterRateLimiter = + RateLimiter.create(subscription.getSerialSubscriptionPolicy().getRate()); + } + + @Override + public void initialize() { + outputRateCalculator.start(); + adjustConsumerRate(); + metrics + .maxRate() + .registerOutputRateGauge( + subscription.getQualifiedName(), rateLimiter, RateLimiter::getRate); + rateLimitSupervisor.register(this); + } + + @Override + public void shutdown() { + rateLimitSupervisor.unregister(this); + outputRateCalculator.shutdown(); + } + + @Override + public void acquire() { + rateLimiter.acquire(); + sendCounters.incrementAttempted(); + } + + @Override + public void acquireFiltered() { + filterRateLimiter.acquire(); + } + + @Override + public void adjustConsumerRate() { + OutputRateCalculationResult result = recalculate(); + rateLimiter.setRate(result.rate()); + currentMode = result.mode(); + sendCounters.reset(); + } + + private OutputRateCalculationResult calculateInitialRate() { + return outputRateCalculator.recalculateRate(sendCounters, currentMode, 0.0); + } + + private OutputRateCalculationResult recalculate() { + return outputRateCalculator.recalculateRate(sendCounters, currentMode, rateLimiter.getRate()); + } + + @Override + public void updateSubscription(Subscription newSubscription) { + this.subscription = newSubscription; + this.filterRateLimiter.setRate(newSubscription.getSerialSubscriptionPolicy().getRate()); + this.outputRateCalculator.updateSubscription(newSubscription); + } + + @Override + public void registerSuccessfulSending() { + sendCounters.incrementSuccesses(); + } + + @Override + public void registerFailedSending() { + sendCounters.incrementFailures(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - @Override - public void initialize() { - outputRateCalculator.start(); - adjustConsumerRate(); - metrics.maxRate().registerOutputRateGauge(subscription.getQualifiedName(), rateLimiter, RateLimiter::getRate); - rateLimitSupervisor.register(this); + if (o == null || getClass() != o.getClass()) { + return false; } - @Override - public void shutdown() { - rateLimitSupervisor.unregister(this); - outputRateCalculator.shutdown(); - } + SerialConsumerRateLimiter that = (SerialConsumerRateLimiter) o; - @Override - public void acquire() { - rateLimiter.acquire(); - sendCounters.incrementAttempted(); - } - - @Override - public void acquireFiltered() { - filterRateLimiter.acquire(); - } + return Objects.equals(subscription.getQualifiedName(), that.subscription.getQualifiedName()); + } - @Override - public void adjustConsumerRate() { - OutputRateCalculationResult result = recalculate(); - rateLimiter.setRate(result.rate()); - currentMode = result.mode(); - sendCounters.reset(); - } - - private OutputRateCalculationResult calculateInitialRate() { - return outputRateCalculator.recalculateRate(sendCounters, currentMode, 0.0); - } - - private OutputRateCalculationResult recalculate() { - return outputRateCalculator.recalculateRate(sendCounters, currentMode, rateLimiter.getRate()); - } - - @Override - public void updateSubscription(Subscription newSubscription) { - this.subscription = newSubscription; - this.filterRateLimiter.setRate(newSubscription.getSerialSubscriptionPolicy().getRate()); - this.outputRateCalculator.updateSubscription(newSubscription); - } - - @Override - public void registerSuccessfulSending() { - sendCounters.incrementSuccesses(); - } - - @Override - public void registerFailedSending() { - sendCounters.incrementFailures(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - - if (o == null || getClass() != o.getClass()) { - return false; - } - - SerialConsumerRateLimiter that = (SerialConsumerRateLimiter) o; - - return Objects.equals(subscription.getQualifiedName(), that.subscription.getQualifiedName()); - } - - @Override - public int hashCode() { - return Objects.hashCode(subscription.getQualifiedName()); - } + @Override + public int hashCode() { + return Objects.hashCode(subscription.getQualifiedName()); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/HeartbeatModeOutputRateCalculator.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/HeartbeatModeOutputRateCalculator.java index b6d4ee467e..e8254b8601 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/HeartbeatModeOutputRateCalculator.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/HeartbeatModeOutputRateCalculator.java @@ -4,19 +4,18 @@ class HeartbeatModeOutputRateCalculator implements ModeOutputRateCalculator { - private final double slowModeRate; + private final double slowModeRate; - HeartbeatModeOutputRateCalculator(double slowModeRate) { - this.slowModeRate = slowModeRate; - } + HeartbeatModeOutputRateCalculator(double slowModeRate) { + this.slowModeRate = slowModeRate; + } - @Override - public OutputRateCalculationResult calculateOutputRate(double currentRate, - double maximumOutputRate, - SendCounters counters) { - if (counters.onlySuccessess()) { - return new OutputRateCalculationResult(slowModeRate, OutputRateCalculator.Mode.SLOW); - } - return new OutputRateCalculationResult(currentRate, OutputRateCalculator.Mode.HEARTBEAT); + @Override + public OutputRateCalculationResult calculateOutputRate( + double currentRate, double maximumOutputRate, SendCounters counters) { + if (counters.onlySuccessess()) { + return new OutputRateCalculationResult(slowModeRate, OutputRateCalculator.Mode.SLOW); } + return new OutputRateCalculationResult(currentRate, OutputRateCalculator.Mode.HEARTBEAT); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/ModeOutputRateCalculator.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/ModeOutputRateCalculator.java index 9a865f7514..cf0c748634 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/ModeOutputRateCalculator.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/ModeOutputRateCalculator.java @@ -4,8 +4,6 @@ interface ModeOutputRateCalculator { - OutputRateCalculationResult calculateOutputRate(double currentRate, - double maximumOutputRate, - SendCounters counters); - + OutputRateCalculationResult calculateOutputRate( + double currentRate, double maximumOutputRate, SendCounters counters); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/NormalModeOutputRateCalculator.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/NormalModeOutputRateCalculator.java index a0a9449d62..9b695949f7 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/NormalModeOutputRateCalculator.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/NormalModeOutputRateCalculator.java @@ -4,39 +4,39 @@ class NormalModeOutputRateCalculator implements ModeOutputRateCalculator { - private final double rateConvergenceFactor; - private final double slowModeRate; - private final double failuresSpeedupToleranceRatio; - private final double failuresNochangeToleranceRatio; - - NormalModeOutputRateCalculator(double rateConvergenceFactor, - double slowModeRate, - double failuresSpeedupToleranceRatio, - double failuresNochangeToleranceRatio) { - this.rateConvergenceFactor = rateConvergenceFactor; - this.slowModeRate = slowModeRate; - this.failuresSpeedupToleranceRatio = failuresSpeedupToleranceRatio; - this.failuresNochangeToleranceRatio = failuresNochangeToleranceRatio; - } - - @Override - public OutputRateCalculationResult calculateOutputRate(double currentRate, - double maximumOutputRate, - SendCounters counters) { - double calculatedRate = currentRate; - OutputRateCalculator.Mode calculatedMode = OutputRateCalculator.Mode.NORMAL; - - if (!counters.failuresRatioExceeds(failuresSpeedupToleranceRatio) && currentRate < maximumOutputRate) { - double rateAddOn = (maximumOutputRate - currentRate) * rateConvergenceFactor; - calculatedRate = Math.min(maximumOutputRate, currentRate + rateAddOn); - } else if (counters.majorityOfFailures()) { - calculatedRate = slowModeRate; - calculatedMode = OutputRateCalculator.Mode.SLOW; - } else if (counters.failuresRatioExceeds(failuresNochangeToleranceRatio)) { - calculatedRate = Math.max(slowModeRate, currentRate * (1 - rateConvergenceFactor)); - } - - return new OutputRateCalculationResult(calculatedRate, calculatedMode); + private final double rateConvergenceFactor; + private final double slowModeRate; + private final double failuresSpeedupToleranceRatio; + private final double failuresNochangeToleranceRatio; + + NormalModeOutputRateCalculator( + double rateConvergenceFactor, + double slowModeRate, + double failuresSpeedupToleranceRatio, + double failuresNochangeToleranceRatio) { + this.rateConvergenceFactor = rateConvergenceFactor; + this.slowModeRate = slowModeRate; + this.failuresSpeedupToleranceRatio = failuresSpeedupToleranceRatio; + this.failuresNochangeToleranceRatio = failuresNochangeToleranceRatio; + } + + @Override + public OutputRateCalculationResult calculateOutputRate( + double currentRate, double maximumOutputRate, SendCounters counters) { + double calculatedRate = currentRate; + OutputRateCalculator.Mode calculatedMode = OutputRateCalculator.Mode.NORMAL; + + if (!counters.failuresRatioExceeds(failuresSpeedupToleranceRatio) + && currentRate < maximumOutputRate) { + double rateAddOn = (maximumOutputRate - currentRate) * rateConvergenceFactor; + calculatedRate = Math.min(maximumOutputRate, currentRate + rateAddOn); + } else if (counters.majorityOfFailures()) { + calculatedRate = slowModeRate; + calculatedMode = OutputRateCalculator.Mode.SLOW; + } else if (counters.failuresRatioExceeds(failuresNochangeToleranceRatio)) { + calculatedRate = Math.max(slowModeRate, currentRate * (1 - rateConvergenceFactor)); } + return new OutputRateCalculationResult(calculatedRate, calculatedMode); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/OutputRateCalculationResult.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/OutputRateCalculationResult.java index 5e0325c4b8..e834a370e3 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/OutputRateCalculationResult.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/OutputRateCalculationResult.java @@ -2,25 +2,25 @@ public class OutputRateCalculationResult { - private final double rate; + private final double rate; - private final OutputRateCalculator.Mode mode; + private final OutputRateCalculator.Mode mode; - public OutputRateCalculationResult(double rate, OutputRateCalculator.Mode mode) { - this.rate = rate; - this.mode = mode; - } + public OutputRateCalculationResult(double rate, OutputRateCalculator.Mode mode) { + this.rate = rate; + this.mode = mode; + } - public static OutputRateCalculationResult adjustRate(OutputRateCalculationResult result, double adjustedRate) { - return new OutputRateCalculationResult(adjustedRate, result.mode()); - } + public static OutputRateCalculationResult adjustRate( + OutputRateCalculationResult result, double adjustedRate) { + return new OutputRateCalculationResult(adjustedRate, result.mode()); + } - public OutputRateCalculator.Mode mode() { - return mode; - } - - public double rate() { - return rate; - } + public OutputRateCalculator.Mode mode() { + return mode; + } + public double rate() { + return rate; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/OutputRateCalculator.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/OutputRateCalculator.java index 3e8f3ac77c..cb633a9624 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/OutputRateCalculator.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/OutputRateCalculator.java @@ -1,66 +1,69 @@ package pl.allegro.tech.hermes.consumers.consumer.rate.calculator; +import java.util.EnumMap; +import java.util.Map; import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.consumers.consumer.rate.SendCounters; import pl.allegro.tech.hermes.consumers.consumer.rate.maxrate.MaxRateProvider; -import java.util.EnumMap; -import java.util.Map; - public class OutputRateCalculator { - public enum Mode { - NORMAL, - SLOW, - HEARTBEAT - } - - private final Map modeCalculators = new EnumMap<>(Mode.class); + public enum Mode { + NORMAL, + SLOW, + HEARTBEAT + } - private final MaxRateProvider maxRateProvider; + private final Map modeCalculators = new EnumMap<>(Mode.class); - public OutputRateCalculator(RateCalculatorParameters rateCalculatorParameters, MaxRateProvider maxRateProvider) { - this.maxRateProvider = maxRateProvider; + private final MaxRateProvider maxRateProvider; - modeCalculators.put(Mode.NORMAL, - new NormalModeOutputRateCalculator( - rateCalculatorParameters.getConvergenceFactor(), - 1.0 / rateCalculatorParameters.getLimiterSlowModeDelay().toSeconds(), - rateCalculatorParameters.getFailuresSpeedUpToleranceRatio(), - rateCalculatorParameters.getFailuresNoChangeToleranceRatio()) - ); - modeCalculators.put(Mode.SLOW, - new SlowModeOutputRateCalculator( - 1.0 / rateCalculatorParameters.getLimiterHeartbeatModeDelay().toSeconds()) - ); - modeCalculators.put(Mode.HEARTBEAT, new HeartbeatModeOutputRateCalculator( - 1.0 / rateCalculatorParameters.getLimiterSlowModeDelay().toSeconds()) - ); - } + public OutputRateCalculator( + RateCalculatorParameters rateCalculatorParameters, MaxRateProvider maxRateProvider) { + this.maxRateProvider = maxRateProvider; - public OutputRateCalculationResult recalculateRate(SendCounters counters, - Mode currentMode, double currentRateLimit) { + modeCalculators.put( + Mode.NORMAL, + new NormalModeOutputRateCalculator( + rateCalculatorParameters.getConvergenceFactor(), + 1.0 / rateCalculatorParameters.getLimiterSlowModeDelay().toSeconds(), + rateCalculatorParameters.getFailuresSpeedUpToleranceRatio(), + rateCalculatorParameters.getFailuresNoChangeToleranceRatio())); + modeCalculators.put( + Mode.SLOW, + new SlowModeOutputRateCalculator( + 1.0 / rateCalculatorParameters.getLimiterHeartbeatModeDelay().toSeconds())); + modeCalculators.put( + Mode.HEARTBEAT, + new HeartbeatModeOutputRateCalculator( + 1.0 / rateCalculatorParameters.getLimiterSlowModeDelay().toSeconds())); + } - double maximumRate = maxRateProvider.get(); - OutputRateCalculationResult recalculatedResult - = modeCalculators.get(currentMode).calculateOutputRate(currentRateLimit, maximumRate, counters); + public OutputRateCalculationResult recalculateRate( + SendCounters counters, Mode currentMode, double currentRateLimit) { - if (recalculatedResult.rate() > maximumRate) { - recalculatedResult = OutputRateCalculationResult.adjustRate(recalculatedResult, maximumRate); - } + double maximumRate = maxRateProvider.get(); + OutputRateCalculationResult recalculatedResult = + modeCalculators + .get(currentMode) + .calculateOutputRate(currentRateLimit, maximumRate, counters); - return recalculatedResult; + if (recalculatedResult.rate() > maximumRate) { + recalculatedResult = OutputRateCalculationResult.adjustRate(recalculatedResult, maximumRate); } - public void updateSubscription(Subscription newSubscription) { - maxRateProvider.updateSubscription(newSubscription); - } + return recalculatedResult; + } - public void start() { - maxRateProvider.start(); - } + public void updateSubscription(Subscription newSubscription) { + maxRateProvider.updateSubscription(newSubscription); + } - public void shutdown() { - maxRateProvider.shutdown(); - } + public void start() { + maxRateProvider.start(); + } + + public void shutdown() { + maxRateProvider.shutdown(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/OutputRateCalculatorFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/OutputRateCalculatorFactory.java index 52d4892b08..6cbb578de2 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/OutputRateCalculatorFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/OutputRateCalculatorFactory.java @@ -8,19 +8,20 @@ public class OutputRateCalculatorFactory { - private final RateCalculatorParameters rateCalculatorParameters; - private final MaxRateProviderFactory maxRateProviderFactory; + private final RateCalculatorParameters rateCalculatorParameters; + private final MaxRateProviderFactory maxRateProviderFactory; - public OutputRateCalculatorFactory(RateCalculatorParameters rateCalculatorParameters, - MaxRateProviderFactory maxRateProviderFactory) { - this.rateCalculatorParameters = rateCalculatorParameters; - this.maxRateProviderFactory = maxRateProviderFactory; - } + public OutputRateCalculatorFactory( + RateCalculatorParameters rateCalculatorParameters, + MaxRateProviderFactory maxRateProviderFactory) { + this.rateCalculatorParameters = rateCalculatorParameters; + this.maxRateProviderFactory = maxRateProviderFactory; + } - public OutputRateCalculator createCalculator(Subscription subscription, - SendCounters sendCounters, - MetricsFacade metrics) { - MaxRateProvider maxRateProvider = maxRateProviderFactory.create(subscription, sendCounters, metrics); - return new OutputRateCalculator(rateCalculatorParameters, maxRateProvider); - } + public OutputRateCalculator createCalculator( + Subscription subscription, SendCounters sendCounters, MetricsFacade metrics) { + MaxRateProvider maxRateProvider = + maxRateProviderFactory.create(subscription, sendCounters, metrics); + return new OutputRateCalculator(rateCalculatorParameters, maxRateProvider); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/RateCalculatorParameters.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/RateCalculatorParameters.java index 7249c6577c..453b43aefc 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/RateCalculatorParameters.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/RateCalculatorParameters.java @@ -4,13 +4,13 @@ public interface RateCalculatorParameters { - Duration getLimiterHeartbeatModeDelay(); + Duration getLimiterHeartbeatModeDelay(); - Duration getLimiterSlowModeDelay(); + Duration getLimiterSlowModeDelay(); - double getConvergenceFactor(); + double getConvergenceFactor(); - double getFailuresNoChangeToleranceRatio(); + double getFailuresNoChangeToleranceRatio(); - double getFailuresSpeedUpToleranceRatio(); + double getFailuresSpeedUpToleranceRatio(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/SlowModeOutputRateCalculator.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/SlowModeOutputRateCalculator.java index 54d8eac575..ee0533171e 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/SlowModeOutputRateCalculator.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/SlowModeOutputRateCalculator.java @@ -4,22 +4,21 @@ class SlowModeOutputRateCalculator implements ModeOutputRateCalculator { - private final double heartbeatModeRate; + private final double heartbeatModeRate; - SlowModeOutputRateCalculator(double heartbeatModeRate) { - this.heartbeatModeRate = heartbeatModeRate; - } + SlowModeOutputRateCalculator(double heartbeatModeRate) { + this.heartbeatModeRate = heartbeatModeRate; + } - @Override - public OutputRateCalculationResult calculateOutputRate(double currentRate, - double maximumOutputRate, - SendCounters counters) { - if (counters.majorityOfFailures()) { - return new OutputRateCalculationResult(heartbeatModeRate, OutputRateCalculator.Mode.HEARTBEAT); - } else if (counters.onlySuccessess()) { - return new OutputRateCalculationResult(currentRate, OutputRateCalculator.Mode.NORMAL); - } - return new OutputRateCalculationResult(currentRate, OutputRateCalculator.Mode.SLOW); + @Override + public OutputRateCalculationResult calculateOutputRate( + double currentRate, double maximumOutputRate, SendCounters counters) { + if (counters.majorityOfFailures()) { + return new OutputRateCalculationResult( + heartbeatModeRate, OutputRateCalculator.Mode.HEARTBEAT); + } else if (counters.onlySuccessess()) { + return new OutputRateCalculationResult(currentRate, OutputRateCalculator.Mode.NORMAL); } - + return new OutputRateCalculationResult(currentRate, OutputRateCalculator.Mode.SLOW); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerInstance.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerInstance.java index 07adf0684d..3bec1cd15d 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerInstance.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerInstance.java @@ -1,50 +1,52 @@ package pl.allegro.tech.hermes.consumers.consumer.rate.maxrate; -import pl.allegro.tech.hermes.api.SubscriptionName; - import java.util.Objects; +import pl.allegro.tech.hermes.api.SubscriptionName; final class ConsumerInstance { - private final String consumerId; - private final SubscriptionName subscription; + private final String consumerId; + private final SubscriptionName subscription; - ConsumerInstance(String consumerId, SubscriptionName subscription) { - this.consumerId = consumerId; - this.subscription = subscription; - } + ConsumerInstance(String consumerId, SubscriptionName subscription) { + this.consumerId = consumerId; + this.subscription = subscription; + } - public String getConsumerId() { - return consumerId; - } + public String getConsumerId() { + return consumerId; + } - public SubscriptionName getSubscription() { - return subscription; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ConsumerInstance that = (ConsumerInstance) o; - return Objects.equals(consumerId, that.consumerId) - && Objects.equals(subscription, that.subscription); - } + public SubscriptionName getSubscription() { + return subscription; + } - @Override - public int hashCode() { - return Objects.hash(consumerId, subscription); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public String toString() { - return "ConsumerInstance{" - + "consumerId='" + consumerId + '\'' - + ", subscription=" + subscription - + '}'; + if (o == null || getClass() != o.getClass()) { + return false; } + ConsumerInstance that = (ConsumerInstance) o; + return Objects.equals(consumerId, that.consumerId) + && Objects.equals(subscription, that.subscription); + } + + @Override + public int hashCode() { + return Objects.hash(consumerId, subscription); + } + + @Override + public String toString() { + return "ConsumerInstance{" + + "consumerId='" + + consumerId + + '\'' + + ", subscription=" + + subscription + + '}'; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerMaxRates.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerMaxRates.java index 1cd556dc52..f59b465c8a 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerMaxRates.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerMaxRates.java @@ -1,66 +1,67 @@ package pl.allegro.tech.hermes.consumers.consumer.rate.maxrate; -import pl.allegro.tech.hermes.api.SubscriptionName; -import pl.allegro.tech.hermes.consumers.subscription.id.SubscriptionId; - import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; +import pl.allegro.tech.hermes.api.SubscriptionName; +import pl.allegro.tech.hermes.consumers.subscription.id.SubscriptionId; class ConsumerMaxRates { - private final Map maxRates; + private final Map maxRates; - ConsumerMaxRates() { - this.maxRates = new ConcurrentHashMap<>(); - } + ConsumerMaxRates() { + this.maxRates = new ConcurrentHashMap<>(); + } - void setAllMaxRates(ConsumerMaxRates newMaxRates) { - maxRates.clear(); - maxRates.putAll(newMaxRates.maxRates); - } + void setAllMaxRates(ConsumerMaxRates newMaxRates) { + maxRates.clear(); + maxRates.putAll(newMaxRates.maxRates); + } - Optional getMaxRate(SubscriptionName subscription) { - return Optional.ofNullable(maxRates.get(subscription)); - } + Optional getMaxRate(SubscriptionName subscription) { + return Optional.ofNullable(maxRates.get(subscription)); + } - void setMaxRate(SubscriptionName subscription, MaxRate maxRate) { - maxRates.put(subscription, maxRate); - } + void setMaxRate(SubscriptionName subscription, MaxRate maxRate) { + maxRates.put(subscription, maxRate); + } - public void cleanup(Set subscriptions) { - this.maxRates.entrySet() - .removeIf(entry -> !subscriptions.contains(entry.getKey())); - } + public void cleanup(Set subscriptions) { + this.maxRates.entrySet().removeIf(entry -> !subscriptions.contains(entry.getKey())); + } - int size() { - return maxRates.size(); - } + int size() { + return maxRates.size(); + } - Map toSubscriptionsIdsMap(SubscriptionIdMapper subscriptionIdMapping) { - return maxRates.keySet().stream() - .map(subscriptionIdMapping::mapToSubscriptionId) - .filter(Optional::isPresent) - .collect(Collectors.toMap(Optional::get, subscriptionId -> maxRates.get(subscriptionId.get().getSubscriptionName()))); - } + Map toSubscriptionsIdsMap(SubscriptionIdMapper subscriptionIdMapping) { + return maxRates.keySet().stream() + .map(subscriptionIdMapping::mapToSubscriptionId) + .filter(Optional::isPresent) + .collect( + Collectors.toMap( + Optional::get, + subscriptionId -> maxRates.get(subscriptionId.get().getSubscriptionName()))); + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ConsumerMaxRates that = (ConsumerMaxRates) o; - return Objects.equals(maxRates, that.maxRates); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(maxRates); + if (o == null || getClass() != o.getClass()) { + return false; } + ConsumerMaxRates that = (ConsumerMaxRates) o; + return Objects.equals(maxRates, that.maxRates); + } + + @Override + public int hashCode() { + return Objects.hash(maxRates); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerMaxRatesDecoder.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerMaxRatesDecoder.java index ee5d3beebf..2af80e2c52 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerMaxRatesDecoder.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerMaxRatesDecoder.java @@ -7,32 +7,37 @@ class ConsumerMaxRatesDecoder { - private final SubscriptionIds subscriptionIds; + private final SubscriptionIds subscriptionIds; - ConsumerMaxRatesDecoder(SubscriptionIds subscriptionIds) { - this.subscriptionIds = subscriptionIds; - } + ConsumerMaxRatesDecoder(SubscriptionIds subscriptionIds) { + this.subscriptionIds = subscriptionIds; + } + + ConsumerMaxRates decode(byte[] data) { + MessageHeaderDecoder header = new MessageHeaderDecoder(); + MaxRateDecoder body = new MaxRateDecoder(); - ConsumerMaxRates decode(byte[] data) { - MessageHeaderDecoder header = new MessageHeaderDecoder(); - MaxRateDecoder body = new MaxRateDecoder(); - - UnsafeBuffer buffer = new UnsafeBuffer(data); - header.wrap(buffer, 0); - - if (header.templateId() != MaxRateDecoder.TEMPLATE_ID) { - throw new IllegalStateException(String.format("MaxRatesDecoder TEMPLATE_ID=%d does not match encoded TEMPLATE_ID=%d", - MaxRateDecoder.TEMPLATE_ID, header.templateId())); - } - body.wrap(buffer, header.encodedLength(), header.blockLength(), header.version()); - - ConsumerMaxRates result = new ConsumerMaxRates(); - for (MaxRateDecoder.SubscriptionsDecoder subscriptionDecoder : body.subscriptions()) { - long id = subscriptionDecoder.id(); - double maxRate = subscriptionDecoder.maxRate(); - subscriptionIds.getSubscriptionId(id) - .ifPresent(subscriptionId -> result.setMaxRate(subscriptionId.getSubscriptionName(), new MaxRate(maxRate))); - } - return result; + UnsafeBuffer buffer = new UnsafeBuffer(data); + header.wrap(buffer, 0); + + if (header.templateId() != MaxRateDecoder.TEMPLATE_ID) { + throw new IllegalStateException( + String.format( + "MaxRatesDecoder TEMPLATE_ID=%d does not match encoded TEMPLATE_ID=%d", + MaxRateDecoder.TEMPLATE_ID, header.templateId())); + } + body.wrap(buffer, header.encodedLength(), header.blockLength(), header.version()); + + ConsumerMaxRates result = new ConsumerMaxRates(); + for (MaxRateDecoder.SubscriptionsDecoder subscriptionDecoder : body.subscriptions()) { + long id = subscriptionDecoder.id(); + double maxRate = subscriptionDecoder.maxRate(); + subscriptionIds + .getSubscriptionId(id) + .ifPresent( + subscriptionId -> + result.setMaxRate(subscriptionId.getSubscriptionName(), new MaxRate(maxRate))); } + return result; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerMaxRatesEncoder.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerMaxRatesEncoder.java index 6683d5ab91..326ed52ca6 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerMaxRatesEncoder.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerMaxRatesEncoder.java @@ -1,5 +1,6 @@ package pl.allegro.tech.hermes.consumers.consumer.rate.maxrate; +import java.util.Map; import org.agrona.ExpandableDirectByteBuffer; import org.agrona.MutableDirectBuffer; import pl.allegro.tech.hermes.consumers.consumer.rate.sbe.stubs.MaxRateEncoder; @@ -7,37 +8,35 @@ import pl.allegro.tech.hermes.consumers.subscription.id.SubscriptionId; import pl.allegro.tech.hermes.consumers.subscription.id.SubscriptionIds; -import java.util.Map; - class ConsumerMaxRatesEncoder { - private final SubscriptionIds subscriptionIds; - private final MutableDirectBuffer buffer; + private final SubscriptionIds subscriptionIds; + private final MutableDirectBuffer buffer; - ConsumerMaxRatesEncoder(SubscriptionIds subscriptionIds, int bufferSize) { - this.subscriptionIds = subscriptionIds; - this.buffer = new ExpandableDirectByteBuffer(bufferSize); - } + ConsumerMaxRatesEncoder(SubscriptionIds subscriptionIds, int bufferSize) { + this.subscriptionIds = subscriptionIds; + this.buffer = new ExpandableDirectByteBuffer(bufferSize); + } - byte[] encode(ConsumerMaxRates consumerMaxRates) { - MessageHeaderEncoder headerEncoder = new MessageHeaderEncoder(); - MaxRateEncoder body = new MaxRateEncoder(); + byte[] encode(ConsumerMaxRates consumerMaxRates) { + MessageHeaderEncoder headerEncoder = new MessageHeaderEncoder(); + MaxRateEncoder body = new MaxRateEncoder(); - Map filteredRates = consumerMaxRates.toSubscriptionsIdsMap(subscriptionIds::getSubscriptionId); + Map filteredRates = + consumerMaxRates.toSubscriptionsIdsMap(subscriptionIds::getSubscriptionId); - MaxRateEncoder.SubscriptionsEncoder subscriptionsEncoder = body.wrapAndApplyHeader(buffer, 0, headerEncoder) - .subscriptionsCount(filteredRates.size()); + MaxRateEncoder.SubscriptionsEncoder subscriptionsEncoder = + body.wrapAndApplyHeader(buffer, 0, headerEncoder).subscriptionsCount(filteredRates.size()); - filteredRates.forEach((id, maxRate) -> { - subscriptionsEncoder.next() - .id(id.getValue()) - .maxRate(maxRate.getMaxRate()); + filteredRates.forEach( + (id, maxRate) -> { + subscriptionsEncoder.next().id(id.getValue()).maxRate(maxRate.getMaxRate()); }); - int len = headerEncoder.encodedLength() + body.encodedLength(); + int len = headerEncoder.encodedLength() + body.encodedLength(); - byte[] dst = new byte[len]; - buffer.getBytes(0, dst); - return dst; - } + byte[] dst = new byte[len]; + buffer.getBytes(0, dst); + return dst; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerRateHistoriesDecoder.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerRateHistoriesDecoder.java index b79e13c6a7..0991fa6a8d 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerRateHistoriesDecoder.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerRateHistoriesDecoder.java @@ -1,49 +1,53 @@ package pl.allegro.tech.hermes.consumers.consumer.rate.maxrate; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import org.agrona.concurrent.UnsafeBuffer; import pl.allegro.tech.hermes.consumers.consumer.rate.sbe.stubs.MessageHeaderDecoder; import pl.allegro.tech.hermes.consumers.consumer.rate.sbe.stubs.RateHistoryDecoder; import pl.allegro.tech.hermes.consumers.consumer.rate.sbe.stubs.RateHistoryDecoder.SubscriptionsDecoder.RatesDecoder; import pl.allegro.tech.hermes.consumers.subscription.id.SubscriptionIds; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; - class ConsumerRateHistoriesDecoder { - private final SubscriptionIds subscriptionIds; + private final SubscriptionIds subscriptionIds; + + ConsumerRateHistoriesDecoder(SubscriptionIds subscriptionIds) { + this.subscriptionIds = subscriptionIds; + } + + ConsumerRateHistory decode(byte[] data) { + MessageHeaderDecoder header = new MessageHeaderDecoder(); + RateHistoryDecoder body = new RateHistoryDecoder(); + + UnsafeBuffer buffer = new UnsafeBuffer(data); + header.wrap(buffer, 0); - ConsumerRateHistoriesDecoder(SubscriptionIds subscriptionIds) { - this.subscriptionIds = subscriptionIds; + if (header.templateId() != RateHistoryDecoder.TEMPLATE_ID) { + throw new IllegalStateException( + String.format( + "RateHistoryDecoder TEMPLATE_ID=%d does not match encoded TEMPLATE_ID=%d", + RateHistoryDecoder.TEMPLATE_ID, header.templateId())); } - ConsumerRateHistory decode(byte[] data) { - MessageHeaderDecoder header = new MessageHeaderDecoder(); - RateHistoryDecoder body = new RateHistoryDecoder(); - - UnsafeBuffer buffer = new UnsafeBuffer(data); - header.wrap(buffer, 0); - - if (header.templateId() != RateHistoryDecoder.TEMPLATE_ID) { - throw new IllegalStateException(String.format("RateHistoryDecoder TEMPLATE_ID=%d does not match encoded TEMPLATE_ID=%d", - RateHistoryDecoder.TEMPLATE_ID, header.templateId())); - } - - body.wrap(buffer, header.encodedLength(), header.blockLength(), header.version()); - - ConsumerRateHistory result = new ConsumerRateHistory(); - for (RateHistoryDecoder.SubscriptionsDecoder subscriptionDecoder : body.subscriptions()) { - long id = subscriptionDecoder.id(); - List rates = StreamSupport.stream(subscriptionDecoder.rates().spliterator(), false) - .map(RatesDecoder::rate) - .collect(Collectors.toList()); - - subscriptionIds.getSubscriptionId(id) - .ifPresent(subscriptionId -> { - result.setRateHistory(subscriptionId.getSubscriptionName(), new RateHistory(rates)); - }); - } - return result; + body.wrap(buffer, header.encodedLength(), header.blockLength(), header.version()); + + ConsumerRateHistory result = new ConsumerRateHistory(); + for (RateHistoryDecoder.SubscriptionsDecoder subscriptionDecoder : body.subscriptions()) { + long id = subscriptionDecoder.id(); + List rates = + StreamSupport.stream(subscriptionDecoder.rates().spliterator(), false) + .map(RatesDecoder::rate) + .collect(Collectors.toList()); + + subscriptionIds + .getSubscriptionId(id) + .ifPresent( + subscriptionId -> { + result.setRateHistory(subscriptionId.getSubscriptionName(), new RateHistory(rates)); + }); } + return result; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerRateHistoriesEncoder.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerRateHistoriesEncoder.java index f84e4ae114..23b0618439 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerRateHistoriesEncoder.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerRateHistoriesEncoder.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.consumers.consumer.rate.maxrate; +import java.util.List; +import java.util.Map; import org.agrona.ExpandableDirectByteBuffer; import org.agrona.MutableDirectBuffer; import pl.allegro.tech.hermes.consumers.consumer.rate.sbe.stubs.MessageHeaderEncoder; @@ -7,41 +9,38 @@ import pl.allegro.tech.hermes.consumers.subscription.id.SubscriptionId; import pl.allegro.tech.hermes.consumers.subscription.id.SubscriptionIds; -import java.util.List; -import java.util.Map; - class ConsumerRateHistoriesEncoder { - private final MutableDirectBuffer buffer; - private final SubscriptionIds subscriptionIds; + private final MutableDirectBuffer buffer; + private final SubscriptionIds subscriptionIds; - ConsumerRateHistoriesEncoder(SubscriptionIds subscriptionIds, int bufferSize) { - this.subscriptionIds = subscriptionIds; - this.buffer = new ExpandableDirectByteBuffer(bufferSize); - } + ConsumerRateHistoriesEncoder(SubscriptionIds subscriptionIds, int bufferSize) { + this.subscriptionIds = subscriptionIds; + this.buffer = new ExpandableDirectByteBuffer(bufferSize); + } - byte[] encode(ConsumerRateHistory consumerRateHistory) { - MessageHeaderEncoder header = new MessageHeaderEncoder(); - RateHistoryEncoder body = new RateHistoryEncoder(); + byte[] encode(ConsumerRateHistory consumerRateHistory) { + MessageHeaderEncoder header = new MessageHeaderEncoder(); + RateHistoryEncoder body = new RateHistoryEncoder(); - Map historiesFiltered = consumerRateHistory.toSubscriptionIdsMap(subscriptionIds::getSubscriptionId); + Map historiesFiltered = + consumerRateHistory.toSubscriptionIdsMap(subscriptionIds::getSubscriptionId); - RateHistoryEncoder.SubscriptionsEncoder subscriptionsEncoder = body.wrapAndApplyHeader(buffer, 0, header) - .subscriptionsCount(historiesFiltered.size()); + RateHistoryEncoder.SubscriptionsEncoder subscriptionsEncoder = + body.wrapAndApplyHeader(buffer, 0, header).subscriptionsCount(historiesFiltered.size()); - historiesFiltered.forEach((id, rateHistory) -> { - List rates = rateHistory.getRates(); - RateHistoryEncoder.SubscriptionsEncoder.RatesEncoder ratesEncoder = subscriptionsEncoder.next() - .id(id.getValue()) - .ratesCount(rates.size()); + historiesFiltered.forEach( + (id, rateHistory) -> { + List rates = rateHistory.getRates(); + RateHistoryEncoder.SubscriptionsEncoder.RatesEncoder ratesEncoder = + subscriptionsEncoder.next().id(id.getValue()).ratesCount(rates.size()); - rates.forEach(rate -> ratesEncoder.next() - .rate(rate)); + rates.forEach(rate -> ratesEncoder.next().rate(rate)); }); - int len = header.encodedLength() + body.encodedLength(); + int len = header.encodedLength() + body.encodedLength(); - byte[] dst = new byte[len]; - buffer.getBytes(0, dst); - return dst; - } + byte[] dst = new byte[len]; + buffer.getBytes(0, dst); + return dst; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerRateHistory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerRateHistory.java index 39c404e253..f54f18dfb6 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerRateHistory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerRateHistory.java @@ -1,61 +1,63 @@ package pl.allegro.tech.hermes.consumers.consumer.rate.maxrate; -import pl.allegro.tech.hermes.api.SubscriptionName; -import pl.allegro.tech.hermes.consumers.subscription.id.SubscriptionId; - import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import pl.allegro.tech.hermes.api.SubscriptionName; +import pl.allegro.tech.hermes.consumers.subscription.id.SubscriptionId; class ConsumerRateHistory { - private final Map rateHistories; - - ConsumerRateHistory() { - this.rateHistories = new HashMap<>(); - } - - RateHistory getRateHistory(SubscriptionName subscription) { - return rateHistories.getOrDefault(subscription, RateHistory.empty()); - } - - void setRateHistory(SubscriptionName subscription, RateHistory rateHistory) { - rateHistories.put(subscription, rateHistory); - } - - void cleanup(Set subscriptions) { - rateHistories.entrySet() - .removeIf(entry -> !subscriptions.contains(entry.getKey())); - } - - int size() { - return rateHistories.size(); - } - - Map toSubscriptionIdsMap(SubscriptionIdMapper subscriptionIdMapping) { - return rateHistories.keySet().stream() - .map(subscriptionIdMapping::mapToSubscriptionId) - .filter(Optional::isPresent) - .collect(Collectors.toMap(Optional::get, subscriptionId -> rateHistories.get(subscriptionId.get().getSubscriptionName()))); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ConsumerRateHistory that = (ConsumerRateHistory) o; - return Objects.equals(rateHistories, that.rateHistories); - } - - @Override - public int hashCode() { - return Objects.hash(rateHistories); - } + private final Map rateHistories; + + ConsumerRateHistory() { + this.rateHistories = new HashMap<>(); + } + + RateHistory getRateHistory(SubscriptionName subscription) { + return rateHistories.getOrDefault(subscription, RateHistory.empty()); + } + + void setRateHistory(SubscriptionName subscription, RateHistory rateHistory) { + rateHistories.put(subscription, rateHistory); + } + + void cleanup(Set subscriptions) { + rateHistories.entrySet().removeIf(entry -> !subscriptions.contains(entry.getKey())); + } + + int size() { + return rateHistories.size(); + } + + Map toSubscriptionIdsMap( + SubscriptionIdMapper subscriptionIdMapping) { + return rateHistories.keySet().stream() + .map(subscriptionIdMapping::mapToSubscriptionId) + .filter(Optional::isPresent) + .collect( + Collectors.toMap( + Optional::get, + subscriptionId -> rateHistories.get(subscriptionId.get().getSubscriptionName()))); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ConsumerRateHistory that = (ConsumerRateHistory) o; + return Objects.equals(rateHistories, that.rateHistories); + } + + @Override + public int hashCode() { + return Objects.hash(rateHistories); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerRateInfo.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerRateInfo.java index e48e1a6b5d..ad0c5c3eb1 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerRateInfo.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ConsumerRateInfo.java @@ -5,51 +5,55 @@ final class ConsumerRateInfo { - private final String consumerId; - private final Optional maxRate; - private final RateHistory history; - - ConsumerRateInfo(String consumerId, RateInfo rateInfo) { - this.consumerId = consumerId; - this.maxRate = rateInfo.getMaxRate(); - this.history = rateInfo.getRateHistory(); + private final String consumerId; + private final Optional maxRate; + private final RateHistory history; + + ConsumerRateInfo(String consumerId, RateInfo rateInfo) { + this.consumerId = consumerId; + this.maxRate = rateInfo.getMaxRate(); + this.history = rateInfo.getRateHistory(); + } + + Optional getMaxRate() { + return maxRate; + } + + RateHistory getHistory() { + return history; + } + + String getConsumerId() { + return consumerId; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - Optional getMaxRate() { - return maxRate; - } - - RateHistory getHistory() { - return history; - } - - String getConsumerId() { - return consumerId; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ConsumerRateInfo that = (ConsumerRateInfo) o; - return Objects.equals(consumerId, that.consumerId); - } - - @Override - public int hashCode() { - return Objects.hash(consumerId); - } - - @Override - public String toString() { - return "ConsumerRateInfo{" - + "consumerId='" + consumerId + '\'' - + ", maxRate=" + maxRate - + ", history=" + history - + '}'; + if (o == null || getClass() != o.getClass()) { + return false; } + ConsumerRateInfo that = (ConsumerRateInfo) o; + return Objects.equals(consumerId, that.consumerId); + } + + @Override + public int hashCode() { + return Objects.hash(consumerId); + } + + @Override + public String toString() { + return "ConsumerRateInfo{" + + "consumerId='" + + consumerId + + '\'' + + ", maxRate=" + + maxRate + + ", history=" + + history + + '}'; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRate.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRate.java index c04d2fd044..a3d9abdc0e 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRate.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRate.java @@ -5,38 +5,36 @@ final class MaxRate { - private final double maxRate; + private final double maxRate; - @ConstructorProperties({"maxRate"}) - MaxRate(double maxRate) { - this.maxRate = maxRate; - } + @ConstructorProperties({"maxRate"}) + MaxRate(double maxRate) { + this.maxRate = maxRate; + } - public double getMaxRate() { - return maxRate; - } + public double getMaxRate() { + return maxRate; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - MaxRate maxRate1 = (MaxRate) o; - return Double.compare(maxRate1.maxRate, maxRate) == 0; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(maxRate); - } - - @Override - public String toString() { - return "MaxRate{" - + "maxRate=" + maxRate - + '}'; + if (o == null || getClass() != o.getClass()) { + return false; } + MaxRate maxRate1 = (MaxRate) o; + return Double.compare(maxRate1.maxRate, maxRate) == 0; + } + + @Override + public int hashCode() { + return Objects.hash(maxRate); + } + + @Override + public String toString() { + return "MaxRate{" + "maxRate=" + maxRate + '}'; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateBalancer.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateBalancer.java index 8e33f6af0b..1fe0aa5886 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateBalancer.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateBalancer.java @@ -10,331 +10,353 @@ class MaxRateBalancer { - private static final double ALLOWED_DISTRIBUTION_ERROR = 1.0d; + private static final double ALLOWED_DISTRIBUTION_ERROR = 1.0d; - private final double busyTolerance; - private final double minMax; - private final double minAllowedChangePercent; + private final double busyTolerance; + private final double minMax; + private final double minAllowedChangePercent; - MaxRateBalancer(double busyTolerance, double minMax, double minAllowedChangePercent) { - this.busyTolerance = busyTolerance; - this.minMax = minMax; - this.minAllowedChangePercent = minAllowedChangePercent; - } + MaxRateBalancer(double busyTolerance, double minMax, double minAllowedChangePercent) { + this.busyTolerance = busyTolerance; + this.minMax = minMax; + this.minAllowedChangePercent = minAllowedChangePercent; + } + + Optional> balance(double subscriptionMax, Set rateInfos) { + double defaultRate = Math.max(minMax, subscriptionMax / Math.max(1, rateInfos.size())); - Optional> balance(double subscriptionMax, Set rateInfos) { - double defaultRate = Math.max(minMax, subscriptionMax / Math.max(1, rateInfos.size())); - - if (shouldResortToDefaults(subscriptionMax, rateInfos)) { - return Optional.of(balanceDefault(defaultRate, rateInfos)); - } - - List activeConsumerInfos = - rateInfos.stream().map(ActiveConsumerInfo::convert).collect(Collectors.toList()); - - if (subscriptionRateChanged(activeConsumerInfos, subscriptionMax)) { - return Optional.of(balanceDefault(defaultRate, rateInfos)); - } - - Map> busyOrNot = busyOrNot(activeConsumerInfos); - List busy = busyOrNot.get(true); - List notBusy = busyOrNot.get(false) - .stream() - .filter(consumer -> !consumer.getRateHistory().getRates().isEmpty()) - .collect(Collectors.toList()); - - if (busy.isEmpty()) { - return Optional.empty(); - } - - double minChange = (minAllowedChangePercent / 100) * subscriptionMax; - NotBusyBalancer.Result notBusyChanges = handleNotBusy(notBusy, minChange); - Map busyUpdates = - handleBusy(minChange, busy, notBusyChanges.getReleasedRate()).calculateNewMaxRates(); - Map notBusyUpdates = notBusyChanges.calculateNewMaxRates(); - - return Optional.of(Stream.of(busyUpdates, notBusyUpdates) - .map(Map::entrySet) - .flatMap(Collection::stream) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); + if (shouldResortToDefaults(subscriptionMax, rateInfos)) { + return Optional.of(balanceDefault(defaultRate, rateInfos)); } - private boolean shouldResortToDefaults(double subscriptionMax, Set rateInfos) { - return anyNewConsumers(rateInfos) || insufficientSubscriptionRate(subscriptionMax, rateInfos.size()); + List activeConsumerInfos = + rateInfos.stream().map(ActiveConsumerInfo::convert).collect(Collectors.toList()); + + if (subscriptionRateChanged(activeConsumerInfos, subscriptionMax)) { + return Optional.of(balanceDefault(defaultRate, rateInfos)); } - private Map balanceDefault(double defaultRate, Set rateInfos) { - return rateInfos.stream() - .collect(Collectors.toMap(ConsumerRateInfo::getConsumerId, - rateInfo -> new MaxRate(defaultRate))); + Map> busyOrNot = busyOrNot(activeConsumerInfos); + List busy = busyOrNot.get(true); + List notBusy = + busyOrNot.get(false).stream() + .filter(consumer -> !consumer.getRateHistory().getRates().isEmpty()) + .collect(Collectors.toList()); + + if (busy.isEmpty()) { + return Optional.empty(); } - private boolean anyNewConsumers(Set rateInfos) { - return rateInfos.stream().anyMatch(this::isUnassigned); + double minChange = (minAllowedChangePercent / 100) * subscriptionMax; + NotBusyBalancer.Result notBusyChanges = handleNotBusy(notBusy, minChange); + Map busyUpdates = + handleBusy(minChange, busy, notBusyChanges.getReleasedRate()).calculateNewMaxRates(); + Map notBusyUpdates = notBusyChanges.calculateNewMaxRates(); + + return Optional.of( + Stream.of(busyUpdates, notBusyUpdates) + .map(Map::entrySet) + .flatMap(Collection::stream) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); + } + + private boolean shouldResortToDefaults(double subscriptionMax, Set rateInfos) { + return anyNewConsumers(rateInfos) + || insufficientSubscriptionRate(subscriptionMax, rateInfos.size()); + } + + private Map balanceDefault(double defaultRate, Set rateInfos) { + return rateInfos.stream() + .collect( + Collectors.toMap( + ConsumerRateInfo::getConsumerId, rateInfo -> new MaxRate(defaultRate))); + } + + private boolean anyNewConsumers(Set rateInfos) { + return rateInfos.stream().anyMatch(this::isUnassigned); + } + + private boolean insufficientSubscriptionRate(double subscriptionMax, int consumersCount) { + return subscriptionMax / consumersCount <= 1.0d; + } + + private boolean subscriptionRateChanged( + List activeConsumerInfos, double subscriptionMax) { + double sum = activeConsumerInfos.stream().mapToDouble(ActiveConsumerInfo::getMax).sum(); + return Math.abs(sum - subscriptionMax) > ALLOWED_DISTRIBUTION_ERROR; + } + + private boolean isUnassigned(ConsumerRateInfo rateInfo) { + return !rateInfo.getMaxRate().isPresent(); + } + + private Map> busyOrNot(List infos) { + return infos.stream().collect(Collectors.partitioningBy(this::isBusy)); + } + + private boolean isBusy(ActiveConsumerInfo info) { + return info.getRateHistory().getRates().stream() + .mapToDouble(Double::doubleValue) + .average() + .orElse(0) + > 1.0 - busyTolerance; + } + + private NotBusyBalancer.Result handleNotBusy(List notBusy, double minChange) { + return new NotBusyBalancer(notBusy, minChange, minMax).balance(); + } + + private BusyBalancer.Result handleBusy( + double minChange, List busy, double freedByNotBusy) { + return new BusyBalancer(busy, freedByNotBusy, minChange).balance(); + } + + private static class ActiveConsumerInfo { + + private final String consumerId; + private final RateHistory rateHistory; + private final Double max; + + ActiveConsumerInfo(String consumerId, RateHistory rateHistory, Double max) { + this.consumerId = consumerId; + this.rateHistory = rateHistory; + this.max = max; } - private boolean insufficientSubscriptionRate(double subscriptionMax, int consumersCount) { - return subscriptionMax / consumersCount <= 1.0d; + static ActiveConsumerInfo convert(ConsumerRateInfo rateInfo) { + return new ActiveConsumerInfo( + rateInfo.getConsumerId(), + rateInfo.getHistory(), + rateInfo.getMaxRate().get().getMaxRate()); } - private boolean subscriptionRateChanged(List activeConsumerInfos, double subscriptionMax) { - double sum = activeConsumerInfos.stream().mapToDouble(ActiveConsumerInfo::getMax).sum(); - return Math.abs(sum - subscriptionMax) > ALLOWED_DISTRIBUTION_ERROR; + String getConsumerId() { + return consumerId; } - private boolean isUnassigned(ConsumerRateInfo rateInfo) { - return !rateInfo.getMaxRate().isPresent(); + RateHistory getRateHistory() { + return rateHistory; } - private Map> busyOrNot(List infos) { - return infos.stream().collect(Collectors.partitioningBy(this::isBusy)); + Double getMax() { + return max; } + } + + private static class NotBusyBalancer { - private boolean isBusy(ActiveConsumerInfo info) { - return info.getRateHistory().getRates().stream() - .mapToDouble(Double::doubleValue).average().orElse(0) > 1.0 - busyTolerance; + private final List consumerInfos; + private final double minChange; + private final double minMax; + + NotBusyBalancer(List consumerInfos, double minChange, double minMax) { + this.consumerInfos = consumerInfos; + this.minChange = minChange; + this.minMax = minMax; } - private NotBusyBalancer.Result handleNotBusy(List notBusy, double minChange) { - return new NotBusyBalancer(notBusy, minChange, minMax).balance(); + Result balance() { + List changes = + consumerInfos.stream() + .map( + ri -> { + double currentMax = ri.getMax(); + double toDistribute = takeAwayFromNotBusy(ri.getRateHistory(), currentMax); + return new ConsumerRateChange(ri.getConsumerId(), currentMax, -toDistribute); + }) + .collect(Collectors.toList()); + + return new Result(changes); } - private BusyBalancer.Result handleBusy(double minChange, List busy, double freedByNotBusy) { - return new BusyBalancer(busy, freedByNotBusy, minChange).balance(); + private double takeAwayFromNotBusy(RateHistory history, double currentMax) { + double usedRatio = history.getRates().get(0); // rate history must be present + double scalingFactor = 2 / (usedRatio + 1.0) - 1; + + double proposedChange = scalingFactor * currentMax; + double actualChange = proposedChange > minChange ? proposedChange : 0.0d; + + return currentMax - actualChange > minMax ? actualChange : currentMax - minMax; } - private static class ActiveConsumerInfo { + private static class Result { - private final String consumerId; - private final RateHistory rateHistory; - private final Double max; + private final List changes; + private final double releasedRate; - ActiveConsumerInfo(String consumerId, RateHistory rateHistory, Double max) { - this.consumerId = consumerId; - this.rateHistory = rateHistory; - this.max = max; - } + Result(List changes) { + this.changes = changes; + this.releasedRate = -changes.stream().mapToDouble(ConsumerRateChange::getRateChange).sum(); + } - static ActiveConsumerInfo convert(ConsumerRateInfo rateInfo) { - return new ActiveConsumerInfo( - rateInfo.getConsumerId(), - rateInfo.getHistory(), - rateInfo.getMaxRate().get().getMaxRate()); - } + double getReleasedRate() { + return releasedRate; + } - String getConsumerId() { - return consumerId; - } + Map calculateNewMaxRates() { + return changes.stream() + .collect( + Collectors.toMap( + ConsumerRateChange::getConsumerId, + change -> new MaxRate(change.getCurrentMax() + change.getRateChange()))); + } + } + } - RateHistory getRateHistory() { - return rateHistory; - } + private static class BusyBalancer { - Double getMax() { - return max; - } + private final List consumerInfos; + private final double freedByNotBusy; + private final double minChange; + + BusyBalancer(List consumerInfos, double freedByNotBusy, double minChange) { + this.consumerInfos = consumerInfos; + this.freedByNotBusy = freedByNotBusy; + this.minChange = minChange; } - private static class NotBusyBalancer { + Result balance() { + double busyMaxSum = consumerInfos.stream().mapToDouble(ActiveConsumerInfo::getMax).sum(); + + List shares = + consumerInfos.stream() + .map( + info -> + new ConsumerMaxShare( + info.getConsumerId(), info.getMax(), info.getMax() / busyMaxSum)) + .collect(Collectors.toList()); + + if (shares.size() == 1) { + return new Result(distribute(freedByNotBusy, shares)); + } + + double equalShare = busyMaxSum / consumerInfos.size(); + + Map> greedyOrNot = + shares.stream() + .collect( + Collectors.partitioningBy( + share -> share.getCurrentMax() > equalShare + minChange)); + + List greedy = greedyOrNot.get(true); + List notGreedy = greedyOrNot.get(false); + + List greedySubtracts = + greedy.stream() + .map( + share -> { + double toDistribute = + takeAwayFromGreedy(share.getCurrentMax(), share.getShare(), equalShare); + return new ConsumerRateChange( + share.getConsumerId(), share.currentMax, -toDistribute); + }) + .collect(Collectors.toList()); + + double toDistribute = + freedByNotBusy + - greedySubtracts.stream().mapToDouble(ConsumerRateChange::getRateChange).sum(); + + List notGreedyShares = recalculateShare(notGreedy); + List notGreedyAdds = distribute(toDistribute, notGreedyShares); + + return new Result( + Stream.concat(notGreedyAdds.stream(), greedySubtracts.stream()) + .collect(Collectors.toList())); + } - private final List consumerInfos; - private final double minChange; - private final double minMax; + private double takeAwayFromGreedy(double currentMax, double share, double equalShare) { + double scale = 2 / (share + 1.0) - 1; + double changeProposal = (currentMax / 2) * scale; + double actualChange = Math.max(changeProposal, minChange); + return (currentMax - actualChange) > equalShare ? actualChange : currentMax - equalShare; + } - NotBusyBalancer(List consumerInfos, double minChange, double minMax) { - this.consumerInfos = consumerInfos; - this.minChange = minChange; - this.minMax = minMax; - } + private List recalculateShare(List shares) { + double sum = shares.stream().mapToDouble(ConsumerMaxShare::getCurrentMax).sum(); + return shares.stream() + .map( + previous -> + new ConsumerMaxShare( + previous.getConsumerId(), + previous.getCurrentMax(), + previous.currentMax / sum)) + .collect(Collectors.toList()); + } - Result balance() { - List changes = consumerInfos.stream() - .map(ri -> { - double currentMax = ri.getMax(); - double toDistribute = takeAwayFromNotBusy(ri.getRateHistory(), currentMax); - return new ConsumerRateChange(ri.getConsumerId(), currentMax, -toDistribute); - }).collect(Collectors.toList()); + private List distribute(double maxAmount, List shares) { + return shares.stream() + .map( + share -> + new ConsumerRateChange( + share.consumerId, share.getCurrentMax(), share.getShare() * maxAmount)) + .collect(Collectors.toList()); + } - return new Result(changes); - } + private static class Result { - private double takeAwayFromNotBusy(RateHistory history, double currentMax) { - double usedRatio = history.getRates().get(0); // rate history must be present - double scalingFactor = 2 / (usedRatio + 1.0) - 1; + private final List changes; - double proposedChange = scalingFactor * currentMax; - double actualChange = proposedChange > minChange ? proposedChange : 0.0d; + Result(List changes) { + this.changes = changes; + } - return currentMax - actualChange > minMax ? actualChange : currentMax - minMax; - } + Map calculateNewMaxRates() { + return changes.stream() + .collect( + Collectors.toMap( + ConsumerRateChange::getConsumerId, + change -> new MaxRate(change.getCurrentMax() + change.getRateChange()))); + } + } - private static class Result { + private static class ConsumerMaxShare { - private final List changes; - private final double releasedRate; + private final String consumerId; + private final double currentMax; + private final double share; - Result(List changes) { - this.changes = changes; - this.releasedRate = -changes.stream().mapToDouble(ConsumerRateChange::getRateChange).sum(); - } + ConsumerMaxShare(String consumerId, double currentMax, double share) { + this.consumerId = consumerId; + this.currentMax = currentMax; + this.share = share; + } - double getReleasedRate() { - return releasedRate; - } + String getConsumerId() { + return consumerId; + } - Map calculateNewMaxRates() { - return changes.stream() - .collect(Collectors.toMap( - ConsumerRateChange::getConsumerId, - change -> new MaxRate(change.getCurrentMax() + change.getRateChange()))); - } - } - } + double getCurrentMax() { + return currentMax; + } - private static class BusyBalancer { - - private final List consumerInfos; - private final double freedByNotBusy; - private final double minChange; - - BusyBalancer(List consumerInfos, double freedByNotBusy, double minChange) { - this.consumerInfos = consumerInfos; - this.freedByNotBusy = freedByNotBusy; - this.minChange = minChange; - } - - Result balance() { - double busyMaxSum = consumerInfos.stream().mapToDouble(ActiveConsumerInfo::getMax).sum(); - - List shares = consumerInfos.stream() - .map(info -> - new ConsumerMaxShare(info.getConsumerId(), info.getMax(), info.getMax() / busyMaxSum)) - .collect(Collectors.toList()); - - if (shares.size() == 1) { - return new Result(distribute(freedByNotBusy, shares)); - } - - double equalShare = busyMaxSum / consumerInfos.size(); - - Map> greedyOrNot = - shares.stream().collect( - Collectors.partitioningBy(share -> share.getCurrentMax() > equalShare + minChange)); - - List greedy = greedyOrNot.get(true); - List notGreedy = greedyOrNot.get(false); - - List greedySubtracts = greedy.stream() - .map(share -> { - double toDistribute = - takeAwayFromGreedy(share.getCurrentMax(), share.getShare(), equalShare); - return new ConsumerRateChange(share.getConsumerId(), share.currentMax, -toDistribute); - }) - .collect(Collectors.toList()); - - double toDistribute = freedByNotBusy - greedySubtracts.stream() - .mapToDouble(ConsumerRateChange::getRateChange) - .sum(); - - List notGreedyShares = recalculateShare(notGreedy); - List notGreedyAdds = distribute(toDistribute, notGreedyShares); - - return new Result( - Stream.concat(notGreedyAdds.stream(), greedySubtracts.stream()) - .collect(Collectors.toList()) - ); - } - - private double takeAwayFromGreedy(double currentMax, double share, double equalShare) { - double scale = 2 / (share + 1.0) - 1; - double changeProposal = (currentMax / 2) * scale; - double actualChange = Math.max(changeProposal, minChange); - return (currentMax - actualChange) > equalShare ? actualChange : currentMax - equalShare; - } - - private List recalculateShare(List shares) { - double sum = shares.stream().mapToDouble(ConsumerMaxShare::getCurrentMax).sum(); - return shares.stream() - .map(previous -> new ConsumerMaxShare( - previous.getConsumerId(), - previous.getCurrentMax(), - previous.currentMax / sum)) - .collect(Collectors.toList()); - } - - private List distribute(double maxAmount, List shares) { - return shares.stream() - .map(share -> new ConsumerRateChange( - share.consumerId, - share.getCurrentMax(), - share.getShare() * maxAmount)) - .collect(Collectors.toList()); - } - - private static class Result { - - private final List changes; - - Result(List changes) { - this.changes = changes; - } - - Map calculateNewMaxRates() { - return changes.stream() - .collect(Collectors.toMap( - ConsumerRateChange::getConsumerId, - change -> new MaxRate(change.getCurrentMax() + change.getRateChange()))); - } - } - - private static class ConsumerMaxShare { - - private final String consumerId; - private final double currentMax; - private final double share; - - ConsumerMaxShare(String consumerId, double currentMax, double share) { - this.consumerId = consumerId; - this.currentMax = currentMax; - this.share = share; - } - - String getConsumerId() { - return consumerId; - } - - double getCurrentMax() { - return currentMax; - } - - double getShare() { - return share; - } - } + double getShare() { + return share; + } } + } - private static class ConsumerRateChange { + private static class ConsumerRateChange { - private String consumerId; - private double currentMax; - private double rateChange; + private String consumerId; + private double currentMax; + private double rateChange; - ConsumerRateChange(String consumerId, double currentMax, double rateChange) { - this.consumerId = consumerId; - this.currentMax = currentMax; - this.rateChange = rateChange; - } + ConsumerRateChange(String consumerId, double currentMax, double rateChange) { + this.consumerId = consumerId; + this.currentMax = currentMax; + this.rateChange = rateChange; + } - String getConsumerId() { - return consumerId; - } + String getConsumerId() { + return consumerId; + } - double getCurrentMax() { - return currentMax; - } + double getCurrentMax() { + return currentMax; + } - double getRateChange() { - return rateChange; - } + double getRateChange() { + return rateChange; } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateCalculator.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateCalculator.java index 536f815fbc..d52bdcf285 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateCalculator.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateCalculator.java @@ -1,5 +1,9 @@ package pl.allegro.tech.hermes.consumers.consumer.rate.maxrate; +import java.time.Clock; +import java.util.Map; +import java.util.Optional; +import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.Subscription; @@ -7,72 +11,81 @@ import pl.allegro.tech.hermes.consumers.subscription.cache.SubscriptionsCache; import pl.allegro.tech.hermes.consumers.supervisor.workload.ClusterAssignmentCache; -import java.time.Clock; -import java.util.Map; -import java.util.Optional; -import java.util.Set; - class MaxRateCalculator { - private static final Logger logger = LoggerFactory.getLogger(MaxRateCalculator.class); - - private final ClusterAssignmentCache clusterAssignmentCache; - private final SubscriptionsCache subscriptionsCache; - private final MaxRateBalancer balancer; - private final MaxRateRegistry maxRateRegistry; - private final Clock clock; - - private volatile long lastUpdateDurationMillis = 0; - - MaxRateCalculator(ClusterAssignmentCache clusterAssignmentCache, - SubscriptionsCache subscriptionsCache, - MaxRateBalancer balancer, - MaxRateRegistry maxRateRegistry, - MetricsFacade metrics, - Clock clock) { - this.clusterAssignmentCache = clusterAssignmentCache; - this.subscriptionsCache = subscriptionsCache; - this.balancer = balancer; - this.maxRateRegistry = maxRateRegistry; - this.clock = clock; - metrics.maxRate().registerCalculationDurationInMillisGauge(this, calculator -> calculator.lastUpdateDurationMillis); - } - - void calculate() { - try { - logger.info("Max rate calculation started"); - - final long start = clock.millis(); - maxRateRegistry.onBeforeMaxRateCalculation(); - - clusterAssignmentCache.getSubscriptionConsumers().forEach((subscriptionName, consumerIds) -> { + private static final Logger logger = LoggerFactory.getLogger(MaxRateCalculator.class); + + private final ClusterAssignmentCache clusterAssignmentCache; + private final SubscriptionsCache subscriptionsCache; + private final MaxRateBalancer balancer; + private final MaxRateRegistry maxRateRegistry; + private final Clock clock; + + private volatile long lastUpdateDurationMillis = 0; + + MaxRateCalculator( + ClusterAssignmentCache clusterAssignmentCache, + SubscriptionsCache subscriptionsCache, + MaxRateBalancer balancer, + MaxRateRegistry maxRateRegistry, + MetricsFacade metrics, + Clock clock) { + this.clusterAssignmentCache = clusterAssignmentCache; + this.subscriptionsCache = subscriptionsCache; + this.balancer = balancer; + this.maxRateRegistry = maxRateRegistry; + this.clock = clock; + metrics + .maxRate() + .registerCalculationDurationInMillisGauge( + this, calculator -> calculator.lastUpdateDurationMillis); + } + + void calculate() { + try { + logger.info("Max rate calculation started"); + + final long start = clock.millis(); + maxRateRegistry.onBeforeMaxRateCalculation(); + + clusterAssignmentCache + .getSubscriptionConsumers() + .forEach( + (subscriptionName, consumerIds) -> { try { - Subscription subscription = subscriptionsCache.getSubscription(subscriptionName); - if (!subscription.isBatchSubscription()) { + Subscription subscription = subscriptionsCache.getSubscription(subscriptionName); + if (!subscription.isBatchSubscription()) { - Set rateInfos = maxRateRegistry.ensureCorrectAssignments( - subscription.getQualifiedName(), consumerIds); + Set rateInfos = + maxRateRegistry.ensureCorrectAssignments( + subscription.getQualifiedName(), consumerIds); - Optional> newRates - = balancer.balance(subscription.getSerialSubscriptionPolicy().getRate(), rateInfos); + Optional> newRates = + balancer.balance( + subscription.getSerialSubscriptionPolicy().getRate(), rateInfos); - newRates.ifPresent(rates -> { - logger.debug("Calculated new max rates for {}: {}", subscription.getQualifiedName(), rates); + newRates.ifPresent( + rates -> { + logger.debug( + "Calculated new max rates for {}: {}", + subscription.getQualifiedName(), + rates); - maxRateRegistry.update(subscription.getQualifiedName(), rates); + maxRateRegistry.update(subscription.getQualifiedName(), rates); }); - } + } } catch (Exception e) { - logger.error("Problem calculating max rates for subscription {}", subscriptionName, e); + logger.error( + "Problem calculating max rates for subscription {}", subscriptionName, e); } - }); + }); - maxRateRegistry.onAfterMaxRateCalculation(); + maxRateRegistry.onAfterMaxRateCalculation(); - lastUpdateDurationMillis = clock.millis() - start; - logger.info("Max rate calculation done in {} ms", lastUpdateDurationMillis); - } catch (Exception e) { - logger.error("Problem calculating max rate", e); - } + lastUpdateDurationMillis = clock.millis() - start; + logger.info("Max rate calculation done in {} ms", lastUpdateDurationMillis); + } catch (Exception e) { + logger.error("Problem calculating max rate", e); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateCalculatorJob.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateCalculatorJob.java index 0ceebc5814..32fc1422f5 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateCalculatorJob.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateCalculatorJob.java @@ -1,80 +1,83 @@ package pl.allegro.tech.hermes.consumers.consumer.rate.maxrate; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.apache.curator.framework.recipes.leader.LeaderLatchListener; -import pl.allegro.tech.hermes.common.metric.MetricsFacade; -import pl.allegro.tech.hermes.consumers.registry.ConsumerNodesRegistry; -import pl.allegro.tech.hermes.consumers.subscription.cache.SubscriptionsCache; -import pl.allegro.tech.hermes.consumers.supervisor.workload.ClusterAssignmentCache; - import java.time.Clock; import java.time.Duration; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import org.apache.curator.framework.recipes.leader.LeaderLatchListener; +import pl.allegro.tech.hermes.common.metric.MetricsFacade; +import pl.allegro.tech.hermes.consumers.registry.ConsumerNodesRegistry; +import pl.allegro.tech.hermes.consumers.subscription.cache.SubscriptionsCache; +import pl.allegro.tech.hermes.consumers.supervisor.workload.ClusterAssignmentCache; class MaxRateCalculatorJob implements LeaderLatchListener, Runnable { - private final Duration interval; - private final ScheduledExecutorService executorService; - private final MaxRateCalculator maxRateCalculator; - private final ConsumerNodesRegistry consumerNodesRegistry; + private final Duration interval; + private final ScheduledExecutorService executorService; + private final MaxRateCalculator maxRateCalculator; + private final ConsumerNodesRegistry consumerNodesRegistry; - private ScheduledFuture job; + private ScheduledFuture job; - MaxRateCalculatorJob(Duration internal, - ClusterAssignmentCache clusterAssignmentCache, - ConsumerNodesRegistry consumerNodesRegistry, - MaxRateBalancer balancer, - MaxRateRegistry maxRateRegistry, - SubscriptionsCache subscriptionsCache, - MetricsFacade metrics, - Clock clock) { - this.consumerNodesRegistry = consumerNodesRegistry; - this.interval = internal; - this.executorService = Executors.newSingleThreadScheduledExecutor( - new ThreadFactoryBuilder().setNameFormat("max-rate-calculator-%d").build()); - this.maxRateCalculator = new MaxRateCalculator( - clusterAssignmentCache, subscriptionsCache, balancer, maxRateRegistry, metrics, clock - ); - } + MaxRateCalculatorJob( + Duration internal, + ClusterAssignmentCache clusterAssignmentCache, + ConsumerNodesRegistry consumerNodesRegistry, + MaxRateBalancer balancer, + MaxRateRegistry maxRateRegistry, + SubscriptionsCache subscriptionsCache, + MetricsFacade metrics, + Clock clock) { + this.consumerNodesRegistry = consumerNodesRegistry; + this.interval = internal; + this.executorService = + Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder().setNameFormat("max-rate-calculator-%d").build()); + this.maxRateCalculator = + new MaxRateCalculator( + clusterAssignmentCache, subscriptionsCache, balancer, maxRateRegistry, metrics, clock); + } - public void start() { - consumerNodesRegistry.addLeaderLatchListener(this); - startJobIfAlreadyBeingLeader(); - } + public void start() { + consumerNodesRegistry.addLeaderLatchListener(this); + startJobIfAlreadyBeingLeader(); + } - private void startJobIfAlreadyBeingLeader() { - if (consumerNodesRegistry.isLeader()) { - isLeader(); - } + private void startJobIfAlreadyBeingLeader() { + if (consumerNodesRegistry.isLeader()) { + isLeader(); } + } - public void stop() throws InterruptedException { - consumerNodesRegistry.removeLeaderLatchListener(this); + public void stop() throws InterruptedException { + consumerNodesRegistry.removeLeaderLatchListener(this); - if (job != null) { - job.cancel(false); - } - executorService.shutdown(); - executorService.awaitTermination(1, TimeUnit.MINUTES); + if (job != null) { + job.cancel(false); } + executorService.shutdown(); + executorService.awaitTermination(1, TimeUnit.MINUTES); + } - @Override - public void run() { - if (consumerNodesRegistry.isLeader()) { - maxRateCalculator.calculate(); - } + @Override + public void run() { + if (consumerNodesRegistry.isLeader()) { + maxRateCalculator.calculate(); } + } - @Override - public void isLeader() { - job = executorService.scheduleAtFixedRate(this, interval.toSeconds(), interval.toSeconds(), TimeUnit.SECONDS); - } + @Override + public void isLeader() { + job = + executorService.scheduleAtFixedRate( + this, interval.toSeconds(), interval.toSeconds(), TimeUnit.SECONDS); + } - @Override - public void notLeader() { - job.cancel(false); - } + @Override + public void notLeader() { + job.cancel(false); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateParameters.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateParameters.java index 70e8a73346..d655f0f077 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateParameters.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateParameters.java @@ -4,17 +4,17 @@ public interface MaxRateParameters { - Duration getBalanceInterval(); + Duration getBalanceInterval(); - Duration getUpdateInterval(); + Duration getUpdateInterval(); - int getHistorySize(); + int getHistorySize(); - double getBusyTolerance(); + double getBusyTolerance(); - double getMinMaxRate(); + double getMinMaxRate(); - double getMinAllowedChangePercent(); + double getMinAllowedChangePercent(); - double getMinSignificantUpdatePercent(); + double getMinSignificantUpdatePercent(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRatePathSerializer.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRatePathSerializer.java index df2e4fc9f6..30872c65e0 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRatePathSerializer.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRatePathSerializer.java @@ -1,37 +1,43 @@ package pl.allegro.tech.hermes.consumers.consumer.rate.maxrate; -import pl.allegro.tech.hermes.api.SubscriptionName; - import static com.google.common.base.Preconditions.checkArgument; +import pl.allegro.tech.hermes.api.SubscriptionName; + public class MaxRatePathSerializer { - ConsumerInstance consumerInstanceFromContentPath(String path) { - String[] paths = splitContentPath(path); - return new ConsumerInstance(paths[paths.length - 2], SubscriptionName.fromString(paths[paths.length - 3])); - } - - ConsumerInstance consumerInstanceFromConsumerPath(String path) { - String[] paths = splitConsumerPath(path); - return new ConsumerInstance(paths[paths.length - 1], SubscriptionName.fromString(paths[paths.length - 2])); - } - - String content(String path) { - String[] paths = splitContentPath(path); - return paths[paths.length - 1]; - } - - private String[] splitContentPath(String path) { - String[] paths = path.split("/"); - checkArgument(paths.length > 2, - "Incorrect path format. Expected:'/base/subscription/consumerId/content'. Found:'%s'", path); - return paths; - } - - private String[] splitConsumerPath(String path) { - String[] paths = path.split("/"); - checkArgument(paths.length > 2, - "Incorrect path format. Expected:'/base/subscription/consumerId'. Found:'%s'", path); - return paths; - } + ConsumerInstance consumerInstanceFromContentPath(String path) { + String[] paths = splitContentPath(path); + return new ConsumerInstance( + paths[paths.length - 2], SubscriptionName.fromString(paths[paths.length - 3])); + } + + ConsumerInstance consumerInstanceFromConsumerPath(String path) { + String[] paths = splitConsumerPath(path); + return new ConsumerInstance( + paths[paths.length - 1], SubscriptionName.fromString(paths[paths.length - 2])); + } + + String content(String path) { + String[] paths = splitContentPath(path); + return paths[paths.length - 1]; + } + + private String[] splitContentPath(String path) { + String[] paths = path.split("/"); + checkArgument( + paths.length > 2, + "Incorrect path format. Expected:'/base/subscription/consumerId/content'. Found:'%s'", + path); + return paths; + } + + private String[] splitConsumerPath(String path) { + String[] paths = path.split("/"); + checkArgument( + paths.length > 2, + "Incorrect path format. Expected:'/base/subscription/consumerId'. Found:'%s'", + path); + return paths; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateProvider.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateProvider.java index c95834dc6e..00982826dd 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateProvider.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateProvider.java @@ -3,11 +3,11 @@ import pl.allegro.tech.hermes.api.Subscription; public interface MaxRateProvider { - double get(); + double get(); - default void start() {} + default void start() {} - default void shutdown() {} + default void shutdown() {} - default void updateSubscription(Subscription newSubscription) {} + default void updateSubscription(Subscription newSubscription) {} } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateProviderFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateProviderFactory.java index 40fed0de87..e3f09aacde 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateProviderFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateProviderFactory.java @@ -7,34 +7,48 @@ public class MaxRateProviderFactory { - private final Creator providerCreator; - - public MaxRateProviderFactory(MaxRateParameters maxRateParameters, - String nodeId, - MaxRateRegistry maxRateRegistry, - MaxRateSupervisor maxRateSupervisor) { - double minSignificantChange = maxRateParameters.getMinSignificantUpdatePercent() / 100; - checkNegotiatedSettings(minSignificantChange, maxRateParameters.getBusyTolerance()); - providerCreator = (subscription, sendCounters, metrics) -> { - int historyLimit = maxRateParameters.getHistorySize(); - double initialMaxRate = maxRateParameters.getMinMaxRate(); - - return new NegotiatedMaxRateProvider(nodeId, maxRateRegistry, maxRateSupervisor, - subscription, sendCounters, metrics, initialMaxRate, minSignificantChange, historyLimit); + private final Creator providerCreator; + + public MaxRateProviderFactory( + MaxRateParameters maxRateParameters, + String nodeId, + MaxRateRegistry maxRateRegistry, + MaxRateSupervisor maxRateSupervisor) { + double minSignificantChange = maxRateParameters.getMinSignificantUpdatePercent() / 100; + checkNegotiatedSettings(minSignificantChange, maxRateParameters.getBusyTolerance()); + providerCreator = + (subscription, sendCounters, metrics) -> { + int historyLimit = maxRateParameters.getHistorySize(); + double initialMaxRate = maxRateParameters.getMinMaxRate(); + + return new NegotiatedMaxRateProvider( + nodeId, + maxRateRegistry, + maxRateSupervisor, + subscription, + sendCounters, + metrics, + initialMaxRate, + minSignificantChange, + historyLimit); }; - } - - public MaxRateProvider create(Subscription subscription, SendCounters sendCounters, MetricsFacade metrics) { - return providerCreator.create(subscription, sendCounters, metrics); - } - - private interface Creator { - MaxRateProvider create(Subscription subscription, SendCounters sendCounters, MetricsFacade metrics); - } - - private void checkNegotiatedSettings(double minSignificantChange, double busyTolerance) { - Preconditions.checkArgument(busyTolerance > minSignificantChange, - "Significant rate change (%s) can't be higher than busy tolerance (%s)", - minSignificantChange, busyTolerance); - } + } + + public MaxRateProvider create( + Subscription subscription, SendCounters sendCounters, MetricsFacade metrics) { + return providerCreator.create(subscription, sendCounters, metrics); + } + + private interface Creator { + MaxRateProvider create( + Subscription subscription, SendCounters sendCounters, MetricsFacade metrics); + } + + private void checkNegotiatedSettings(double minSignificantChange, double busyTolerance) { + Preconditions.checkArgument( + busyTolerance > minSignificantChange, + "Significant rate change (%s) can't be higher than busy tolerance (%s)", + minSignificantChange, + busyTolerance); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateRegistry.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateRegistry.java index 1aa24f0dac..466285a26a 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateRegistry.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateRegistry.java @@ -1,6 +1,16 @@ package pl.allegro.tech.hermes.consumers.consumer.rate.maxrate; +import static org.slf4j.LoggerFactory.getLogger; + import com.google.common.base.Preconditions; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.NodeCache; @@ -12,238 +22,258 @@ import pl.allegro.tech.hermes.consumers.supervisor.workload.ConsumerAssignmentCache; import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; - -import static org.slf4j.LoggerFactory.getLogger; - public class MaxRateRegistry implements NodeCacheListener { - private static final Logger logger = getLogger(MaxRateRegistry.class); - - private final ZookeeperOperations zookeeper; - - private final Map consumersMaxRates = new HashMap<>(); - private final Map consumersRateHistories = new HashMap<>(); - - private final ConsumerRateHistory currentConsumerRateHistories; - private final ConsumerMaxRates currentConsumerMaxRates; - - private final String consumerId; - private final ClusterAssignmentCache clusterAssignmentCache; - private final ConsumerAssignmentCache consumerAssignmentCache; - - private final ConsumerRateHistoriesEncoder consumerRateHistoriesEncoder; - private final ConsumerRateHistoriesDecoder consumerRateHistoriesDecoder; - private final ConsumerMaxRatesDecoder consumerMaxRatesDecoder; - private final ConsumerMaxRatesEncoder consumerMaxRatesEncoder; - - private final NodeCache maxRateNodeCache; - - private final MaxRateRegistryPaths registryPaths; - - public MaxRateRegistry(int historiesEncoderBufferSize, - int maxRateEncoderBufferSize, - String nodeId, - String clusterName, - ClusterAssignmentCache clusterAssignmentCache, - ConsumerAssignmentCache consumerAssignmentCache, - CuratorFramework curator, - ZookeeperPaths zookeeperPaths, - SubscriptionIds subscriptionIds) { - this.consumerId = nodeId; - this.clusterAssignmentCache = clusterAssignmentCache; - this.consumerAssignmentCache = consumerAssignmentCache; - - this.currentConsumerRateHistories = new ConsumerRateHistory(); - this.currentConsumerMaxRates = new ConsumerMaxRates(); - - this.registryPaths = new MaxRateRegistryPaths(zookeeperPaths, consumerId, clusterName); - this.zookeeper = new ZookeeperOperations(curator); - - this.consumerRateHistoriesEncoder = new ConsumerRateHistoriesEncoder(subscriptionIds, historiesEncoderBufferSize); - this.consumerRateHistoriesDecoder = new ConsumerRateHistoriesDecoder(subscriptionIds); - - this.consumerMaxRatesEncoder = new ConsumerMaxRatesEncoder(subscriptionIds, maxRateEncoderBufferSize); - this.consumerMaxRatesDecoder = new ConsumerMaxRatesDecoder(subscriptionIds); - - this.maxRateNodeCache = new NodeCache(curator, registryPaths.consumerMaxRatePath(consumerId)); - maxRateNodeCache.getListenable().addListener(this); + private static final Logger logger = getLogger(MaxRateRegistry.class); + + private final ZookeeperOperations zookeeper; + + private final Map consumersMaxRates = new HashMap<>(); + private final Map consumersRateHistories = new HashMap<>(); + + private final ConsumerRateHistory currentConsumerRateHistories; + private final ConsumerMaxRates currentConsumerMaxRates; + + private final String consumerId; + private final ClusterAssignmentCache clusterAssignmentCache; + private final ConsumerAssignmentCache consumerAssignmentCache; + + private final ConsumerRateHistoriesEncoder consumerRateHistoriesEncoder; + private final ConsumerRateHistoriesDecoder consumerRateHistoriesDecoder; + private final ConsumerMaxRatesDecoder consumerMaxRatesDecoder; + private final ConsumerMaxRatesEncoder consumerMaxRatesEncoder; + + private final NodeCache maxRateNodeCache; + + private final MaxRateRegistryPaths registryPaths; + + public MaxRateRegistry( + int historiesEncoderBufferSize, + int maxRateEncoderBufferSize, + String nodeId, + String clusterName, + ClusterAssignmentCache clusterAssignmentCache, + ConsumerAssignmentCache consumerAssignmentCache, + CuratorFramework curator, + ZookeeperPaths zookeeperPaths, + SubscriptionIds subscriptionIds) { + this.consumerId = nodeId; + this.clusterAssignmentCache = clusterAssignmentCache; + this.consumerAssignmentCache = consumerAssignmentCache; + + this.currentConsumerRateHistories = new ConsumerRateHistory(); + this.currentConsumerMaxRates = new ConsumerMaxRates(); + + this.registryPaths = new MaxRateRegistryPaths(zookeeperPaths, consumerId, clusterName); + this.zookeeper = new ZookeeperOperations(curator); + + this.consumerRateHistoriesEncoder = + new ConsumerRateHistoriesEncoder(subscriptionIds, historiesEncoderBufferSize); + this.consumerRateHistoriesDecoder = new ConsumerRateHistoriesDecoder(subscriptionIds); + + this.consumerMaxRatesEncoder = + new ConsumerMaxRatesEncoder(subscriptionIds, maxRateEncoderBufferSize); + this.consumerMaxRatesDecoder = new ConsumerMaxRatesDecoder(subscriptionIds); + + this.maxRateNodeCache = new NodeCache(curator, registryPaths.consumerMaxRatePath(consumerId)); + maxRateNodeCache.getListenable().addListener(this); + } + + public void start() { + try { + logger.info( + "Starting binary max rate registry at {}, watching current consumer path at {}", + registryPaths.consumersRateCurrentClusterRuntimeBinaryPath(), + registryPaths.consumerMaxRatePath(consumerId)); + maxRateNodeCache.start(); + } catch (Exception e) { + throw new IllegalStateException("Could not start node cache for consumer max rate", e); } - - public void start() { - try { - logger.info("Starting binary max rate registry at {}, watching current consumer path at {}", - registryPaths.consumersRateCurrentClusterRuntimeBinaryPath(), registryPaths.consumerMaxRatePath(consumerId)); - maxRateNodeCache.start(); - } catch (Exception e) { - throw new IllegalStateException("Could not start node cache for consumer max rate", e); - } - refreshConsumerMaxRates(); + refreshConsumerMaxRates(); + } + + private void refreshConsumerMaxRates() { + ChildData nodeData = maxRateNodeCache.getCurrentData(); + if (nodeData != null) { + byte[] data = nodeData.getData(); + ConsumerMaxRates decodedMaxRates = consumerMaxRatesDecoder.decode(data); + currentConsumerMaxRates.setAllMaxRates(decodedMaxRates); } - - private void refreshConsumerMaxRates() { - ChildData nodeData = maxRateNodeCache.getCurrentData(); - if (nodeData != null) { - byte[] data = nodeData.getData(); - ConsumerMaxRates decodedMaxRates = consumerMaxRatesDecoder.decode(data); - currentConsumerMaxRates.setAllMaxRates(decodedMaxRates); - } - } - - public void stop() { - try { - logger.info("Stopping binary max rate registry"); - maxRateNodeCache.close(); - } catch (IOException e) { - throw new RuntimeException("Could not stop node cache for consumer max rate", e); - } - } - - public void onBeforeMaxRateCalculation() { - Set assignedConsumers = clusterAssignmentCache.getAssignedConsumers(); - clearCacheFromInactiveConsumers(assignedConsumers); - refreshRateCachesOfConsumers(assignedConsumers); - } - - private void clearCacheFromInactiveConsumers(Set assignedConsumers) { - consumersMaxRates.entrySet().removeIf(entry -> !assignedConsumers.contains(entry.getKey())); - consumersRateHistories.entrySet().removeIf(entry -> !assignedConsumers.contains(entry.getKey())); + } + + public void stop() { + try { + logger.info("Stopping binary max rate registry"); + maxRateNodeCache.close(); + } catch (IOException e) { + throw new RuntimeException("Could not stop node cache for consumer max rate", e); } - - private void refreshRateCachesOfConsumers(Set assignedConsumers) { - getMaxRateConsumerNodes().forEach(consumerId -> { - if (assignedConsumers.contains(consumerId)) { + } + + public void onBeforeMaxRateCalculation() { + Set assignedConsumers = clusterAssignmentCache.getAssignedConsumers(); + clearCacheFromInactiveConsumers(assignedConsumers); + refreshRateCachesOfConsumers(assignedConsumers); + } + + private void clearCacheFromInactiveConsumers(Set assignedConsumers) { + consumersMaxRates.entrySet().removeIf(entry -> !assignedConsumers.contains(entry.getKey())); + consumersRateHistories + .entrySet() + .removeIf(entry -> !assignedConsumers.contains(entry.getKey())); + } + + private void refreshRateCachesOfConsumers(Set assignedConsumers) { + getMaxRateConsumerNodes() + .forEach( + consumerId -> { + if (assignedConsumers.contains(consumerId)) { refreshConsumerRateHistory(consumerId); refreshConsumerMaxRate(consumerId); - } else { + } else { removeConsumerRateRootNode(consumerId); - } - }); - } - - private List getMaxRateConsumerNodes() { - String path = registryPaths.consumersRateCurrentClusterRuntimeBinaryPath(); - try { - if (zookeeper.exists(path)) { - return zookeeper.getNodeChildren(path); - } - } catch (Exception e) { - logger.warn("Could not get max rate consumer nodes list", e); - } - return Collections.emptyList(); + } + }); + } + + private List getMaxRateConsumerNodes() { + String path = registryPaths.consumersRateCurrentClusterRuntimeBinaryPath(); + try { + if (zookeeper.exists(path)) { + return zookeeper.getNodeChildren(path); + } + } catch (Exception e) { + logger.warn("Could not get max rate consumer nodes list", e); } - - private void refreshConsumerMaxRate(String consumerId) { - logger.debug("Refreshing max rate of {}", consumerId); - String consumerMaxRatePath = registryPaths.consumerMaxRatePath(consumerId); - zookeeper.getNodeData(consumerMaxRatePath) - .map(consumerMaxRatesDecoder::decode) - .ifPresent(maxRates -> { - int decodedSize = maxRates.size(); - maxRates.cleanup(clusterAssignmentCache.getConsumerSubscriptions(consumerId)); - int cleanedSize = maxRates.size(); - if (decodedSize > cleanedSize) { - logger.debug("Refreshed max rates of {} with {} subscriptions ({} stale entries omitted)", - consumerId, cleanedSize, decodedSize - cleanedSize); - } else { - logger.debug("Refreshed max rates of {} with {} subscriptions", consumerId, cleanedSize); - } - consumersMaxRates.put(consumerId, maxRates); - }); - } - - private void refreshConsumerRateHistory(String consumerId) { - logger.debug("Refreshing rate history of {}", consumerId); - String consumerRateHistoryPath = registryPaths.consumerRateHistoryPath(consumerId); - zookeeper.getNodeData(consumerRateHistoryPath) - .map(consumerRateHistoriesDecoder::decode) - .ifPresent(rateHistories -> { - logger.debug("Refreshed rate history of {} with {} subscriptions", consumerId, rateHistories.size()); - consumersRateHistories.put(consumerId, rateHistories); - }); + return Collections.emptyList(); + } + + private void refreshConsumerMaxRate(String consumerId) { + logger.debug("Refreshing max rate of {}", consumerId); + String consumerMaxRatePath = registryPaths.consumerMaxRatePath(consumerId); + zookeeper + .getNodeData(consumerMaxRatePath) + .map(consumerMaxRatesDecoder::decode) + .ifPresent( + maxRates -> { + int decodedSize = maxRates.size(); + maxRates.cleanup(clusterAssignmentCache.getConsumerSubscriptions(consumerId)); + int cleanedSize = maxRates.size(); + if (decodedSize > cleanedSize) { + logger.debug( + "Refreshed max rates of {} with {} subscriptions ({} stale entries omitted)", + consumerId, + cleanedSize, + decodedSize - cleanedSize); + } else { + logger.debug( + "Refreshed max rates of {} with {} subscriptions", consumerId, cleanedSize); + } + consumersMaxRates.put(consumerId, maxRates); + }); + } + + private void refreshConsumerRateHistory(String consumerId) { + logger.debug("Refreshing rate history of {}", consumerId); + String consumerRateHistoryPath = registryPaths.consumerRateHistoryPath(consumerId); + zookeeper + .getNodeData(consumerRateHistoryPath) + .map(consumerRateHistoriesDecoder::decode) + .ifPresent( + rateHistories -> { + logger.debug( + "Refreshed rate history of {} with {} subscriptions", + consumerId, + rateHistories.size()); + consumersRateHistories.put(consumerId, rateHistories); + }); + } + + private void removeConsumerRateRootNode(String consumerId) { + logger.info("Deleting max rate node of stale consumer {}", consumerId); + String path = registryPaths.consumerRateParentRuntimePath(consumerId); + try { + zookeeper.deleteNodeRecursively(path); + } catch (Exception e) { + logger.warn("Could not delete stale consumer max rate node {}", path, e); } - - private void removeConsumerRateRootNode(String consumerId) { - logger.info("Deleting max rate node of stale consumer {}", consumerId); - String path = registryPaths.consumerRateParentRuntimePath(consumerId); - try { - zookeeper.deleteNodeRecursively(path); - } catch (Exception e) { - logger.warn("Could not delete stale consumer max rate node {}", path, e); - } - } - - public void onAfterMaxRateCalculation() { - persistMaxRatesForAllConsumers(); - } - - private void persistMaxRatesForAllConsumers() { - consumersMaxRates.forEach((consumerId, maxRates) -> { - byte[] encoded = consumerMaxRatesEncoder.encode(maxRates); - String consumerMaxRatePath = registryPaths.consumerMaxRatePath(consumerId); - try { - zookeeper.writeOrCreatePersistent(consumerMaxRatePath, encoded); - } catch (Exception e) { - logger.warn("Could not write max rates for consumer {}", consumerId, e); - } + } + + public void onAfterMaxRateCalculation() { + persistMaxRatesForAllConsumers(); + } + + private void persistMaxRatesForAllConsumers() { + consumersMaxRates.forEach( + (consumerId, maxRates) -> { + byte[] encoded = consumerMaxRatesEncoder.encode(maxRates); + String consumerMaxRatePath = registryPaths.consumerMaxRatePath(consumerId); + try { + zookeeper.writeOrCreatePersistent(consumerMaxRatePath, encoded); + } catch (Exception e) { + logger.warn("Could not write max rates for consumer {}", consumerId, e); + } }); + } + + public Set ensureCorrectAssignments( + SubscriptionName subscriptionName, Set currentConsumers) { + Set rateInfos = new HashSet<>(); + for (String consumerId : currentConsumers) { + Optional maxRate = + Optional.ofNullable(consumersMaxRates.get(consumerId)) + .flatMap(rates -> rates.getMaxRate(subscriptionName)); + RateHistory rateHistory = + Optional.ofNullable(consumersRateHistories.get(consumerId)) + .map(histories -> histories.getRateHistory(subscriptionName)) + .orElse(RateHistory.empty()); + rateInfos.add(new ConsumerRateInfo(consumerId, new RateInfo(maxRate, rateHistory))); } - - public Set ensureCorrectAssignments(SubscriptionName subscriptionName, Set currentConsumers) { - Set rateInfos = new HashSet<>(); - for (String consumerId : currentConsumers) { - Optional maxRate = Optional.ofNullable(consumersMaxRates.get(consumerId)) - .flatMap(rates -> rates.getMaxRate(subscriptionName)); - RateHistory rateHistory = Optional.ofNullable(consumersRateHistories.get(consumerId)) - .map(histories -> histories.getRateHistory(subscriptionName)) - .orElse(RateHistory.empty()); - rateInfos.add(new ConsumerRateInfo(consumerId, new RateInfo(maxRate, rateHistory))); - } - return rateInfos; - } - - public void update(SubscriptionName subscriptionName, Map newMaxRates) { - newMaxRates.forEach((consumerId, maxRate) -> { - consumersMaxRates.putIfAbsent(consumerId, new ConsumerMaxRates()); - consumersMaxRates.get(consumerId).setMaxRate(subscriptionName, maxRate); + return rateInfos; + } + + public void update(SubscriptionName subscriptionName, Map newMaxRates) { + newMaxRates.forEach( + (consumerId, maxRate) -> { + consumersMaxRates.putIfAbsent(consumerId, new ConsumerMaxRates()); + consumersMaxRates.get(consumerId).setMaxRate(subscriptionName, maxRate); }); + } + + public Optional getMaxRate(ConsumerInstance consumer) { + Preconditions.checkState( + consumer.getConsumerId().equals(consumerId), + "Reading max rate is allowed only for current consumer"); + return currentConsumerMaxRates.getMaxRate(consumer.getSubscription()); + } + + public RateHistory getRateHistory(ConsumerInstance consumer) { + Preconditions.checkState( + consumer.getConsumerId().equals(consumerId), + "Reading rate history is allowed only for current consumer"); + return currentConsumerRateHistories.getRateHistory(consumer.getSubscription()); + } + + public void writeRateHistory(ConsumerInstance consumer, RateHistory rateHistory) { + Preconditions.checkState( + consumer.getConsumerId().equals(consumerId), + "Saving rate history is allowed only for current consumer"); + currentConsumerRateHistories.setRateHistory(consumer.getSubscription(), rateHistory); + } + + public void onAfterWriteRateHistories() { + Set subscriptions = consumerAssignmentCache.getConsumerSubscriptions(); + currentConsumerRateHistories.cleanup(subscriptions); + byte[] encoded = consumerRateHistoriesEncoder.encode(currentConsumerRateHistories); + try { + zookeeper.writeOrCreatePersistent(registryPaths.currentConsumerRateHistoryPath(), encoded); + } catch (Exception e) { + logger.error("An error while saving consumers rate histories"); } + } - public Optional getMaxRate(ConsumerInstance consumer) { - Preconditions.checkState(consumer.getConsumerId().equals(consumerId), "Reading max rate is allowed only for current consumer"); - return currentConsumerMaxRates.getMaxRate(consumer.getSubscription()); - } - - public RateHistory getRateHistory(ConsumerInstance consumer) { - Preconditions.checkState(consumer.getConsumerId().equals(consumerId), "Reading rate history is allowed only for current consumer"); - return currentConsumerRateHistories.getRateHistory(consumer.getSubscription()); - } - - public void writeRateHistory(ConsumerInstance consumer, RateHistory rateHistory) { - Preconditions.checkState(consumer.getConsumerId().equals(consumerId), "Saving rate history is allowed only for current consumer"); - currentConsumerRateHistories.setRateHistory(consumer.getSubscription(), rateHistory); - } - - public void onAfterWriteRateHistories() { - Set subscriptions = consumerAssignmentCache.getConsumerSubscriptions(); - currentConsumerRateHistories.cleanup(subscriptions); - byte[] encoded = consumerRateHistoriesEncoder.encode(currentConsumerRateHistories); - try { - zookeeper.writeOrCreatePersistent(registryPaths.currentConsumerRateHistoryPath(), encoded); - } catch (Exception e) { - logger.error("An error while saving consumers rate histories"); - } - } - - @Override - public void nodeChanged() { - refreshConsumerMaxRates(); - } + @Override + public void nodeChanged() { + refreshConsumerMaxRates(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateRegistryPaths.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateRegistryPaths.java index 2999a54f54..f96dc45b6f 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateRegistryPaths.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateRegistryPaths.java @@ -1,42 +1,44 @@ package pl.allegro.tech.hermes.consumers.consumer.rate.maxrate; -import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; - import static pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths.CONSUMERS_RATE_PATH; import static pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths.MAX_RATE_HISTORY_PATH; import static pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths.MAX_RATE_PATH; +import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; + class MaxRateRegistryPaths { - private static final String RATE_RUNTIME_PATH = "runtime-bin"; + private static final String RATE_RUNTIME_PATH = "runtime-bin"; - private final ZookeeperPaths zookeeperPaths; - private final String clusterName; - private final String currentConsumerRateHistoryPath; + private final ZookeeperPaths zookeeperPaths; + private final String clusterName; + private final String currentConsumerRateHistoryPath; - MaxRateRegistryPaths(ZookeeperPaths zookeeperPaths, String currentConsumerId, String clusterName) { - this.zookeeperPaths = zookeeperPaths; - this.clusterName = clusterName; - this.currentConsumerRateHistoryPath = consumerRateHistoryPath(currentConsumerId); - } + MaxRateRegistryPaths( + ZookeeperPaths zookeeperPaths, String currentConsumerId, String clusterName) { + this.zookeeperPaths = zookeeperPaths; + this.clusterName = clusterName; + this.currentConsumerRateHistoryPath = consumerRateHistoryPath(currentConsumerId); + } - String consumerMaxRatePath(String consumerId) { - return zookeeperPaths.join(consumerRateParentRuntimePath(consumerId), MAX_RATE_PATH); - } + String consumerMaxRatePath(String consumerId) { + return zookeeperPaths.join(consumerRateParentRuntimePath(consumerId), MAX_RATE_PATH); + } - String consumerRateHistoryPath(String consumerId) { - return zookeeperPaths.join(consumerRateParentRuntimePath(consumerId), MAX_RATE_HISTORY_PATH); - } + String consumerRateHistoryPath(String consumerId) { + return zookeeperPaths.join(consumerRateParentRuntimePath(consumerId), MAX_RATE_HISTORY_PATH); + } - String consumerRateParentRuntimePath(String consumerId) { - return zookeeperPaths.join(consumersRateCurrentClusterRuntimeBinaryPath(), consumerId); - } + String consumerRateParentRuntimePath(String consumerId) { + return zookeeperPaths.join(consumersRateCurrentClusterRuntimeBinaryPath(), consumerId); + } - String consumersRateCurrentClusterRuntimeBinaryPath() { - return zookeeperPaths.join(zookeeperPaths.basePath(), CONSUMERS_RATE_PATH, clusterName, RATE_RUNTIME_PATH); - } + String consumersRateCurrentClusterRuntimeBinaryPath() { + return zookeeperPaths.join( + zookeeperPaths.basePath(), CONSUMERS_RATE_PATH, clusterName, RATE_RUNTIME_PATH); + } - String currentConsumerRateHistoryPath() { - return currentConsumerRateHistoryPath; - } + String currentConsumerRateHistoryPath() { + return currentConsumerRateHistoryPath; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateSupervisor.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateSupervisor.java index 069356e9dd..5966d27651 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateSupervisor.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateSupervisor.java @@ -1,11 +1,6 @@ package pl.allegro.tech.hermes.consumers.consumer.rate.maxrate; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import pl.allegro.tech.hermes.common.metric.MetricsFacade; -import pl.allegro.tech.hermes.consumers.registry.ConsumerNodesRegistry; -import pl.allegro.tech.hermes.consumers.subscription.cache.SubscriptionsCache; -import pl.allegro.tech.hermes.consumers.supervisor.workload.ClusterAssignmentCache; - import java.time.Clock; import java.time.Duration; import java.util.Collections; @@ -15,79 +10,86 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import pl.allegro.tech.hermes.common.metric.MetricsFacade; +import pl.allegro.tech.hermes.consumers.registry.ConsumerNodesRegistry; +import pl.allegro.tech.hermes.consumers.subscription.cache.SubscriptionsCache; +import pl.allegro.tech.hermes.consumers.supervisor.workload.ClusterAssignmentCache; public class MaxRateSupervisor implements Runnable { - private final Set providers = Collections.newSetFromMap(new ConcurrentHashMap<>()); - private final Duration selfUpdateInterval; - private final ScheduledExecutorService selfUpdateExecutor; - private final MaxRateCalculatorJob calculatorJob; - private final MaxRateRegistry maxRateRegistry; - private ScheduledFuture updateJob; + private final Set providers = + Collections.newSetFromMap(new ConcurrentHashMap<>()); + private final Duration selfUpdateInterval; + private final ScheduledExecutorService selfUpdateExecutor; + private final MaxRateCalculatorJob calculatorJob; + private final MaxRateRegistry maxRateRegistry; + private ScheduledFuture updateJob; - public MaxRateSupervisor(MaxRateParameters maxRateParameters, - ClusterAssignmentCache clusterAssignmentCache, - MaxRateRegistry maxRateRegistry, - ConsumerNodesRegistry consumerNodesRegistry, - SubscriptionsCache subscriptionsCache, - MetricsFacade metrics, - Clock clock) { - this.maxRateRegistry = maxRateRegistry; - this.selfUpdateInterval = maxRateParameters.getUpdateInterval(); + public MaxRateSupervisor( + MaxRateParameters maxRateParameters, + ClusterAssignmentCache clusterAssignmentCache, + MaxRateRegistry maxRateRegistry, + ConsumerNodesRegistry consumerNodesRegistry, + SubscriptionsCache subscriptionsCache, + MetricsFacade metrics, + Clock clock) { + this.maxRateRegistry = maxRateRegistry; + this.selfUpdateInterval = maxRateParameters.getUpdateInterval(); - this.selfUpdateExecutor = Executors.newSingleThreadScheduledExecutor( - new ThreadFactoryBuilder().setNameFormat("max-rate-provider-%d").build() - ); + this.selfUpdateExecutor = + Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder().setNameFormat("max-rate-provider-%d").build()); - MaxRateBalancer balancer = new MaxRateBalancer( - maxRateParameters.getBusyTolerance(), - maxRateParameters.getMinMaxRate(), - maxRateParameters.getMinAllowedChangePercent()); + MaxRateBalancer balancer = + new MaxRateBalancer( + maxRateParameters.getBusyTolerance(), + maxRateParameters.getMinMaxRate(), + maxRateParameters.getMinAllowedChangePercent()); - this.calculatorJob = new MaxRateCalculatorJob( - maxRateParameters.getBalanceInterval(), - clusterAssignmentCache, - consumerNodesRegistry, - balancer, - maxRateRegistry, - subscriptionsCache, - metrics, - clock - ); - } + this.calculatorJob = + new MaxRateCalculatorJob( + maxRateParameters.getBalanceInterval(), + clusterAssignmentCache, + consumerNodesRegistry, + balancer, + maxRateRegistry, + subscriptionsCache, + metrics, + clock); + } - public void start() throws Exception { - maxRateRegistry.start(); - calculatorJob.start(); - updateJob = startSelfUpdate(); - } + public void start() throws Exception { + maxRateRegistry.start(); + calculatorJob.start(); + updateJob = startSelfUpdate(); + } - public void stop() throws Exception { - maxRateRegistry.stop(); - calculatorJob.stop(); - if (updateJob != null) { - updateJob.cancel(false); - } - selfUpdateExecutor.shutdown(); - selfUpdateExecutor.awaitTermination(10, TimeUnit.SECONDS); + public void stop() throws Exception { + maxRateRegistry.stop(); + calculatorJob.stop(); + if (updateJob != null) { + updateJob.cancel(false); } + selfUpdateExecutor.shutdown(); + selfUpdateExecutor.awaitTermination(10, TimeUnit.SECONDS); + } - private ScheduledFuture startSelfUpdate() { - return selfUpdateExecutor.scheduleAtFixedRate( - this, 0, selfUpdateInterval.toSeconds(), TimeUnit.SECONDS); - } + private ScheduledFuture startSelfUpdate() { + return selfUpdateExecutor.scheduleAtFixedRate( + this, 0, selfUpdateInterval.toSeconds(), TimeUnit.SECONDS); + } - @Override - public void run() { - providers.forEach(NegotiatedMaxRateProvider::tickForHistory); - maxRateRegistry.onAfterWriteRateHistories(); - } + @Override + public void run() { + providers.forEach(NegotiatedMaxRateProvider::tickForHistory); + maxRateRegistry.onAfterWriteRateHistories(); + } - public void register(NegotiatedMaxRateProvider maxRateProvider) { - providers.add(maxRateProvider); - } + public void register(NegotiatedMaxRateProvider maxRateProvider) { + providers.add(maxRateProvider); + } - public void unregister(NegotiatedMaxRateProvider maxRateProvider) { - providers.remove(maxRateProvider); - } + public void unregister(NegotiatedMaxRateProvider maxRateProvider) { + providers.remove(maxRateProvider); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/NegotiatedMaxRateProvider.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/NegotiatedMaxRateProvider.java index 705c2c1e2c..0fef710608 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/NegotiatedMaxRateProvider.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/NegotiatedMaxRateProvider.java @@ -1,5 +1,6 @@ package pl.allegro.tech.hermes.consumers.consumer.rate.maxrate; +import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.Subscription; @@ -7,91 +8,98 @@ import pl.allegro.tech.hermes.consumers.consumer.rate.SendCounters; import pl.allegro.tech.hermes.metrics.HermesCounter; -import java.util.Optional; - public class NegotiatedMaxRateProvider implements MaxRateProvider { - private static final Logger logger = LoggerFactory.getLogger(NegotiatedMaxRateProvider.class); + private static final Logger logger = LoggerFactory.getLogger(NegotiatedMaxRateProvider.class); - private final ConsumerInstance consumer; - private final MaxRateRegistry registry; - private final MaxRateSupervisor maxRateSupervisor; - private final SendCounters sendCounters; - private final MetricsFacade metrics; - private final HermesCounter fetchFailuresCounter; - private final HermesCounter historyUpdateFailuresCounter; - private final double minSignificantChange; - private final int historyLimit; - private volatile double maxRate; - private volatile double previousRecordedRate = -1; + private final ConsumerInstance consumer; + private final MaxRateRegistry registry; + private final MaxRateSupervisor maxRateSupervisor; + private final SendCounters sendCounters; + private final MetricsFacade metrics; + private final HermesCounter fetchFailuresCounter; + private final HermesCounter historyUpdateFailuresCounter; + private final double minSignificantChange; + private final int historyLimit; + private volatile double maxRate; + private volatile double previousRecordedRate = -1; - NegotiatedMaxRateProvider(String consumerId, - MaxRateRegistry registry, - MaxRateSupervisor maxRateSupervisor, - Subscription subscription, - SendCounters sendCounters, - MetricsFacade metrics, - double initialMaxRate, - double minSignificantChange, - int historyLimit) { - this.consumer = new ConsumerInstance(consumerId, subscription.getQualifiedName()); - this.registry = registry; - this.maxRateSupervisor = maxRateSupervisor; - this.sendCounters = sendCounters; - this.metrics = metrics; - this.fetchFailuresCounter = metrics.maxRate().fetchFailuresCounter(subscription.getQualifiedName()); - this.historyUpdateFailuresCounter = metrics.maxRate().historyUpdateFailuresCounter(subscription.getQualifiedName()); - this.minSignificantChange = minSignificantChange; - this.historyLimit = historyLimit; - this.maxRate = initialMaxRate; - } + NegotiatedMaxRateProvider( + String consumerId, + MaxRateRegistry registry, + MaxRateSupervisor maxRateSupervisor, + Subscription subscription, + SendCounters sendCounters, + MetricsFacade metrics, + double initialMaxRate, + double minSignificantChange, + int historyLimit) { + this.consumer = new ConsumerInstance(consumerId, subscription.getQualifiedName()); + this.registry = registry; + this.maxRateSupervisor = maxRateSupervisor; + this.sendCounters = sendCounters; + this.metrics = metrics; + this.fetchFailuresCounter = + metrics.maxRate().fetchFailuresCounter(subscription.getQualifiedName()); + this.historyUpdateFailuresCounter = + metrics.maxRate().historyUpdateFailuresCounter(subscription.getQualifiedName()); + this.minSignificantChange = minSignificantChange; + this.historyLimit = historyLimit; + this.maxRate = initialMaxRate; + } - @Override - public double get() { - return maxRate; - } + @Override + public double get() { + return maxRate; + } - void tickForHistory() { - recordCurrentRate(sendCounters.getRate()); - fetchCurrentMaxRate().ifPresent(currentMaxRate -> maxRate = currentMaxRate.getMaxRate()); - } + void tickForHistory() { + recordCurrentRate(sendCounters.getRate()); + fetchCurrentMaxRate().ifPresent(currentMaxRate -> maxRate = currentMaxRate.getMaxRate()); + } - private void recordCurrentRate(double actualRate) { - double usedRate = Math.min(actualRate / Math.max(maxRate, 1), 1.0d); - if (shouldRecordHistory(usedRate)) { - try { - RateHistory rateHistory = registry.getRateHistory(consumer); - RateHistory updatedHistory = RateHistory.updatedRates(rateHistory, usedRate, historyLimit); - registry.writeRateHistory(consumer, updatedHistory); - previousRecordedRate = usedRate; - } catch (Exception e) { - logger.warn("Encountered problem updating max rate for {}", consumer, e); - historyUpdateFailuresCounter.increment(); - } - } + private void recordCurrentRate(double actualRate) { + double usedRate = Math.min(actualRate / Math.max(maxRate, 1), 1.0d); + if (shouldRecordHistory(usedRate)) { + try { + RateHistory rateHistory = registry.getRateHistory(consumer); + RateHistory updatedHistory = RateHistory.updatedRates(rateHistory, usedRate, historyLimit); + registry.writeRateHistory(consumer, updatedHistory); + previousRecordedRate = usedRate; + } catch (Exception e) { + logger.warn("Encountered problem updating max rate for {}", consumer, e); + historyUpdateFailuresCounter.increment(); + } } + } - private boolean shouldRecordHistory(double usedRate) { - return previousRecordedRate < 0 || Math.abs(previousRecordedRate - usedRate) > minSignificantChange; - } + private boolean shouldRecordHistory(double usedRate) { + return previousRecordedRate < 0 + || Math.abs(previousRecordedRate - usedRate) > minSignificantChange; + } - private Optional fetchCurrentMaxRate() { - try { - return registry.getMaxRate(consumer); - } catch (Exception e) { - logger.warn("Encountered problem fetching max rate for {}", consumer); - fetchFailuresCounter.increment(); - return Optional.empty(); - } + private Optional fetchCurrentMaxRate() { + try { + return registry.getMaxRate(consumer); + } catch (Exception e) { + logger.warn("Encountered problem fetching max rate for {}", consumer); + fetchFailuresCounter.increment(); + return Optional.empty(); } + } - public void start() { - maxRateSupervisor.register(this); - metrics.maxRate().registerCalculatedRateGauge(consumer.getSubscription(), this, NegotiatedMaxRateProvider::get); - metrics.maxRate().registerActualRateGauge(consumer.getSubscription(), sendCounters, SendCounters::getRate); - } + public void start() { + maxRateSupervisor.register(this); + metrics + .maxRate() + .registerCalculatedRateGauge( + consumer.getSubscription(), this, NegotiatedMaxRateProvider::get); + metrics + .maxRate() + .registerActualRateGauge(consumer.getSubscription(), sendCounters, SendCounters::getRate); + } - public void shutdown() { - maxRateSupervisor.unregister(this); - } + public void shutdown() { + maxRateSupervisor.unregister(this); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/RateHistory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/RateHistory.java index 17bc2f68e7..b2a380eecc 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/RateHistory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/RateHistory.java @@ -1,7 +1,6 @@ package pl.allegro.tech.hermes.consumers.consumer.rate.maxrate; import com.google.common.base.Preconditions; - import java.beans.ConstructorProperties; import java.util.Arrays; import java.util.Collections; @@ -11,54 +10,52 @@ import java.util.stream.Stream; public final class RateHistory { - private final List rates; - - @ConstructorProperties({"rates"}) - public RateHistory(List rates) { - this.rates = rates; - } - - static RateHistory updatedRates(RateHistory history, double newRate, int limit) { - Preconditions.checkArgument(limit > 0); - List rates = Stream.concat( - Stream.of(newRate), history.getRates().stream().limit(limit - 1)).collect(Collectors.toList()); - return new RateHistory(rates); - } - - static RateHistory create(double rate) { - return new RateHistory(Arrays.asList(rate)); - } - - static RateHistory empty() { - return new RateHistory(Collections.emptyList()); - } - - public List getRates() { - return rates; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - RateHistory that = (RateHistory) o; - return Objects.equals(rates, that.rates); - } - - @Override - public int hashCode() { - return Objects.hash(rates); - } - - @Override - public String toString() { - return "RateHistory{" - + "rates=" - + rates - + '}'; - } + private final List rates; + + @ConstructorProperties({"rates"}) + public RateHistory(List rates) { + this.rates = rates; + } + + static RateHistory updatedRates(RateHistory history, double newRate, int limit) { + Preconditions.checkArgument(limit > 0); + List rates = + Stream.concat(Stream.of(newRate), history.getRates().stream().limit(limit - 1)) + .collect(Collectors.toList()); + return new RateHistory(rates); + } + + static RateHistory create(double rate) { + return new RateHistory(Arrays.asList(rate)); + } + + static RateHistory empty() { + return new RateHistory(Collections.emptyList()); + } + + public List getRates() { + return rates; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RateHistory that = (RateHistory) o; + return Objects.equals(rates, that.rates); + } + + @Override + public int hashCode() { + return Objects.hash(rates); + } + + @Override + public String toString() { + return "RateHistory{" + "rates=" + rates + '}'; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/RateInfo.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/RateInfo.java index 1f9dc87b37..4084df7202 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/RateInfo.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/RateInfo.java @@ -4,50 +4,47 @@ class RateInfo { - private static final RateInfo empty = new RateInfo(Optional.empty(), RateHistory.empty()); + private static final RateInfo empty = new RateInfo(Optional.empty(), RateHistory.empty()); - private final Optional maxRate; + private final Optional maxRate; - private final RateHistory rateHistory; + private final RateHistory rateHistory; - RateInfo(Optional maxRate, RateHistory rateHistory) { - this.maxRate = maxRate; - this.rateHistory = rateHistory; - } + RateInfo(Optional maxRate, RateHistory rateHistory) { + this.maxRate = maxRate; + this.rateHistory = rateHistory; + } - static RateInfo empty() { - return empty; - } + static RateInfo empty() { + return empty; + } - static RateInfo withNoHistory(MaxRate maxRate) { - return new RateInfo(Optional.of(maxRate), RateHistory.empty()); - } + static RateInfo withNoHistory(MaxRate maxRate) { + return new RateInfo(Optional.of(maxRate), RateHistory.empty()); + } - static RateInfo withNoMaxRate(RateHistory rateHistory) { - return new RateInfo(Optional.empty(), rateHistory); - } + static RateInfo withNoMaxRate(RateHistory rateHistory) { + return new RateInfo(Optional.empty(), rateHistory); + } - Optional getMaxRate() { - return maxRate; - } + Optional getMaxRate() { + return maxRate; + } - RateHistory getRateHistory() { - return rateHistory; - } + RateHistory getRateHistory() { + return rateHistory; + } - RateInfo copyWithNewMaxRate(MaxRate maxRate) { - return new RateInfo(Optional.of(maxRate), this.rateHistory); - } + RateInfo copyWithNewMaxRate(MaxRate maxRate) { + return new RateInfo(Optional.of(maxRate), this.rateHistory); + } - RateInfo copyWithNewRateHistory(RateHistory rateHistory) { - return new RateInfo(this.maxRate, rateHistory); - } + RateInfo copyWithNewRateHistory(RateHistory rateHistory) { + return new RateInfo(this.maxRate, rateHistory); + } - @Override - public String toString() { - return "RateInfo{" - + "maxRate=" + maxRate - + ", rateHistory=" + rateHistory - + '}'; - } + @Override + public String toString() { + return "RateInfo{" + "maxRate=" + maxRate + ", rateHistory=" + rateHistory + '}'; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/SubscriptionIdMapper.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/SubscriptionIdMapper.java index efd63ed2b2..388507559a 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/SubscriptionIdMapper.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/SubscriptionIdMapper.java @@ -1,12 +1,11 @@ package pl.allegro.tech.hermes.consumers.consumer.rate.maxrate; +import java.util.Optional; import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.consumers.subscription.id.SubscriptionId; -import java.util.Optional; - @FunctionalInterface interface SubscriptionIdMapper { - Optional mapToSubscriptionId(SubscriptionName subscriptionName); + Optional mapToSubscriptionId(SubscriptionName subscriptionName); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ZookeeperOperations.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ZookeeperOperations.java index 7d954066a5..b263c42ec5 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ZookeeperOperations.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/ZookeeperOperations.java @@ -1,69 +1,71 @@ package pl.allegro.tech.hermes.consumers.consumer.rate.maxrate; +import java.util.List; +import java.util.Optional; import org.apache.curator.framework.CuratorFramework; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import pl.allegro.tech.hermes.common.exception.InternalProcessingException; -import java.util.List; -import java.util.Optional; - class ZookeeperOperations { - private final CuratorFramework curator; + private final CuratorFramework curator; - ZookeeperOperations(CuratorFramework curator) { - this.curator = curator; - } + ZookeeperOperations(CuratorFramework curator) { + this.curator = curator; + } - void writeOrCreatePersistent(String path, byte[] serializedData) throws Exception { - try { - curator.setData().forPath(path, serializedData); - } catch (KeeperException.NoNodeException e) { - try { - curator.create().creatingParentContainersIfNeeded() - .withMode(CreateMode.PERSISTENT) - .forPath(path, serializedData); - } catch (KeeperException.NodeExistsException ex) { - // ignore - } - } + void writeOrCreatePersistent(String path, byte[] serializedData) throws Exception { + try { + curator.setData().forPath(path, serializedData); + } catch (KeeperException.NoNodeException e) { + try { + curator + .create() + .creatingParentContainersIfNeeded() + .withMode(CreateMode.PERSISTENT) + .forPath(path, serializedData); + } catch (KeeperException.NodeExistsException ex) { + // ignore + } } + } - Optional getNodeData(String path) { - try { - if (curator.checkExists().forPath(path) != null) { - return Optional.of(curator.getData().forPath(path)); - } - } catch (Exception e) { - throw new InternalProcessingException(String.format("Could not read node data on path %s", path), e); - } - return Optional.empty(); + Optional getNodeData(String path) { + try { + if (curator.checkExists().forPath(path) != null) { + return Optional.of(curator.getData().forPath(path)); + } + } catch (Exception e) { + throw new InternalProcessingException( + String.format("Could not read node data on path %s", path), e); } + return Optional.empty(); + } - void deleteNodeRecursively(String path) { - try { - if (curator.checkExists().forPath(path) != null) { - curator.delete().deletingChildrenIfNeeded().forPath(path); - } - } catch (Exception e) { - throw new InternalProcessingException("Could not delete node " + path, e); - } + void deleteNodeRecursively(String path) { + try { + if (curator.checkExists().forPath(path) != null) { + curator.delete().deletingChildrenIfNeeded().forPath(path); + } + } catch (Exception e) { + throw new InternalProcessingException("Could not delete node " + path, e); } + } - List getNodeChildren(String path) { - try { - return curator.getChildren().forPath(path); - } catch (Exception e) { - throw new InternalProcessingException("Could not get children of node " + path, e); - } + List getNodeChildren(String path) { + try { + return curator.getChildren().forPath(path); + } catch (Exception e) { + throw new InternalProcessingException("Could not get children of node " + path, e); } + } - boolean exists(String path) { - try { - return curator.checkExists().forPath(path) != null; - } catch (Exception e) { - throw new InternalProcessingException("Unable to check existence of node " + path, e); - } + boolean exists(String path) { + try { + return curator.checkExists().forPath(path) != null; + } catch (Exception e) { + throw new InternalProcessingException("Unable to check existence of node " + path, e); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/ConsumerNotInitializedException.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/ConsumerNotInitializedException.java index ed7e3561c8..a5219dc1ad 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/ConsumerNotInitializedException.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/ConsumerNotInitializedException.java @@ -3,7 +3,7 @@ import pl.allegro.tech.hermes.common.exception.InternalProcessingException; public class ConsumerNotInitializedException extends InternalProcessingException { - public ConsumerNotInitializedException() { - super("Please make sure that you call initialize first."); - } + public ConsumerNotInitializedException() { + super("Please make sure that you call initialize first."); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/MessageReceiver.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/MessageReceiver.java index b875bd8b40..797539028a 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/MessageReceiver.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/MessageReceiver.java @@ -1,37 +1,37 @@ package pl.allegro.tech.hermes.consumers.consumer.receiver; +import java.util.Optional; +import java.util.Set; import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.common.kafka.offset.PartitionOffset; import pl.allegro.tech.hermes.consumers.consumer.Message; import pl.allegro.tech.hermes.consumers.consumer.offset.SubscriptionPartitionOffset; -import java.util.Optional; -import java.util.Set; - public interface MessageReceiver { - /** - * Retrieves the next available message from the queue. - * - *

Depending on the context, the returned {@link Optional} can contain: - *

    - *
  • A {@link Message} that contains a valid message ready to be sent.
  • - *
  • A {@link Message} with the `isFiltered` flag set, indicating that the message - * has been filtered and should be skipped during processing or sending.
  • - *
  • {@code null}, indicating that there are no messages currently available in the queue.
  • - *
- * - * @return an {@link Optional} containing the next {@link Message} if available; - * an {@link Optional} containing a filtered message if it should be skipped; - * or an empty {@link Optional} if there are no messages in the queue. - */ - Optional next(); - - default void stop() {} - - default void update(Subscription newSubscription) {} - - void commit(Set offsets); - - boolean moveOffset(PartitionOffset offset); + /** + * Retrieves the next available message from the queue. + * + *

Depending on the context, the returned {@link Optional} can contain: + * + *

    + *
  • A {@link Message} that contains a valid message ready to be sent. + *
  • A {@link Message} with the `isFiltered` flag set, indicating that the message has been + * filtered and should be skipped during processing or sending. + *
  • {@code null}, indicating that there are no messages currently available in the queue. + *
+ * + * @return an {@link Optional} containing the next {@link Message} if available; an {@link + * Optional} containing a filtered message if it should be skipped; or an empty {@link + * Optional} if there are no messages in the queue. + */ + Optional next(); + + default void stop() {} + + default void update(Subscription newSubscription) {} + + void commit(Set offsets); + + boolean moveOffset(PartitionOffset offset); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/ReceiverFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/ReceiverFactory.java index c3dce357a3..75c379a8c7 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/ReceiverFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/ReceiverFactory.java @@ -9,11 +9,11 @@ public interface ReceiverFactory { - MessageReceiver createMessageReceiver(Topic receivingTopic, - Subscription subscription, - ConsumerRateLimiter consumerRateLimiter, - SubscriptionLoadRecorder subscriptionLoadRecorder, - MetricsFacade metrics, - PendingOffsetsAppender pendingOffsetsAppender); - + MessageReceiver createMessageReceiver( + Topic receivingTopic, + Subscription subscription, + ConsumerRateLimiter consumerRateLimiter, + SubscriptionLoadRecorder subscriptionLoadRecorder, + MetricsFacade metrics, + PendingOffsetsAppender pendingOffsetsAppender); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/RetryableReceiverError.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/RetryableReceiverError.java index 56b013ee41..8cd5065d5d 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/RetryableReceiverError.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/RetryableReceiverError.java @@ -3,7 +3,7 @@ import pl.allegro.tech.hermes.common.exception.InternalProcessingException; public class RetryableReceiverError extends InternalProcessingException { - public RetryableReceiverError(String message, Throwable throwable) { - super(message, throwable); - } + public RetryableReceiverError(String message, Throwable throwable) { + super(message, throwable); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/ThrottlingMessageReceiver.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/ThrottlingMessageReceiver.java index 41d959780f..7abe865d26 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/ThrottlingMessageReceiver.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/ThrottlingMessageReceiver.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.consumers.consumer.receiver; +import java.util.Optional; +import java.util.Set; import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.common.kafka.offset.PartitionOffset; @@ -10,61 +12,58 @@ import pl.allegro.tech.hermes.metrics.HermesTimer; import pl.allegro.tech.hermes.metrics.HermesTimerContext; -import java.util.Optional; -import java.util.Set; - - public class ThrottlingMessageReceiver implements MessageReceiver { - private final MessageReceiver receiver; - private final IdleTimeCalculator idleTimeCalculator; - private final HermesTimer idleTimer; + private final MessageReceiver receiver; + private final IdleTimeCalculator idleTimeCalculator; + private final HermesTimer idleTimer; - public ThrottlingMessageReceiver(MessageReceiver receiver, - IdleTimeCalculator idleTimeCalculator, - SubscriptionName subscriptionName, - MetricsFacade metrics) { - this.receiver = receiver; - this.idleTimeCalculator = idleTimeCalculator; - this.idleTimer = metrics.subscriptions().consumerIdleTimer(subscriptionName); - } + public ThrottlingMessageReceiver( + MessageReceiver receiver, + IdleTimeCalculator idleTimeCalculator, + SubscriptionName subscriptionName, + MetricsFacade metrics) { + this.receiver = receiver; + this.idleTimeCalculator = idleTimeCalculator; + this.idleTimer = metrics.subscriptions().consumerIdleTimer(subscriptionName); + } - @Override - public Optional next() { - Optional next = receiver.next(); - if (next.isPresent()) { - idleTimeCalculator.reset(); - } else { - awaitUntilNextPoll(); - } - return next; + @Override + public Optional next() { + Optional next = receiver.next(); + if (next.isPresent()) { + idleTimeCalculator.reset(); + } else { + awaitUntilNextPoll(); } + return next; + } - private void awaitUntilNextPoll() { - try (HermesTimerContext ignored = idleTimer.time()) { - Thread.sleep(idleTimeCalculator.increaseIdleTime()); - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - } + private void awaitUntilNextPoll() { + try (HermesTimerContext ignored = idleTimer.time()) { + Thread.sleep(idleTimeCalculator.increaseIdleTime()); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); } + } - @Override - public void commit(Set offsets) { - receiver.commit(offsets); - } + @Override + public void commit(Set offsets) { + receiver.commit(offsets); + } - @Override - public boolean moveOffset(PartitionOffset offset) { - return receiver.moveOffset(offset); - } + @Override + public boolean moveOffset(PartitionOffset offset) { + return receiver.moveOffset(offset); + } - @Override - public void stop() { - receiver.stop(); - } + @Override + public void stop() { + receiver.stop(); + } - @Override - public void update(Subscription newSubscription) { - this.receiver.update(newSubscription); - } + @Override + public void update(Subscription newSubscription) { + this.receiver.update(newSubscription); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/UninitializedMessageReceiver.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/UninitializedMessageReceiver.java index cfb9b4a18a..591a24384d 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/UninitializedMessageReceiver.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/UninitializedMessageReceiver.java @@ -1,25 +1,24 @@ package pl.allegro.tech.hermes.consumers.consumer.receiver; +import java.util.Optional; +import java.util.Set; import pl.allegro.tech.hermes.common.kafka.offset.PartitionOffset; import pl.allegro.tech.hermes.consumers.consumer.Message; import pl.allegro.tech.hermes.consumers.consumer.offset.SubscriptionPartitionOffset; -import java.util.Optional; -import java.util.Set; - public class UninitializedMessageReceiver implements MessageReceiver { - @Override - public Optional next() { - throw new ConsumerNotInitializedException(); - } + @Override + public Optional next() { + throw new ConsumerNotInitializedException(); + } - @Override - public void commit(Set offsets) { - throw new ConsumerNotInitializedException(); - } + @Override + public void commit(Set offsets) { + throw new ConsumerNotInitializedException(); + } - @Override - public boolean moveOffset(PartitionOffset offset) { - throw new ConsumerNotInitializedException(); - } + @Override + public boolean moveOffset(PartitionOffset offset) { + throw new ConsumerNotInitializedException(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/BasicMessageContentReader.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/BasicMessageContentReader.java index 8bd23d7c87..9e41a1041d 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/BasicMessageContentReader.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/BasicMessageContentReader.java @@ -13,43 +13,47 @@ class BasicMessageContentReader implements MessageContentReader { - private final CompositeMessageContentWrapper compositeMessageContentWrapper; - private final KafkaHeaderExtractor kafkaHeaderExtractor; - private final Topic topic; - private final SchemaExistenceEnsurer schemaExistenceEnsurer; + private final CompositeMessageContentWrapper compositeMessageContentWrapper; + private final KafkaHeaderExtractor kafkaHeaderExtractor; + private final Topic topic; + private final SchemaExistenceEnsurer schemaExistenceEnsurer; - BasicMessageContentReader(CompositeMessageContentWrapper compositeMessageContentWrapper, - KafkaHeaderExtractor kafkaHeaderExtractor, - Topic topic, SchemaExistenceEnsurer schemaExistenceEnsurer) { - this.compositeMessageContentWrapper = compositeMessageContentWrapper; - this.kafkaHeaderExtractor = kafkaHeaderExtractor; - this.topic = topic; - this.schemaExistenceEnsurer = schemaExistenceEnsurer; - } + BasicMessageContentReader( + CompositeMessageContentWrapper compositeMessageContentWrapper, + KafkaHeaderExtractor kafkaHeaderExtractor, + Topic topic, + SchemaExistenceEnsurer schemaExistenceEnsurer) { + this.compositeMessageContentWrapper = compositeMessageContentWrapper; + this.kafkaHeaderExtractor = kafkaHeaderExtractor; + this.topic = topic; + this.schemaExistenceEnsurer = schemaExistenceEnsurer; + } - @Override - public UnwrappedMessageContent read(ConsumerRecord message, ContentType contentType) { - if (contentType == ContentType.AVRO) { - Integer schemaVersion = kafkaHeaderExtractor.extractSchemaVersion(message.headers()); - Integer schemaId = kafkaHeaderExtractor.extractSchemaId(message.headers()); - ensureExistence(schemaVersion, schemaId); - return compositeMessageContentWrapper.unwrapAvro(message.value(), topic, schemaId, schemaVersion); - } else if (contentType == ContentType.JSON) { - return compositeMessageContentWrapper.unwrapJson(message.value()); - } - throw new UnsupportedContentTypeException(topic); + @Override + public UnwrappedMessageContent read( + ConsumerRecord message, ContentType contentType) { + if (contentType == ContentType.AVRO) { + Integer schemaVersion = kafkaHeaderExtractor.extractSchemaVersion(message.headers()); + Integer schemaId = kafkaHeaderExtractor.extractSchemaId(message.headers()); + ensureExistence(schemaVersion, schemaId); + return compositeMessageContentWrapper.unwrapAvro( + message.value(), topic, schemaId, schemaVersion); + } else if (contentType == ContentType.JSON) { + return compositeMessageContentWrapper.unwrapJson(message.value()); } + throw new UnsupportedContentTypeException(topic); + } - private void ensureExistence(Integer schemaVersion, Integer schemaId) { - try { - if (schemaVersion != null) { - schemaExistenceEnsurer.ensureSchemaExists(topic, SchemaVersion.valueOf(schemaVersion)); - } - if (schemaId != null) { - schemaExistenceEnsurer.ensureSchemaExists(topic, SchemaId.valueOf(schemaId)); - } - } catch (SchemaExistenceEnsurer.SchemaNotLoaded ex) { - throw new RetryableReceiverError("Requested schema not present yet...", ex); - } + private void ensureExistence(Integer schemaVersion, Integer schemaId) { + try { + if (schemaVersion != null) { + schemaExistenceEnsurer.ensureSchemaExists(topic, SchemaVersion.valueOf(schemaVersion)); + } + if (schemaId != null) { + schemaExistenceEnsurer.ensureSchemaExists(topic, SchemaId.valueOf(schemaId)); + } + } catch (SchemaExistenceEnsurer.SchemaNotLoaded ex) { + throw new RetryableReceiverError("Requested schema not present yet...", ex); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/BasicMessageContentReaderFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/BasicMessageContentReaderFactory.java index ff2e996501..708da15eb1 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/BasicMessageContentReaderFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/BasicMessageContentReaderFactory.java @@ -7,20 +7,23 @@ public class BasicMessageContentReaderFactory implements MessageContentReaderFactory { - private final CompositeMessageContentWrapper compositeMessageContentWrapper; - private final KafkaHeaderExtractor kafkaHeaderExtractor; - private final SchemaRepository schemaRepository; + private final CompositeMessageContentWrapper compositeMessageContentWrapper; + private final KafkaHeaderExtractor kafkaHeaderExtractor; + private final SchemaRepository schemaRepository; - public BasicMessageContentReaderFactory(CompositeMessageContentWrapper compositeMessageContentWrapper, - KafkaHeaderExtractor kafkaHeaderExtractor, SchemaRepository schemaRepository) { - this.compositeMessageContentWrapper = compositeMessageContentWrapper; - this.kafkaHeaderExtractor = kafkaHeaderExtractor; - this.schemaRepository = schemaRepository; - } + public BasicMessageContentReaderFactory( + CompositeMessageContentWrapper compositeMessageContentWrapper, + KafkaHeaderExtractor kafkaHeaderExtractor, + SchemaRepository schemaRepository) { + this.compositeMessageContentWrapper = compositeMessageContentWrapper; + this.kafkaHeaderExtractor = kafkaHeaderExtractor; + this.schemaRepository = schemaRepository; + } - @Override - public MessageContentReader provide(Topic topic) { - SchemaExistenceEnsurer schemaExistenceEnsurer = new SchemaExistenceEnsurer(schemaRepository); - return new BasicMessageContentReader(compositeMessageContentWrapper, kafkaHeaderExtractor, topic, schemaExistenceEnsurer); - } + @Override + public MessageContentReader provide(Topic topic) { + SchemaExistenceEnsurer schemaExistenceEnsurer = new SchemaExistenceEnsurer(schemaRepository); + return new BasicMessageContentReader( + compositeMessageContentWrapper, kafkaHeaderExtractor, topic, schemaExistenceEnsurer); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/FilteringMessageReceiver.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/FilteringMessageReceiver.java index d2a3634458..67ac4f0b6a 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/FilteringMessageReceiver.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/FilteringMessageReceiver.java @@ -1,5 +1,8 @@ package pl.allegro.tech.hermes.consumers.consumer.receiver.kafka; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.common.kafka.offset.PartitionOffset; import pl.allegro.tech.hermes.consumers.consumer.Message; @@ -10,62 +13,59 @@ import pl.allegro.tech.hermes.domain.filtering.chain.FilterChainFactory; import pl.allegro.tech.hermes.domain.filtering.chain.FilterResult; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; - public class FilteringMessageReceiver implements MessageReceiver { - private final MessageReceiver receiver; - private final FilteredMessageHandler filteredMessageHandler; - private final FilterChainFactory filterChainFactory; + private final MessageReceiver receiver; + private final FilteredMessageHandler filteredMessageHandler; + private final FilterChainFactory filterChainFactory; - private volatile FilterChain filterChain; - private Subscription subscription; + private volatile FilterChain filterChain; + private Subscription subscription; - public FilteringMessageReceiver(MessageReceiver receiver, - FilteredMessageHandler filteredMessageHandler, - FilterChainFactory filterChainFactory, - Subscription subscription) { - this.receiver = receiver; - this.filteredMessageHandler = filteredMessageHandler; - this.filterChainFactory = filterChainFactory; - this.subscription = subscription; - this.filterChain = filterChainFactory.create(subscription.getFilters()); - } + public FilteringMessageReceiver( + MessageReceiver receiver, + FilteredMessageHandler filteredMessageHandler, + FilterChainFactory filterChainFactory, + Subscription subscription) { + this.receiver = receiver; + this.filteredMessageHandler = filteredMessageHandler; + this.filterChainFactory = filterChainFactory; + this.subscription = subscription; + this.filterChain = filterChainFactory.create(subscription.getFilters()); + } - @Override - public Optional next() { - return receiver.next().map(this::filter); - } + @Override + public Optional next() { + return receiver.next().map(this::filter); + } - private Message filter(Message message) { - FilterResult result = filterChain.apply(message); - filteredMessageHandler.handle(result, message, subscription); - message.setFiltered(result.isFiltered()); - return message; - } + private Message filter(Message message) { + FilterResult result = filterChain.apply(message); + filteredMessageHandler.handle(result, message, subscription); + message.setFiltered(result.isFiltered()); + return message; + } - @Override - public void stop() { - receiver.stop(); - } + @Override + public void stop() { + receiver.stop(); + } - @Override - public void update(Subscription newSubscription) { - if (!Objects.equals(subscription.getFilters(), newSubscription.getFilters())) { - this.filterChain = filterChainFactory.create(newSubscription.getFilters()); - } - this.subscription = newSubscription; - this.receiver.update(newSubscription); + @Override + public void update(Subscription newSubscription) { + if (!Objects.equals(subscription.getFilters(), newSubscription.getFilters())) { + this.filterChain = filterChainFactory.create(newSubscription.getFilters()); } + this.subscription = newSubscription; + this.receiver.update(newSubscription); + } - @Override - public void commit(Set offsets) { - receiver.commit(offsets); - } + @Override + public void commit(Set offsets) { + receiver.commit(offsets); + } - @Override - public boolean moveOffset(PartitionOffset offset) { - return receiver.moveOffset(offset); - } + @Override + public boolean moveOffset(PartitionOffset offset) { + return receiver.moveOffset(offset); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaConsumerParameters.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaConsumerParameters.java index ce07cf65f6..0647f43282 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaConsumerParameters.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaConsumerParameters.java @@ -5,43 +5,43 @@ public interface KafkaConsumerParameters { - int getSendBufferBytes(); + int getSendBufferBytes(); - int getReceiveBufferBytes(); + int getReceiveBufferBytes(); - int getFetchMinBytes(); + int getFetchMinBytes(); - Duration getFetchMaxWait(); + Duration getFetchMaxWait(); - Duration getReconnectBackoff(); + Duration getReconnectBackoff(); - Duration getRetryBackoff(); + Duration getRetryBackoff(); - boolean isCheckCrcs(); + boolean isCheckCrcs(); - Duration getMetricsSampleWindow(); + Duration getMetricsSampleWindow(); - int getMetricsNumSamples(); + int getMetricsNumSamples(); - Duration getRequestTimeout(); + Duration getRequestTimeout(); - Duration getConnectionsMaxIdle(); + Duration getConnectionsMaxIdle(); - int getMaxPollRecords(); + int getMaxPollRecords(); - Duration getMaxPollInterval(); + Duration getMaxPollInterval(); - String getAutoOffsetReset(); + String getAutoOffsetReset(); - Duration getSessionTimeout(); + Duration getSessionTimeout(); - Duration getHeartbeatInterval(); + Duration getHeartbeatInterval(); - Duration getMetadataMaxAge(); + Duration getMetadataMaxAge(); - int getMaxPartitionFetchMin(); + int getMaxPartitionFetchMin(); - int getMaxPartitionFetchMax(); + int getMaxPartitionFetchMax(); - List getPartitionAssignmentStrategies(); + List getPartitionAssignmentStrategies(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaConsumerRecordToMessageConverter.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaConsumerRecordToMessageConverter.java index a6c6f2297a..d70311d68c 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaConsumerRecordToMessageConverter.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaConsumerRecordToMessageConverter.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.consumers.consumer.receiver.kafka; +import java.time.Clock; +import java.util.Map; import org.apache.kafka.clients.consumer.ConsumerRecord; import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.api.Topic; @@ -8,58 +10,58 @@ import pl.allegro.tech.hermes.common.message.wrapper.UnwrappedMessageContent; import pl.allegro.tech.hermes.consumers.consumer.Message; -import java.time.Clock; -import java.util.Map; - public class KafkaConsumerRecordToMessageConverter { - private final Topic topic; - private volatile Subscription subscription; - private final Map topics; - private final MessageContentReader messageContentReader; - private final KafkaHeaderExtractor kafkaHeaderExtractor; - private final Clock clock; - - public KafkaConsumerRecordToMessageConverter(Topic topic, - Subscription subscription, - Map topics, - MessageContentReader messageContentReader, - KafkaHeaderExtractor kafkaHeaderExtractor, - Clock clock) { - this.topic = topic; - this.subscription = subscription; - this.topics = topics; - this.messageContentReader = messageContentReader; - this.kafkaHeaderExtractor = kafkaHeaderExtractor; - this.clock = clock; - } + private final Topic topic; + private volatile Subscription subscription; + private final Map topics; + private final MessageContentReader messageContentReader; + private final KafkaHeaderExtractor kafkaHeaderExtractor; + private final Clock clock; - public Message convertToMessage(ConsumerRecord record, long partitionAssignmentTerm) { - KafkaTopic kafkaTopic = topics.get(record.topic()); - UnwrappedMessageContent unwrappedContent = messageContentReader.read(record, kafkaTopic.contentType()); + public KafkaConsumerRecordToMessageConverter( + Topic topic, + Subscription subscription, + Map topics, + MessageContentReader messageContentReader, + KafkaHeaderExtractor kafkaHeaderExtractor, + Clock clock) { + this.topic = topic; + this.subscription = subscription; + this.topics = topics; + this.messageContentReader = messageContentReader; + this.kafkaHeaderExtractor = kafkaHeaderExtractor; + this.clock = clock; + } - Map externalMetadataFromBody = unwrappedContent.getMessageMetadata().getExternalMetadata(); - Map externalMetadata = kafkaHeaderExtractor.extractExternalMetadata(record.headers(), externalMetadataFromBody); + public Message convertToMessage( + ConsumerRecord record, long partitionAssignmentTerm) { + KafkaTopic kafkaTopic = topics.get(record.topic()); + UnwrappedMessageContent unwrappedContent = + messageContentReader.read(record, kafkaTopic.contentType()); - return new Message( - kafkaHeaderExtractor.extractMessageId(record.headers()), - topic.getQualifiedName(), - unwrappedContent.getContent(), - kafkaTopic.contentType(), - unwrappedContent.getSchema(), - record.timestamp(), - clock.millis(), - new PartitionOffset(kafkaTopic.name(), record.offset(), record.partition()), - partitionAssignmentTerm, - externalMetadata, - subscription.getHeaders(), - subscription.getName(), - subscription.isSubscriptionIdentityHeadersEnabled() - ); - } + Map externalMetadataFromBody = + unwrappedContent.getMessageMetadata().getExternalMetadata(); + Map externalMetadata = + kafkaHeaderExtractor.extractExternalMetadata(record.headers(), externalMetadataFromBody); - public void update(Subscription newSubscription) { - this.subscription = newSubscription; - } + return new Message( + kafkaHeaderExtractor.extractMessageId(record.headers()), + topic.getQualifiedName(), + unwrappedContent.getContent(), + kafkaTopic.contentType(), + unwrappedContent.getSchema(), + record.timestamp(), + clock.millis(), + new PartitionOffset(kafkaTopic.name(), record.offset(), record.partition()), + partitionAssignmentTerm, + externalMetadata, + subscription.getHeaders(), + subscription.getName(), + subscription.isSubscriptionIdentityHeadersEnabled()); + } + public void update(Subscription newSubscription) { + this.subscription = newSubscription; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaConsumerRecordToMessageConverterFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaConsumerRecordToMessageConverterFactory.java index e748983102..3aa47993e3 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaConsumerRecordToMessageConverterFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaConsumerRecordToMessageConverterFactory.java @@ -1,28 +1,34 @@ package pl.allegro.tech.hermes.consumers.consumer.receiver.kafka; +import java.time.Clock; +import java.util.Map; import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.common.kafka.KafkaTopic; -import java.time.Clock; -import java.util.Map; - public class KafkaConsumerRecordToMessageConverterFactory { - private final MessageContentReaderFactory messageContentReaderFactory; - private final KafkaHeaderExtractor kafkaHeaderExtractor; - private final Clock clock; - - public KafkaConsumerRecordToMessageConverterFactory(MessageContentReaderFactory messageContentReaderFactory, - KafkaHeaderExtractor kafkaHeaderExtractor, Clock clock) { - this.messageContentReaderFactory = messageContentReaderFactory; - this.kafkaHeaderExtractor = kafkaHeaderExtractor; - this.clock = clock; - } + private final MessageContentReaderFactory messageContentReaderFactory; + private final KafkaHeaderExtractor kafkaHeaderExtractor; + private final Clock clock; - public KafkaConsumerRecordToMessageConverter create(Topic topic, Subscription subscription, Map topics) { - return new KafkaConsumerRecordToMessageConverter(topic, subscription, topics, messageContentReaderFactory.provide(topic), - kafkaHeaderExtractor, clock); - } + public KafkaConsumerRecordToMessageConverterFactory( + MessageContentReaderFactory messageContentReaderFactory, + KafkaHeaderExtractor kafkaHeaderExtractor, + Clock clock) { + this.messageContentReaderFactory = messageContentReaderFactory; + this.kafkaHeaderExtractor = kafkaHeaderExtractor; + this.clock = clock; + } + public KafkaConsumerRecordToMessageConverter create( + Topic topic, Subscription subscription, Map topics) { + return new KafkaConsumerRecordToMessageConverter( + topic, + subscription, + topics, + messageContentReaderFactory.provide(topic), + kafkaHeaderExtractor, + clock); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaHeaderExtractor.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaHeaderExtractor.java index 01d2952e73..36149b0599 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaHeaderExtractor.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaHeaderExtractor.java @@ -1,77 +1,84 @@ package pl.allegro.tech.hermes.consumers.consumer.receiver.kafka; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.stream.Collectors.toMap; +import static java.util.stream.StreamSupport.stream; + import com.google.common.primitives.Ints; +import java.util.HashMap; +import java.util.Map; import org.apache.kafka.common.header.Header; import org.apache.kafka.common.header.Headers; import pl.allegro.tech.hermes.common.kafka.HTTPHeadersPropagationAsKafkaHeadersProperties; import pl.allegro.tech.hermes.common.message.wrapper.AvroMetadataMarker; import pl.allegro.tech.hermes.consumers.config.KafkaHeaderNameProperties; -import java.util.HashMap; -import java.util.Map; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.stream.Collectors.toMap; -import static java.util.stream.StreamSupport.stream; - public class KafkaHeaderExtractor { - private final KafkaHeaderNameProperties kafkaHeaderNameProperties; - private final boolean isHTTPheadersPropagationAsKafkaHeadersEnabled; - private final String httpHeadersPrefix; + private final KafkaHeaderNameProperties kafkaHeaderNameProperties; + private final boolean isHTTPheadersPropagationAsKafkaHeadersEnabled; + private final String httpHeadersPrefix; - public KafkaHeaderExtractor(KafkaHeaderNameProperties kafkaHeaderNameProperties, - HTTPHeadersPropagationAsKafkaHeadersProperties httpHeadersPropagationAsKafkaHeadersProperties) { + public KafkaHeaderExtractor( + KafkaHeaderNameProperties kafkaHeaderNameProperties, + HTTPHeadersPropagationAsKafkaHeadersProperties + httpHeadersPropagationAsKafkaHeadersProperties) { - this.kafkaHeaderNameProperties = kafkaHeaderNameProperties; - this.isHTTPheadersPropagationAsKafkaHeadersEnabled = httpHeadersPropagationAsKafkaHeadersProperties.isEnabled(); - this.httpHeadersPrefix = httpHeadersPropagationAsKafkaHeadersProperties.getPrefix(); - } + this.kafkaHeaderNameProperties = kafkaHeaderNameProperties; + this.isHTTPheadersPropagationAsKafkaHeadersEnabled = + httpHeadersPropagationAsKafkaHeadersProperties.isEnabled(); + this.httpHeadersPrefix = httpHeadersPropagationAsKafkaHeadersProperties.getPrefix(); + } - public Integer extractSchemaVersion(Headers headers) { - Header header = headers.lastHeader(kafkaHeaderNameProperties.getSchemaVersion()); - return extract(header); - } + public Integer extractSchemaVersion(Headers headers) { + Header header = headers.lastHeader(kafkaHeaderNameProperties.getSchemaVersion()); + return extract(header); + } - public Integer extractSchemaId(Headers headers) { - Header header = headers.lastHeader(kafkaHeaderNameProperties.getSchemaId()); - return extract(header); - } + public Integer extractSchemaId(Headers headers) { + Header header = headers.lastHeader(kafkaHeaderNameProperties.getSchemaId()); + return extract(header); + } - private Integer extract(Header header) { - if (header != null) { - return Ints.fromByteArray(header.value()); - } else { - return null; - } + private Integer extract(Header header) { + if (header != null) { + return Ints.fromByteArray(header.value()); + } else { + return null; } + } - public String extractMessageId(Headers headers) { - Header header = headers.lastHeader(kafkaHeaderNameProperties.getMessageId()); - if (header == null) { - return ""; - } - return new String(header.value(), UTF_8); + public String extractMessageId(Headers headers) { + Header header = headers.lastHeader(kafkaHeaderNameProperties.getMessageId()); + if (header == null) { + return ""; } + return new String(header.value(), UTF_8); + } - public Map extractExternalMetadata(Headers headers, Map defaultExternalMetadata) { - if (isHTTPheadersPropagationAsKafkaHeadersEnabled) { - Map httpHeaders = stream(headers.spliterator(), false) - .filter(h -> h.key().startsWith(httpHeadersPrefix)) - .collect(toMap( - h -> h.key().substring(httpHeadersPrefix.length()), - h -> new String(h.value(), UTF_8)) - ); - if (httpHeaders.isEmpty()) { - // After completing the migration to the approach with Kafka headers, we should remove this condition. - return defaultExternalMetadata; - } - // The following is necessary to be compatible with building external metadata based on the message body. - // See: pl.allegro.tech.hermes.common.message.wrapper.AvroMessageContentWrapper.getMetadata - Map externalMetadata = new HashMap<>(httpHeaders); - externalMetadata.put(AvroMetadataMarker.METADATA_MESSAGE_ID_KEY.toString(), extractMessageId(headers)); - return externalMetadata; - } + public Map extractExternalMetadata( + Headers headers, Map defaultExternalMetadata) { + if (isHTTPheadersPropagationAsKafkaHeadersEnabled) { + Map httpHeaders = + stream(headers.spliterator(), false) + .filter(h -> h.key().startsWith(httpHeadersPrefix)) + .collect( + toMap( + h -> h.key().substring(httpHeadersPrefix.length()), + h -> new String(h.value(), UTF_8))); + if (httpHeaders.isEmpty()) { + // After completing the migration to the approach with Kafka headers, we should remove this + // condition. return defaultExternalMetadata; + } + // The following is necessary to be compatible with building external metadata based on the + // message body. + // See: pl.allegro.tech.hermes.common.message.wrapper.AvroMessageContentWrapper.getMetadata + Map externalMetadata = new HashMap<>(httpHeaders); + externalMetadata.put( + AvroMetadataMarker.METADATA_MESSAGE_ID_KEY.toString(), extractMessageId(headers)); + return externalMetadata; } + return defaultExternalMetadata; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaMessageReceiverFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaMessageReceiverFactory.java index 26a58cfd4a..e3a6b4a093 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaMessageReceiverFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaMessageReceiverFactory.java @@ -1,29 +1,5 @@ package pl.allegro.tech.hermes.consumers.consumer.receiver.kafka; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import pl.allegro.tech.hermes.api.Subscription; -import pl.allegro.tech.hermes.api.Topic; -import pl.allegro.tech.hermes.common.kafka.ConsumerGroupId; -import pl.allegro.tech.hermes.common.kafka.KafkaNamesMapper; -import pl.allegro.tech.hermes.common.kafka.KafkaParameters; -import pl.allegro.tech.hermes.common.metric.MetricsFacade; -import pl.allegro.tech.hermes.consumers.CommonConsumerParameters; -import pl.allegro.tech.hermes.consumers.consumer.filtering.FilteredMessageHandler; -import pl.allegro.tech.hermes.consumers.consumer.idletime.ExponentiallyGrowingIdleTimeCalculator; -import pl.allegro.tech.hermes.consumers.consumer.idletime.IdleTimeCalculator; -import pl.allegro.tech.hermes.consumers.consumer.load.SubscriptionLoadRecorder; -import pl.allegro.tech.hermes.consumers.consumer.offset.ConsumerPartitionAssignmentState; -import pl.allegro.tech.hermes.consumers.consumer.offset.PendingOffsetsAppender; -import pl.allegro.tech.hermes.consumers.consumer.rate.ConsumerRateLimiter; -import pl.allegro.tech.hermes.consumers.consumer.receiver.MessageReceiver; -import pl.allegro.tech.hermes.consumers.consumer.receiver.ReceiverFactory; -import pl.allegro.tech.hermes.consumers.consumer.receiver.ThrottlingMessageReceiver; -import pl.allegro.tech.hermes.domain.filtering.chain.FilterChainFactory; -import pl.allegro.tech.hermes.tracker.consumers.Trackers; - -import java.util.List; -import java.util.Properties; - import static java.util.stream.Collectors.toList; import static org.apache.kafka.clients.CommonClientConfigs.SECURITY_PROTOCOL_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.AUTO_OFFSET_RESET_CONFIG; @@ -52,165 +28,208 @@ import static org.apache.kafka.common.config.SaslConfigs.SASL_JAAS_CONFIG; import static org.apache.kafka.common.config.SaslConfigs.SASL_MECHANISM; -public class KafkaMessageReceiverFactory implements ReceiverFactory { - - private final CommonConsumerParameters commonConsumerParameters; - private final KafkaParameters kafkaParameters; - private final KafkaReceiverParameters consumerReceiverParameters; - private final KafkaConsumerParameters kafkaConsumerParameters; - private final KafkaConsumerRecordToMessageConverterFactory messageConverterFactory; - private final MetricsFacade metricsFacade; - private final KafkaNamesMapper kafkaNamesMapper; - private final FilterChainFactory filterChainFactory; - private final Trackers trackers; - private final ConsumerPartitionAssignmentState consumerPartitionAssignmentState; - - public KafkaMessageReceiverFactory(CommonConsumerParameters commonConsumerParameters, - KafkaReceiverParameters consumerReceiverParameters, - KafkaConsumerParameters kafkaConsumerParameters, - KafkaParameters kafkaParameters, - KafkaConsumerRecordToMessageConverterFactory messageConverterFactory, - MetricsFacade metricsFacade, - KafkaNamesMapper kafkaNamesMapper, - FilterChainFactory filterChainFactory, - Trackers trackers, - ConsumerPartitionAssignmentState consumerPartitionAssignmentState) { - this.commonConsumerParameters = commonConsumerParameters; - this.consumerReceiverParameters = consumerReceiverParameters; - this.kafkaConsumerParameters = kafkaConsumerParameters; - this.kafkaParameters = kafkaParameters; - this.messageConverterFactory = messageConverterFactory; - this.metricsFacade = metricsFacade; - this.kafkaNamesMapper = kafkaNamesMapper; - this.filterChainFactory = filterChainFactory; - this.trackers = trackers; - this.consumerPartitionAssignmentState = consumerPartitionAssignmentState; - } - - @Override - public MessageReceiver createMessageReceiver(Topic topic, - Subscription subscription, - ConsumerRateLimiter consumerRateLimiter, - SubscriptionLoadRecorder loadReporter, - MetricsFacade metrics, - PendingOffsetsAppender pendingOffsetsAppender) { - - MessageReceiver receiver = createKafkaSingleThreadedMessageReceiver(topic, subscription, loadReporter); - - if (consumerReceiverParameters.isWaitBetweenUnsuccessfulPolls()) { - receiver = createThrottlingMessageReceiver(receiver, subscription, metrics); - } - - if (consumerReceiverParameters.isFilteringEnabled()) { - receiver = createFilteringMessageReceiver(receiver, consumerRateLimiter, subscription, metrics, pendingOffsetsAppender); - } - - return receiver; - } - - private MessageReceiver createKafkaSingleThreadedMessageReceiver(Topic topic, - Subscription subscription, - SubscriptionLoadRecorder loadReporter) { - return new KafkaSingleThreadedMessageReceiver( - createKafkaConsumer(topic, subscription), - messageConverterFactory, - metricsFacade, - kafkaNamesMapper, - topic, - subscription, - consumerReceiverParameters.getPoolTimeout(), - consumerReceiverParameters.getReadQueueCapacity(), - loadReporter, - consumerPartitionAssignmentState - ); - } - - private MessageReceiver createThrottlingMessageReceiver(MessageReceiver receiver, - Subscription subscription, - MetricsFacade metrics) { - IdleTimeCalculator idleTimeCalculator = new ExponentiallyGrowingIdleTimeCalculator( - consumerReceiverParameters.getInitialIdleTime().toMillis(), - consumerReceiverParameters.getMaxIdleTime().toMillis()); - - return new ThrottlingMessageReceiver(receiver, idleTimeCalculator, subscription.getQualifiedName(), metrics); - } - - private MessageReceiver createFilteringMessageReceiver(MessageReceiver receiver, - ConsumerRateLimiter consumerRateLimiter, - Subscription subscription, - MetricsFacade metrics, - PendingOffsetsAppender pendingOffsetsAppender) { - boolean filteringRateLimitEnabled = consumerReceiverParameters.isFilteringRateLimiterEnabled(); - FilteredMessageHandler filteredMessageHandler = new FilteredMessageHandler( - filteringRateLimitEnabled ? consumerRateLimiter : null, - pendingOffsetsAppender, - trackers, - metrics, - subscription.getQualifiedName() - ); - return new FilteringMessageReceiver(receiver, filteredMessageHandler, filterChainFactory, subscription); - } +import java.util.List; +import java.util.Properties; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import pl.allegro.tech.hermes.api.Subscription; +import pl.allegro.tech.hermes.api.Topic; +import pl.allegro.tech.hermes.common.kafka.ConsumerGroupId; +import pl.allegro.tech.hermes.common.kafka.KafkaNamesMapper; +import pl.allegro.tech.hermes.common.kafka.KafkaParameters; +import pl.allegro.tech.hermes.common.metric.MetricsFacade; +import pl.allegro.tech.hermes.consumers.CommonConsumerParameters; +import pl.allegro.tech.hermes.consumers.consumer.filtering.FilteredMessageHandler; +import pl.allegro.tech.hermes.consumers.consumer.idletime.ExponentiallyGrowingIdleTimeCalculator; +import pl.allegro.tech.hermes.consumers.consumer.idletime.IdleTimeCalculator; +import pl.allegro.tech.hermes.consumers.consumer.load.SubscriptionLoadRecorder; +import pl.allegro.tech.hermes.consumers.consumer.offset.ConsumerPartitionAssignmentState; +import pl.allegro.tech.hermes.consumers.consumer.offset.PendingOffsetsAppender; +import pl.allegro.tech.hermes.consumers.consumer.rate.ConsumerRateLimiter; +import pl.allegro.tech.hermes.consumers.consumer.receiver.MessageReceiver; +import pl.allegro.tech.hermes.consumers.consumer.receiver.ReceiverFactory; +import pl.allegro.tech.hermes.consumers.consumer.receiver.ThrottlingMessageReceiver; +import pl.allegro.tech.hermes.domain.filtering.chain.FilterChainFactory; +import pl.allegro.tech.hermes.tracker.consumers.Trackers; - private KafkaConsumer createKafkaConsumer(Topic topic, Subscription subscription) { - ConsumerGroupId groupId = kafkaNamesMapper.toConsumerGroupId(subscription.getQualifiedName()); - Properties props = new Properties(); - props.put(BOOTSTRAP_SERVERS_CONFIG, kafkaParameters.getBrokerList()); - props.put(CLIENT_ID_CONFIG, consumerReceiverParameters.getClientId() + "_" + groupId.asString()); - props.put(GROUP_ID_CONFIG, groupId.asString()); - props.put(ENABLE_AUTO_COMMIT_CONFIG, "false"); - props.put("key.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer"); - props.put("value.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer"); - - addKafkaAuthorizationParameters(props); - addKafkaConsumerParameters(props, topic); - return new KafkaConsumer<>(props); - } +public class KafkaMessageReceiverFactory implements ReceiverFactory { - private void addKafkaAuthorizationParameters(Properties props) { - if (kafkaParameters.isAuthenticationEnabled()) { - props.put(SASL_MECHANISM, kafkaParameters.getAuthenticationMechanism()); - props.put(SECURITY_PROTOCOL_CONFIG, kafkaParameters.getAuthenticationProtocol()); - props.put(SASL_JAAS_CONFIG, kafkaParameters.getJaasConfig()); - } + private final CommonConsumerParameters commonConsumerParameters; + private final KafkaParameters kafkaParameters; + private final KafkaReceiverParameters consumerReceiverParameters; + private final KafkaConsumerParameters kafkaConsumerParameters; + private final KafkaConsumerRecordToMessageConverterFactory messageConverterFactory; + private final MetricsFacade metricsFacade; + private final KafkaNamesMapper kafkaNamesMapper; + private final FilterChainFactory filterChainFactory; + private final Trackers trackers; + private final ConsumerPartitionAssignmentState consumerPartitionAssignmentState; + + public KafkaMessageReceiverFactory( + CommonConsumerParameters commonConsumerParameters, + KafkaReceiverParameters consumerReceiverParameters, + KafkaConsumerParameters kafkaConsumerParameters, + KafkaParameters kafkaParameters, + KafkaConsumerRecordToMessageConverterFactory messageConverterFactory, + MetricsFacade metricsFacade, + KafkaNamesMapper kafkaNamesMapper, + FilterChainFactory filterChainFactory, + Trackers trackers, + ConsumerPartitionAssignmentState consumerPartitionAssignmentState) { + this.commonConsumerParameters = commonConsumerParameters; + this.consumerReceiverParameters = consumerReceiverParameters; + this.kafkaConsumerParameters = kafkaConsumerParameters; + this.kafkaParameters = kafkaParameters; + this.messageConverterFactory = messageConverterFactory; + this.metricsFacade = metricsFacade; + this.kafkaNamesMapper = kafkaNamesMapper; + this.filterChainFactory = filterChainFactory; + this.trackers = trackers; + this.consumerPartitionAssignmentState = consumerPartitionAssignmentState; + } + + @Override + public MessageReceiver createMessageReceiver( + Topic topic, + Subscription subscription, + ConsumerRateLimiter consumerRateLimiter, + SubscriptionLoadRecorder loadReporter, + MetricsFacade metrics, + PendingOffsetsAppender pendingOffsetsAppender) { + + MessageReceiver receiver = + createKafkaSingleThreadedMessageReceiver(topic, subscription, loadReporter); + + if (consumerReceiverParameters.isWaitBetweenUnsuccessfulPolls()) { + receiver = createThrottlingMessageReceiver(receiver, subscription, metrics); } - private void addKafkaConsumerParameters(Properties props, Topic topic) { - props.put(AUTO_OFFSET_RESET_CONFIG, kafkaConsumerParameters.getAutoOffsetReset()); - props.put(SESSION_TIMEOUT_MS_CONFIG, (int) kafkaConsumerParameters.getSessionTimeout().toMillis()); - props.put(HEARTBEAT_INTERVAL_MS_CONFIG, (int) kafkaConsumerParameters.getHeartbeatInterval().toMillis()); - props.put(METADATA_MAX_AGE_CONFIG, (int) kafkaConsumerParameters.getMetadataMaxAge().toMillis()); - props.put(MAX_PARTITION_FETCH_BYTES_CONFIG, getMaxPartitionFetch(topic)); - props.put(SEND_BUFFER_CONFIG, kafkaConsumerParameters.getSendBufferBytes()); - props.put(RECEIVE_BUFFER_CONFIG, kafkaConsumerParameters.getReceiveBufferBytes()); - props.put(FETCH_MIN_BYTES_CONFIG, kafkaConsumerParameters.getFetchMinBytes()); - props.put(FETCH_MAX_WAIT_MS_CONFIG, (int) kafkaConsumerParameters.getFetchMaxWait().toMillis()); - props.put(RECONNECT_BACKOFF_MS_CONFIG, (int) kafkaConsumerParameters.getReconnectBackoff().toMillis()); - props.put(RETRY_BACKOFF_MS_CONFIG, (int) kafkaConsumerParameters.getRetryBackoff().toMillis()); - props.put(CHECK_CRCS_CONFIG, kafkaConsumerParameters.isCheckCrcs()); - props.put(METRICS_SAMPLE_WINDOW_MS_CONFIG, (int) kafkaConsumerParameters.getMetricsSampleWindow().toMillis()); - props.put(METRICS_NUM_SAMPLES_CONFIG, kafkaConsumerParameters.getMetricsNumSamples()); - props.put(REQUEST_TIMEOUT_MS_CONFIG, (int) kafkaConsumerParameters.getRequestTimeout().toMillis()); - props.put(CONNECTIONS_MAX_IDLE_MS_CONFIG, (int) kafkaConsumerParameters.getConnectionsMaxIdle().toMillis()); - props.put(MAX_POLL_RECORDS_CONFIG, kafkaConsumerParameters.getMaxPollRecords()); - props.put(MAX_POLL_INTERVAL_MS_CONFIG, (int) kafkaConsumerParameters.getMaxPollInterval().toMillis()); - props.put(PARTITION_ASSIGNMENT_STRATEGY_CONFIG, getPartitionAssignmentStrategies()); + if (consumerReceiverParameters.isFilteringEnabled()) { + receiver = + createFilteringMessageReceiver( + receiver, consumerRateLimiter, subscription, metrics, pendingOffsetsAppender); } - private int getMaxPartitionFetch(Topic topic) { - if (commonConsumerParameters.isUseTopicMessageSizeEnabled()) { - int topicMessageSize = topic.getMaxMessageSize(); - int min = kafkaConsumerParameters.getMaxPartitionFetchMin(); - int max = kafkaConsumerParameters.getMaxPartitionFetchMax(); - return Math.max(Math.min(topicMessageSize, max), min); - } else { - return kafkaConsumerParameters.getMaxPartitionFetchMax(); - } + return receiver; + } + + private MessageReceiver createKafkaSingleThreadedMessageReceiver( + Topic topic, Subscription subscription, SubscriptionLoadRecorder loadReporter) { + return new KafkaSingleThreadedMessageReceiver( + createKafkaConsumer(topic, subscription), + messageConverterFactory, + metricsFacade, + kafkaNamesMapper, + topic, + subscription, + consumerReceiverParameters.getPoolTimeout(), + consumerReceiverParameters.getReadQueueCapacity(), + loadReporter, + consumerPartitionAssignmentState); + } + + private MessageReceiver createThrottlingMessageReceiver( + MessageReceiver receiver, Subscription subscription, MetricsFacade metrics) { + IdleTimeCalculator idleTimeCalculator = + new ExponentiallyGrowingIdleTimeCalculator( + consumerReceiverParameters.getInitialIdleTime().toMillis(), + consumerReceiverParameters.getMaxIdleTime().toMillis()); + + return new ThrottlingMessageReceiver( + receiver, idleTimeCalculator, subscription.getQualifiedName(), metrics); + } + + private MessageReceiver createFilteringMessageReceiver( + MessageReceiver receiver, + ConsumerRateLimiter consumerRateLimiter, + Subscription subscription, + MetricsFacade metrics, + PendingOffsetsAppender pendingOffsetsAppender) { + boolean filteringRateLimitEnabled = consumerReceiverParameters.isFilteringRateLimiterEnabled(); + FilteredMessageHandler filteredMessageHandler = + new FilteredMessageHandler( + filteringRateLimitEnabled ? consumerRateLimiter : null, + pendingOffsetsAppender, + trackers, + metrics, + subscription.getQualifiedName()); + return new FilteringMessageReceiver( + receiver, filteredMessageHandler, filterChainFactory, subscription); + } + + private KafkaConsumer createKafkaConsumer( + Topic topic, Subscription subscription) { + ConsumerGroupId groupId = kafkaNamesMapper.toConsumerGroupId(subscription.getQualifiedName()); + Properties props = new Properties(); + props.put(BOOTSTRAP_SERVERS_CONFIG, kafkaParameters.getBrokerList()); + props.put( + CLIENT_ID_CONFIG, consumerReceiverParameters.getClientId() + "_" + groupId.asString()); + props.put(GROUP_ID_CONFIG, groupId.asString()); + props.put(ENABLE_AUTO_COMMIT_CONFIG, "false"); + props.put("key.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer"); + props.put("value.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer"); + + addKafkaAuthorizationParameters(props); + addKafkaConsumerParameters(props, topic); + return new KafkaConsumer<>(props); + } + + private void addKafkaAuthorizationParameters(Properties props) { + if (kafkaParameters.isAuthenticationEnabled()) { + props.put(SASL_MECHANISM, kafkaParameters.getAuthenticationMechanism()); + props.put(SECURITY_PROTOCOL_CONFIG, kafkaParameters.getAuthenticationProtocol()); + props.put(SASL_JAAS_CONFIG, kafkaParameters.getJaasConfig()); } - - private List getPartitionAssignmentStrategies() { - return kafkaConsumerParameters.getPartitionAssignmentStrategies().stream() - .map(PartitionAssignmentStrategy::getAssignorClass) - .map(Class::getName) - .collect(toList()); + } + + private void addKafkaConsumerParameters(Properties props, Topic topic) { + props.put(AUTO_OFFSET_RESET_CONFIG, kafkaConsumerParameters.getAutoOffsetReset()); + props.put( + SESSION_TIMEOUT_MS_CONFIG, (int) kafkaConsumerParameters.getSessionTimeout().toMillis()); + props.put( + HEARTBEAT_INTERVAL_MS_CONFIG, + (int) kafkaConsumerParameters.getHeartbeatInterval().toMillis()); + props.put( + METADATA_MAX_AGE_CONFIG, (int) kafkaConsumerParameters.getMetadataMaxAge().toMillis()); + props.put(MAX_PARTITION_FETCH_BYTES_CONFIG, getMaxPartitionFetch(topic)); + props.put(SEND_BUFFER_CONFIG, kafkaConsumerParameters.getSendBufferBytes()); + props.put(RECEIVE_BUFFER_CONFIG, kafkaConsumerParameters.getReceiveBufferBytes()); + props.put(FETCH_MIN_BYTES_CONFIG, kafkaConsumerParameters.getFetchMinBytes()); + props.put(FETCH_MAX_WAIT_MS_CONFIG, (int) kafkaConsumerParameters.getFetchMaxWait().toMillis()); + props.put( + RECONNECT_BACKOFF_MS_CONFIG, + (int) kafkaConsumerParameters.getReconnectBackoff().toMillis()); + props.put(RETRY_BACKOFF_MS_CONFIG, (int) kafkaConsumerParameters.getRetryBackoff().toMillis()); + props.put(CHECK_CRCS_CONFIG, kafkaConsumerParameters.isCheckCrcs()); + props.put( + METRICS_SAMPLE_WINDOW_MS_CONFIG, + (int) kafkaConsumerParameters.getMetricsSampleWindow().toMillis()); + props.put(METRICS_NUM_SAMPLES_CONFIG, kafkaConsumerParameters.getMetricsNumSamples()); + props.put( + REQUEST_TIMEOUT_MS_CONFIG, (int) kafkaConsumerParameters.getRequestTimeout().toMillis()); + props.put( + CONNECTIONS_MAX_IDLE_MS_CONFIG, + (int) kafkaConsumerParameters.getConnectionsMaxIdle().toMillis()); + props.put(MAX_POLL_RECORDS_CONFIG, kafkaConsumerParameters.getMaxPollRecords()); + props.put( + MAX_POLL_INTERVAL_MS_CONFIG, (int) kafkaConsumerParameters.getMaxPollInterval().toMillis()); + props.put(PARTITION_ASSIGNMENT_STRATEGY_CONFIG, getPartitionAssignmentStrategies()); + } + + private int getMaxPartitionFetch(Topic topic) { + if (commonConsumerParameters.isUseTopicMessageSizeEnabled()) { + int topicMessageSize = topic.getMaxMessageSize(); + int min = kafkaConsumerParameters.getMaxPartitionFetchMin(); + int max = kafkaConsumerParameters.getMaxPartitionFetchMax(); + return Math.max(Math.min(topicMessageSize, max), min); + } else { + return kafkaConsumerParameters.getMaxPartitionFetchMax(); } + } + + private List getPartitionAssignmentStrategies() { + return kafkaConsumerParameters.getPartitionAssignmentStrategies().stream() + .map(PartitionAssignmentStrategy::getAssignorClass) + .map(Class::getName) + .collect(toList()); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaReceiverParameters.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaReceiverParameters.java index 37d246f450..39e6bf90a0 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaReceiverParameters.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaReceiverParameters.java @@ -4,19 +4,19 @@ public interface KafkaReceiverParameters { - Duration getPoolTimeout(); + Duration getPoolTimeout(); - int getReadQueueCapacity(); + int getReadQueueCapacity(); - boolean isWaitBetweenUnsuccessfulPolls(); + boolean isWaitBetweenUnsuccessfulPolls(); - Duration getInitialIdleTime(); + Duration getInitialIdleTime(); - Duration getMaxIdleTime(); + Duration getMaxIdleTime(); - String getClientId(); + String getClientId(); - boolean isFilteringRateLimiterEnabled(); + boolean isFilteringRateLimiterEnabled(); - boolean isFilteringEnabled(); + boolean isFilteringEnabled(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaSingleThreadedMessageReceiver.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaSingleThreadedMessageReceiver.java index cc95936347..e256371bc3 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaSingleThreadedMessageReceiver.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/KafkaSingleThreadedMessageReceiver.java @@ -1,6 +1,16 @@ package pl.allegro.tech.hermes.consumers.consumer.receiver.kafka; import com.google.common.collect.ImmutableList; +import java.time.Duration; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.function.Function; +import java.util.stream.Collectors; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; @@ -27,189 +37,193 @@ import pl.allegro.tech.hermes.consumers.consumer.receiver.RetryableReceiverError; import pl.allegro.tech.hermes.metrics.HermesCounter; -import java.time.Duration; -import java.util.Collection; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; -import java.util.function.Function; -import java.util.stream.Collectors; - public class KafkaSingleThreadedMessageReceiver implements MessageReceiver { - private static final Logger logger = LoggerFactory.getLogger(KafkaSingleThreadedMessageReceiver.class); - - private final KafkaConsumer consumer; - private final KafkaConsumerRecordToMessageConverter messageConverter; - - private final BlockingQueue> readQueue; - private final KafkaConsumerOffsetMover offsetMover; - - private final HermesCounter skippedCounter; - private final HermesCounter failuresCounter; - private final SubscriptionLoadRecorder loadReporter; - private volatile Subscription subscription; - - private final Duration poolTimeout; - private final ConsumerPartitionAssignmentState partitionAssignmentState; - - public KafkaSingleThreadedMessageReceiver(KafkaConsumer consumer, - KafkaConsumerRecordToMessageConverterFactory messageConverterFactory, - MetricsFacade metrics, - KafkaNamesMapper kafkaNamesMapper, - Topic topic, - Subscription subscription, - Duration poolTimeout, - int readQueueCapacity, - SubscriptionLoadRecorder loadReporter, - ConsumerPartitionAssignmentState partitionAssignmentState) { - this.skippedCounter = metrics.offsetCommits().skippedCounter(); - this.failuresCounter = metrics.offsetCommits().failuresCounter(); - this.subscription = subscription; - this.poolTimeout = poolTimeout; - this.loadReporter = loadReporter; - this.partitionAssignmentState = partitionAssignmentState; - this.consumer = consumer; - this.readQueue = new ArrayBlockingQueue<>(readQueueCapacity); - this.offsetMover = new KafkaConsumerOffsetMover(subscription.getQualifiedName(), consumer); - Map topics = getKafkaTopics(topic, kafkaNamesMapper).stream() - .collect(Collectors.toMap(t -> t.name().asString(), Function.identity())); - this.messageConverter = messageConverterFactory.create(topic, subscription, topics); - this.consumer.subscribe(topics.keySet(), - new OffsetCommitterConsumerRebalanceListener(subscription.getQualifiedName(), partitionAssignmentState)); - } - - private Collection getKafkaTopics(Topic topic, KafkaNamesMapper kafkaNamesMapper) { - KafkaTopics kafkaTopics = kafkaNamesMapper.toKafkaTopics(topic); - ImmutableList.Builder topicsBuilder = new ImmutableList.Builder().add(kafkaTopics.getPrimary()); - kafkaTopics.getSecondary().ifPresent(topicsBuilder::add); - return topicsBuilder.build(); - } - - @Override - public Optional next() { - try { - supplyReadQueue(); - return getMessageFromReadQueue(); - } catch (InterruptException ex) { - // Despite that Thread.currentThread().interrupt() is called in InterruptException's constructor - // Thread.currentThread().isInterrupted() somehow returns false so we reset it. - logger.info("Kafka consumer thread interrupted", ex); - Thread.currentThread().interrupt(); - return Optional.empty(); - } catch (KafkaException ex) { - logger.error("Error while reading message for subscription {}", subscription.getQualifiedName(), ex); - return Optional.empty(); - } catch (Exception ex) { - logger.error("Failed to read message for subscription {}, readQueueSize {}", - subscription.getQualifiedName(), - readQueue.size(), - ex); - return Optional.empty(); - } + private static final Logger logger = + LoggerFactory.getLogger(KafkaSingleThreadedMessageReceiver.class); + + private final KafkaConsumer consumer; + private final KafkaConsumerRecordToMessageConverter messageConverter; + + private final BlockingQueue> readQueue; + private final KafkaConsumerOffsetMover offsetMover; + + private final HermesCounter skippedCounter; + private final HermesCounter failuresCounter; + private final SubscriptionLoadRecorder loadReporter; + private volatile Subscription subscription; + + private final Duration poolTimeout; + private final ConsumerPartitionAssignmentState partitionAssignmentState; + + public KafkaSingleThreadedMessageReceiver( + KafkaConsumer consumer, + KafkaConsumerRecordToMessageConverterFactory messageConverterFactory, + MetricsFacade metrics, + KafkaNamesMapper kafkaNamesMapper, + Topic topic, + Subscription subscription, + Duration poolTimeout, + int readQueueCapacity, + SubscriptionLoadRecorder loadReporter, + ConsumerPartitionAssignmentState partitionAssignmentState) { + this.skippedCounter = metrics.offsetCommits().skippedCounter(); + this.failuresCounter = metrics.offsetCommits().failuresCounter(); + this.subscription = subscription; + this.poolTimeout = poolTimeout; + this.loadReporter = loadReporter; + this.partitionAssignmentState = partitionAssignmentState; + this.consumer = consumer; + this.readQueue = new ArrayBlockingQueue<>(readQueueCapacity); + this.offsetMover = new KafkaConsumerOffsetMover(subscription.getQualifiedName(), consumer); + Map topics = + getKafkaTopics(topic, kafkaNamesMapper).stream() + .collect(Collectors.toMap(t -> t.name().asString(), Function.identity())); + this.messageConverter = messageConverterFactory.create(topic, subscription, topics); + this.consumer.subscribe( + topics.keySet(), + new OffsetCommitterConsumerRebalanceListener( + subscription.getQualifiedName(), partitionAssignmentState)); + } + + private Collection getKafkaTopics(Topic topic, KafkaNamesMapper kafkaNamesMapper) { + KafkaTopics kafkaTopics = kafkaNamesMapper.toKafkaTopics(topic); + ImmutableList.Builder topicsBuilder = + new ImmutableList.Builder().add(kafkaTopics.getPrimary()); + kafkaTopics.getSecondary().ifPresent(topicsBuilder::add); + return topicsBuilder.build(); + } + + @Override + public Optional next() { + try { + supplyReadQueue(); + return getMessageFromReadQueue(); + } catch (InterruptException ex) { + // Despite that Thread.currentThread().interrupt() is called in InterruptException's + // constructor + // Thread.currentThread().isInterrupted() somehow returns false so we reset it. + logger.info("Kafka consumer thread interrupted", ex); + Thread.currentThread().interrupt(); + return Optional.empty(); + } catch (KafkaException ex) { + logger.error( + "Error while reading message for subscription {}", subscription.getQualifiedName(), ex); + return Optional.empty(); + } catch (Exception ex) { + logger.error( + "Failed to read message for subscription {}, readQueueSize {}", + subscription.getQualifiedName(), + readQueue.size(), + ex); + return Optional.empty(); } - - private void supplyReadQueue() { - if (readQueue.isEmpty()) { - ConsumerRecords records = consumer.poll(poolTimeout); - try { - for (ConsumerRecord record : records) { - loadReporter.recordSingleOperation(); - readQueue.add(record); - } - } catch (Exception ex) { - logger.error("Failed to read message for subscription {}, readQueueSize {}, records {}", - subscription.getQualifiedName(), - readQueue.size(), - records.count(), - ex); - } + } + + private void supplyReadQueue() { + if (readQueue.isEmpty()) { + ConsumerRecords records = consumer.poll(poolTimeout); + try { + for (ConsumerRecord record : records) { + loadReporter.recordSingleOperation(); + readQueue.add(record); } + } catch (Exception ex) { + logger.error( + "Failed to read message for subscription {}, readQueueSize {}, records {}", + subscription.getQualifiedName(), + readQueue.size(), + records.count(), + ex); + } } - - private Optional getMessageFromReadQueue() { - if (!readQueue.isEmpty()) { - ConsumerRecord record = readQueue.element(); - try { - Message message = convertToMessage(record); - readQueue.poll(); - return Optional.of(message); - } catch (RetryableReceiverError ex) { - logger.warn("Cannot convert record to message... Operation will be delayed", ex); - return Optional.empty(); - } - } + } + + private Optional getMessageFromReadQueue() { + if (!readQueue.isEmpty()) { + ConsumerRecord record = readQueue.element(); + try { + Message message = convertToMessage(record); + readQueue.poll(); + return Optional.of(message); + } catch (RetryableReceiverError ex) { + logger.warn("Cannot convert record to message... Operation will be delayed", ex); return Optional.empty(); + } } - - private Message convertToMessage(ConsumerRecord record) { - long currentTerm = partitionAssignmentState.currentTerm(subscription.getQualifiedName()); - return messageConverter.convertToMessage(record, currentTerm); + return Optional.empty(); + } + + private Message convertToMessage(ConsumerRecord record) { + long currentTerm = partitionAssignmentState.currentTerm(subscription.getQualifiedName()); + return messageConverter.convertToMessage(record, currentTerm); + } + + @Override + public void stop() { + try { + consumer.close(); + } catch (IllegalStateException ex) { + // means it was already closed + } catch (InterruptException ex) { + // means that the thread was interrupted + } catch (KafkaException ex) { + logger.warn("KafkaException occurred during closing consumer.", ex); + } finally { + partitionAssignmentState.revokeAll(subscription.getQualifiedName()); } - - @Override - public void stop() { - try { - consumer.close(); - } catch (IllegalStateException ex) { - // means it was already closed - } catch (InterruptException ex) { - // means that the thread was interrupted - } catch (KafkaException ex) { - logger.warn("KafkaException occurred during closing consumer.", ex); - } finally { - partitionAssignmentState.revokeAll(subscription.getQualifiedName()); - } + } + + @Override + public void update(Subscription newSubscription) { + this.subscription = newSubscription; + messageConverter.update(subscription); + } + + @Override + public void commit(Set offsets) { + try { + consumer.commitSync(createOffset(offsets)); + } catch (InterruptException ex) { + logger.info("Kafka consumer thread interrupted", ex); + Thread.currentThread().interrupt(); + } catch (Exception ex) { + logger.error( + "Error while committing offset for subscription {}", subscription.getQualifiedName(), ex); + failuresCounter.increment(); } - - @Override - public void update(Subscription newSubscription) { - this.subscription = newSubscription; - messageConverter.update(subscription); - } - - @Override - public void commit(Set offsets) { - try { - consumer.commitSync(createOffset(offsets)); - } catch (InterruptException ex) { - logger.info("Kafka consumer thread interrupted", ex); - Thread.currentThread().interrupt(); - } catch (Exception ex) { - logger.error("Error while committing offset for subscription {}", subscription.getQualifiedName(), ex); - failuresCounter.increment(); + } + + private Map createOffset( + Set partitionOffsets) { + Map offsetsData = new LinkedHashMap<>(); + for (SubscriptionPartitionOffset partitionOffset : partitionOffsets) { + TopicPartition topicAndPartition = + new TopicPartition( + partitionOffset.getKafkaTopicName().asString(), partitionOffset.getPartition()); + + if (partitionAssignmentState.isAssignedPartitionAtCurrentTerm( + partitionOffset.getSubscriptionPartition())) { + if (consumer.position(topicAndPartition) >= partitionOffset.getOffset()) { + offsetsData.put(topicAndPartition, new OffsetAndMetadata(partitionOffset.getOffset())); + } else { + skippedCounter.increment(); } + } else { + logger.warn( + "Consumer is not assigned to partition {} of subscription {} at current term {}," + + " ignoring offset {} from term {} to commit", + partitionOffset.getPartition(), + partitionOffset.getSubscriptionName(), + partitionAssignmentState.currentTerm(partitionOffset.getSubscriptionName()), + partitionOffset.getOffset(), + partitionOffset.getPartitionAssignmentTerm()); + } } + return offsetsData; + } - private Map createOffset(Set partitionOffsets) { - Map offsetsData = new LinkedHashMap<>(); - for (SubscriptionPartitionOffset partitionOffset : partitionOffsets) { - TopicPartition topicAndPartition = new TopicPartition( - partitionOffset.getKafkaTopicName().asString(), - partitionOffset.getPartition()); - - if (partitionAssignmentState.isAssignedPartitionAtCurrentTerm(partitionOffset.getSubscriptionPartition())) { - if (consumer.position(topicAndPartition) >= partitionOffset.getOffset()) { - offsetsData.put(topicAndPartition, new OffsetAndMetadata(partitionOffset.getOffset())); - } else { - skippedCounter.increment(); - } - } else { - logger.warn( - "Consumer is not assigned to partition {} of subscription {} at current term {}," - + " ignoring offset {} from term {} to commit", - partitionOffset.getPartition(), partitionOffset.getSubscriptionName(), - partitionAssignmentState.currentTerm(partitionOffset.getSubscriptionName()), - partitionOffset.getOffset(), partitionOffset.getPartitionAssignmentTerm()); - } - } - return offsetsData; - } - - @Override - public boolean moveOffset(PartitionOffset offset) { - return offsetMover.move(offset); - } + @Override + public boolean moveOffset(PartitionOffset offset) { + return offsetMover.move(offset); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/MessageContentReader.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/MessageContentReader.java index 1930f97296..dd7a7b3b8a 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/MessageContentReader.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/MessageContentReader.java @@ -5,5 +5,5 @@ import pl.allegro.tech.hermes.common.message.wrapper.UnwrappedMessageContent; public interface MessageContentReader { - UnwrappedMessageContent read(ConsumerRecord message, ContentType contentType); + UnwrappedMessageContent read(ConsumerRecord message, ContentType contentType); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/MessageContentReaderFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/MessageContentReaderFactory.java index bf159d9638..f24c9536c9 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/MessageContentReaderFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/MessageContentReaderFactory.java @@ -3,5 +3,5 @@ import pl.allegro.tech.hermes.api.Topic; public interface MessageContentReaderFactory { - MessageContentReader provide(Topic topic); + MessageContentReader provide(Topic topic); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/PartitionAssignmentStrategy.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/PartitionAssignmentStrategy.java index a30e2b2869..462553dba6 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/PartitionAssignmentStrategy.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/receiver/kafka/PartitionAssignmentStrategy.java @@ -6,17 +6,17 @@ import org.apache.kafka.clients.consumer.StickyAssignor; public enum PartitionAssignmentStrategy { - RANGE(RangeAssignor.class), - STICKY(StickyAssignor.class), - COOPERATIVE(CooperativeStickyAssignor.class); + RANGE(RangeAssignor.class), + STICKY(StickyAssignor.class), + COOPERATIVE(CooperativeStickyAssignor.class); - private final Class assignorClass; + private final Class assignorClass; - PartitionAssignmentStrategy(Class assignorClass) { - this.assignorClass = assignorClass; - } + PartitionAssignmentStrategy(Class assignorClass) { + this.assignorClass = assignorClass; + } - Class getAssignorClass() { - return assignorClass; - } + Class getAssignorClass() { + return assignorClass; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/result/DefaultErrorHandler.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/result/DefaultErrorHandler.java index 945cd8df01..c842b514e3 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/result/DefaultErrorHandler.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/result/DefaultErrorHandler.java @@ -1,5 +1,11 @@ package pl.allegro.tech.hermes.consumers.consumer.result; +import static pl.allegro.tech.hermes.api.SentMessageTrace.Builder.undeliveredMessage; +import static pl.allegro.tech.hermes.consumers.consumer.message.MessageConverter.toMessageMetadata; + +import java.time.Clock; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.Subscription; @@ -12,67 +18,68 @@ import pl.allegro.tech.hermes.metrics.HermesHistogram; import pl.allegro.tech.hermes.tracker.consumers.Trackers; -import java.time.Clock; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import static pl.allegro.tech.hermes.api.SentMessageTrace.Builder.undeliveredMessage; -import static pl.allegro.tech.hermes.consumers.consumer.message.MessageConverter.toMessageMetadata; - public class DefaultErrorHandler implements ErrorHandler { - private static final Logger logger = LoggerFactory.getLogger(DefaultErrorHandler.class); - - private final MetricsFacade metrics; - private final UndeliveredMessageLog undeliveredMessageLog; - private final Clock clock; - private final Trackers trackers; - private final String cluster; - private final SubscriptionName subscriptionName; - private final HermesCounter failures; - private final HermesCounter timeouts; - private final HermesCounter otherErrors; - private final HermesCounter discarded; - private final HermesHistogram inflightTime; - private final HermesCounter throughputInBytes; - private final Map httpStatusCodes = new ConcurrentHashMap<>(); - - - public DefaultErrorHandler(MetricsFacade metrics, - UndeliveredMessageLog undeliveredMessageLog, - Clock clock, - Trackers trackers, - String cluster, - SubscriptionName subscriptionName) { - this.metrics = metrics; - this.undeliveredMessageLog = undeliveredMessageLog; - this.clock = clock; - this.trackers = trackers; - this.cluster = cluster; - this.subscriptionName = subscriptionName; - this.failures = metrics.subscriptions().failuresCounter(subscriptionName); - this.timeouts = metrics.subscriptions().timeoutsCounter(subscriptionName); - this.otherErrors = metrics.subscriptions().otherErrorsCounter(subscriptionName); - this.discarded = metrics.subscriptions().discarded(subscriptionName); - this.inflightTime = metrics.subscriptions().inflightTimeInMillisHistogram(subscriptionName); - this.throughputInBytes = metrics.subscriptions().throughputInBytes(subscriptionName); - } - - @Override - public void handleDiscarded(Message message, Subscription subscription, MessageSendingResult result) { - logResult(message, subscription, result); - - discarded.increment(); - inflightTime.record(System.currentTimeMillis() - message.getReadingTimestamp()); - - addToMessageLog(message, subscription, result); - - trackers.get(subscription).logDiscarded(toMessageMetadata(message, subscription), result.getRootCause()); - } - - private void addToMessageLog(Message message, Subscription subscription, MessageSendingResult result) { - result.getLogInfo().forEach(logInfo -> undeliveredMessageLog.add( - undeliveredMessage() + private static final Logger logger = LoggerFactory.getLogger(DefaultErrorHandler.class); + + private final MetricsFacade metrics; + private final UndeliveredMessageLog undeliveredMessageLog; + private final Clock clock; + private final Trackers trackers; + private final String cluster; + private final SubscriptionName subscriptionName; + private final HermesCounter failures; + private final HermesCounter timeouts; + private final HermesCounter otherErrors; + private final HermesCounter discarded; + private final HermesHistogram inflightTime; + private final HermesCounter throughputInBytes; + private final Map httpStatusCodes = new ConcurrentHashMap<>(); + + public DefaultErrorHandler( + MetricsFacade metrics, + UndeliveredMessageLog undeliveredMessageLog, + Clock clock, + Trackers trackers, + String cluster, + SubscriptionName subscriptionName) { + this.metrics = metrics; + this.undeliveredMessageLog = undeliveredMessageLog; + this.clock = clock; + this.trackers = trackers; + this.cluster = cluster; + this.subscriptionName = subscriptionName; + this.failures = metrics.subscriptions().failuresCounter(subscriptionName); + this.timeouts = metrics.subscriptions().timeoutsCounter(subscriptionName); + this.otherErrors = metrics.subscriptions().otherErrorsCounter(subscriptionName); + this.discarded = metrics.subscriptions().discarded(subscriptionName); + this.inflightTime = metrics.subscriptions().inflightTimeInMillisHistogram(subscriptionName); + this.throughputInBytes = metrics.subscriptions().throughputInBytes(subscriptionName); + } + + @Override + public void handleDiscarded( + Message message, Subscription subscription, MessageSendingResult result) { + logResult(message, subscription, result); + + discarded.increment(); + inflightTime.record(System.currentTimeMillis() - message.getReadingTimestamp()); + + addToMessageLog(message, subscription, result); + + trackers + .get(subscription) + .logDiscarded(toMessageMetadata(message, subscription), result.getRootCause()); + } + + private void addToMessageLog( + Message message, Subscription subscription, MessageSendingResult result) { + result + .getLogInfo() + .forEach( + logInfo -> + undeliveredMessageLog.add( + undeliveredMessage() .withSubscription(subscription.getName()) .withTopicName(subscription.getQualifiedTopicName()) .withMessage(new String(message.getData())) @@ -81,41 +88,51 @@ private void addToMessageLog(Message message, Subscription subscription, Message .withPartition(message.getPartition()) .withOffset(message.getOffset()) .withCluster(cluster) - .build() - )); - } - - private void logResult(Message message, Subscription subscription, MessageSendingResult result) { - if (result.isLoggable()) { - result.getLogInfo().forEach(logInfo -> - logger.warn( - "Abnormal delivery failure: " - + "subscription: {}; cause: {}; endpoint: {}; messageId: {}; partition: {}; offset: {}", - subscription.getQualifiedName(), logInfo.getRootCause(), logInfo.getUrlString(), message.getId(), - message.getPartition(), message.getOffset(), logInfo.getFailure() - ) - ); - } + .build())); + } + + private void logResult(Message message, Subscription subscription, MessageSendingResult result) { + if (result.isLoggable()) { + result + .getLogInfo() + .forEach( + logInfo -> + logger.warn( + "Abnormal delivery failure: " + + "subscription: {}; cause: {}; endpoint: {}; messageId: {}; partition: {}; offset: {}", + subscription.getQualifiedName(), + logInfo.getRootCause(), + logInfo.getUrlString(), + message.getId(), + message.getPartition(), + message.getOffset(), + logInfo.getFailure())); } - - @Override - public void handleFailed(Message message, Subscription subscription, MessageSendingResult result) { - failures.increment(); - if (result.hasHttpAnswer()) { - markHttpStatusCode(result.getStatusCode()); - } else if (result.isTimeout()) { - timeouts.increment(); - } else { - otherErrors.increment(); - } - throughputInBytes.increment(message.getSize()); - trackers.get(subscription).logFailed(toMessageMetadata(message, subscription), result.getRootCause(), result.getHostname()); - } - - private void markHttpStatusCode(int statusCode) { - httpStatusCodes.computeIfAbsent( - statusCode, - integer -> metrics.subscriptions().httpAnswerCounter(subscriptionName, statusCode) - ).increment(); + } + + @Override + public void handleFailed( + Message message, Subscription subscription, MessageSendingResult result) { + failures.increment(); + if (result.hasHttpAnswer()) { + markHttpStatusCode(result.getStatusCode()); + } else if (result.isTimeout()) { + timeouts.increment(); + } else { + otherErrors.increment(); } + throughputInBytes.increment(message.getSize()); + trackers + .get(subscription) + .logFailed( + toMessageMetadata(message, subscription), result.getRootCause(), result.getHostname()); + } + + private void markHttpStatusCode(int statusCode) { + httpStatusCodes + .computeIfAbsent( + statusCode, + integer -> metrics.subscriptions().httpAnswerCounter(subscriptionName, statusCode)) + .increment(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/result/DefaultSuccessHandler.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/result/DefaultSuccessHandler.java index 2d74f9cd2f..c5e8968abd 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/result/DefaultSuccessHandler.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/result/DefaultSuccessHandler.java @@ -1,5 +1,9 @@ package pl.allegro.tech.hermes.consumers.consumer.result; +import static pl.allegro.tech.hermes.consumers.consumer.message.MessageConverter.toMessageMetadata; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.common.metric.MetricsFacade; @@ -9,49 +13,47 @@ import pl.allegro.tech.hermes.metrics.HermesHistogram; import pl.allegro.tech.hermes.tracker.consumers.Trackers; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import static pl.allegro.tech.hermes.consumers.consumer.message.MessageConverter.toMessageMetadata; - public class DefaultSuccessHandler implements SuccessHandler { - private final Trackers trackers; - private final SubscriptionName subscriptionName; - private final MetricsFacade metrics; - private final Map httpStatusCodes = new ConcurrentHashMap<>(); - private final HermesCounter throughputInBytes; - private final HermesCounter successes; - private final HermesHistogram inflightTime; - - public DefaultSuccessHandler(MetricsFacade metrics, - Trackers trackers, - SubscriptionName subscriptionName) { - this.metrics = metrics; - this.trackers = trackers; - this.subscriptionName = subscriptionName; - this.throughputInBytes = metrics.subscriptions().throughputInBytes(subscriptionName); - this.successes = metrics.subscriptions().successes(subscriptionName); - this.inflightTime = metrics.subscriptions().inflightTimeInMillisHistogram(subscriptionName); - } - - @Override - public void handleSuccess(Message message, Subscription subscription, MessageSendingResult result) { - markSuccess(message, result); - trackers.get(subscription).logSent(toMessageMetadata(message, subscription), result.getHostname()); - } - - private void markSuccess(Message message, MessageSendingResult result) { - successes.increment(); - throughputInBytes.increment(message.getSize()); - markHttpStatusCode(result.getStatusCode()); - inflightTime.record(System.currentTimeMillis() - message.getReadingTimestamp()); - } - - private void markHttpStatusCode(int statusCode) { - httpStatusCodes.computeIfAbsent( - statusCode, - integer -> metrics.subscriptions().httpAnswerCounter(subscriptionName, statusCode) - ).increment(); - } + private final Trackers trackers; + private final SubscriptionName subscriptionName; + private final MetricsFacade metrics; + private final Map httpStatusCodes = new ConcurrentHashMap<>(); + private final HermesCounter throughputInBytes; + private final HermesCounter successes; + private final HermesHistogram inflightTime; + + public DefaultSuccessHandler( + MetricsFacade metrics, Trackers trackers, SubscriptionName subscriptionName) { + this.metrics = metrics; + this.trackers = trackers; + this.subscriptionName = subscriptionName; + this.throughputInBytes = metrics.subscriptions().throughputInBytes(subscriptionName); + this.successes = metrics.subscriptions().successes(subscriptionName); + this.inflightTime = metrics.subscriptions().inflightTimeInMillisHistogram(subscriptionName); + } + + @Override + public void handleSuccess( + Message message, Subscription subscription, MessageSendingResult result) { + markSuccess(message, result); + trackers + .get(subscription) + .logSent(toMessageMetadata(message, subscription), result.getHostname()); + } + + private void markSuccess(Message message, MessageSendingResult result) { + successes.increment(); + throughputInBytes.increment(message.getSize()); + markHttpStatusCode(result.getStatusCode()); + inflightTime.record(System.currentTimeMillis() - message.getReadingTimestamp()); + } + + private void markHttpStatusCode(int statusCode) { + httpStatusCodes + .computeIfAbsent( + statusCode, + integer -> metrics.subscriptions().httpAnswerCounter(subscriptionName, statusCode)) + .increment(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/result/ErrorHandler.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/result/ErrorHandler.java index 5dcdf0052d..e571ad86d3 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/result/ErrorHandler.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/result/ErrorHandler.java @@ -5,7 +5,7 @@ import pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult; public interface ErrorHandler { - void handleDiscarded(Message message, Subscription subscription, MessageSendingResult result); + void handleDiscarded(Message message, Subscription subscription, MessageSendingResult result); - void handleFailed(Message message, Subscription subscription, MessageSendingResult result); + void handleFailed(Message message, Subscription subscription, MessageSendingResult result); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/result/SuccessHandler.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/result/SuccessHandler.java index d6b9b23e26..350353b31f 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/result/SuccessHandler.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/result/SuccessHandler.java @@ -5,5 +5,5 @@ import pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult; public interface SuccessHandler { - void handleSuccess(Message message, Subscription subscription, MessageSendingResult result); + void handleSuccess(Message message, Subscription subscription, MessageSendingResult result); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/CompletableFutureAwareMessageSender.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/CompletableFutureAwareMessageSender.java index 0e7824b81c..fd4317085b 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/CompletableFutureAwareMessageSender.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/CompletableFutureAwareMessageSender.java @@ -1,12 +1,11 @@ package pl.allegro.tech.hermes.consumers.consumer.sender; -import pl.allegro.tech.hermes.consumers.consumer.Message; - import java.util.concurrent.CompletableFuture; +import pl.allegro.tech.hermes.consumers.consumer.Message; public interface CompletableFutureAwareMessageSender { - void send(Message message, CompletableFuture resultFuture); + void send(Message message, CompletableFuture resultFuture); - void stop(); + void stop(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageBatchSender.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageBatchSender.java index 33692b1e15..fa8b795c3c 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageBatchSender.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageBatchSender.java @@ -5,5 +5,9 @@ import pl.allegro.tech.hermes.consumers.consumer.batch.MessageBatch; public interface MessageBatchSender { - MessageSendingResult send(MessageBatch message, EndpointAddress address, EndpointAddressResolverMetadata metadata, int requestTimeout); + MessageSendingResult send( + MessageBatch message, + EndpointAddress address, + EndpointAddressResolverMetadata metadata, + int requestTimeout); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageBatchSenderFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageBatchSenderFactory.java index 293be4a3dc..0ed2a44dc5 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageBatchSenderFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageBatchSenderFactory.java @@ -3,5 +3,5 @@ import pl.allegro.tech.hermes.api.Subscription; public interface MessageBatchSenderFactory { - MessageBatchSender create(Subscription subscription); + MessageBatchSender create(Subscription subscription); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageSender.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageSender.java index 0f2bde7a4a..f2df875196 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageSender.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageSender.java @@ -1,12 +1,11 @@ package pl.allegro.tech.hermes.consumers.consumer.sender; -import pl.allegro.tech.hermes.consumers.consumer.Message; - import java.util.concurrent.CompletableFuture; +import pl.allegro.tech.hermes.consumers.consumer.Message; public interface MessageSender { - CompletableFuture send(Message message); + CompletableFuture send(Message message); - void stop(); + void stop(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageSenderFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageSenderFactory.java index 5fdc0acc9f..22e8a49169 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageSenderFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageSenderFactory.java @@ -1,59 +1,64 @@ package pl.allegro.tech.hermes.consumers.consumer.sender; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import pl.allegro.tech.hermes.api.EndpointAddress; import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.common.exception.EndpointProtocolNotSupportedException; import pl.allegro.tech.hermes.common.exception.InternalProcessingException; import pl.allegro.tech.hermes.consumers.consumer.ResilientMessageSender; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - public class MessageSenderFactory { - private final Map protocolProviders = new HashMap<>(); + private final Map protocolProviders = new HashMap<>(); - public MessageSenderFactory(List providers) { - for (ProtocolMessageSenderProvider provider : providers) { - for (String protocol : provider.getSupportedProtocols()) { - addSupportedProtocol(protocol, provider); - } - } + public MessageSenderFactory(List providers) { + for (ProtocolMessageSenderProvider provider : providers) { + for (String protocol : provider.getSupportedProtocols()) { + addSupportedProtocol(protocol, provider); + } } + } - public MessageSender create(Subscription subscription, ResilientMessageSender resilientMessageSender) { - EndpointAddress endpoint = subscription.getEndpoint(); + public MessageSender create( + Subscription subscription, ResilientMessageSender resilientMessageSender) { + EndpointAddress endpoint = subscription.getEndpoint(); - ProtocolMessageSenderProvider provider = protocolProviders.get(endpoint.getProtocol()); - if (provider == null) { - throw new EndpointProtocolNotSupportedException(endpoint); - } - return provider.create(subscription, resilientMessageSender); + ProtocolMessageSenderProvider provider = protocolProviders.get(endpoint.getProtocol()); + if (provider == null) { + throw new EndpointProtocolNotSupportedException(endpoint); } + return provider.create(subscription, resilientMessageSender); + } - private void addSupportedProtocol(String protocol, ProtocolMessageSenderProvider provider) { - if (!protocolProviders.containsKey(protocol)) { - startProvider(provider); - protocolProviders.put(protocol, provider); - } + private void addSupportedProtocol(String protocol, ProtocolMessageSenderProvider provider) { + if (!protocolProviders.containsKey(protocol)) { + startProvider(provider); + protocolProviders.put(protocol, provider); } + } - private void startProvider(ProtocolMessageSenderProvider provider) { - try { - provider.start(); - } catch (Exception e) { - throw new InternalProcessingException("Something went wrong while starting message sender provider", e); - } + private void startProvider(ProtocolMessageSenderProvider provider) { + try { + provider.start(); + } catch (Exception e) { + throw new InternalProcessingException( + "Something went wrong while starting message sender provider", e); } + } - public void closeProviders() { - protocolProviders.values().forEach(provider -> { - try { + public void closeProviders() { + protocolProviders + .values() + .forEach( + provider -> { + try { provider.stop(); - } catch (Exception e) { - throw new InternalProcessingException("Something went wrong while stopping message sender provider", e); - } - }); - } + } catch (Exception e) { + throw new InternalProcessingException( + "Something went wrong while stopping message sender provider", e); + } + }); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageSendingResult.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageSendingResult.java index 6ce56cfa0a..45cf4a7496 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageSendingResult.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageSendingResult.java @@ -1,92 +1,93 @@ package pl.allegro.tech.hermes.consumers.consumer.sender; -import org.eclipse.jetty.client.Result; -import pl.allegro.tech.hermes.consumers.consumer.sender.resolver.EndpointAddressResolutionException; +import static io.netty.handler.codec.http.HttpResponseStatus.TOO_MANY_REQUESTS; +import static jakarta.ws.rs.core.Response.Status.OK; +import static jakarta.ws.rs.core.Response.Status.SERVICE_UNAVAILABLE; +import static java.util.stream.Collectors.joining; import java.net.URI; import java.util.List; import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.function.Predicate; - -import static io.netty.handler.codec.http.HttpResponseStatus.TOO_MANY_REQUESTS; -import static jakarta.ws.rs.core.Response.Status.OK; -import static jakarta.ws.rs.core.Response.Status.SERVICE_UNAVAILABLE; -import static java.util.stream.Collectors.joining; +import org.eclipse.jetty.client.Result; +import pl.allegro.tech.hermes.consumers.consumer.sender.resolver.EndpointAddressResolutionException; public interface MessageSendingResult { - String CAUSE_UNKNOWN = "unknown"; + String CAUSE_UNKNOWN = "unknown"; - static SingleMessageSendingResult succeededResult() { - return new SingleMessageSendingResult(OK.getStatusCode()); - } + static SingleMessageSendingResult succeededResult() { + return new SingleMessageSendingResult(OK.getStatusCode()); + } - static SingleMessageSendingResult succeededResult(URI requestURI) { - return new SingleMessageSendingResult(OK.getStatusCode(), requestURI); - } + static SingleMessageSendingResult succeededResult(URI requestURI) { + return new SingleMessageSendingResult(OK.getStatusCode(), requestURI); + } - static SingleMessageSendingResult failedResult(Throwable cause) { - return new SingleMessageSendingResult(cause); - } + static SingleMessageSendingResult failedResult(Throwable cause) { + return new SingleMessageSendingResult(cause); + } - static SingleMessageSendingResult failedResult(EndpointAddressResolutionException cause) { - return new SingleMessageSendingResult(cause, cause.isIgnoreInRateCalculation()); - } + static SingleMessageSendingResult failedResult(EndpointAddressResolutionException cause) { + return new SingleMessageSendingResult(cause, cause.isIgnoreInRateCalculation()); + } - static SingleMessageSendingResult failedResult(int statusCode) { - return new SingleMessageSendingResult(statusCode); - } + static SingleMessageSendingResult failedResult(int statusCode) { + return new SingleMessageSendingResult(statusCode); + } - static SingleMessageSendingResult ofStatusCode(int statusCode) { - return new SingleMessageSendingResult(statusCode); - } + static SingleMessageSendingResult ofStatusCode(int statusCode) { + return new SingleMessageSendingResult(statusCode); + } - static SingleMessageSendingResult retryAfter(int seconds) { - return new SingleMessageSendingResult(SERVICE_UNAVAILABLE.getStatusCode(), TimeUnit.SECONDS.toMillis(seconds)); - } + static SingleMessageSendingResult retryAfter(int seconds) { + return new SingleMessageSendingResult( + SERVICE_UNAVAILABLE.getStatusCode(), TimeUnit.SECONDS.toMillis(seconds)); + } - static SingleMessageSendingResult tooManyRequests(int seconds) { - return new SingleMessageSendingResult(TOO_MANY_REQUESTS.code(), TimeUnit.SECONDS.toMillis(seconds)); - } + static SingleMessageSendingResult tooManyRequests(int seconds) { + return new SingleMessageSendingResult( + TOO_MANY_REQUESTS.code(), TimeUnit.SECONDS.toMillis(seconds)); + } - static SingleMessageSendingResult of(Result result) { - return new SingleMessageSendingResult(result); - } + static SingleMessageSendingResult of(Result result) { + return new SingleMessageSendingResult(result); + } - static SingleMessageSendingResult ofResultWithUri(Result result, URI uri) { - return new SingleMessageSendingResult(result, uri); - } + static SingleMessageSendingResult ofResultWithUri(Result result, URI uri) { + return new SingleMessageSendingResult(result, uri); + } - String getRootCause(); + String getRootCause(); - int getStatusCode(); + int getStatusCode(); - boolean isLoggable(); + boolean isLoggable(); - Optional getRetryAfterMillis(); + Optional getRetryAfterMillis(); - boolean isClientError(); + boolean isClientError(); - boolean isTimeout(); + boolean isTimeout(); - boolean succeeded(); + boolean succeeded(); - boolean ignoreInRateCalculation(boolean retryClientErrors, boolean isOAuthSecuredSubscription); + boolean ignoreInRateCalculation(boolean retryClientErrors, boolean isOAuthSecuredSubscription); - default boolean hasHttpAnswer() { - return getStatusCode() != 0; - } + default boolean hasHttpAnswer() { + return getStatusCode() != 0; + } - boolean isRetryLater(); + boolean isRetryLater(); - List getLogInfo(); + List getLogInfo(); - List getSucceededUris(Predicate filter); + List getSucceededUris(Predicate filter); - default String getHostname() { - return getLogInfo().stream() - .filter(logInfo -> logInfo.getUrl().isPresent()) - .map(logInfo -> logInfo.getUrl().get().getHost()) - .collect(joining(",")); - } + default String getHostname() { + return getLogInfo().stream() + .filter(logInfo -> logInfo.getUrl().isPresent()) + .map(logInfo -> logInfo.getUrl().get().getHost()) + .collect(joining(",")); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageSendingResultLogInfo.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageSendingResultLogInfo.java index 4b8cde9f75..656e1e7544 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageSendingResultLogInfo.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageSendingResultLogInfo.java @@ -4,29 +4,29 @@ import java.util.Optional; public class MessageSendingResultLogInfo { - private final Optional url; - private final String rootCause; - private final Throwable failure; + private final Optional url; + private final String rootCause; + private final Throwable failure; - public MessageSendingResultLogInfo(Optional url, Throwable failure, String rootCause) { - this.url = url; - this.failure = failure; - this.rootCause = rootCause; - } + public MessageSendingResultLogInfo(Optional url, Throwable failure, String rootCause) { + this.url = url; + this.failure = failure; + this.rootCause = rootCause; + } - public Optional getUrl() { - return url; - } + public Optional getUrl() { + return url; + } - public String getUrlString() { - return url.isPresent() ? url.get().toString() : ""; - } + public String getUrlString() { + return url.isPresent() ? url.get().toString() : ""; + } - public String getRootCause() { - return rootCause; - } + public String getRootCause() { + return rootCause; + } - public Throwable getFailure() { - return failure; - } + public Throwable getFailure() { + return failure; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MultiMessageSendingResult.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MultiMessageSendingResult.java index 6f7f18caec..9b0e0234a5 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MultiMessageSendingResult.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/MultiMessageSendingResult.java @@ -1,8 +1,8 @@ package pl.allegro.tech.hermes.consumers.consumer.sender; +import static java.util.stream.Collectors.joining; import com.google.common.collect.ImmutableList; - import java.net.URI; import java.util.Comparator; import java.util.List; @@ -10,97 +10,106 @@ import java.util.function.Predicate; import java.util.stream.Collectors; -import static java.util.stream.Collectors.joining; - public class MultiMessageSendingResult implements MessageSendingResult { - private final List children; - - public MultiMessageSendingResult(List children) { - this.children = ImmutableList.copyOf(children); - } - - @Override - public int getStatusCode() { - if (succeeded()) { - return children.stream().mapToInt(MessageSendingResult::getStatusCode).max().orElse(0); - } else { - return children.stream().filter(child -> !child.succeeded()).mapToInt(MessageSendingResult::getStatusCode).max().orElse(0); - } - } - - @Override - public boolean isLoggable() { - return children.stream().anyMatch(MessageSendingResult::isLoggable); - } - - @Override - public boolean ignoreInRateCalculation(boolean retryClientErrors, boolean isOAuthSecuredSubscription) { - return children.stream().allMatch(r -> r.ignoreInRateCalculation(retryClientErrors, isOAuthSecuredSubscription)); - } - - @Override - public Optional getRetryAfterMillis() { - return children.stream() - .map(MessageSendingResult::getRetryAfterMillis) - .filter(Optional::isPresent).map(Optional::get) - .min(Comparator.naturalOrder()); - } - - @Override - public boolean succeeded() { - return !children.isEmpty() && children.stream().allMatch(MessageSendingResult::succeeded); + private final List children; + + public MultiMessageSendingResult(List children) { + this.children = ImmutableList.copyOf(children); + } + + @Override + public int getStatusCode() { + if (succeeded()) { + return children.stream().mapToInt(MessageSendingResult::getStatusCode).max().orElse(0); + } else { + return children.stream() + .filter(child -> !child.succeeded()) + .mapToInt(MessageSendingResult::getStatusCode) + .max() + .orElse(0); } - - @Override - public boolean isClientError() { - List failed = children.stream().filter(child -> !child.succeeded()).collect(Collectors.toList()); - return !failed.isEmpty() - && failed.stream().allMatch(MessageSendingResult::isClientError); - } - - @Override - public boolean isTimeout() { - return !children.isEmpty() && children.stream().anyMatch(MessageSendingResult::isTimeout); - } - - @Override - public boolean isRetryLater() { - return children.isEmpty() || children.stream().anyMatch(MessageSendingResult::isRetryLater); - } - - @Override - public List getLogInfo() { - return children.stream().map(child -> - new MessageSendingResultLogInfo(child.getRequestUri(), child.getFailure(), child.getRootCause())) - .collect(Collectors.toList()); - - } - - @Override - public List getSucceededUris(Predicate filter) { - return children.stream() - .filter(filter) - .map(SingleMessageSendingResult::getRequestUri) - .filter(Optional::isPresent) - .map(Optional::get) - .collect(Collectors.toList()); - - } - - public List getChildren() { - return children; - } - - @Override - public String getRootCause() { - if (children.isEmpty()) { - return "Empty children message results"; - } else { - return children.stream() - .map(child -> child.getRequestUri().map(Object::toString).orElse("") + ":" + child.getRootCause()) - .collect(joining(";")); - } + } + + @Override + public boolean isLoggable() { + return children.stream().anyMatch(MessageSendingResult::isLoggable); + } + + @Override + public boolean ignoreInRateCalculation( + boolean retryClientErrors, boolean isOAuthSecuredSubscription) { + return children.stream() + .allMatch(r -> r.ignoreInRateCalculation(retryClientErrors, isOAuthSecuredSubscription)); + } + + @Override + public Optional getRetryAfterMillis() { + return children.stream() + .map(MessageSendingResult::getRetryAfterMillis) + .filter(Optional::isPresent) + .map(Optional::get) + .min(Comparator.naturalOrder()); + } + + @Override + public boolean succeeded() { + return !children.isEmpty() && children.stream().allMatch(MessageSendingResult::succeeded); + } + + @Override + public boolean isClientError() { + List failed = + children.stream().filter(child -> !child.succeeded()).collect(Collectors.toList()); + return !failed.isEmpty() && failed.stream().allMatch(MessageSendingResult::isClientError); + } + + @Override + public boolean isTimeout() { + return !children.isEmpty() && children.stream().anyMatch(MessageSendingResult::isTimeout); + } + + @Override + public boolean isRetryLater() { + return children.isEmpty() || children.stream().anyMatch(MessageSendingResult::isRetryLater); + } + + @Override + public List getLogInfo() { + return children.stream() + .map( + child -> + new MessageSendingResultLogInfo( + child.getRequestUri(), child.getFailure(), child.getRootCause())) + .collect(Collectors.toList()); + } + + @Override + public List getSucceededUris(Predicate filter) { + return children.stream() + .filter(filter) + .map(SingleMessageSendingResult::getRequestUri) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } + + public List getChildren() { + return children; + } + + @Override + public String getRootCause() { + if (children.isEmpty()) { + return "Empty children message results"; + } else { + return children.stream() + .map( + child -> + child.getRequestUri().map(Object::toString).orElse("") + + ":" + + child.getRootCause()) + .collect(joining(";")); } + } } - diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/ProtocolMessageSenderProvider.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/ProtocolMessageSenderProvider.java index 6743bd8272..cd3c4fbd61 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/ProtocolMessageSenderProvider.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/ProtocolMessageSenderProvider.java @@ -1,17 +1,16 @@ package pl.allegro.tech.hermes.consumers.consumer.sender; +import java.util.Set; import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.consumers.consumer.ResilientMessageSender; -import java.util.Set; - public interface ProtocolMessageSenderProvider { - MessageSender create(Subscription subscription, ResilientMessageSender resilientMessageSender); + MessageSender create(Subscription subscription, ResilientMessageSender resilientMessageSender); - Set getSupportedProtocols(); + Set getSupportedProtocols(); - void start() throws Exception; + void start() throws Exception; - void stop() throws Exception; + void stop() throws Exception; } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/SingleMessageSendingResult.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/SingleMessageSendingResult.java index 5ca3c26236..df8533b2b2 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/SingleMessageSendingResult.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/SingleMessageSendingResult.java @@ -1,13 +1,15 @@ package pl.allegro.tech.hermes.consumers.consumer.sender; +import static io.netty.handler.codec.http.HttpResponseStatus.SERVICE_UNAVAILABLE; +import static io.netty.handler.codec.http.HttpResponseStatus.TOO_MANY_REQUESTS; +import static io.netty.handler.codec.http.HttpResponseStatus.UNAUTHORIZED; +import static jakarta.ws.rs.core.Response.Status.Family.CLIENT_ERROR; +import static jakarta.ws.rs.core.Response.Status.Family.SUCCESSFUL; +import static jakarta.ws.rs.core.Response.Status.Family.familyOf; + import com.google.common.base.Preconditions; import com.google.common.base.Throwables; import jakarta.ws.rs.core.Response; -import org.eclipse.jetty.client.Result; -import org.eclipse.jetty.http.HttpFields; -import org.eclipse.jetty.http.HttpHeader; -import pl.allegro.tech.hermes.common.exception.InternalProcessingException; - import java.net.URI; import java.util.Collections; import java.util.List; @@ -15,173 +17,174 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.function.Predicate; - -import static io.netty.handler.codec.http.HttpResponseStatus.SERVICE_UNAVAILABLE; -import static io.netty.handler.codec.http.HttpResponseStatus.TOO_MANY_REQUESTS; -import static io.netty.handler.codec.http.HttpResponseStatus.UNAUTHORIZED; -import static jakarta.ws.rs.core.Response.Status.Family.CLIENT_ERROR; -import static jakarta.ws.rs.core.Response.Status.Family.SUCCESSFUL; -import static jakarta.ws.rs.core.Response.Status.Family.familyOf; +import org.eclipse.jetty.client.Result; +import org.eclipse.jetty.http.HttpFields; +import org.eclipse.jetty.http.HttpHeader; +import pl.allegro.tech.hermes.common.exception.InternalProcessingException; public class SingleMessageSendingResult implements MessageSendingResult { - private Throwable failure; - private boolean loggable; - private boolean ignoreInRateCalculation = false; - - private Optional retryAfterMillis = Optional.empty(); - private Optional requestUri = Optional.empty(); - private int statusCode; - private Response.Status.Family responseFamily; - - SingleMessageSendingResult(Throwable failure, boolean ignoreInRateCalculation) { - this.failure = failure; - this.loggable = !isTimeout(); - this.ignoreInRateCalculation = ignoreInRateCalculation; - } - - SingleMessageSendingResult(Throwable failure) { - this(failure, false); - } - - SingleMessageSendingResult(Result result) { - this.failure = result.getFailure(); - - if (result.getResponse() != null) { - initializeForStatusCode(result.getResponse().getStatus()); - if (shouldRetryOnStatusCode()) { - initializeRetryAfterMillis(result); - } - } - - this.loggable = !isTimeout() && !hasHttpAnswer(); - this.requestUri = Optional.ofNullable(result.getRequest().getURI()); - } - - SingleMessageSendingResult(int statusCode) { - initializeForStatusCode(statusCode); - } - - SingleMessageSendingResult(int statusCode, URI requestURI) { - this(statusCode); - this.requestUri = Optional.of(requestURI); - } - - SingleMessageSendingResult(int statusCode, long retryAfterMillis) { - initializeForStatusCode(statusCode); - if (shouldRetryOnStatusCode() && retryAfterMillis >= 0) { - this.retryAfterMillis = Optional.of(retryAfterMillis); + private Throwable failure; + private boolean loggable; + private boolean ignoreInRateCalculation = false; + + private Optional retryAfterMillis = Optional.empty(); + private Optional requestUri = Optional.empty(); + private int statusCode; + private Response.Status.Family responseFamily; + + SingleMessageSendingResult(Throwable failure, boolean ignoreInRateCalculation) { + this.failure = failure; + this.loggable = !isTimeout(); + this.ignoreInRateCalculation = ignoreInRateCalculation; + } + + SingleMessageSendingResult(Throwable failure) { + this(failure, false); + } + + SingleMessageSendingResult(Result result) { + this.failure = result.getFailure(); + + if (result.getResponse() != null) { + initializeForStatusCode(result.getResponse().getStatus()); + if (shouldRetryOnStatusCode()) { + initializeRetryAfterMillis(result); + } + } + + this.loggable = !isTimeout() && !hasHttpAnswer(); + this.requestUri = Optional.ofNullable(result.getRequest().getURI()); + } + + SingleMessageSendingResult(int statusCode) { + initializeForStatusCode(statusCode); + } + + SingleMessageSendingResult(int statusCode, URI requestURI) { + this(statusCode); + this.requestUri = Optional.of(requestURI); + } + + SingleMessageSendingResult(int statusCode, long retryAfterMillis) { + initializeForStatusCode(statusCode); + if (shouldRetryOnStatusCode() && retryAfterMillis >= 0) { + this.retryAfterMillis = Optional.of(retryAfterMillis); + } + } + + public SingleMessageSendingResult(Result result, URI uri) { + this(result); + this.requestUri = Optional.of(uri); + } + + private void initializeForStatusCode(int statusCode) { + this.statusCode = statusCode; + responseFamily = familyOf(statusCode); + if (this.failure == null && !isInFamily(SUCCESSFUL)) { + this.failure = + new InternalProcessingException("Message sending failed with status code: " + statusCode); + } + } + + private void initializeRetryAfterMillis(Result result) { + HttpFields headers = result.getResponse().getHeaders(); + if (headers.contains(HttpHeader.RETRY_AFTER)) { + try { + int seconds = Integer.parseInt(headers.get(HttpHeader.RETRY_AFTER)); + if (seconds >= 0) { + retryAfterMillis = Optional.of(TimeUnit.SECONDS.toMillis(seconds)); } - } - - public SingleMessageSendingResult(Result result, URI uri) { - this(result); - this.requestUri = Optional.of(uri); - } - - private void initializeForStatusCode(int statusCode) { - this.statusCode = statusCode; - responseFamily = familyOf(statusCode); - if (this.failure == null && !isInFamily(SUCCESSFUL)) { - this.failure = new InternalProcessingException("Message sending failed with status code: " + statusCode); - } - } - - private void initializeRetryAfterMillis(Result result) { - HttpFields headers = result.getResponse().getHeaders(); - if (headers.contains(HttpHeader.RETRY_AFTER)) { - try { - int seconds = Integer.parseInt(headers.get(HttpHeader.RETRY_AFTER)); - if (seconds >= 0) { - retryAfterMillis = Optional.of(TimeUnit.SECONDS.toMillis(seconds)); - } - } catch (NumberFormatException e) { - // retryAfterMillis stays empty - } - } - } - - private boolean isInFamily(Response.Status.Family family) { - Preconditions.checkNotNull(family); - return family.equals(responseFamily); - } - - @Override - public boolean isRetryLater() { - return shouldRetryOnStatusCode() && retryAfterMillis.isPresent(); - } - - private boolean shouldRetryOnStatusCode() { - return getStatusCode() == SERVICE_UNAVAILABLE.code() - || getStatusCode() == TOO_MANY_REQUESTS.code(); - } - - @Override - public boolean succeeded() { - return getFailure() == null; - } - - Throwable getFailure() { - return failure; - } - - @Override - public String getRootCause() { - return failure != null ? Throwables.getRootCause(failure).getMessage() : CAUSE_UNKNOWN; - } - - @Override - public int getStatusCode() { - return statusCode; - } - - @Override - public boolean isLoggable() { - return loggable; - } - - @Override - public Optional getRetryAfterMillis() { - return retryAfterMillis; - } - - public Optional getRequestUri() { - return requestUri; - } - - @Override - public boolean isClientError() { - return isInFamily(CLIENT_ERROR); - } - - @Override - public boolean ignoreInRateCalculation(boolean retryClientErrors, boolean isOAuthSecuredSubscription) { - return isRetryLater() - || this.ignoreInRateCalculation - || (isClientError() && !retryClientErrors && !(isOAuthSecuredSubscription && isUnauthorized())); - } - - private boolean isUnauthorized() { - return getStatusCode() == UNAUTHORIZED.code(); - } - - @Override - public boolean isTimeout() { - return failure instanceof TimeoutException; - } - - @Override - public List getLogInfo() { - return Collections.singletonList(new MessageSendingResultLogInfo(getRequestUri(), failure, getRootCause())); - } - - @Override - public List getSucceededUris(Predicate filter) { - if (filter.test(this) && requestUri.isPresent()) { - return Collections.singletonList(getRequestUri().get()); - } else { - return Collections.emptyList(); - } - } - + } catch (NumberFormatException e) { + // retryAfterMillis stays empty + } + } + } + + private boolean isInFamily(Response.Status.Family family) { + Preconditions.checkNotNull(family); + return family.equals(responseFamily); + } + + @Override + public boolean isRetryLater() { + return shouldRetryOnStatusCode() && retryAfterMillis.isPresent(); + } + + private boolean shouldRetryOnStatusCode() { + return getStatusCode() == SERVICE_UNAVAILABLE.code() + || getStatusCode() == TOO_MANY_REQUESTS.code(); + } + + @Override + public boolean succeeded() { + return getFailure() == null; + } + + Throwable getFailure() { + return failure; + } + + @Override + public String getRootCause() { + return failure != null ? Throwables.getRootCause(failure).getMessage() : CAUSE_UNKNOWN; + } + + @Override + public int getStatusCode() { + return statusCode; + } + + @Override + public boolean isLoggable() { + return loggable; + } + + @Override + public Optional getRetryAfterMillis() { + return retryAfterMillis; + } + + public Optional getRequestUri() { + return requestUri; + } + + @Override + public boolean isClientError() { + return isInFamily(CLIENT_ERROR); + } + + @Override + public boolean ignoreInRateCalculation( + boolean retryClientErrors, boolean isOAuthSecuredSubscription) { + return isRetryLater() + || this.ignoreInRateCalculation + || (isClientError() + && !retryClientErrors + && !(isOAuthSecuredSubscription && isUnauthorized())); + } + + private boolean isUnauthorized() { + return getStatusCode() == UNAUTHORIZED.code(); + } + + @Override + public boolean isTimeout() { + return failure instanceof TimeoutException; + } + + @Override + public List getLogInfo() { + return Collections.singletonList( + new MessageSendingResultLogInfo(getRequestUri(), failure, getRootCause())); + } + + @Override + public List getSucceededUris(Predicate filter) { + if (filter.test(this) && requestUri.isPresent()) { + return Collections.singletonList(getRequestUri().get()); + } else { + return Collections.emptyList(); + } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/SingleRecipientMessageSenderAdapter.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/SingleRecipientMessageSenderAdapter.java index 47b8dd0c77..72c9fed511 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/SingleRecipientMessageSenderAdapter.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/SingleRecipientMessageSenderAdapter.java @@ -1,32 +1,31 @@ package pl.allegro.tech.hermes.consumers.consumer.sender; -import pl.allegro.tech.hermes.consumers.consumer.Message; -import pl.allegro.tech.hermes.consumers.consumer.ResilientMessageSender; - import java.util.concurrent.CompletableFuture; import java.util.function.Function; +import pl.allegro.tech.hermes.consumers.consumer.Message; +import pl.allegro.tech.hermes.consumers.consumer.ResilientMessageSender; public class SingleRecipientMessageSenderAdapter implements MessageSender { - private final CompletableFutureAwareMessageSender adaptee; - private final Function exceptionMapper = MessageSendingResult::failedResult; + private final CompletableFutureAwareMessageSender adaptee; + private final Function exceptionMapper = + MessageSendingResult::failedResult; - private final ResilientMessageSender resilientMessageSender; + private final ResilientMessageSender resilientMessageSender; - public SingleRecipientMessageSenderAdapter(CompletableFutureAwareMessageSender adaptee, ResilientMessageSender resilientMessageSender) { - this.resilientMessageSender = resilientMessageSender; - this.adaptee = adaptee; - } + public SingleRecipientMessageSenderAdapter( + CompletableFutureAwareMessageSender adaptee, ResilientMessageSender resilientMessageSender) { + this.resilientMessageSender = resilientMessageSender; + this.adaptee = adaptee; + } - @Override - public CompletableFuture send(Message message) { - return resilientMessageSender.send( - resultFuture -> adaptee.send(message, resultFuture), - exceptionMapper - ); - } + @Override + public CompletableFuture send(Message message) { + return resilientMessageSender.send( + resultFuture -> adaptee.send(message, resultFuture), exceptionMapper); + } - @Override - public void stop() { - adaptee.stop(); - } + @Override + public void stop() { + adaptee.stop(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/CompressionCodec.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/CompressionCodec.java index d03d08c931..b8223d1651 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/CompressionCodec.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/CompressionCodec.java @@ -1,21 +1,20 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.googlepubsub; enum CompressionCodec { - DEFLATE("df"), - BZIP2("bz2"), - ZSTANDARD("zstd"), - EMPTY; + DEFLATE("df"), + BZIP2("bz2"), + ZSTANDARD("zstd"), + EMPTY; - private String header; + private String header; - CompressionCodec(String header) { - this.header = header; - } + CompressionCodec(String header) { + this.header = header; + } - CompressionCodec() { - } + CompressionCodec() {} - String getHeader() { - return header; - } -} \ No newline at end of file + String getHeader() { + return header; + } +} diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/CompressionCodecFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/CompressionCodecFactory.java index b6152b1b8d..4866b6212b 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/CompressionCodecFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/CompressionCodecFactory.java @@ -7,84 +7,86 @@ abstract class CompressionCodecFactory { - enum CompressionLevel { - LOW(1), - MEDIUM(3), - HIGH(9); + enum CompressionLevel { + LOW(1), + MEDIUM(3), + HIGH(9); - private final int levelId; + private final int levelId; - CompressionLevel(int levelId) { - this.levelId = levelId; - } - - int getLevelId() { - return levelId; - } + CompressionLevel(int levelId) { + this.levelId = levelId; } - private final String name; - - static CompressionCodecFactory of(CompressionCodec codec, CompressionLevel level) { - switch (codec) { - case DEFLATE: return new DeflateCodecFactory(codec.name(), level.getLevelId()); - case BZIP2: return new CompressionCodecFactory.Bzip2CodecFactory(codec.name()); - case ZSTANDARD: return new CompressionCodecFactory.ZstandardCodecFactory(codec.name(), level.getLevelId()); - default: return null; - } + int getLevelId() { + return levelId; } - - abstract Codec createInstance(); - - CompressionCodecFactory(String name) { - this.name = name; + } + + private final String name; + + static CompressionCodecFactory of(CompressionCodec codec, CompressionLevel level) { + switch (codec) { + case DEFLATE: + return new DeflateCodecFactory(codec.name(), level.getLevelId()); + case BZIP2: + return new CompressionCodecFactory.Bzip2CodecFactory(codec.name()); + case ZSTANDARD: + return new CompressionCodecFactory.ZstandardCodecFactory(codec.name(), level.getLevelId()); + default: + return null; } + } - String codecName() { - return this.name; - } + abstract Codec createInstance(); - static class DeflateCodecFactory extends CompressionCodecFactory { + CompressionCodecFactory(String name) { + this.name = name; + } - private final int compressionLevel; + String codecName() { + return this.name; + } - DeflateCodecFactory(String name, int compressionLevel) { - super(name); - this.compressionLevel = compressionLevel; - } + static class DeflateCodecFactory extends CompressionCodecFactory { - @Override - Codec createInstance() { - return new DeflateCodec(this.compressionLevel); - } - } + private final int compressionLevel; + DeflateCodecFactory(String name, int compressionLevel) { + super(name); + this.compressionLevel = compressionLevel; + } - static class Bzip2CodecFactory extends CompressionCodecFactory { + @Override + Codec createInstance() { + return new DeflateCodec(this.compressionLevel); + } + } - Bzip2CodecFactory(String name) { - super(name); - } + static class Bzip2CodecFactory extends CompressionCodecFactory { - @Override - Codec createInstance() { - return new BZip2Codec(); - } + Bzip2CodecFactory(String name) { + super(name); } + @Override + Codec createInstance() { + return new BZip2Codec(); + } + } - static class ZstandardCodecFactory extends CompressionCodecFactory { + static class ZstandardCodecFactory extends CompressionCodecFactory { - private final int compressionLevel; + private final int compressionLevel; - ZstandardCodecFactory(String name, int compressionLevel) { - super(name); - this.compressionLevel = compressionLevel; - } + ZstandardCodecFactory(String name, int compressionLevel) { + super(name); + this.compressionLevel = compressionLevel; + } - @Override - Codec createInstance() { - return new ZstandardCodec(compressionLevel, true, ZstandardCodec.DEFAULT_USE_BUFFERPOOL); - } + @Override + Codec createInstance() { + return new ZstandardCodec(compressionLevel, true, ZstandardCodec.DEFAULT_USE_BUFFERPOOL); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubClient.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubClient.java index aeb77eeb17..ef39f1da70 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubClient.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubClient.java @@ -5,37 +5,37 @@ import com.google.cloud.pubsub.v1.Publisher; import com.google.common.util.concurrent.MoreExecutors; import com.google.pubsub.v1.PubsubMessage; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult; - import java.io.IOException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult; class GooglePubSubClient { - private static final Logger logger = LoggerFactory.getLogger(GooglePubSubClient.class); + private static final Logger logger = LoggerFactory.getLogger(GooglePubSubClient.class); - private final Publisher publisher; + private final Publisher publisher; - GooglePubSubClient(Publisher publisher) { - this.publisher = publisher; - } + GooglePubSubClient(Publisher publisher) { + this.publisher = publisher; + } - void publish(PubsubMessage pubsubMessage, CompletableFuture resultFuture) - throws IOException, ExecutionException, InterruptedException { - ApiFuture future = publisher.publish(pubsubMessage); - ApiFutures.addCallback(future, new GooglePubSubMessageSentCallback(resultFuture), MoreExecutors.directExecutor()); - } + void publish(PubsubMessage pubsubMessage, CompletableFuture resultFuture) + throws IOException, ExecutionException, InterruptedException { + ApiFuture future = publisher.publish(pubsubMessage); + ApiFutures.addCallback( + future, new GooglePubSubMessageSentCallback(resultFuture), MoreExecutors.directExecutor()); + } - void shutdown() { - publisher.shutdown(); - try { - publisher.awaitTermination(30, TimeUnit.SECONDS); - } catch (InterruptedException e) { - logger.error("Interrupted termination of the PubSub publisher."); - } + void shutdown() { + publisher.shutdown(); + try { + publisher.awaitTermination(30, TimeUnit.SECONDS); + } catch (InterruptedException e) { + logger.error("Interrupted termination of the PubSub publisher."); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubClientsPool.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubClientsPool.java index aa4e1f0aba..531d660b5d 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubClientsPool.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubClientsPool.java @@ -6,81 +6,84 @@ import com.google.api.gax.retrying.RetrySettings; import com.google.api.gax.rpc.TransportChannelProvider; import com.google.cloud.pubsub.v1.Publisher; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.util.HashMap; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; class GooglePubSubClientsPool { - private static final Logger logger = LoggerFactory.getLogger(GooglePubSubClientsPool.class); + private static final Logger logger = LoggerFactory.getLogger(GooglePubSubClientsPool.class); - private final CredentialsProvider credentialsProvider; - private final ExecutorProvider publishingExecutorProvider; - private final RetrySettings retrySettings; - private final BatchingSettings batchingSettings; - private final Map clients = new HashMap<>(); - private final Map counters = new HashMap<>(); + private final CredentialsProvider credentialsProvider; + private final ExecutorProvider publishingExecutorProvider; + private final RetrySettings retrySettings; + private final BatchingSettings batchingSettings; + private final Map clients = new HashMap<>(); + private final Map counters = new HashMap<>(); - private final TransportChannelProvider transportChannelProvider; + private final TransportChannelProvider transportChannelProvider; - GooglePubSubClientsPool(CredentialsProvider credentialsProvider, - ExecutorProvider publishingExecutorProvider, - RetrySettings retrySettings, - BatchingSettings batchingSettings, - TransportChannelProvider transportChannelProvider) { - this.credentialsProvider = credentialsProvider; - this.publishingExecutorProvider = publishingExecutorProvider; - this.retrySettings = retrySettings; - this.batchingSettings = batchingSettings; - this.transportChannelProvider = transportChannelProvider; - } + GooglePubSubClientsPool( + CredentialsProvider credentialsProvider, + ExecutorProvider publishingExecutorProvider, + RetrySettings retrySettings, + BatchingSettings batchingSettings, + TransportChannelProvider transportChannelProvider) { + this.credentialsProvider = credentialsProvider; + this.publishingExecutorProvider = publishingExecutorProvider; + this.retrySettings = retrySettings; + this.batchingSettings = batchingSettings; + this.transportChannelProvider = transportChannelProvider; + } - synchronized GooglePubSubClient acquire(GooglePubSubSenderTarget resolvedTarget) throws IOException { - GooglePubSubClient client = clients.get(resolvedTarget); - if (client == null) { - client = createClient(resolvedTarget); - } - clients.put(resolvedTarget, client); - Integer counter = counters.getOrDefault(resolvedTarget, 0); - counters.put(resolvedTarget, ++counter); - return client; + synchronized GooglePubSubClient acquire(GooglePubSubSenderTarget resolvedTarget) + throws IOException { + GooglePubSubClient client = clients.get(resolvedTarget); + if (client == null) { + client = createClient(resolvedTarget); } + clients.put(resolvedTarget, client); + Integer counter = counters.getOrDefault(resolvedTarget, 0); + counters.put(resolvedTarget, ++counter); + return client; + } - synchronized void release(GooglePubSubSenderTarget resolvedTarget) { - Integer counter = counters.getOrDefault(resolvedTarget, 0); - if (counter == 0) { - logger.warn("Attempt to release GooglePubSubClient that is not acquired"); - } else if (counter == 1) { - counters.remove(resolvedTarget); - GooglePubSubClient client = clients.remove(resolvedTarget); - client.shutdown(); - } else if (counter > 1) { - counters.put(resolvedTarget, --counter); - } + synchronized void release(GooglePubSubSenderTarget resolvedTarget) { + Integer counter = counters.getOrDefault(resolvedTarget, 0); + if (counter == 0) { + logger.warn("Attempt to release GooglePubSubClient that is not acquired"); + } else if (counter == 1) { + counters.remove(resolvedTarget); + GooglePubSubClient client = clients.remove(resolvedTarget); + client.shutdown(); + } else if (counter > 1) { + counters.put(resolvedTarget, --counter); } + } - synchronized void shutdown() { - clients.values().forEach(GooglePubSubClient::shutdown); - clients.clear(); - counters.clear(); - } + synchronized void shutdown() { + clients.values().forEach(GooglePubSubClient::shutdown); + clients.clear(); + counters.clear(); + } - protected GooglePubSubClient createClient(GooglePubSubSenderTarget resolvedTarget) throws IOException { - final Publisher.Builder builder = Publisher.newBuilder(resolvedTarget.getTopicName()) - .setEndpoint(resolvedTarget.getPubSubEndpoint()) - .setCredentialsProvider(credentialsProvider) - .setRetrySettings(retrySettings) - .setBatchingSettings(batchingSettings) - .setExecutorProvider(publishingExecutorProvider); + protected GooglePubSubClient createClient(GooglePubSubSenderTarget resolvedTarget) + throws IOException { + final Publisher.Builder builder = + Publisher.newBuilder(resolvedTarget.getTopicName()) + .setEndpoint(resolvedTarget.getPubSubEndpoint()) + .setCredentialsProvider(credentialsProvider) + .setRetrySettings(retrySettings) + .setBatchingSettings(batchingSettings) + .setExecutorProvider(publishingExecutorProvider); - Publisher publisher; - if (transportChannelProvider == null) { - publisher = builder.build(); - } else { - publisher = builder.setChannelProvider(transportChannelProvider).build(); - } - return new GooglePubSubClient(publisher); + Publisher publisher; + if (transportChannelProvider == null) { + publisher = builder.build(); + } else { + publisher = builder.setChannelProvider(transportChannelProvider).build(); } + return new GooglePubSubClient(publisher); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageCompressionException.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageCompressionException.java index cbc34af1cc..c6dade4fdf 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageCompressionException.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageCompressionException.java @@ -2,7 +2,7 @@ public class GooglePubSubMessageCompressionException extends RuntimeException { - public GooglePubSubMessageCompressionException(String message, Throwable e) { - super(message, e); - } + public GooglePubSubMessageCompressionException(String message, Throwable e) { + super(message, e); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageSender.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageSender.java index 3fed4f81f8..6c41a524bc 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageSender.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageSender.java @@ -1,44 +1,45 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.googlepubsub; -import com.google.pubsub.v1.PubsubMessage; -import pl.allegro.tech.hermes.consumers.consumer.Message; -import pl.allegro.tech.hermes.consumers.consumer.sender.CompletableFutureAwareMessageSender; -import pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult; +import static pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult.failedResult; +import com.google.pubsub.v1.PubsubMessage; import java.io.IOException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; - -import static pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult.failedResult; +import pl.allegro.tech.hermes.consumers.consumer.Message; +import pl.allegro.tech.hermes.consumers.consumer.sender.CompletableFutureAwareMessageSender; +import pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult; class GooglePubSubMessageSender implements CompletableFutureAwareMessageSender { - private final GooglePubSubClient googlePubSubClient; - private final GooglePubSubSenderTarget resolvedTarget; - private final GooglePubSubClientsPool clientsPool; - private final GooglePubSubMessageTransformer messageTransformer; - - GooglePubSubMessageSender(GooglePubSubSenderTarget resolvedTarget, - GooglePubSubClientsPool clientsPool, - GooglePubSubMessageTransformer messageTransformer) throws IOException { - this.googlePubSubClient = clientsPool.acquire(resolvedTarget); - this.resolvedTarget = resolvedTarget; - this.clientsPool = clientsPool; - this.messageTransformer = messageTransformer; + private final GooglePubSubClient googlePubSubClient; + private final GooglePubSubSenderTarget resolvedTarget; + private final GooglePubSubClientsPool clientsPool; + private final GooglePubSubMessageTransformer messageTransformer; + + GooglePubSubMessageSender( + GooglePubSubSenderTarget resolvedTarget, + GooglePubSubClientsPool clientsPool, + GooglePubSubMessageTransformer messageTransformer) + throws IOException { + this.googlePubSubClient = clientsPool.acquire(resolvedTarget); + this.resolvedTarget = resolvedTarget; + this.clientsPool = clientsPool; + this.messageTransformer = messageTransformer; + } + + @Override + public void send(Message message, CompletableFuture resultFuture) { + try { + PubsubMessage pubsubMessage = messageTransformer.fromHermesMessage(message); + googlePubSubClient.publish(pubsubMessage, resultFuture); + } catch (IOException | ExecutionException | InterruptedException exception) { + resultFuture.complete(failedResult(exception)); } + } - @Override - public void send(Message message, CompletableFuture resultFuture) { - try { - PubsubMessage pubsubMessage = messageTransformer.fromHermesMessage(message); - googlePubSubClient.publish(pubsubMessage, resultFuture); - } catch (IOException | ExecutionException | InterruptedException exception) { - resultFuture.complete(failedResult(exception)); - } - } - - @Override - public void stop() { - clientsPool.release(resolvedTarget); - } -} \ No newline at end of file + @Override + public void stop() { + clientsPool.release(resolvedTarget); + } +} diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageSenderProvider.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageSenderProvider.java index fb0302c04b..4b66a1ce3e 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageSenderProvider.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageSenderProvider.java @@ -6,65 +6,67 @@ import com.google.api.gax.retrying.RetrySettings; import com.google.api.gax.rpc.TransportChannelProvider; import com.google.common.collect.ImmutableSet; +import java.io.IOException; +import java.util.Set; import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.consumers.consumer.ResilientMessageSender; import pl.allegro.tech.hermes.consumers.consumer.sender.MessageSender; import pl.allegro.tech.hermes.consumers.consumer.sender.ProtocolMessageSenderProvider; import pl.allegro.tech.hermes.consumers.consumer.sender.SingleRecipientMessageSenderAdapter; -import java.io.IOException; -import java.util.Set; - public class GooglePubSubMessageSenderProvider implements ProtocolMessageSenderProvider { - public static final String SUPPORTED_PROTOCOL = "googlepubsub"; + public static final String SUPPORTED_PROTOCOL = "googlepubsub"; - private final GooglePubSubSenderTargetResolver resolver; - private final GooglePubSubMessageTransformerCreator messageTransformerCreator; - private final GooglePubSubClientsPool clientsPool; + private final GooglePubSubSenderTargetResolver resolver; + private final GooglePubSubMessageTransformerCreator messageTransformerCreator; + private final GooglePubSubClientsPool clientsPool; - public GooglePubSubMessageSenderProvider(GooglePubSubSenderTargetResolver resolver, - CredentialsProvider credentialsProvider, - ExecutorProvider executorProvider, - RetrySettings retrySettings, - BatchingSettings batchingSettings, - TransportChannelProvider transportChannelProvider, - GooglePubSubMessageTransformerCreator messageTransformerCreator) { + public GooglePubSubMessageSenderProvider( + GooglePubSubSenderTargetResolver resolver, + CredentialsProvider credentialsProvider, + ExecutorProvider executorProvider, + RetrySettings retrySettings, + BatchingSettings batchingSettings, + TransportChannelProvider transportChannelProvider, + GooglePubSubMessageTransformerCreator messageTransformerCreator) { - this.resolver = resolver; - this.messageTransformerCreator = messageTransformerCreator; - this.clientsPool = new GooglePubSubClientsPool( - credentialsProvider, - executorProvider, - retrySettings, - batchingSettings, - transportChannelProvider - ); - } + this.resolver = resolver; + this.messageTransformerCreator = messageTransformerCreator; + this.clientsPool = + new GooglePubSubClientsPool( + credentialsProvider, + executorProvider, + retrySettings, + batchingSettings, + transportChannelProvider); + } - @Override - public MessageSender create(final Subscription subscription, ResilientMessageSender resilientMessageSender) { - final GooglePubSubSenderTarget resolvedTarget = resolver.resolve(subscription.getEndpoint()); - try { - GooglePubSubMessageTransformer messageTransformer = messageTransformerCreator.getTransformerForTargetEndpoint(resolvedTarget); - GooglePubSubMessageSender sender = new GooglePubSubMessageSender(resolvedTarget, clientsPool, messageTransformer); - return new SingleRecipientMessageSenderAdapter(sender, resilientMessageSender); - } catch (IOException e) { - throw new RuntimeException("Cannot create Google PubSub publishers cache", e); - } + @Override + public MessageSender create( + final Subscription subscription, ResilientMessageSender resilientMessageSender) { + final GooglePubSubSenderTarget resolvedTarget = resolver.resolve(subscription.getEndpoint()); + try { + GooglePubSubMessageTransformer messageTransformer = + messageTransformerCreator.getTransformerForTargetEndpoint(resolvedTarget); + GooglePubSubMessageSender sender = + new GooglePubSubMessageSender(resolvedTarget, clientsPool, messageTransformer); + return new SingleRecipientMessageSenderAdapter(sender, resilientMessageSender); + } catch (IOException e) { + throw new RuntimeException("Cannot create Google PubSub publishers cache", e); } + } - @Override - public Set getSupportedProtocols() { - return ImmutableSet.of(SUPPORTED_PROTOCOL); - } + @Override + public Set getSupportedProtocols() { + return ImmutableSet.of(SUPPORTED_PROTOCOL); + } - @Override - public void start() throws Exception { - } + @Override + public void start() throws Exception {} - @Override - public void stop() throws Exception { - clientsPool.shutdown(); - } + @Override + public void stop() throws Exception { + clientsPool.shutdown(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageSentCallback.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageSentCallback.java index 5a29e23e42..a6dd28731d 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageSentCallback.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageSentCallback.java @@ -1,27 +1,26 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.googlepubsub; -import com.google.api.core.ApiFutureCallback; -import pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult; - -import java.util.concurrent.CompletableFuture; - import static pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult.failedResult; import static pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult.succeededResult; +import com.google.api.core.ApiFutureCallback; +import java.util.concurrent.CompletableFuture; +import pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult; + class GooglePubSubMessageSentCallback implements ApiFutureCallback { - private final CompletableFuture resultFuture; + private final CompletableFuture resultFuture; - GooglePubSubMessageSentCallback(CompletableFuture resultFuture) { - this.resultFuture = resultFuture; - } + GooglePubSubMessageSentCallback(CompletableFuture resultFuture) { + this.resultFuture = resultFuture; + } - @Override - public void onFailure(Throwable throwable) { - resultFuture.complete(failedResult(throwable)); - } + @Override + public void onFailure(Throwable throwable) { + resultFuture.complete(failedResult(throwable)); + } - @Override - public void onSuccess(String messageId) { - resultFuture.complete(succeededResult()); - } + @Override + public void onSuccess(String messageId) { + resultFuture.complete(succeededResult()); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageTransformer.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageTransformer.java index 40e9b7bfa8..fd547141a0 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageTransformer.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageTransformer.java @@ -5,6 +5,5 @@ interface GooglePubSubMessageTransformer { - PubsubMessage fromHermesMessage(Message message); - + PubsubMessage fromHermesMessage(Message message); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageTransformerCompression.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageTransformerCompression.java index d8260a964e..e2dd61e599 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageTransformerCompression.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageTransformerCompression.java @@ -2,49 +2,48 @@ import com.google.protobuf.ByteString; import com.google.pubsub.v1.PubsubMessage; +import java.io.IOException; import pl.allegro.tech.hermes.consumers.consumer.Message; import pl.allegro.tech.hermes.consumers.consumer.trace.MetadataAppender; -import java.io.IOException; - class GooglePubSubMessageTransformerCompression implements GooglePubSubMessageTransformer { - private final MessageCompressor compressor; - private final Long compressionThresholdBytes; - private final GooglePubSubMessageTransformerRaw rawTransformer; - private final MetadataAppender metadataAppender; - - GooglePubSubMessageTransformerCompression( - Long compressionThresholdBytes, - GooglePubSubMessageTransformerRaw rawTransformer, - MetadataAppender metadataAppender, - MessageCompressor compressor) { - this.compressor = compressor; - this.compressionThresholdBytes = compressionThresholdBytes; - this.rawTransformer = rawTransformer; - this.metadataAppender = metadataAppender; + private final MessageCompressor compressor; + private final Long compressionThresholdBytes; + private final GooglePubSubMessageTransformerRaw rawTransformer; + private final MetadataAppender metadataAppender; + + GooglePubSubMessageTransformerCompression( + Long compressionThresholdBytes, + GooglePubSubMessageTransformerRaw rawTransformer, + MetadataAppender metadataAppender, + MessageCompressor compressor) { + this.compressor = compressor; + this.compressionThresholdBytes = compressionThresholdBytes; + this.rawTransformer = rawTransformer; + this.metadataAppender = metadataAppender; + } + + @Override + public PubsubMessage fromHermesMessage(Message message) { + byte[] data = message.getData(); + if (data.length > compressionThresholdBytes) { + return compressHermesMessage(message); + } else { + return rawTransformer.fromHermesMessage(message); } - - @Override - public PubsubMessage fromHermesMessage(Message message) { - byte[] data = message.getData(); - if (data.length > compressionThresholdBytes) { - return compressHermesMessage(message); - } else { - return rawTransformer.fromHermesMessage(message); - } - } - - private PubsubMessage compressHermesMessage(Message message) { - try { - final PubsubMessage pubsubMessage = PubsubMessage.newBuilder() - .setData(ByteString.copyFrom( - compressor.compress(message.getData()) - )).build(); - - return metadataAppender.append(pubsubMessage, message); - } catch (IOException e) { - throw new GooglePubSubMessageCompressionException("Error on PubSub message compression", e); - } + } + + private PubsubMessage compressHermesMessage(Message message) { + try { + final PubsubMessage pubsubMessage = + PubsubMessage.newBuilder() + .setData(ByteString.copyFrom(compressor.compress(message.getData()))) + .build(); + + return metadataAppender.append(pubsubMessage, message); + } catch (IOException e) { + throw new GooglePubSubMessageCompressionException("Error on PubSub message compression", e); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageTransformerCreator.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageTransformerCreator.java index cff2b884ff..bf373e234c 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageTransformerCreator.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageTransformerCreator.java @@ -1,71 +1,77 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.googlepubsub; +import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Optional; - public class GooglePubSubMessageTransformerCreator { - private boolean compressionEnabled; - private Long compressionThresholdBytes; - private CompressionCodecFactory.CompressionLevel compressionLevel; - private final GooglePubSubMessageTransformerRaw messageRawTransformer; + private boolean compressionEnabled; + private Long compressionThresholdBytes; + private CompressionCodecFactory.CompressionLevel compressionLevel; + private final GooglePubSubMessageTransformerRaw messageRawTransformer; - private static final Logger logger = LoggerFactory.getLogger(GooglePubSubMessageTransformerCreator.class); + private static final Logger logger = + LoggerFactory.getLogger(GooglePubSubMessageTransformerCreator.class); - public static GooglePubSubMessageTransformerCreator creator() { - return new GooglePubSubMessageTransformerCreator(); - } + public static GooglePubSubMessageTransformerCreator creator() { + return new GooglePubSubMessageTransformerCreator(); + } - private GooglePubSubMessageTransformerCreator() { - this.messageRawTransformer = new GooglePubSubMessageTransformerRaw(new GooglePubSubMetadataAppender()); - } + private GooglePubSubMessageTransformerCreator() { + this.messageRawTransformer = + new GooglePubSubMessageTransformerRaw(new GooglePubSubMetadataAppender()); + } - public GooglePubSubMessageTransformerCreator withCompressionEnabled(boolean compressionEnabled) { - this.compressionEnabled = compressionEnabled; - return this; - } + public GooglePubSubMessageTransformerCreator withCompressionEnabled(boolean compressionEnabled) { + this.compressionEnabled = compressionEnabled; + return this; + } - public GooglePubSubMessageTransformerCreator withCompressionLevel(String compressionLevel) { - try { - this.compressionLevel = Optional.ofNullable(compressionLevel) - .map(String::toUpperCase) - .map(CompressionCodecFactory.CompressionLevel::valueOf) - .orElse(CompressionCodecFactory.CompressionLevel.MEDIUM); - } catch (IllegalArgumentException ex) { - logger.warn("Unsupported compression level: {}, setting it to default", compressionLevel); - this.compressionLevel = CompressionCodecFactory.CompressionLevel.MEDIUM; - } - return this; + public GooglePubSubMessageTransformerCreator withCompressionLevel(String compressionLevel) { + try { + this.compressionLevel = + Optional.ofNullable(compressionLevel) + .map(String::toUpperCase) + .map(CompressionCodecFactory.CompressionLevel::valueOf) + .orElse(CompressionCodecFactory.CompressionLevel.MEDIUM); + } catch (IllegalArgumentException ex) { + logger.warn("Unsupported compression level: {}, setting it to default", compressionLevel); + this.compressionLevel = CompressionCodecFactory.CompressionLevel.MEDIUM; } + return this; + } - public GooglePubSubMessageTransformerCreator withCompressionThresholdBytes(Long compressionThresholdBytes) { - this.compressionThresholdBytes = compressionThresholdBytes; - return this; - } + public GooglePubSubMessageTransformerCreator withCompressionThresholdBytes( + Long compressionThresholdBytes) { + this.compressionThresholdBytes = compressionThresholdBytes; + return this; + } - GooglePubSubMessageTransformer getTransformerForTargetEndpoint(GooglePubSubSenderTarget pubSubTarget) { - if (this.compressionEnabled && pubSubTarget.isCompressionRequested()) { - Optional compressingTransformer = - transformerForCodec(pubSubTarget.getCompressionCodec()); - if (compressingTransformer.isPresent()) { - return compressingTransformer.get(); - } else { - logger.warn("Unsupported codec, switching to raw transfer for {}.", pubSubTarget.getTopicName()); - } - } - return this.messageRawTransformer; + GooglePubSubMessageTransformer getTransformerForTargetEndpoint( + GooglePubSubSenderTarget pubSubTarget) { + if (this.compressionEnabled && pubSubTarget.isCompressionRequested()) { + Optional compressingTransformer = + transformerForCodec(pubSubTarget.getCompressionCodec()); + if (compressingTransformer.isPresent()) { + return compressingTransformer.get(); + } else { + logger.warn( + "Unsupported codec, switching to raw transfer for {}.", pubSubTarget.getTopicName()); + } } + return this.messageRawTransformer; + } - private Optional transformerForCodec(CompressionCodec codec) { - return Optional.ofNullable(codec) - .map(it -> CompressionCodecFactory.of(it, compressionLevel)) - .map(codecFactory -> - new GooglePubSubMessageTransformerCompression( - this.compressionThresholdBytes, - this.messageRawTransformer, - new GooglePubSubMetadataCompressionAppender(codec), - new MessageCompressor(codecFactory))); - } + private Optional transformerForCodec(CompressionCodec codec) { + return Optional.ofNullable(codec) + .map(it -> CompressionCodecFactory.of(it, compressionLevel)) + .map( + codecFactory -> + new GooglePubSubMessageTransformerCompression( + this.compressionThresholdBytes, + this.messageRawTransformer, + new GooglePubSubMetadataCompressionAppender(codec), + new MessageCompressor(codecFactory))); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageTransformerRaw.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageTransformerRaw.java index 407d9c5231..6bdc3c9da9 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageTransformerRaw.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMessageTransformerRaw.java @@ -7,18 +7,17 @@ class GooglePubSubMessageTransformerRaw implements GooglePubSubMessageTransformer { - private final MetadataAppender metadataAppender; + private final MetadataAppender metadataAppender; - GooglePubSubMessageTransformerRaw(MetadataAppender metadataAppender) { - this.metadataAppender = metadataAppender; - } + GooglePubSubMessageTransformerRaw(MetadataAppender metadataAppender) { + this.metadataAppender = metadataAppender; + } - @Override - public PubsubMessage fromHermesMessage(Message message) { - final PubsubMessage pubsubMessage = PubsubMessage.newBuilder() - .setData(ByteString.copyFrom(message.getData())) - .build(); + @Override + public PubsubMessage fromHermesMessage(Message message) { + final PubsubMessage pubsubMessage = + PubsubMessage.newBuilder().setData(ByteString.copyFrom(message.getData())).build(); - return metadataAppender.append(pubsubMessage, message); - } + return metadataAppender.append(pubsubMessage, message); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMetadataAppender.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMetadataAppender.java index 8d23df04bc..f219d1144d 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMetadataAppender.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMetadataAppender.java @@ -2,48 +2,55 @@ import com.google.common.collect.ImmutableMap; import com.google.pubsub.v1.PubsubMessage; -import org.apache.commons.lang3.tuple.Pair; -import pl.allegro.tech.hermes.consumers.consumer.Message; -import pl.allegro.tech.hermes.consumers.consumer.trace.MetadataAppender; - import java.util.HashMap; import java.util.Map; import java.util.Optional; +import org.apache.commons.lang3.tuple.Pair; +import pl.allegro.tech.hermes.consumers.consumer.Message; +import pl.allegro.tech.hermes.consumers.consumer.trace.MetadataAppender; public class GooglePubSubMetadataAppender implements MetadataAppender { - public static final String HEADER_NAME_TOPIC_NAME = "tn"; - public static final String HEADER_NAME_MESSAGE_ID = "id"; - public static final String HEADER_NAME_SUBSCRIPTION_NAME = "sn"; - public static final String HEADER_NAME_TIMESTAMP = "ts"; - public static final String HEADER_NAME_SCHEMA_ID = "sid"; - public static final String HEADER_NAME_SCHEMA_VERSION = "sv"; - - @Override - public PubsubMessage append(PubsubMessage target, Message message) { - return PubsubMessage.newBuilder(target) - .putAllAttributes(createMessageAttributes(message)) - .build(); - } - - protected Map createMessageAttributes(Message message) { - Optional> schemaIdAndVersion = message.getSchema().map(s -> - Pair.of(String.valueOf(s.getId().value()), String.valueOf(s.getVersion().value()))); - - final Map headers = new HashMap<>(ImmutableMap.of( + public static final String HEADER_NAME_TOPIC_NAME = "tn"; + public static final String HEADER_NAME_MESSAGE_ID = "id"; + public static final String HEADER_NAME_SUBSCRIPTION_NAME = "sn"; + public static final String HEADER_NAME_TIMESTAMP = "ts"; + public static final String HEADER_NAME_SCHEMA_ID = "sid"; + public static final String HEADER_NAME_SCHEMA_VERSION = "sv"; + + @Override + public PubsubMessage append(PubsubMessage target, Message message) { + return PubsubMessage.newBuilder(target) + .putAllAttributes(createMessageAttributes(message)) + .build(); + } + + protected Map createMessageAttributes(Message message) { + Optional> schemaIdAndVersion = + message + .getSchema() + .map( + s -> + Pair.of( + String.valueOf(s.getId().value()), String.valueOf(s.getVersion().value()))); + + final Map headers = + new HashMap<>( + ImmutableMap.of( HEADER_NAME_TOPIC_NAME, message.getTopic(), HEADER_NAME_MESSAGE_ID, message.getId(), HEADER_NAME_TIMESTAMP, String.valueOf(message.getPublishingTimestamp()))); - if (message.hasSubscriptionIdentityHeaders()) { - headers.put(HEADER_NAME_SUBSCRIPTION_NAME, message.getSubscription()); - } + if (message.hasSubscriptionIdentityHeaders()) { + headers.put(HEADER_NAME_SUBSCRIPTION_NAME, message.getSubscription()); + } - schemaIdAndVersion.ifPresent(sv -> { - headers.put(HEADER_NAME_SCHEMA_ID, sv.getLeft()); - headers.put(HEADER_NAME_SCHEMA_VERSION, sv.getRight()); + schemaIdAndVersion.ifPresent( + sv -> { + headers.put(HEADER_NAME_SCHEMA_ID, sv.getLeft()); + headers.put(HEADER_NAME_SCHEMA_VERSION, sv.getRight()); }); - return headers; - } + return headers; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMetadataCompressionAppender.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMetadataCompressionAppender.java index 03fa6e3305..2557e2bcd3 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMetadataCompressionAppender.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubMetadataCompressionAppender.java @@ -1,22 +1,21 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.googlepubsub; -import pl.allegro.tech.hermes.consumers.consumer.Message; - import java.util.Map; +import pl.allegro.tech.hermes.consumers.consumer.Message; class GooglePubSubMetadataCompressionAppender extends GooglePubSubMetadataAppender { - public static final String HEADER_NAME_CODEC_NAME = "cn"; - private final String codecHeader; + public static final String HEADER_NAME_CODEC_NAME = "cn"; + private final String codecHeader; - GooglePubSubMetadataCompressionAppender(CompressionCodec codec) { - this.codecHeader = codec.getHeader(); - } + GooglePubSubMetadataCompressionAppender(CompressionCodec codec) { + this.codecHeader = codec.getHeader(); + } - @Override - protected Map createMessageAttributes(Message message) { - Map messageAttributes = super.createMessageAttributes(message); - messageAttributes.put(HEADER_NAME_CODEC_NAME, codecHeader); - return messageAttributes; - } + @Override + protected Map createMessageAttributes(Message message) { + Map messageAttributes = super.createMessageAttributes(message); + messageAttributes.put(HEADER_NAME_CODEC_NAME, codecHeader); + return messageAttributes; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubSenderTarget.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubSenderTarget.java index 4a179d8ac2..f984c6f559 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubSenderTarget.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubSenderTarget.java @@ -1,91 +1,90 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.googlepubsub; import com.google.pubsub.v1.TopicName; - import java.util.Objects; class GooglePubSubSenderTarget { - private final TopicName topicName; - private final String pubSubEndpoint; - private final CompressionCodec compressionCodec; - - private GooglePubSubSenderTarget(TopicName topicName, String pubSubEndpoint, CompressionCodec compressionCodec) { - this.topicName = topicName; - this.pubSubEndpoint = pubSubEndpoint; - this.compressionCodec = compressionCodec; + private final TopicName topicName; + private final String pubSubEndpoint; + private final CompressionCodec compressionCodec; + + private GooglePubSubSenderTarget( + TopicName topicName, String pubSubEndpoint, CompressionCodec compressionCodec) { + this.topicName = topicName; + this.pubSubEndpoint = pubSubEndpoint; + this.compressionCodec = compressionCodec; + } + + TopicName getTopicName() { + return topicName; + } + + String getPubSubEndpoint() { + return pubSubEndpoint; + } + + CompressionCodec getCompressionCodec() { + return compressionCodec; + } + + boolean isCompressionRequested() { + return compressionCodec != CompressionCodec.EMPTY; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - TopicName getTopicName() { - return topicName; + if (o == null || getClass() != o.getClass()) { + return false; } + return skipCompressionCodecEquals((GooglePubSubSenderTarget) o); + } - String getPubSubEndpoint() { - return pubSubEndpoint; - } + @Override + public int hashCode() { + return skipCompressionCodecHashCode(); + } - CompressionCodec getCompressionCodec() { - return compressionCodec; - } + private boolean skipCompressionCodecEquals(GooglePubSubSenderTarget that) { + return Objects.equals(topicName, that.topicName) + && Objects.equals(pubSubEndpoint, that.pubSubEndpoint); + } - boolean isCompressionRequested() { - return compressionCodec != CompressionCodec.EMPTY; - } + private int skipCompressionCodecHashCode() { + return Objects.hash(topicName, pubSubEndpoint); + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - return skipCompressionCodecEquals((GooglePubSubSenderTarget) o); - } + static PubSubTargetBuilder builder() { + return new PubSubTargetBuilder(); + } - @Override - public int hashCode() { - return skipCompressionCodecHashCode(); - } + static final class PubSubTargetBuilder { + private TopicName topicName; + private String pubSubEndpoint; + private CompressionCodec compressionCodec; - private boolean skipCompressionCodecEquals(GooglePubSubSenderTarget that) { - return Objects.equals(topicName, that.topicName) - && Objects.equals(pubSubEndpoint, that.pubSubEndpoint); - } + private PubSubTargetBuilder() {} - private int skipCompressionCodecHashCode() { - return Objects.hash(topicName, pubSubEndpoint); + PubSubTargetBuilder withTopicName(TopicName topicName) { + this.topicName = topicName; + return this; } - static PubSubTargetBuilder builder() { - return new PubSubTargetBuilder(); + PubSubTargetBuilder withPubSubEndpoint(String pubSubEndpoint) { + this.pubSubEndpoint = pubSubEndpoint; + return this; } - static final class PubSubTargetBuilder { - private TopicName topicName; - private String pubSubEndpoint; - private CompressionCodec compressionCodec; - - private PubSubTargetBuilder() { - } - - PubSubTargetBuilder withTopicName(TopicName topicName) { - this.topicName = topicName; - return this; - } - - PubSubTargetBuilder withPubSubEndpoint(String pubSubEndpoint) { - this.pubSubEndpoint = pubSubEndpoint; - return this; - } - - PubSubTargetBuilder withCompressionCodec(CompressionCodec compressionCodec) { - this.compressionCodec = compressionCodec; - return this; - } + PubSubTargetBuilder withCompressionCodec(CompressionCodec compressionCodec) { + this.compressionCodec = compressionCodec; + return this; + } - GooglePubSubSenderTarget build() { - return new GooglePubSubSenderTarget(topicName, pubSubEndpoint, compressionCodec); - } + GooglePubSubSenderTarget build() { + return new GooglePubSubSenderTarget(topicName, pubSubEndpoint, compressionCodec); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubSenderTargetResolver.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubSenderTargetResolver.java index 0d3231fd80..17fad97fe6 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubSenderTargetResolver.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/GooglePubSubSenderTargetResolver.java @@ -2,8 +2,6 @@ import com.google.common.base.Preconditions; import com.google.pubsub.v1.TopicName; -import pl.allegro.tech.hermes.api.EndpointAddress; - import java.net.URI; import java.util.Arrays; import java.util.Collections; @@ -11,42 +9,46 @@ import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; +import pl.allegro.tech.hermes.api.EndpointAddress; public class GooglePubSubSenderTargetResolver { - GooglePubSubSenderTarget resolve(EndpointAddress address) { - try { - final URI uri = URI.create(address.getRawEndpoint()); - Preconditions.checkArgument(uri.getScheme().equals("googlepubsub")); - Preconditions.checkArgument(uri.getPort() > 0); + GooglePubSubSenderTarget resolve(EndpointAddress address) { + try { + final URI uri = URI.create(address.getRawEndpoint()); + Preconditions.checkArgument(uri.getScheme().equals("googlepubsub")); + Preconditions.checkArgument(uri.getPort() > 0); - return GooglePubSubSenderTarget.builder() - .withPubSubEndpoint(uri.getAuthority()) - .withTopicName(TopicName.parse(uri.getPath().substring(1))) - .withCompressionCodec(findCompressionCodec(uri.getQuery())) - .build(); - } catch (RuntimeException e) { - throw new IllegalArgumentException("Given endpoint is invalid", e); - } + return GooglePubSubSenderTarget.builder() + .withPubSubEndpoint(uri.getAuthority()) + .withTopicName(TopicName.parse(uri.getPath().substring(1))) + .withCompressionCodec(findCompressionCodec(uri.getQuery())) + .build(); + } catch (RuntimeException e) { + throw new IllegalArgumentException("Given endpoint is invalid", e); } + } - private CompressionCodec findCompressionCodec(String params) { - try { - Map> paramListMap = Optional.ofNullable(params) - .map(q -> Arrays.stream(q.split("&")) - .map(p -> p.split("=")) - .filter(p -> p.length > 1) - .collect(Collectors.groupingBy(c -> c[0]))) - .orElse(Collections.emptyMap()); + private CompressionCodec findCompressionCodec(String params) { + try { + Map> paramListMap = + Optional.ofNullable(params) + .map( + q -> + Arrays.stream(q.split("&")) + .map(p -> p.split("=")) + .filter(p -> p.length > 1) + .collect(Collectors.groupingBy(c -> c[0]))) + .orElse(Collections.emptyMap()); - return Optional.ofNullable(paramListMap.get("compression")) - .flatMap(p -> p.stream().findFirst()) - .flatMap(p -> Optional.ofNullable(p[1])) - .map(String::toUpperCase) - .map(CompressionCodec::valueOf) - .orElse(CompressionCodec.EMPTY); - } catch (RuntimeException ex) { - throw new IllegalArgumentException("Unsupported compression codec", ex); - } + return Optional.ofNullable(paramListMap.get("compression")) + .flatMap(p -> p.stream().findFirst()) + .flatMap(p -> Optional.ofNullable(p[1])) + .map(String::toUpperCase) + .map(CompressionCodec::valueOf) + .orElse(CompressionCodec.EMPTY); + } catch (RuntimeException ex) { + throw new IllegalArgumentException("Unsupported compression codec", ex); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/MessageCompressor.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/MessageCompressor.java index ae2ed0579e..b0017f77b0 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/MessageCompressor.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/googlepubsub/MessageCompressor.java @@ -1,34 +1,33 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.googlepubsub; -import org.apache.avro.file.Codec; - import java.io.IOException; import java.nio.ByteBuffer; +import org.apache.avro.file.Codec; class MessageCompressor { - private final CompressionCodecFactory codecFactory; + private final CompressionCodecFactory codecFactory; - MessageCompressor(CompressionCodecFactory codecFactory) { - this.codecFactory = codecFactory; - } + MessageCompressor(CompressionCodecFactory codecFactory) { + this.codecFactory = codecFactory; + } - byte[] compress(byte[] data) throws IOException { - Codec codec = codecFactory.createInstance(); + byte[] compress(byte[] data) throws IOException { + Codec codec = codecFactory.createInstance(); - ByteBuffer compressed = codec.compress(ByteBuffer.wrap(data)); - byte[] compressedBytes = new byte[compressed.limit()]; - compressed.get(compressedBytes); + ByteBuffer compressed = codec.compress(ByteBuffer.wrap(data)); + byte[] compressedBytes = new byte[compressed.limit()]; + compressed.get(compressedBytes); - return compressedBytes; - } + return compressedBytes; + } - byte[] decompress(byte[] data) throws IOException { - Codec codec = codecFactory.createInstance(); + byte[] decompress(byte[] data) throws IOException { + Codec codec = codecFactory.createInstance(); - ByteBuffer decompressed = codec.decompress(ByteBuffer.wrap(data)); - byte[] decompressedBytes = new byte[decompressed.limit()]; - decompressed.get(decompressedBytes); - return decompressedBytes; - } + ByteBuffer decompressed = codec.decompress(ByteBuffer.wrap(data)); + byte[] decompressedBytes = new byte[decompressed.limit()]; + decompressed.get(decompressedBytes); + return decompressedBytes; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/BatchHttpRequestFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/BatchHttpRequestFactory.java index 737576ba15..5db38ef1af 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/BatchHttpRequestFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/BatchHttpRequestFactory.java @@ -1,13 +1,13 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http; +import java.net.URI; import org.eclipse.jetty.client.Request; import pl.allegro.tech.hermes.consumers.consumer.batch.MessageBatch; import pl.allegro.tech.hermes.consumers.consumer.sender.http.headers.HttpRequestHeaders; -import java.net.URI; - public interface BatchHttpRequestFactory { - Request buildRequest(MessageBatch message, URI uri, HttpRequestHeaders headers, int requestTimeout); + Request buildRequest( + MessageBatch message, URI uri, HttpRequestHeaders headers, int requestTimeout); - void stop(); + void stop(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/DefaultBatchHttpRequestFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/DefaultBatchHttpRequestFactory.java index 5a76d1646d..4925aec082 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/DefaultBatchHttpRequestFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/DefaultBatchHttpRequestFactory.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http; +import java.net.URI; +import java.util.concurrent.TimeUnit; import org.eclipse.jetty.client.ByteBufferRequestContent; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.Request; @@ -8,39 +10,37 @@ import pl.allegro.tech.hermes.consumers.consumer.batch.MessageBatch; import pl.allegro.tech.hermes.consumers.consumer.sender.http.headers.HttpRequestHeaders; -import java.net.URI; -import java.util.concurrent.TimeUnit; - public class DefaultBatchHttpRequestFactory implements BatchHttpRequestFactory { - private final HttpClient client; - - public DefaultBatchHttpRequestFactory(HttpClient client) { - this.client = started(client); + private final HttpClient client; + + public DefaultBatchHttpRequestFactory(HttpClient client) { + this.client = started(client); + } + + public Request buildRequest( + MessageBatch message, URI uri, HttpRequestHeaders headers, int requestTimeout) { + return client + .newRequest(uri) + .method(HttpMethod.POST) + .timeout(requestTimeout, TimeUnit.MILLISECONDS) + .body(new ByteBufferRequestContent(message.getContent())) + .headers(httpHeaders -> headers.asMap().forEach(httpHeaders::add)); + } + + private static HttpClient started(HttpClient httpClient) { + try { + httpClient.start(); + return httpClient; + } catch (Exception e) { + throw new InternalProcessingException("Failed to start http batch client", e); } + } - public Request buildRequest(MessageBatch message, URI uri, HttpRequestHeaders headers, int requestTimeout) { - return client.newRequest(uri) - .method(HttpMethod.POST) - .timeout(requestTimeout, TimeUnit.MILLISECONDS) - .body(new ByteBufferRequestContent(message.getContent())) - .headers(httpHeaders -> headers.asMap().forEach(httpHeaders::add)); - } - - private static HttpClient started(HttpClient httpClient) { - try { - httpClient.start(); - return httpClient; - } catch (Exception e) { - throw new InternalProcessingException("Failed to start http batch client", e); - } - } - - - public void stop() { - try { - client.stop(); - } catch (Exception e) { - throw new InternalProcessingException("Failed to stop http batch client", e); - } + public void stop() { + try { + client.stop(); + } catch (Exception e) { + throw new InternalProcessingException("Failed to stop http batch client", e); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/DefaultHttpMetadataAppender.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/DefaultHttpMetadataAppender.java index 804eebe17c..368753ecbd 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/DefaultHttpMetadataAppender.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/DefaultHttpMetadataAppender.java @@ -6,9 +6,9 @@ public class DefaultHttpMetadataAppender implements MetadataAppender { - @Override - public Request append(Request target, Message message) { - target.headers(httpFields -> message.getExternalMetadata().forEach(httpFields::add)); - return target; - } + @Override + public Request append(Request target, Message message) { + target.headers(httpFields -> message.getExternalMetadata().forEach(httpFields::add)); + return target; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/DefaultHttpRequestFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/DefaultHttpRequestFactory.java index db1a88d7c9..dfb2c1cc88 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/DefaultHttpRequestFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/DefaultHttpRequestFactory.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http; +import java.net.URI; +import java.util.concurrent.TimeUnit; import org.eclipse.jetty.client.BytesRequestContent; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.Request; @@ -8,59 +10,57 @@ import pl.allegro.tech.hermes.consumers.consumer.sender.http.headers.HttpRequestHeaders; import pl.allegro.tech.hermes.consumers.consumer.trace.MetadataAppender; -import java.net.URI; -import java.util.concurrent.TimeUnit; - class DefaultHttpRequestFactory implements HttpRequestFactory { - private final HttpClient client; - private final long timeout; - private final long socketTimeout; - private final MetadataAppender metadataAppender; + private final HttpClient client; + private final long timeout; + private final long socketTimeout; + private final MetadataAppender metadataAppender; - public DefaultHttpRequestFactory(HttpClient client, - long timeout, - long socketTimeout, - MetadataAppender metadataAppender) { - this.client = client; - this.timeout = timeout; - this.socketTimeout = socketTimeout; - this.metadataAppender = metadataAppender; - } - - @Override - public Request buildRequest(Message message, URI uri, HttpRequestHeaders headers) { - return new HttpRequestBuilder(message, uri, headers).build(); - } + public DefaultHttpRequestFactory( + HttpClient client, + long timeout, + long socketTimeout, + MetadataAppender metadataAppender) { + this.client = client; + this.timeout = timeout; + this.socketTimeout = socketTimeout; + this.metadataAppender = metadataAppender; + } - private class HttpRequestBuilder { + @Override + public Request buildRequest(Message message, URI uri, HttpRequestHeaders headers) { + return new HttpRequestBuilder(message, uri, headers).build(); + } - private final Message message; - private final URI uri; - private final HttpRequestHeaders headers; + private class HttpRequestBuilder { - HttpRequestBuilder(Message message, URI uri, HttpRequestHeaders headers) { - this.message = message; - this.uri = uri; - this.headers = headers; - } + private final Message message; + private final URI uri; + private final HttpRequestHeaders headers; - Request build() { - return baseRequest() - .headers(httpHeaders -> headers.asMap().forEach(httpHeaders::add)); - } + HttpRequestBuilder(Message message, URI uri, HttpRequestHeaders headers) { + this.message = message; + this.uri = uri; + this.headers = headers; + } - private Request baseRequest() { - Request baseRequest = client.newRequest(uri) - .method(HttpMethod.POST) - .timeout(timeout, TimeUnit.MILLISECONDS) - .idleTimeout(socketTimeout, TimeUnit.MILLISECONDS) - .body(new BytesRequestContent(message.getData())); + Request build() { + return baseRequest().headers(httpHeaders -> headers.asMap().forEach(httpHeaders::add)); + } - metadataAppender.append(baseRequest, message); + private Request baseRequest() { + Request baseRequest = + client + .newRequest(uri) + .method(HttpMethod.POST) + .timeout(timeout, TimeUnit.MILLISECONDS) + .idleTimeout(socketTimeout, TimeUnit.MILLISECONDS) + .body(new BytesRequestContent(message.getData())); - return baseRequest; - } + metadataAppender.append(baseRequest, message); + return baseRequest; } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/DefaultHttpRequestFactoryProvider.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/DefaultHttpRequestFactoryProvider.java index 6cba998e03..1ff86235e2 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/DefaultHttpRequestFactoryProvider.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/DefaultHttpRequestFactoryProvider.java @@ -6,17 +6,14 @@ import pl.allegro.tech.hermes.consumers.consumer.trace.MetadataAppender; public class DefaultHttpRequestFactoryProvider implements HttpRequestFactoryProvider { - @Override - public HttpRequestFactory provideRequestFactory(Subscription subscription, - HttpClient httpClient, - MetadataAppender metadataAppender) { - int requestTimeout = subscription.getSerialSubscriptionPolicy().getRequestTimeout(); - int socketTimeout = subscription.getSerialSubscriptionPolicy().getSocketTimeout(); - return new DefaultHttpRequestFactory( - httpClient, - requestTimeout, - socketTimeout, - metadataAppender - ); - } + @Override + public HttpRequestFactory provideRequestFactory( + Subscription subscription, + HttpClient httpClient, + MetadataAppender metadataAppender) { + int requestTimeout = subscription.getSerialSubscriptionPolicy().getRequestTimeout(); + int socketTimeout = subscription.getSerialSubscriptionPolicy().getSocketTimeout(); + return new DefaultHttpRequestFactory( + httpClient, requestTimeout, socketTimeout, metadataAppender); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/DefaultSendingResultHandlers.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/DefaultSendingResultHandlers.java index 1d435a62f2..679ef249dd 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/DefaultSendingResultHandlers.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/DefaultSendingResultHandlers.java @@ -1,25 +1,26 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http; +import java.util.concurrent.CompletableFuture; import org.eclipse.jetty.client.ContentResponse; import org.eclipse.jetty.client.Response; import pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult; import pl.allegro.tech.hermes.consumers.consumer.sender.SingleMessageSendingResult; -import java.util.concurrent.CompletableFuture; - public class DefaultSendingResultHandlers implements SendingResultHandlers { - @Override - public Response.CompleteListener handleSendingResultForSerial(CompletableFuture resultFuture) { - return new JettyResponseListener(resultFuture); - } + @Override + public Response.CompleteListener handleSendingResultForSerial( + CompletableFuture resultFuture) { + return new JettyResponseListener(resultFuture); + } - @Override - public Response.CompleteListener handleSendingResultForBroadcast(CompletableFuture resultFuture) { - return new JettyBroadCastResponseListener(resultFuture); - } + @Override + public Response.CompleteListener handleSendingResultForBroadcast( + CompletableFuture resultFuture) { + return new JettyBroadCastResponseListener(resultFuture); + } - @Override - public MessageSendingResult handleSendingResultForBatch(ContentResponse response) { - return MessageSendingResult.ofStatusCode(response.getStatus()); - } + @Override + public MessageSendingResult handleSendingResultForBatch(ContentResponse response) { + return MessageSendingResult.ofStatusCode(response.getStatus()); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/EmptyHttpHeadersProvidersFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/EmptyHttpHeadersProvidersFactory.java index 959dfa66c7..eab054bd53 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/EmptyHttpHeadersProvidersFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/EmptyHttpHeadersProvidersFactory.java @@ -1,15 +1,14 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http; -import pl.allegro.tech.hermes.consumers.consumer.sender.http.headers.HttpHeadersProvider; +import static java.util.Collections.emptySet; import java.util.Collection; - -import static java.util.Collections.emptySet; +import pl.allegro.tech.hermes.consumers.consumer.sender.http.headers.HttpHeadersProvider; public class EmptyHttpHeadersProvidersFactory implements HttpHeadersProvidersFactory { - @Override - public Collection createAll() { - return emptySet(); - } + @Override + public Collection createAll() { + return emptySet(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/Http1ClientParameters.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/Http1ClientParameters.java index 41412d908c..a576b29b58 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/Http1ClientParameters.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/Http1ClientParameters.java @@ -2,6 +2,5 @@ public interface Http1ClientParameters extends HttpClientParameters { - int getMaxConnectionsPerDestination(); - + int getMaxConnectionsPerDestination(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/Http2ClientHolder.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/Http2ClientHolder.java index 3514906f16..f42ae7882c 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/Http2ClientHolder.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/Http2ClientHolder.java @@ -1,18 +1,17 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http; -import org.eclipse.jetty.client.HttpClient; - import java.util.Optional; +import org.eclipse.jetty.client.HttpClient; public class Http2ClientHolder { - private final HttpClient http2Client; + private final HttpClient http2Client; - public Http2ClientHolder(HttpClient http2Client) { - this.http2Client = http2Client; - } + public Http2ClientHolder(HttpClient http2Client) { + this.http2Client = http2Client; + } - Optional getHttp2Client() { - return Optional.ofNullable(http2Client); - } + Optional getHttp2Client() { + return Optional.ofNullable(http2Client); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/Http2ClientParameters.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/Http2ClientParameters.java index 3e3c9c3170..e14ead01f5 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/Http2ClientParameters.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/Http2ClientParameters.java @@ -1,6 +1,3 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http; - -public interface Http2ClientParameters extends HttpClientParameters { - -} +public interface Http2ClientParameters extends HttpClientParameters {} diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpBatchSenderException.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpBatchSenderException.java index 6be23ce272..4776e143f4 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpBatchSenderException.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpBatchSenderException.java @@ -1,7 +1,7 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http; class HttpBatchSenderException extends RuntimeException { - HttpBatchSenderException(String message, Throwable cause) { - super(message, cause); - } + HttpBatchSenderException(String message, Throwable cause) { + super(message, cause); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpClientParameters.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpClientParameters.java index 63da1e530f..416686b143 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpClientParameters.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpClientParameters.java @@ -4,15 +4,15 @@ public interface HttpClientParameters { - int getThreadPoolSize(); + int getThreadPoolSize(); - boolean isThreadPoolMonitoringEnabled(); + boolean isThreadPoolMonitoringEnabled(); - boolean isFollowRedirectsEnabled(); + boolean isFollowRedirectsEnabled(); - Duration getIdleTimeout(); + Duration getIdleTimeout(); - int getMaxRequestsQueuedPerDestination(); + int getMaxRequestsQueuedPerDestination(); - Duration getConnectionTimeout(); + Duration getConnectionTimeout(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpClientsFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpClientsFactory.java index b6b79becf1..daef1ad234 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpClientsFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpClientsFactory.java @@ -1,5 +1,6 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http; +import java.util.concurrent.ExecutorService; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.transport.HttpClientTransportOverHTTP; import org.eclipse.jetty.http.HttpCookieStore; @@ -8,64 +9,70 @@ import org.eclipse.jetty.io.ClientConnector; import pl.allegro.tech.hermes.common.metric.executor.InstrumentedExecutorServiceFactory; -import java.util.concurrent.ExecutorService; - public class HttpClientsFactory { - private final InstrumentedExecutorServiceFactory executorFactory; - private final SslContextFactoryProvider sslContextFactoryProvider; + private final InstrumentedExecutorServiceFactory executorFactory; + private final SslContextFactoryProvider sslContextFactoryProvider; - public HttpClientsFactory( - InstrumentedExecutorServiceFactory executorFactory, - SslContextFactoryProvider sslContextFactoryProvider) { - this.executorFactory = executorFactory; - this.sslContextFactoryProvider = sslContextFactoryProvider; - } + public HttpClientsFactory( + InstrumentedExecutorServiceFactory executorFactory, + SslContextFactoryProvider sslContextFactoryProvider) { + this.executorFactory = executorFactory; + this.sslContextFactoryProvider = sslContextFactoryProvider; + } - public HttpClient createClientForHttp1(String name, Http1ClientParameters http1ClientParameters) { - ClientConnector clientConnector = new ClientConnector(); - sslContextFactoryProvider.provideSslContextFactory() - .ifPresent(clientConnector::setSslContextFactory); - HttpClientTransportOverHTTP transport = new HttpClientTransportOverHTTP(clientConnector); - HttpClient client = new HttpClient(transport); + public HttpClient createClientForHttp1(String name, Http1ClientParameters http1ClientParameters) { + ClientConnector clientConnector = new ClientConnector(); + sslContextFactoryProvider + .provideSslContextFactory() + .ifPresent(clientConnector::setSslContextFactory); + HttpClientTransportOverHTTP transport = new HttpClientTransportOverHTTP(clientConnector); + HttpClient client = new HttpClient(transport); - ExecutorService executor = executorFactory.getExecutorService( - name, - http1ClientParameters.getThreadPoolSize(), - http1ClientParameters.isThreadPoolMonitoringEnabled()); - client.setExecutor(executor); - client.setMaxConnectionsPerDestination(http1ClientParameters.getMaxConnectionsPerDestination()); - client.setMaxRequestsQueuedPerDestination(http1ClientParameters.getMaxRequestsQueuedPerDestination()); - client.setHttpCookieStore(new HttpCookieStore.Empty()); - client.setIdleTimeout(http1ClientParameters.getIdleTimeout().toMillis()); - client.setFollowRedirects(http1ClientParameters.isFollowRedirectsEnabled()); - client.setConnectTimeout(http1ClientParameters.getConnectionTimeout().toMillis()); - return client; - } + ExecutorService executor = + executorFactory.getExecutorService( + name, + http1ClientParameters.getThreadPoolSize(), + http1ClientParameters.isThreadPoolMonitoringEnabled()); + client.setExecutor(executor); + client.setMaxConnectionsPerDestination(http1ClientParameters.getMaxConnectionsPerDestination()); + client.setMaxRequestsQueuedPerDestination( + http1ClientParameters.getMaxRequestsQueuedPerDestination()); + client.setHttpCookieStore(new HttpCookieStore.Empty()); + client.setIdleTimeout(http1ClientParameters.getIdleTimeout().toMillis()); + client.setFollowRedirects(http1ClientParameters.isFollowRedirectsEnabled()); + client.setConnectTimeout(http1ClientParameters.getConnectionTimeout().toMillis()); + return client; + } - public HttpClient createClientForHttp2(String name, Http2ClientParameters http2ClientParameters) { - ClientConnector clientConnector = new ClientConnector(); - sslContextFactoryProvider.provideSslContextFactory() - .ifPresentOrElse(clientConnector::setSslContextFactory, - () -> { - throw new IllegalStateException("Cannot create http/2 client due to lack of ssl context factory"); - }); - HTTP2Client http2Client = new HTTP2Client(clientConnector); + public HttpClient createClientForHttp2(String name, Http2ClientParameters http2ClientParameters) { + ClientConnector clientConnector = new ClientConnector(); + sslContextFactoryProvider + .provideSslContextFactory() + .ifPresentOrElse( + clientConnector::setSslContextFactory, + () -> { + throw new IllegalStateException( + "Cannot create http/2 client due to lack of ssl context factory"); + }); + HTTP2Client http2Client = new HTTP2Client(clientConnector); - ExecutorService executor = executorFactory.getExecutorService( - name, - http2ClientParameters.getThreadPoolSize(), - http2ClientParameters.isThreadPoolMonitoringEnabled()); - http2Client.setExecutor(executor); + ExecutorService executor = + executorFactory.getExecutorService( + name, + http2ClientParameters.getThreadPoolSize(), + http2ClientParameters.isThreadPoolMonitoringEnabled()); + http2Client.setExecutor(executor); - HttpClientTransportOverHTTP2 transport = new HttpClientTransportOverHTTP2(http2Client); - HttpClient client = new HttpClient(transport); + HttpClientTransportOverHTTP2 transport = new HttpClientTransportOverHTTP2(http2Client); + HttpClient client = new HttpClient(transport); - client.setMaxRequestsQueuedPerDestination(http2ClientParameters.getMaxRequestsQueuedPerDestination()); - client.setHttpCookieStore(new HttpCookieStore.Empty()); - client.setIdleTimeout(http2ClientParameters.getIdleTimeout().toMillis()); - client.setFollowRedirects(http2ClientParameters.isFollowRedirectsEnabled()); - client.setConnectTimeout(http2ClientParameters.getConnectionTimeout().toMillis()); - return client; - } + client.setMaxRequestsQueuedPerDestination( + http2ClientParameters.getMaxRequestsQueuedPerDestination()); + client.setHttpCookieStore(new HttpCookieStore.Empty()); + client.setIdleTimeout(http2ClientParameters.getIdleTimeout().toMillis()); + client.setFollowRedirects(http2ClientParameters.isFollowRedirectsEnabled()); + client.setConnectTimeout(http2ClientParameters.getConnectionTimeout().toMillis()); + return client; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpClientsWorkloadReporter.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpClientsWorkloadReporter.java index 2692c3d7c5..1effa5c7f0 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpClientsWorkloadReporter.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpClientsWorkloadReporter.java @@ -1,5 +1,8 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http; +import java.util.Queue; +import java.util.function.Function; +import java.util.stream.Stream; import org.eclipse.jetty.client.ConnectionPool; import org.eclipse.jetty.client.DuplexConnectionPool; import org.eclipse.jetty.client.HttpClient; @@ -7,163 +10,172 @@ import org.eclipse.jetty.client.transport.HttpDestination; import pl.allegro.tech.hermes.common.metric.MetricsFacade; -import java.util.Queue; -import java.util.function.Function; -import java.util.stream.Stream; - public class HttpClientsWorkloadReporter { - private final MetricsFacade metrics; - private final HttpClient http1SerialClient; - private final HttpClient http1BatchClient; - private final Http2ClientHolder http2ClientHolder; - private final boolean isRequestQueueMonitoringEnabled; - private final boolean isConnectionPoolMonitoringEnabled; - - public HttpClientsWorkloadReporter( - MetricsFacade metrics, - HttpClient http1SerialClient, - HttpClient http1BatchClient, - Http2ClientHolder http2ClientHolder, - boolean isRequestQueueMonitoringEnabled, - boolean isConnectionPoolMonitoringEnabled - ) { - this.metrics = metrics; - this.http1SerialClient = http1SerialClient; - this.http1BatchClient = http1BatchClient; - this.http2ClientHolder = http2ClientHolder; - this.isRequestQueueMonitoringEnabled = isRequestQueueMonitoringEnabled; - this.isConnectionPoolMonitoringEnabled = isConnectionPoolMonitoringEnabled; - } - - public void start() { - if (isRequestQueueMonitoringEnabled) { - registerRequestQueueSizeGauges(); - } - if (isConnectionPoolMonitoringEnabled) { - registerConnectionGauges(); - } - } - - private void registerRequestQueueSizeGauges() { - metrics.consumerSender() - .registerRequestQueueSizeGauge(this, HttpClientsWorkloadReporter::getQueuesSize); - metrics.consumerSender() - .registerHttp1SerialClientRequestQueueSizeGauge(this, HttpClientsWorkloadReporter::getHttp1SerialQueueSize); - metrics.consumerSender() - .registerHttp1BatchClientRequestQueueSizeGauge(this, HttpClientsWorkloadReporter::getHttp1BatchQueueSize); - metrics.consumerSender() - .registerHttp2RequestQueueSizeGauge(this, HttpClientsWorkloadReporter::getHttp2SerialQueueSize); - } - - private void registerConnectionGauges() { - metrics.consumerSender() - .registerHttp1SerialClientActiveConnectionsGauge(this, HttpClientsWorkloadReporter::getHttp1SerialActiveConnections); - metrics.consumerSender() - .registerHttp1SerialClientIdleConnectionsGauge(this, HttpClientsWorkloadReporter::getHttp1SerialIdleConnections); - metrics.consumerSender() - .registerHttp1BatchClientActiveConnectionsGauge(this, HttpClientsWorkloadReporter::getHttp1BatchActiveConnections); - metrics.consumerSender() - .registerHttp1BatchClientIdleConnectionsGauge(this, HttpClientsWorkloadReporter::getHttp1BatchIdleConnections); - metrics.consumerSender() - .registerHttp2SerialClientConnectionsGauge(this, HttpClientsWorkloadReporter::getHttp2SerialConnections); - metrics.consumerSender() - .registerHttp2SerialClientPendingConnectionsGauge(this, HttpClientsWorkloadReporter::getHttp2SerialPendingConnections); - } - - int getQueuesSize() { - return getHttp1SerialQueueSize() + getHttp1BatchQueueSize() + getHttp2SerialQueueSize(); - } - - int getHttp1SerialQueueSize() { - return getQueueSize.apply(http1SerialClient); - } - - int getHttp1BatchQueueSize() { - return getQueueSize.apply(http1BatchClient); - } - - - int getHttp2SerialQueueSize() { - return http2ClientHolder.getHttp2Client() - .map(getQueueSize) - .orElse(0); - } - - private int getHttp1SerialActiveConnections() { - return getHttp1ActiveConnectionsCount.apply(http1SerialClient); - } - - private int getHttp1SerialIdleConnections() { - return getHttp1IdleConnectionsCount.apply(http1SerialClient); - } - - private int getHttp1BatchActiveConnections() { - return getHttp1ActiveConnectionsCount.apply(http1BatchClient); - } - - private int getHttp1BatchIdleConnections() { - return getHttp1IdleConnectionsCount.apply(http1BatchClient); - } - - private int getHttp2SerialConnections() { - return http2ClientHolder.getHttp2Client() - .map(getHttp2ConnectionsCount) - .orElse(0); - } - - private int getHttp2SerialPendingConnections() { - return http2ClientHolder.getHttp2Client() - .map(getHttp2PendingConnectionsCount) - .orElse(0); - } - - private final Function getQueueSize = httpClient -> - httpClient.getDestinations().stream() - .map(HttpDestination.class::cast) - .map(HttpDestination::getHttpExchanges) - .map(Queue::size) - .mapToInt(i -> i) - .sum(); - - private final Function getHttp1ActiveConnectionsCount = httpClient -> - getHttp1ConnectionPool(httpClient) - .map(DuplexConnectionPool::getActiveConnectionCount) - .mapToInt(i -> i) - .sum(); - - private final Function getHttp1IdleConnectionsCount = httpClient -> - getHttp1ConnectionPool(httpClient) - .map(DuplexConnectionPool::getIdleConnectionCount) - .mapToInt(i -> i) - .sum(); - - private final Function getHttp2ConnectionsCount = http2Client -> - getHttp2ConnectionPool(http2Client) - .map(MultiplexConnectionPool::getConnectionCount) - .mapToInt(i -> i) - .sum(); - - private final Function getHttp2PendingConnectionsCount = http2Client -> - getHttp2ConnectionPool(http2Client) - .map(MultiplexConnectionPool::getPendingConnectionCount) - .mapToInt(i -> i) - .sum(); - - private Stream getConnectionPool(HttpClient httpClient) { - return httpClient.getDestinations().stream() - .map(HttpDestination.class::cast) - .map(HttpDestination::getConnectionPool); - } - - private Stream getHttp1ConnectionPool(HttpClient httpClient) { - return getConnectionPool(httpClient) - .map(DuplexConnectionPool.class::cast); - } - - private Stream getHttp2ConnectionPool(HttpClient http2Client) { - return getConnectionPool(http2Client) - .map(MultiplexConnectionPool.class::cast); - } - + private final MetricsFacade metrics; + private final HttpClient http1SerialClient; + private final HttpClient http1BatchClient; + private final Http2ClientHolder http2ClientHolder; + private final boolean isRequestQueueMonitoringEnabled; + private final boolean isConnectionPoolMonitoringEnabled; + + public HttpClientsWorkloadReporter( + MetricsFacade metrics, + HttpClient http1SerialClient, + HttpClient http1BatchClient, + Http2ClientHolder http2ClientHolder, + boolean isRequestQueueMonitoringEnabled, + boolean isConnectionPoolMonitoringEnabled) { + this.metrics = metrics; + this.http1SerialClient = http1SerialClient; + this.http1BatchClient = http1BatchClient; + this.http2ClientHolder = http2ClientHolder; + this.isRequestQueueMonitoringEnabled = isRequestQueueMonitoringEnabled; + this.isConnectionPoolMonitoringEnabled = isConnectionPoolMonitoringEnabled; + } + + public void start() { + if (isRequestQueueMonitoringEnabled) { + registerRequestQueueSizeGauges(); + } + if (isConnectionPoolMonitoringEnabled) { + registerConnectionGauges(); + } + } + + private void registerRequestQueueSizeGauges() { + metrics + .consumerSender() + .registerRequestQueueSizeGauge(this, HttpClientsWorkloadReporter::getQueuesSize); + metrics + .consumerSender() + .registerHttp1SerialClientRequestQueueSizeGauge( + this, HttpClientsWorkloadReporter::getHttp1SerialQueueSize); + metrics + .consumerSender() + .registerHttp1BatchClientRequestQueueSizeGauge( + this, HttpClientsWorkloadReporter::getHttp1BatchQueueSize); + metrics + .consumerSender() + .registerHttp2RequestQueueSizeGauge( + this, HttpClientsWorkloadReporter::getHttp2SerialQueueSize); + } + + private void registerConnectionGauges() { + metrics + .consumerSender() + .registerHttp1SerialClientActiveConnectionsGauge( + this, HttpClientsWorkloadReporter::getHttp1SerialActiveConnections); + metrics + .consumerSender() + .registerHttp1SerialClientIdleConnectionsGauge( + this, HttpClientsWorkloadReporter::getHttp1SerialIdleConnections); + metrics + .consumerSender() + .registerHttp1BatchClientActiveConnectionsGauge( + this, HttpClientsWorkloadReporter::getHttp1BatchActiveConnections); + metrics + .consumerSender() + .registerHttp1BatchClientIdleConnectionsGauge( + this, HttpClientsWorkloadReporter::getHttp1BatchIdleConnections); + metrics + .consumerSender() + .registerHttp2SerialClientConnectionsGauge( + this, HttpClientsWorkloadReporter::getHttp2SerialConnections); + metrics + .consumerSender() + .registerHttp2SerialClientPendingConnectionsGauge( + this, HttpClientsWorkloadReporter::getHttp2SerialPendingConnections); + } + + int getQueuesSize() { + return getHttp1SerialQueueSize() + getHttp1BatchQueueSize() + getHttp2SerialQueueSize(); + } + + int getHttp1SerialQueueSize() { + return getQueueSize.apply(http1SerialClient); + } + + int getHttp1BatchQueueSize() { + return getQueueSize.apply(http1BatchClient); + } + + int getHttp2SerialQueueSize() { + return http2ClientHolder.getHttp2Client().map(getQueueSize).orElse(0); + } + + private int getHttp1SerialActiveConnections() { + return getHttp1ActiveConnectionsCount.apply(http1SerialClient); + } + + private int getHttp1SerialIdleConnections() { + return getHttp1IdleConnectionsCount.apply(http1SerialClient); + } + + private int getHttp1BatchActiveConnections() { + return getHttp1ActiveConnectionsCount.apply(http1BatchClient); + } + + private int getHttp1BatchIdleConnections() { + return getHttp1IdleConnectionsCount.apply(http1BatchClient); + } + + private int getHttp2SerialConnections() { + return http2ClientHolder.getHttp2Client().map(getHttp2ConnectionsCount).orElse(0); + } + + private int getHttp2SerialPendingConnections() { + return http2ClientHolder.getHttp2Client().map(getHttp2PendingConnectionsCount).orElse(0); + } + + private final Function getQueueSize = + httpClient -> + httpClient.getDestinations().stream() + .map(HttpDestination.class::cast) + .map(HttpDestination::getHttpExchanges) + .map(Queue::size) + .mapToInt(i -> i) + .sum(); + + private final Function getHttp1ActiveConnectionsCount = + httpClient -> + getHttp1ConnectionPool(httpClient) + .map(DuplexConnectionPool::getActiveConnectionCount) + .mapToInt(i -> i) + .sum(); + + private final Function getHttp1IdleConnectionsCount = + httpClient -> + getHttp1ConnectionPool(httpClient) + .map(DuplexConnectionPool::getIdleConnectionCount) + .mapToInt(i -> i) + .sum(); + + private final Function getHttp2ConnectionsCount = + http2Client -> + getHttp2ConnectionPool(http2Client) + .map(MultiplexConnectionPool::getConnectionCount) + .mapToInt(i -> i) + .sum(); + + private final Function getHttp2PendingConnectionsCount = + http2Client -> + getHttp2ConnectionPool(http2Client) + .map(MultiplexConnectionPool::getPendingConnectionCount) + .mapToInt(i -> i) + .sum(); + + private Stream getConnectionPool(HttpClient httpClient) { + return httpClient.getDestinations().stream() + .map(HttpDestination.class::cast) + .map(HttpDestination::getConnectionPool); + } + + private Stream getHttp1ConnectionPool(HttpClient httpClient) { + return getConnectionPool(httpClient).map(DuplexConnectionPool.class::cast); + } + + private Stream getHttp2ConnectionPool(HttpClient http2Client) { + return getConnectionPool(http2Client).map(MultiplexConnectionPool.class::cast); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpHeadersProvidersFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpHeadersProvidersFactory.java index 6906fa59bc..ef51d29151 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpHeadersProvidersFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpHeadersProvidersFactory.java @@ -1,10 +1,9 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http; -import pl.allegro.tech.hermes.consumers.consumer.sender.http.headers.HttpHeadersProvider; - import java.util.Collection; +import pl.allegro.tech.hermes.consumers.consumer.sender.http.headers.HttpHeadersProvider; public interface HttpHeadersProvidersFactory { - Collection createAll(); + Collection createAll(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpMessageBatchSenderFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpMessageBatchSenderFactory.java index 3253e94cc7..9d3e613e78 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpMessageBatchSenderFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpMessageBatchSenderFactory.java @@ -1,35 +1,34 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http; +import static com.google.common.base.Preconditions.checkState; + import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.consumers.consumer.sender.MessageBatchSender; import pl.allegro.tech.hermes.consumers.consumer.sender.MessageBatchSenderFactory; import pl.allegro.tech.hermes.consumers.consumer.sender.http.headers.DefaultBatchHeadersProvider; import pl.allegro.tech.hermes.consumers.consumer.sender.resolver.SimpleEndpointAddressResolver; -import static com.google.common.base.Preconditions.checkState; - public class HttpMessageBatchSenderFactory implements MessageBatchSenderFactory { - private final SendingResultHandlers resultHandlers; - private final BatchHttpRequestFactory batchHttpRequestFactory; - - public HttpMessageBatchSenderFactory(SendingResultHandlers resultHandlers, - BatchHttpRequestFactory batchHttpRequestFactory - ) { - this.resultHandlers = resultHandlers; - this.batchHttpRequestFactory = batchHttpRequestFactory; - } - - - @Override - public MessageBatchSender create(Subscription subscription) { - checkState(subscription.getEndpoint().getProtocol().contains("http"), "Batching is only supported for http/s currently."); - - - return new JettyMessageBatchSender( - batchHttpRequestFactory, - new SimpleEndpointAddressResolver(), - resultHandlers, - new DefaultBatchHeadersProvider()); - } + private final SendingResultHandlers resultHandlers; + private final BatchHttpRequestFactory batchHttpRequestFactory; + + public HttpMessageBatchSenderFactory( + SendingResultHandlers resultHandlers, BatchHttpRequestFactory batchHttpRequestFactory) { + this.resultHandlers = resultHandlers; + this.batchHttpRequestFactory = batchHttpRequestFactory; + } + + @Override + public MessageBatchSender create(Subscription subscription) { + checkState( + subscription.getEndpoint().getProtocol().contains("http"), + "Batching is only supported for http/s currently."); + + return new JettyMessageBatchSender( + batchHttpRequestFactory, + new SimpleEndpointAddressResolver(), + resultHandlers, + new DefaultBatchHeadersProvider()); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpRequestData.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpRequestData.java index 25869cb036..196a98d20d 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpRequestData.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpRequestData.java @@ -4,26 +4,26 @@ public class HttpRequestData { - private EndpointAddress rawAddress; + private EndpointAddress rawAddress; - public HttpRequestData(EndpointAddress rawAddress) { - this.rawAddress = rawAddress; - } + public HttpRequestData(EndpointAddress rawAddress) { + this.rawAddress = rawAddress; + } - public EndpointAddress getRawAddress() { - return rawAddress; - } + public EndpointAddress getRawAddress() { + return rawAddress; + } - public static class HttpRequestDataBuilder { - private EndpointAddress rawAddress; + public static class HttpRequestDataBuilder { + private EndpointAddress rawAddress; - public HttpRequestDataBuilder withRawAddress(EndpointAddress rawAddress) { - this.rawAddress = rawAddress; - return this; - } + public HttpRequestDataBuilder withRawAddress(EndpointAddress rawAddress) { + this.rawAddress = rawAddress; + return this; + } - public HttpRequestData build() { - return new HttpRequestData(rawAddress); - } + public HttpRequestData build() { + return new HttpRequestData(rawAddress); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpRequestFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpRequestFactory.java index d31e7354a8..2970c957b5 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpRequestFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpRequestFactory.java @@ -1,11 +1,10 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http; +import java.net.URI; import org.eclipse.jetty.client.Request; import pl.allegro.tech.hermes.consumers.consumer.Message; import pl.allegro.tech.hermes.consumers.consumer.sender.http.headers.HttpRequestHeaders; -import java.net.URI; - public interface HttpRequestFactory { - Request buildRequest(Message message, URI uri, HttpRequestHeaders headers); + Request buildRequest(Message message, URI uri, HttpRequestHeaders headers); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpRequestFactoryProvider.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpRequestFactoryProvider.java index 7cb84395a9..9ee4f83515 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpRequestFactoryProvider.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/HttpRequestFactoryProvider.java @@ -6,5 +6,6 @@ import pl.allegro.tech.hermes.consumers.consumer.trace.MetadataAppender; public interface HttpRequestFactoryProvider { - HttpRequestFactory provideRequestFactory(Subscription subscription, HttpClient httpClient, MetadataAppender metadataAppender); + HttpRequestFactory provideRequestFactory( + Subscription subscription, HttpClient httpClient, MetadataAppender metadataAppender); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyBroadCastMessageSender.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyBroadCastMessageSender.java index 31e86f7a45..4e515c3338 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyBroadCastMessageSender.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyBroadCastMessageSender.java @@ -1,6 +1,15 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http; import com.google.common.collect.ImmutableList; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.function.Function; +import java.util.stream.Collectors; import org.eclipse.jetty.client.Request; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -16,121 +25,116 @@ import pl.allegro.tech.hermes.consumers.consumer.sender.resolver.EndpointAddressResolutionException; import pl.allegro.tech.hermes.consumers.consumer.sender.resolver.ResolvableEndpointAddress; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.function.Function; -import java.util.stream.Collectors; - public class JettyBroadCastMessageSender implements MessageSender { - private static final Logger logger = LoggerFactory.getLogger(JettyBroadCastMessageSender.class); - - private final HttpRequestFactory requestFactory; - private final ResolvableEndpointAddress endpoint; - private final HttpHeadersProvider requestHeadersProvider; - private final SendingResultHandlers sendingResultHandlers; - private final Function exceptionMapper = MessageSendingResult::failedResult; - private final ResilientMessageSender resilientMessageSender; - - public JettyBroadCastMessageSender(HttpRequestFactory requestFactory, - ResolvableEndpointAddress endpoint, - HttpHeadersProvider requestHeadersProvider, - SendingResultHandlers sendingResultHandlers, - ResilientMessageSender resilientMessageSender - ) { - this.requestFactory = requestFactory; - this.endpoint = endpoint; - this.requestHeadersProvider = requestHeadersProvider; - this.sendingResultHandlers = sendingResultHandlers; - this.resilientMessageSender = resilientMessageSender; - } - - @Override - public CompletableFuture send(Message message) { - try { - return sendMessage(message).thenApply(MultiMessageSendingResult::new); - } catch (Exception exception) { - return CompletableFuture.completedFuture(exceptionMapper.apply(exception)); - } - } - - private CompletableFuture> sendMessage(Message message) { - try { - Set> results = collectResults(message); - return mergeResults(results); - } catch (EndpointAddressResolutionException exception) { - return CompletableFuture.completedFuture(Collections.singletonList(exceptionMapper.apply(exception))); - } - } - - private Set> collectResults( - Message message - ) throws EndpointAddressResolutionException { - var currentResults = sendPendingMessages(message); - var results = new HashSet<>(currentResults); - - // add previously succeeded uris to the result set so that successful uris from all attempts are retained. - // this way a MessageSendingResult can be considered successful even when the last send attempt - // did not send to any uri, e.g. because all uris returned by endpoint resolver were already sent to in the past. - for (String succeededUri : message.getSucceededUris()) { - try { - var uri = new URI(succeededUri); - var result = MessageSendingResult.succeededResult(uri); - results.add(CompletableFuture.completedFuture(result)); - } catch (URISyntaxException exception) { - logger.error("Error while parsing already sent broadcast URI {}", succeededUri, exception); - } - } - return results; - } - - private Set> sendPendingMessages(Message message) throws EndpointAddressResolutionException { - final HttpRequestData requestData = new HttpRequestDataBuilder() - .withRawAddress(endpoint.getRawAddress()) - .build(); - - HttpRequestHeaders headers = requestHeadersProvider.getHeaders(message, requestData); - - List resolvedUris = endpoint.resolveAllFor(message).stream() - .filter(uri -> message.hasNotBeenSentTo(uri.toString())).toList(); - - if (resolvedUris.isEmpty()) { - logger.debug("Empty resolved URIs for message: {}", message.getId()); - return Collections.emptySet(); - } else { - return resolvedUris.stream() - .map(uri -> requestFactory.buildRequest(message, uri, headers)) - .map(this::processResponse) - .collect(Collectors.toSet()); - } + private static final Logger logger = LoggerFactory.getLogger(JettyBroadCastMessageSender.class); + + private final HttpRequestFactory requestFactory; + private final ResolvableEndpointAddress endpoint; + private final HttpHeadersProvider requestHeadersProvider; + private final SendingResultHandlers sendingResultHandlers; + private final Function exceptionMapper = + MessageSendingResult::failedResult; + private final ResilientMessageSender resilientMessageSender; + + public JettyBroadCastMessageSender( + HttpRequestFactory requestFactory, + ResolvableEndpointAddress endpoint, + HttpHeadersProvider requestHeadersProvider, + SendingResultHandlers sendingResultHandlers, + ResilientMessageSender resilientMessageSender) { + this.requestFactory = requestFactory; + this.endpoint = endpoint; + this.requestHeadersProvider = requestHeadersProvider; + this.sendingResultHandlers = sendingResultHandlers; + this.resilientMessageSender = resilientMessageSender; + } + + @Override + public CompletableFuture send(Message message) { + try { + return sendMessage(message).thenApply(MultiMessageSendingResult::new); + } catch (Exception exception) { + return CompletableFuture.completedFuture(exceptionMapper.apply(exception)); } - - private CompletableFuture> mergeResults(Set> results) { - return CompletableFuture.allOf(results.toArray(new CompletableFuture[results.size()])) - .thenApply(v -> results.stream() - .map(CompletableFuture::join) - .reduce( - ImmutableList.builder(), - (builder, element) -> builder.add(element), - (listA, listB) -> listA.addAll(listB.build()) - ).build()); + } + + private CompletableFuture> sendMessage(Message message) { + try { + Set> results = collectResults(message); + return mergeResults(results); + } catch (EndpointAddressResolutionException exception) { + return CompletableFuture.completedFuture( + Collections.singletonList(exceptionMapper.apply(exception))); } - - private CompletableFuture processResponse(Request request) { - return resilientMessageSender.send( - resultFuture -> request.send(sendingResultHandlers.handleSendingResultForBroadcast(resultFuture)), - exceptionMapper); - + } + + private Set> collectResults(Message message) + throws EndpointAddressResolutionException { + var currentResults = sendPendingMessages(message); + var results = new HashSet<>(currentResults); + + // add previously succeeded uris to the result set so that successful uris from all attempts are + // retained. + // this way a MessageSendingResult can be considered successful even when the last send attempt + // did not send to any uri, e.g. because all uris returned by endpoint resolver were already + // sent to in the past. + for (String succeededUri : message.getSucceededUris()) { + try { + var uri = new URI(succeededUri); + var result = MessageSendingResult.succeededResult(uri); + results.add(CompletableFuture.completedFuture(result)); + } catch (URISyntaxException exception) { + logger.error("Error while parsing already sent broadcast URI {}", succeededUri, exception); + } } - - @Override - public void stop() { + return results; + } + + private Set> sendPendingMessages(Message message) + throws EndpointAddressResolutionException { + final HttpRequestData requestData = + new HttpRequestDataBuilder().withRawAddress(endpoint.getRawAddress()).build(); + + HttpRequestHeaders headers = requestHeadersProvider.getHeaders(message, requestData); + + List resolvedUris = + endpoint.resolveAllFor(message).stream() + .filter(uri -> message.hasNotBeenSentTo(uri.toString())) + .toList(); + + if (resolvedUris.isEmpty()) { + logger.debug("Empty resolved URIs for message: {}", message.getId()); + return Collections.emptySet(); + } else { + return resolvedUris.stream() + .map(uri -> requestFactory.buildRequest(message, uri, headers)) + .map(this::processResponse) + .collect(Collectors.toSet()); } - + } + + private CompletableFuture> mergeResults( + Set> results) { + return CompletableFuture.allOf(results.toArray(new CompletableFuture[results.size()])) + .thenApply( + v -> + results.stream() + .map(CompletableFuture::join) + .reduce( + ImmutableList.builder(), + (builder, element) -> builder.add(element), + (listA, listB) -> listA.addAll(listB.build())) + .build()); + } + + private CompletableFuture processResponse(Request request) { + return resilientMessageSender.send( + resultFuture -> + request.send(sendingResultHandlers.handleSendingResultForBroadcast(resultFuture)), + exceptionMapper); + } + + @Override + public void stop() {} } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyBroadCastResponseListener.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyBroadCastResponseListener.java index a0be25467c..f11233f9bb 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyBroadCastResponseListener.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyBroadCastResponseListener.java @@ -1,23 +1,22 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http; +import java.util.concurrent.CompletableFuture; import org.eclipse.jetty.client.BufferingResponseListener; import org.eclipse.jetty.client.Result; import pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult; import pl.allegro.tech.hermes.consumers.consumer.sender.SingleMessageSendingResult; -import java.util.concurrent.CompletableFuture; - public class JettyBroadCastResponseListener extends BufferingResponseListener { - CompletableFuture resultFuture; - - public JettyBroadCastResponseListener(CompletableFuture resultFuture) { - this.resultFuture = resultFuture; - } + CompletableFuture resultFuture; - @Override - public void onComplete(Result result) { - resultFuture.complete(MessageSendingResult.of(result)); - } + public JettyBroadCastResponseListener( + CompletableFuture resultFuture) { + this.resultFuture = resultFuture; + } -} \ No newline at end of file + @Override + public void onComplete(Result result) { + resultFuture.complete(MessageSendingResult.of(result)); + } +} diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyHttpMessageSenderProvider.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyHttpMessageSenderProvider.java index 809e456c4a..8a0abad58b 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyHttpMessageSenderProvider.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyHttpMessageSenderProvider.java @@ -1,6 +1,9 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http; import com.google.common.collect.ImmutableSet; +import java.util.Collection; +import java.util.Optional; +import java.util.Set; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.Request; import org.slf4j.Logger; @@ -24,149 +27,152 @@ import pl.allegro.tech.hermes.consumers.consumer.sender.resolver.ResolvableEndpointAddress; import pl.allegro.tech.hermes.consumers.consumer.trace.MetadataAppender; -import java.util.Collection; -import java.util.Optional; -import java.util.Set; - public class JettyHttpMessageSenderProvider implements ProtocolMessageSenderProvider { - private static final Logger logger = LoggerFactory.getLogger(JettyHttpMessageSenderProvider.class); - - private static final HttpHeadersProvider http1HeadersProvider = new Http1HeadersProvider(); - private static final HttpHeadersProvider http2HeadersProvider = new Http2HeadersProvider(); - - private final HttpClient httpClient; - private final Http2ClientHolder http2ClientHolder; - private final EndpointAddressResolver endpointAddressResolver; - private final MetadataAppender metadataAppender; - private final HttpAuthorizationProviderFactory authorizationProviderFactory; - private final HttpHeadersProvidersFactory httpHeadersProviderFactory; - private final SendingResultHandlers sendingResultHandlers; - private final HttpRequestFactoryProvider requestFactoryProvider; - private final Set supportedProtocols; - - public JettyHttpMessageSenderProvider( - HttpClient httpClient, - Http2ClientHolder http2ClientHolder, - EndpointAddressResolver endpointAddressResolver, - MetadataAppender metadataAppender, - HttpAuthorizationProviderFactory authorizationProviderFactory, - HttpHeadersProvidersFactory httpHeadersProviderFactory, - SendingResultHandlers sendingResultHandlers, - HttpRequestFactoryProvider requestFactoryProvider, - Set supportedProtocols) { - this.httpClient = httpClient; - this.http2ClientHolder = http2ClientHolder; - this.endpointAddressResolver = endpointAddressResolver; - this.metadataAppender = metadataAppender; - this.authorizationProviderFactory = authorizationProviderFactory; - this.httpHeadersProviderFactory = httpHeadersProviderFactory; - this.sendingResultHandlers = sendingResultHandlers; - this.requestFactoryProvider = requestFactoryProvider; - this.supportedProtocols = supportedProtocols; + private static final Logger logger = + LoggerFactory.getLogger(JettyHttpMessageSenderProvider.class); + + private static final HttpHeadersProvider http1HeadersProvider = new Http1HeadersProvider(); + private static final HttpHeadersProvider http2HeadersProvider = new Http2HeadersProvider(); + + private final HttpClient httpClient; + private final Http2ClientHolder http2ClientHolder; + private final EndpointAddressResolver endpointAddressResolver; + private final MetadataAppender metadataAppender; + private final HttpAuthorizationProviderFactory authorizationProviderFactory; + private final HttpHeadersProvidersFactory httpHeadersProviderFactory; + private final SendingResultHandlers sendingResultHandlers; + private final HttpRequestFactoryProvider requestFactoryProvider; + private final Set supportedProtocols; + + public JettyHttpMessageSenderProvider( + HttpClient httpClient, + Http2ClientHolder http2ClientHolder, + EndpointAddressResolver endpointAddressResolver, + MetadataAppender metadataAppender, + HttpAuthorizationProviderFactory authorizationProviderFactory, + HttpHeadersProvidersFactory httpHeadersProviderFactory, + SendingResultHandlers sendingResultHandlers, + HttpRequestFactoryProvider requestFactoryProvider, + Set supportedProtocols) { + this.httpClient = httpClient; + this.http2ClientHolder = http2ClientHolder; + this.endpointAddressResolver = endpointAddressResolver; + this.metadataAppender = metadataAppender; + this.authorizationProviderFactory = authorizationProviderFactory; + this.httpHeadersProviderFactory = httpHeadersProviderFactory; + this.sendingResultHandlers = sendingResultHandlers; + this.requestFactoryProvider = requestFactoryProvider; + this.supportedProtocols = supportedProtocols; + } + + @Override + public MessageSender create( + Subscription subscription, ResilientMessageSender resilientMessageSender) { + EndpointAddress endpoint = subscription.getEndpoint(); + EndpointAddressResolverMetadata endpointAddressResolverMetadata = + subscription.getEndpointAddressResolverMetadata(); + ResolvableEndpointAddress resolvableEndpoint = + new ResolvableEndpointAddress( + endpoint, endpointAddressResolver, endpointAddressResolverMetadata); + HttpRequestFactory requestFactory = + requestFactoryProvider.provideRequestFactory( + subscription, getHttpClient(subscription), metadataAppender); + + if (subscription.getMode() == SubscriptionMode.BROADCAST) { + return new JettyBroadCastMessageSender( + requestFactory, + resolvableEndpoint, + getHttpRequestHeadersProvider(subscription), + sendingResultHandlers, + resilientMessageSender); + } else { + JettyMessageSender jettyMessageSender = + new JettyMessageSender( + requestFactory, + resolvableEndpoint, + getHttpRequestHeadersProvider(subscription), + sendingResultHandlers); + return new SingleRecipientMessageSenderAdapter(jettyMessageSender, resilientMessageSender); } - - @Override - public MessageSender create(Subscription subscription, ResilientMessageSender resilientMessageSender) { - EndpointAddress endpoint = subscription.getEndpoint(); - EndpointAddressResolverMetadata endpointAddressResolverMetadata = subscription.getEndpointAddressResolverMetadata(); - ResolvableEndpointAddress resolvableEndpoint = - new ResolvableEndpointAddress(endpoint, endpointAddressResolver, endpointAddressResolverMetadata); - HttpRequestFactory requestFactory = - requestFactoryProvider.provideRequestFactory(subscription, getHttpClient(subscription), metadataAppender); - - if (subscription.getMode() == SubscriptionMode.BROADCAST) { - return new JettyBroadCastMessageSender( - requestFactory, - resolvableEndpoint, - getHttpRequestHeadersProvider(subscription), - sendingResultHandlers, - resilientMessageSender); - } else { - JettyMessageSender jettyMessageSender = new JettyMessageSender( - requestFactory, - resolvableEndpoint, - getHttpRequestHeadersProvider(subscription), - sendingResultHandlers); - return new SingleRecipientMessageSenderAdapter(jettyMessageSender, resilientMessageSender); - } + } + + @Override + public Set getSupportedProtocols() { + return supportedProtocols; + } + + private HttpHeadersProvider getHttpRequestHeadersProvider(Subscription subscription) { + AuthHeadersProvider authProvider = getAuthHeadersProvider(subscription); + Collection additionalProviders = httpHeadersProviderFactory.createAll(); + Collection providers = + ImmutableSet.builder() + .addAll(additionalProviders) + .add(authProvider) + .build(); + + return new HermesHeadersProvider(providers); + } + + private AuthHeadersProvider getAuthHeadersProvider(Subscription subscription) { + Optional authorizationProvider = + authorizationProviderFactory.create(subscription); + HttpHeadersProvider httpHeadersProvider = + subscription.isHttp2Enabled() ? http2HeadersProvider : http1HeadersProvider; + + return new AuthHeadersProvider(httpHeadersProvider, authorizationProvider.orElse(null)); + } + + private HttpClient getHttpClient(Subscription subscription) { + if (subscription.isHttp2Enabled()) { + return tryToGetHttp2Client(subscription); + } else { + logger.info("Using http/1.1 for {}.", subscription.getQualifiedName()); + return httpClient; } - - @Override - public Set getSupportedProtocols() { - return supportedProtocols; + } + + private HttpClient tryToGetHttp2Client(Subscription subscription) { + if (http2ClientHolder.getHttp2Client().isPresent()) { + logger.info("Using http/2 for {}.", subscription.getQualifiedName()); + return http2ClientHolder.getHttp2Client().get(); + } else { + logger.info( + "Using http/1.1 for {}. Http/2 delivery is not enabled on this server.", + subscription.getQualifiedName()); + return httpClient; } - - private HttpHeadersProvider getHttpRequestHeadersProvider(Subscription subscription) { - AuthHeadersProvider authProvider = getAuthHeadersProvider(subscription); - Collection additionalProviders = httpHeadersProviderFactory.createAll(); - Collection providers = ImmutableSet.builder() - .addAll(additionalProviders) - .add(authProvider) - .build(); - - return new HermesHeadersProvider(providers); + } + + @Override + public void start() throws Exception { + startClient(httpClient); + http2ClientHolder.getHttp2Client().ifPresent(this::startClient); + } + + private void startClient(HttpClient client) { + if (client.isStopped()) { + try { + client.start(); + } catch (Exception ex) { + logger.error("Could not start http client.", ex); + } } - - private AuthHeadersProvider getAuthHeadersProvider(Subscription subscription) { - Optional authorizationProvider = authorizationProviderFactory.create(subscription); - HttpHeadersProvider httpHeadersProvider = subscription.isHttp2Enabled() ? http2HeadersProvider : http1HeadersProvider; - - return new AuthHeadersProvider( - httpHeadersProvider, - authorizationProvider.orElse(null) - ); - } - - private HttpClient getHttpClient(Subscription subscription) { - if (subscription.isHttp2Enabled()) { - return tryToGetHttp2Client(subscription); - } else { - logger.info("Using http/1.1 for {}.", subscription.getQualifiedName()); - return httpClient; - } - } - - private HttpClient tryToGetHttp2Client(Subscription subscription) { - if (http2ClientHolder.getHttp2Client().isPresent()) { - logger.info("Using http/2 for {}.", subscription.getQualifiedName()); - return http2ClientHolder.getHttp2Client().get(); - } else { - logger.info("Using http/1.1 for {}. Http/2 delivery is not enabled on this server.", - subscription.getQualifiedName()); - return httpClient; - } - } - - @Override - public void start() throws Exception { - startClient(httpClient); - http2ClientHolder.getHttp2Client().ifPresent(this::startClient); - } - - private void startClient(HttpClient client) { - if (client.isStopped()) { - try { - client.start(); - } catch (Exception ex) { - logger.error("Could not start http client.", ex); - } - } - } - - @Override - public void stop() throws Exception { - stopClient(httpClient); - http2ClientHolder.getHttp2Client().ifPresent(this::stopClient); - } - - private void stopClient(HttpClient client) { - if (client.isRunning()) { - try { - client.stop(); - } catch (Exception ex) { - logger.error("Could not stop http client", ex); - } - } + } + + @Override + public void stop() throws Exception { + stopClient(httpClient); + http2ClientHolder.getHttp2Client().ifPresent(this::stopClient); + } + + private void stopClient(HttpClient client) { + if (client.isRunning()) { + try { + client.stop(); + } catch (Exception ex) { + logger.error("Could not stop http client", ex); + } } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyMessageBatchSender.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyMessageBatchSender.java index f9abdf262f..d3565dff15 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyMessageBatchSender.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyMessageBatchSender.java @@ -1,5 +1,18 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http; +import static jakarta.ws.rs.core.HttpHeaders.CONTENT_TYPE; +import static pl.allegro.tech.hermes.api.AvroMediaType.AVRO_BINARY; +import static pl.allegro.tech.hermes.api.ContentType.AVRO; +import static pl.allegro.tech.hermes.common.http.MessageMetadataHeaders.BATCH_ID; +import static pl.allegro.tech.hermes.common.http.MessageMetadataHeaders.RETRY_COUNT; +import static pl.allegro.tech.hermes.common.http.MessageMetadataHeaders.SUBSCRIPTION_NAME; +import static pl.allegro.tech.hermes.common.http.MessageMetadataHeaders.TOPIC_NAME; + +import java.net.URI; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; import org.apache.hc.core5.http.ContentType; import org.eclipse.jetty.client.ContentResponse; import org.eclipse.jetty.client.Request; @@ -15,85 +28,78 @@ import pl.allegro.tech.hermes.consumers.consumer.sender.resolver.EndpointAddressResolutionException; import pl.allegro.tech.hermes.consumers.consumer.sender.resolver.EndpointAddressResolver; -import java.net.URI; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeoutException; - -import static jakarta.ws.rs.core.HttpHeaders.CONTENT_TYPE; -import static pl.allegro.tech.hermes.api.AvroMediaType.AVRO_BINARY; -import static pl.allegro.tech.hermes.api.ContentType.AVRO; -import static pl.allegro.tech.hermes.common.http.MessageMetadataHeaders.BATCH_ID; -import static pl.allegro.tech.hermes.common.http.MessageMetadataHeaders.RETRY_COUNT; -import static pl.allegro.tech.hermes.common.http.MessageMetadataHeaders.SUBSCRIPTION_NAME; -import static pl.allegro.tech.hermes.common.http.MessageMetadataHeaders.TOPIC_NAME; - public class JettyMessageBatchSender implements MessageBatchSender { - private static final Logger logger = LoggerFactory.getLogger(JettyMessageBatchSender.class); + private static final Logger logger = LoggerFactory.getLogger(JettyMessageBatchSender.class); - private final BatchHttpRequestFactory requestFactory; - private final EndpointAddressResolver resolver; - private final SendingResultHandlers resultHandlers; - private final BatchHttpHeadersProvider headersProvider; + private final BatchHttpRequestFactory requestFactory; + private final EndpointAddressResolver resolver; + private final SendingResultHandlers resultHandlers; + private final BatchHttpHeadersProvider headersProvider; - public JettyMessageBatchSender(BatchHttpRequestFactory requestFactory, - EndpointAddressResolver resolver, - SendingResultHandlers resultHandlers, - BatchHttpHeadersProvider headersProvider) { - this.requestFactory = requestFactory; - this.resolver = resolver; - this.resultHandlers = resultHandlers; - this.headersProvider = headersProvider; - } + public JettyMessageBatchSender( + BatchHttpRequestFactory requestFactory, + EndpointAddressResolver resolver, + SendingResultHandlers resultHandlers, + BatchHttpHeadersProvider headersProvider) { + this.requestFactory = requestFactory; + this.resolver = resolver; + this.resultHandlers = resultHandlers; + this.headersProvider = headersProvider; + } - @Override - public MessageSendingResult send(MessageBatch batch, - EndpointAddress address, - EndpointAddressResolverMetadata metadata, - int requestTimeout) { - try { - HttpRequestHeaders headers = headersProvider.getHeaders(address); - return send(batch, resolver.resolve(address, batch, metadata), requestTimeout, headers); - } catch (EndpointAddressResolutionException e) { - return MessageSendingResult.failedResult(e); - } + @Override + public MessageSendingResult send( + MessageBatch batch, + EndpointAddress address, + EndpointAddressResolverMetadata metadata, + int requestTimeout) { + try { + HttpRequestHeaders headers = headersProvider.getHeaders(address); + return send(batch, resolver.resolve(address, batch, metadata), requestTimeout, headers); + } catch (EndpointAddressResolutionException e) { + return MessageSendingResult.failedResult(e); } + } - private MessageSendingResult send(MessageBatch batch, URI address, int requestTimeout, HttpRequestHeaders baseHeaders) { - HttpRequestHeaders headers = buildHeaders(batch, baseHeaders); - Request request = requestFactory.buildRequest(batch, address, headers, requestTimeout); - try { - ContentResponse response = request.send(); - return resultHandlers.handleSendingResultForBatch(response); - } catch (TimeoutException | ExecutionException | InterruptedException e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - logger.info("Restoring interrupted status", e); - } - throw new HttpBatchSenderException("Failed to send message batch", e); - } + private MessageSendingResult send( + MessageBatch batch, URI address, int requestTimeout, HttpRequestHeaders baseHeaders) { + HttpRequestHeaders headers = buildHeaders(batch, baseHeaders); + Request request = requestFactory.buildRequest(batch, address, headers, requestTimeout); + try { + ContentResponse response = request.send(); + return resultHandlers.handleSendingResultForBatch(response); + } catch (TimeoutException | ExecutionException | InterruptedException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + logger.info("Restoring interrupted status", e); + } + throw new HttpBatchSenderException("Failed to send message batch", e); } + } - private HttpRequestHeaders buildHeaders(MessageBatch batch, HttpRequestHeaders baseHeaders) { - Map headers = new HashMap<>(baseHeaders.asMap()); + private HttpRequestHeaders buildHeaders(MessageBatch batch, HttpRequestHeaders baseHeaders) { + Map headers = new HashMap<>(baseHeaders.asMap()); - ContentType contentType = getMediaType(batch.getContentType()); - headers.put(BATCH_ID.getName(), batch.getId()); - headers.put(CONTENT_TYPE, contentType.getMimeType()); - headers.put(RETRY_COUNT.getName(), Integer.toString(batch.getRetryCounter())); + ContentType contentType = getMediaType(batch.getContentType()); + headers.put(BATCH_ID.getName(), batch.getId()); + headers.put(CONTENT_TYPE, contentType.getMimeType()); + headers.put(RETRY_COUNT.getName(), Integer.toString(batch.getRetryCounter())); - if (batch.hasSubscriptionIdentityHeaders()) { - headers.put(TOPIC_NAME.getName(), batch.getTopic()); - headers.put(SUBSCRIPTION_NAME.getName(), batch.getSubscription().getName()); - } - - batch.getAdditionalHeaders().forEach(header -> headers.put(header.getName(), header.getValue())); - return new HttpRequestHeaders(headers); + if (batch.hasSubscriptionIdentityHeaders()) { + headers.put(TOPIC_NAME.getName(), batch.getTopic()); + headers.put(SUBSCRIPTION_NAME.getName(), batch.getSubscription().getName()); } - private ContentType getMediaType(pl.allegro.tech.hermes.api.ContentType contentType) { - return AVRO.equals(contentType) ? ContentType.create(AVRO_BINARY) : ContentType.APPLICATION_JSON; - } + batch + .getAdditionalHeaders() + .forEach(header -> headers.put(header.getName(), header.getValue())); + return new HttpRequestHeaders(headers); + } + + private ContentType getMediaType(pl.allegro.tech.hermes.api.ContentType contentType) { + return AVRO.equals(contentType) + ? ContentType.create(AVRO_BINARY) + : ContentType.APPLICATION_JSON; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyMessageSender.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyMessageSender.java index dc2690af06..05e47d988f 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyMessageSender.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyMessageSender.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http; +import java.net.URI; +import java.util.concurrent.CompletableFuture; import org.eclipse.jetty.client.Request; import pl.allegro.tech.hermes.consumers.consumer.Message; import pl.allegro.tech.hermes.consumers.consumer.sender.CompletableFutureAwareMessageSender; @@ -10,45 +12,41 @@ import pl.allegro.tech.hermes.consumers.consumer.sender.resolver.EndpointAddressResolutionException; import pl.allegro.tech.hermes.consumers.consumer.sender.resolver.ResolvableEndpointAddress; -import java.net.URI; -import java.util.concurrent.CompletableFuture; - public class JettyMessageSender implements CompletableFutureAwareMessageSender { - private final HttpRequestFactory requestFactory; - private final ResolvableEndpointAddress addressResolver; - private final HttpHeadersProvider requestHeadersProvider; - private final SendingResultHandlers sendingResultHandlers; - - public JettyMessageSender(HttpRequestFactory requestFactory, - ResolvableEndpointAddress addressResolver, - HttpHeadersProvider headersProvider, - SendingResultHandlers sendingResultHandlers) { - this.requestFactory = requestFactory; - this.addressResolver = addressResolver; - this.requestHeadersProvider = headersProvider; - this.sendingResultHandlers = sendingResultHandlers; + private final HttpRequestFactory requestFactory; + private final ResolvableEndpointAddress addressResolver; + private final HttpHeadersProvider requestHeadersProvider; + private final SendingResultHandlers sendingResultHandlers; + + public JettyMessageSender( + HttpRequestFactory requestFactory, + ResolvableEndpointAddress addressResolver, + HttpHeadersProvider headersProvider, + SendingResultHandlers sendingResultHandlers) { + this.requestFactory = requestFactory; + this.addressResolver = addressResolver; + this.requestHeadersProvider = headersProvider; + this.sendingResultHandlers = sendingResultHandlers; + } + + @Override + public void send(Message message, final CompletableFuture resultFuture) { + try { + final HttpRequestData requestData = + new HttpRequestDataBuilder().withRawAddress(addressResolver.getRawAddress()).build(); + + HttpRequestHeaders headers = requestHeadersProvider.getHeaders(message, requestData); + + URI resolvedUri = addressResolver.resolveFor(message); + Request request = requestFactory.buildRequest(message, resolvedUri, headers); + + request.send(sendingResultHandlers.handleSendingResultForSerial(resultFuture)); + } catch (EndpointAddressResolutionException exception) { + resultFuture.complete(MessageSendingResult.failedResult(exception)); } + } - @Override - public void send(Message message, final CompletableFuture resultFuture) { - try { - final HttpRequestData requestData = new HttpRequestDataBuilder() - .withRawAddress(addressResolver.getRawAddress()) - .build(); - - HttpRequestHeaders headers = requestHeadersProvider.getHeaders(message, requestData); - - URI resolvedUri = addressResolver.resolveFor(message); - Request request = requestFactory.buildRequest(message, resolvedUri, headers); - - request.send(sendingResultHandlers.handleSendingResultForSerial(resultFuture)); - } catch (EndpointAddressResolutionException exception) { - resultFuture.complete(MessageSendingResult.failedResult(exception)); - } - } - - @Override - public void stop() { - } + @Override + public void stop() {} } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyResponseListener.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyResponseListener.java index fd6d7bb6d4..eb725b2cff 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyResponseListener.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyResponseListener.java @@ -1,22 +1,20 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http; +import java.util.concurrent.CompletableFuture; import org.eclipse.jetty.client.BufferingResponseListener; import org.eclipse.jetty.client.Result; import pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult; -import java.util.concurrent.CompletableFuture; - public class JettyResponseListener extends BufferingResponseListener { - CompletableFuture resultFuture; - - public JettyResponseListener(CompletableFuture resultFuture) { - this.resultFuture = resultFuture; - } + CompletableFuture resultFuture; - @Override - public void onComplete(Result result) { - resultFuture.complete(MessageSendingResult.of(result)); - } + public JettyResponseListener(CompletableFuture resultFuture) { + this.resultFuture = resultFuture; + } + @Override + public void onComplete(Result result) { + resultFuture.complete(MessageSendingResult.of(result)); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/SendingResultHandlers.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/SendingResultHandlers.java index 80fdc90f59..3fb38d478d 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/SendingResultHandlers.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/SendingResultHandlers.java @@ -1,16 +1,17 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http; +import java.util.concurrent.CompletableFuture; import org.eclipse.jetty.client.ContentResponse; import org.eclipse.jetty.client.Response.CompleteListener; import pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult; import pl.allegro.tech.hermes.consumers.consumer.sender.SingleMessageSendingResult; -import java.util.concurrent.CompletableFuture; - public interface SendingResultHandlers { - CompleteListener handleSendingResultForSerial(CompletableFuture resultFuture); + CompleteListener handleSendingResultForSerial( + CompletableFuture resultFuture); - CompleteListener handleSendingResultForBroadcast(CompletableFuture resultFuture); + CompleteListener handleSendingResultForBroadcast( + CompletableFuture resultFuture); - MessageSendingResult handleSendingResultForBatch(ContentResponse response); + MessageSendingResult handleSendingResultForBatch(ContentResponse response); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/SslContextFactoryProvider.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/SslContextFactoryProvider.java index 1b8f22b92a..88d67b4b22 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/SslContextFactoryProvider.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/SslContextFactoryProvider.java @@ -1,5 +1,9 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http; +import static pl.allegro.tech.hermes.common.ssl.KeystoreSource.JRE; +import static pl.allegro.tech.hermes.common.ssl.KeystoreSource.PROVIDED; + +import java.util.Optional; import pl.allegro.tech.hermes.common.ssl.DefaultSslContextFactory; import pl.allegro.tech.hermes.common.ssl.KeyManagersProvider; import pl.allegro.tech.hermes.common.ssl.KeystoreConfigurationException; @@ -12,73 +16,70 @@ import pl.allegro.tech.hermes.common.ssl.provided.ProvidedKeyManagersProvider; import pl.allegro.tech.hermes.common.ssl.provided.ProvidedTrustManagersProvider; -import java.util.Optional; - -import static pl.allegro.tech.hermes.common.ssl.KeystoreSource.JRE; -import static pl.allegro.tech.hermes.common.ssl.KeystoreSource.PROVIDED; - public class SslContextFactoryProvider { - private final SslContextFactory sslContextFactory; + private final SslContextFactory sslContextFactory; - private final SslContextParameters sslContextParams; + private final SslContextParameters sslContextParams; - public SslContextFactoryProvider(SslContextFactory sslContextFactory, SslContextParameters sslContextParams) { - this.sslContextFactory = sslContextFactory; - this.sslContextParams = sslContextParams; - } + public SslContextFactoryProvider( + SslContextFactory sslContextFactory, SslContextParameters sslContextParams) { + this.sslContextFactory = sslContextFactory; + this.sslContextParams = sslContextParams; + } - public Optional provideSslContextFactory() { - if (sslContextParams.isEnabled()) { - org.eclipse.jetty.util.ssl.SslContextFactory.Client sslCtx = new org.eclipse.jetty.util.ssl.SslContextFactory.Client(); - sslCtx.setEndpointIdentificationAlgorithm("HTTPS"); - sslCtx.setSslContext(sslContextFactory().create().getSslContext()); - return Optional.of(sslCtx); - } else { - return Optional.empty(); - } + public Optional provideSslContextFactory() { + if (sslContextParams.isEnabled()) { + org.eclipse.jetty.util.ssl.SslContextFactory.Client sslCtx = + new org.eclipse.jetty.util.ssl.SslContextFactory.Client(); + sslCtx.setEndpointIdentificationAlgorithm("HTTPS"); + sslCtx.setSslContext(sslContextFactory().create().getSslContext()); + return Optional.of(sslCtx); + } else { + return Optional.empty(); } + } - private SslContextFactory sslContextFactory() { - return Optional.ofNullable(sslContextFactory).orElseGet(this::defaultSslContextFactory); - } + private SslContextFactory sslContextFactory() { + return Optional.ofNullable(sslContextFactory).orElseGet(this::defaultSslContextFactory); + } - private SslContextFactory defaultSslContextFactory() { - String protocol = sslContextParams.getProtocol(); - KeyManagersProvider keyManagersProvider = createKeyManagersProvider(); - TrustManagersProvider trustManagersProvider = createTrustManagersProvider(); - return new DefaultSslContextFactory(protocol, keyManagersProvider, trustManagersProvider); - } + private SslContextFactory defaultSslContextFactory() { + String protocol = sslContextParams.getProtocol(); + KeyManagersProvider keyManagersProvider = createKeyManagersProvider(); + TrustManagersProvider trustManagersProvider = createTrustManagersProvider(); + return new DefaultSslContextFactory(protocol, keyManagersProvider, trustManagersProvider); + } - private KeyManagersProvider createKeyManagersProvider() { - String keystoreSource = sslContextParams.getKeystoreSource(); - if (PROVIDED.getValue().equals(keystoreSource)) { - KeystoreProperties properties = new KeystoreProperties( - sslContextParams.getKeystoreLocation(), - sslContextParams.getKeystoreFormat(), - sslContextParams.getKeystorePassword() - ); - return new ProvidedKeyManagersProvider(properties); - } - if (JRE.getValue().equals(keystoreSource)) { - return new JvmKeyManagersProvider(); - } - throw new KeystoreConfigurationException(keystoreSource); + private KeyManagersProvider createKeyManagersProvider() { + String keystoreSource = sslContextParams.getKeystoreSource(); + if (PROVIDED.getValue().equals(keystoreSource)) { + KeystoreProperties properties = + new KeystoreProperties( + sslContextParams.getKeystoreLocation(), + sslContextParams.getKeystoreFormat(), + sslContextParams.getKeystorePassword()); + return new ProvidedKeyManagersProvider(properties); + } + if (JRE.getValue().equals(keystoreSource)) { + return new JvmKeyManagersProvider(); } + throw new KeystoreConfigurationException(keystoreSource); + } - public TrustManagersProvider createTrustManagersProvider() { - String truststoreSource = sslContextParams.getTruststoreSource(); - if (PROVIDED.getValue().equals(truststoreSource)) { - KeystoreProperties properties = new KeystoreProperties( - sslContextParams.getTruststoreLocation(), - sslContextParams.getTruststoreFormat(), - sslContextParams.getTruststorePassword() - ); - return new ProvidedTrustManagersProvider(properties); - } - if (JRE.getValue().equals(truststoreSource)) { - return new JvmTrustManagerProvider(); - } - throw new TruststoreConfigurationException(truststoreSource); + public TrustManagersProvider createTrustManagersProvider() { + String truststoreSource = sslContextParams.getTruststoreSource(); + if (PROVIDED.getValue().equals(truststoreSource)) { + KeystoreProperties properties = + new KeystoreProperties( + sslContextParams.getTruststoreLocation(), + sslContextParams.getTruststoreFormat(), + sslContextParams.getTruststorePassword()); + return new ProvidedTrustManagersProvider(properties); + } + if (JRE.getValue().equals(truststoreSource)) { + return new JvmTrustManagerProvider(); } + throw new TruststoreConfigurationException(truststoreSource); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/SslContextParameters.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/SslContextParameters.java index 25d0044a1e..96c7ed79eb 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/SslContextParameters.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/SslContextParameters.java @@ -2,23 +2,23 @@ public interface SslContextParameters { - boolean isEnabled(); + boolean isEnabled(); - String getProtocol(); + String getProtocol(); - String getKeystoreSource(); + String getKeystoreSource(); - String getKeystoreLocation(); + String getKeystoreLocation(); - String getKeystorePassword(); + String getKeystorePassword(); - String getKeystoreFormat(); + String getKeystoreFormat(); - String getTruststoreSource(); + String getTruststoreSource(); - String getTruststoreLocation(); + String getTruststoreLocation(); - String getTruststorePassword(); + String getTruststorePassword(); - String getTruststoreFormat(); + String getTruststoreFormat(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/auth/BasicAuthProvider.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/auth/BasicAuthProvider.java index ddde76854d..b9708f52cf 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/auth/BasicAuthProvider.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/auth/BasicAuthProvider.java @@ -1,23 +1,23 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http.auth; -import pl.allegro.tech.hermes.api.EndpointAddress; - import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.Optional; +import pl.allegro.tech.hermes.api.EndpointAddress; public class BasicAuthProvider implements HttpAuthorizationProvider { - private final String token; + private final String token; - public BasicAuthProvider(EndpointAddress endpoint) { - String credentials = endpoint.getUsername() + ":" + endpoint.getPassword(); - String encodedCredentials = Base64.getEncoder().encodeToString(credentials.getBytes(StandardCharsets.UTF_8)); - this.token = "Basic " + encodedCredentials; - } + public BasicAuthProvider(EndpointAddress endpoint) { + String credentials = endpoint.getUsername() + ":" + endpoint.getPassword(); + String encodedCredentials = + Base64.getEncoder().encodeToString(credentials.getBytes(StandardCharsets.UTF_8)); + this.token = "Basic " + encodedCredentials; + } - @Override - public Optional authorizationToken() { - return Optional.of(token); - } + @Override + public Optional authorizationToken() { + return Optional.of(token); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/auth/HttpAuthorizationProvider.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/auth/HttpAuthorizationProvider.java index af1b119397..a6e58ab1f9 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/auth/HttpAuthorizationProvider.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/auth/HttpAuthorizationProvider.java @@ -4,5 +4,5 @@ public interface HttpAuthorizationProvider { - Optional authorizationToken(); + Optional authorizationToken(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/auth/HttpAuthorizationProviderFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/auth/HttpAuthorizationProviderFactory.java index 010d1f2980..31d334e686 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/auth/HttpAuthorizationProviderFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/auth/HttpAuthorizationProviderFactory.java @@ -1,25 +1,24 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http.auth; +import java.util.Optional; import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.consumers.consumer.oauth.OAuthAccessTokens; -import java.util.Optional; - public class HttpAuthorizationProviderFactory { - private final OAuthAccessTokens accessTokens; + private final OAuthAccessTokens accessTokens; - public HttpAuthorizationProviderFactory(OAuthAccessTokens accessTokens) { - this.accessTokens = accessTokens; - } + public HttpAuthorizationProviderFactory(OAuthAccessTokens accessTokens) { + this.accessTokens = accessTokens; + } - public Optional create(Subscription subscription) { - if (subscription.getEndpoint().containsCredentials()) { - return Optional.of(new BasicAuthProvider(subscription.getEndpoint())); - } else if (subscription.hasOAuthPolicy()) { - return Optional.of(new OAuthHttpAuthorizationProvider(subscription.getQualifiedName(), accessTokens)); - } - return Optional.empty(); + public Optional create(Subscription subscription) { + if (subscription.getEndpoint().containsCredentials()) { + return Optional.of(new BasicAuthProvider(subscription.getEndpoint())); + } else if (subscription.hasOAuthPolicy()) { + return Optional.of( + new OAuthHttpAuthorizationProvider(subscription.getQualifiedName(), accessTokens)); } - + return Optional.empty(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/auth/OAuthHttpAuthorizationProvider.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/auth/OAuthHttpAuthorizationProvider.java index 727a024d4f..7ace474330 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/auth/OAuthHttpAuthorizationProvider.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/auth/OAuthHttpAuthorizationProvider.java @@ -1,28 +1,29 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http.auth; +import java.util.Optional; import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.consumers.consumer.oauth.OAuthAccessToken; import pl.allegro.tech.hermes.consumers.consumer.oauth.OAuthAccessTokens; -import java.util.Optional; - public class OAuthHttpAuthorizationProvider implements HttpAuthorizationProvider { - private static final String BEARER_TOKEN_PREFIX = "Bearer "; + private static final String BEARER_TOKEN_PREFIX = "Bearer "; - private final SubscriptionName subscriptionName; + private final SubscriptionName subscriptionName; - private final OAuthAccessTokens accessTokens; + private final OAuthAccessTokens accessTokens; - public OAuthHttpAuthorizationProvider(SubscriptionName subscriptionName, OAuthAccessTokens accessTokens) { - this.subscriptionName = subscriptionName; - this.accessTokens = accessTokens; - } + public OAuthHttpAuthorizationProvider( + SubscriptionName subscriptionName, OAuthAccessTokens accessTokens) { + this.subscriptionName = subscriptionName; + this.accessTokens = accessTokens; + } - @Override - public Optional authorizationToken() { - return accessTokens.getTokenIfPresent(subscriptionName) - .map(OAuthAccessToken::getTokenValue) - .map(value -> BEARER_TOKEN_PREFIX + value); - } + @Override + public Optional authorizationToken() { + return accessTokens + .getTokenIfPresent(subscriptionName) + .map(OAuthAccessToken::getTokenValue) + .map(value -> BEARER_TOKEN_PREFIX + value); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/AuthHeadersProvider.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/AuthHeadersProvider.java index fcb7cf854e..833c7a4e93 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/AuthHeadersProvider.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/AuthHeadersProvider.java @@ -1,37 +1,36 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http.headers; import com.google.common.collect.ImmutableMap; +import java.util.Optional; import org.eclipse.jetty.http.HttpHeader; import pl.allegro.tech.hermes.consumers.consumer.Message; import pl.allegro.tech.hermes.consumers.consumer.sender.http.HttpRequestData; import pl.allegro.tech.hermes.consumers.consumer.sender.http.auth.HttpAuthorizationProvider; -import java.util.Optional; - public final class AuthHeadersProvider implements HttpHeadersProvider { - private final HttpHeadersProvider headersProvider; - private final HttpAuthorizationProvider authorizationProvider; + private final HttpHeadersProvider headersProvider; + private final HttpAuthorizationProvider authorizationProvider; - public AuthHeadersProvider(HttpHeadersProvider headersProvider, HttpAuthorizationProvider authorizationProvider) { - this.headersProvider = headersProvider; - this.authorizationProvider = authorizationProvider; - } - - @Override - public HttpRequestHeaders getHeaders(Message message, HttpRequestData requestData) { - ImmutableMap.Builder builder = ImmutableMap.builder(); + public AuthHeadersProvider( + HttpHeadersProvider headersProvider, HttpAuthorizationProvider authorizationProvider) { + this.headersProvider = headersProvider; + this.authorizationProvider = authorizationProvider; + } - if (headersProvider != null) { - builder.putAll(headersProvider.getHeaders(message, requestData).asMap()); - } + @Override + public HttpRequestHeaders getHeaders(Message message, HttpRequestData requestData) { + ImmutableMap.Builder builder = ImmutableMap.builder(); - if (authorizationProvider != null) { - Optional token = authorizationProvider.authorizationToken(); - token.ifPresent(s -> builder.put(HttpHeader.AUTHORIZATION.toString(), s)); - } + if (headersProvider != null) { + builder.putAll(headersProvider.getHeaders(message, requestData).asMap()); + } - return new HttpRequestHeaders(builder.build()); + if (authorizationProvider != null) { + Optional token = authorizationProvider.authorizationToken(); + token.ifPresent(s -> builder.put(HttpHeader.AUTHORIZATION.toString(), s)); } + return new HttpRequestHeaders(builder.build()); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/BatchHttpHeadersProvider.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/BatchHttpHeadersProvider.java index d8a9e19022..220dae47f8 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/BatchHttpHeadersProvider.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/BatchHttpHeadersProvider.java @@ -3,5 +3,5 @@ import pl.allegro.tech.hermes.api.EndpointAddress; public interface BatchHttpHeadersProvider { - HttpRequestHeaders getHeaders(EndpointAddress address); + HttpRequestHeaders getHeaders(EndpointAddress address); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/DefaultBatchHeadersProvider.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/DefaultBatchHeadersProvider.java index d655695d60..dd7bbdbed0 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/DefaultBatchHeadersProvider.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/DefaultBatchHeadersProvider.java @@ -1,13 +1,12 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http.headers; -import pl.allegro.tech.hermes.api.EndpointAddress; - import java.util.Collections; +import pl.allegro.tech.hermes.api.EndpointAddress; public final class DefaultBatchHeadersProvider implements BatchHttpHeadersProvider { - @Override - public HttpRequestHeaders getHeaders(EndpointAddress address) { - return new HttpRequestHeaders(Collections.emptyMap()); - } + @Override + public HttpRequestHeaders getHeaders(EndpointAddress address) { + return new HttpRequestHeaders(Collections.emptyMap()); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/HermesHeadersProvider.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/HermesHeadersProvider.java index 77445fe955..322e130e1a 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/HermesHeadersProvider.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/HermesHeadersProvider.java @@ -1,11 +1,5 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http.headers; -import com.google.common.collect.ImmutableMap; -import pl.allegro.tech.hermes.consumers.consumer.Message; -import pl.allegro.tech.hermes.consumers.consumer.sender.http.HttpRequestData; - -import java.util.Collection; - import static java.lang.String.valueOf; import static pl.allegro.tech.hermes.common.http.MessageMetadataHeaders.MESSAGE_ID; import static pl.allegro.tech.hermes.common.http.MessageMetadataHeaders.RETRY_COUNT; @@ -14,33 +8,45 @@ import static pl.allegro.tech.hermes.common.http.MessageMetadataHeaders.SUBSCRIPTION_NAME; import static pl.allegro.tech.hermes.common.http.MessageMetadataHeaders.TOPIC_NAME; -public final class HermesHeadersProvider implements HttpHeadersProvider { - - private final Collection headersProvider; +import com.google.common.collect.ImmutableMap; +import java.util.Collection; +import pl.allegro.tech.hermes.consumers.consumer.Message; +import pl.allegro.tech.hermes.consumers.consumer.sender.http.HttpRequestData; - public HermesHeadersProvider(Collection headersProvider) { - this.headersProvider = headersProvider; - } +public final class HermesHeadersProvider implements HttpHeadersProvider { - @Override - public HttpRequestHeaders getHeaders(Message message, HttpRequestData requestData) { - ImmutableMap.Builder builder = ImmutableMap.builder(); + private final Collection headersProvider; - headersProvider.forEach(provider -> builder.putAll(provider.getHeaders(message, requestData).asMap())); + public HermesHeadersProvider(Collection headersProvider) { + this.headersProvider = headersProvider; + } - builder.put(MESSAGE_ID.getName(), message.getId()); - builder.put(RETRY_COUNT.getName(), Integer.toString(message.getRetryCounter())); + @Override + public HttpRequestHeaders getHeaders(Message message, HttpRequestData requestData) { + ImmutableMap.Builder builder = ImmutableMap.builder(); - if (message.hasSubscriptionIdentityHeaders()) { - builder.put(TOPIC_NAME.getName(), message.getTopic()); - builder.put(SUBSCRIPTION_NAME.getName(), message.getSubscription()); - } + headersProvider.forEach( + provider -> builder.putAll(provider.getHeaders(message, requestData).asMap())); - message.getSchema().ifPresent(schema -> builder.put(SCHEMA_VERSION.getName(), valueOf(schema.getVersion().value()))); - message.getSchema().ifPresent(schema -> builder.put(SCHEMA_ID.getName(), valueOf(schema.getId().value()))); - message.getAdditionalHeaders().forEach(header -> builder.put(header.getName(), header.getValue())); + builder.put(MESSAGE_ID.getName(), message.getId()); + builder.put(RETRY_COUNT.getName(), Integer.toString(message.getRetryCounter())); - return new HttpRequestHeaders(builder.build()); + if (message.hasSubscriptionIdentityHeaders()) { + builder.put(TOPIC_NAME.getName(), message.getTopic()); + builder.put(SUBSCRIPTION_NAME.getName(), message.getSubscription()); } + message + .getSchema() + .ifPresent( + schema -> builder.put(SCHEMA_VERSION.getName(), valueOf(schema.getVersion().value()))); + message + .getSchema() + .ifPresent(schema -> builder.put(SCHEMA_ID.getName(), valueOf(schema.getId().value()))); + message + .getAdditionalHeaders() + .forEach(header -> builder.put(header.getName(), header.getValue())); + + return new HttpRequestHeaders(builder.build()); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/Http1HeadersProvider.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/Http1HeadersProvider.java index a78502454b..f6441b58e6 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/Http1HeadersProvider.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/Http1HeadersProvider.java @@ -1,30 +1,28 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http.headers; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; +import static pl.allegro.tech.hermes.api.AvroMediaType.AVRO_BINARY; +import static pl.allegro.tech.hermes.api.ContentType.AVRO; + import com.google.common.collect.ImmutableMap; +import java.util.function.Function; import org.eclipse.jetty.http.HttpHeader; import pl.allegro.tech.hermes.api.ContentType; import pl.allegro.tech.hermes.consumers.consumer.Message; import pl.allegro.tech.hermes.consumers.consumer.sender.http.HttpRequestData; -import java.util.function.Function; - -import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; -import static pl.allegro.tech.hermes.api.AvroMediaType.AVRO_BINARY; -import static pl.allegro.tech.hermes.api.ContentType.AVRO; - public final class Http1HeadersProvider implements HttpHeadersProvider { - private static final Function contentTypeToMediaType = contentType -> - AVRO.equals(contentType) ? AVRO_BINARY : APPLICATION_JSON; - - @Override - public HttpRequestHeaders getHeaders(Message message, HttpRequestData requestData) { - return new HttpRequestHeaders( - ImmutableMap.of( - HttpHeader.CONTENT_TYPE.toString(), contentTypeToMediaType.apply(message.getContentType()), - HttpHeader.KEEP_ALIVE.toString(), "true" - ) - ); - } + private static final Function contentTypeToMediaType = + contentType -> AVRO.equals(contentType) ? AVRO_BINARY : APPLICATION_JSON; + @Override + public HttpRequestHeaders getHeaders(Message message, HttpRequestData requestData) { + return new HttpRequestHeaders( + ImmutableMap.of( + HttpHeader.CONTENT_TYPE.toString(), + contentTypeToMediaType.apply(message.getContentType()), + HttpHeader.KEEP_ALIVE.toString(), + "true")); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/Http2HeadersProvider.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/Http2HeadersProvider.java index 4d61a75184..d4c34bb75a 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/Http2HeadersProvider.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/Http2HeadersProvider.java @@ -1,27 +1,26 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http.headers; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; +import static pl.allegro.tech.hermes.api.AvroMediaType.AVRO_BINARY; +import static pl.allegro.tech.hermes.api.ContentType.AVRO; + import com.google.common.collect.ImmutableMap; +import java.util.function.Function; import org.eclipse.jetty.http.HttpHeader; import pl.allegro.tech.hermes.api.ContentType; import pl.allegro.tech.hermes.consumers.consumer.Message; import pl.allegro.tech.hermes.consumers.consumer.sender.http.HttpRequestData; -import java.util.function.Function; - -import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; -import static pl.allegro.tech.hermes.api.AvroMediaType.AVRO_BINARY; -import static pl.allegro.tech.hermes.api.ContentType.AVRO; - public final class Http2HeadersProvider implements HttpHeadersProvider { - private static final Function contentTypeToMediaType = contentType -> - AVRO.equals(contentType) ? AVRO_BINARY : APPLICATION_JSON; - - @Override - public HttpRequestHeaders getHeaders(Message message, HttpRequestData requestData) { - return new HttpRequestHeaders( - ImmutableMap.of(HttpHeader.CONTENT_TYPE.toString(), contentTypeToMediaType.apply(message.getContentType())) - ); - } + private static final Function contentTypeToMediaType = + contentType -> AVRO.equals(contentType) ? AVRO_BINARY : APPLICATION_JSON; + @Override + public HttpRequestHeaders getHeaders(Message message, HttpRequestData requestData) { + return new HttpRequestHeaders( + ImmutableMap.of( + HttpHeader.CONTENT_TYPE.toString(), + contentTypeToMediaType.apply(message.getContentType()))); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/HttpHeadersProvider.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/HttpHeadersProvider.java index cfc5107662..380c26e3a9 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/HttpHeadersProvider.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/HttpHeadersProvider.java @@ -5,6 +5,5 @@ public interface HttpHeadersProvider { - HttpRequestHeaders getHeaders(Message message, HttpRequestData requestData); - + HttpRequestHeaders getHeaders(Message message, HttpRequestData requestData); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/HttpRequestHeaders.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/HttpRequestHeaders.java index c0c625b7d9..9195e5b7c2 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/HttpRequestHeaders.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/headers/HttpRequestHeaders.java @@ -1,19 +1,17 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http.headers; import com.google.common.collect.ImmutableMap; - import java.util.Map; public final class HttpRequestHeaders { - private final Map headers; - - public HttpRequestHeaders(Map headers) { - this.headers = headers; - } + private final Map headers; - public Map asMap() { - return ImmutableMap.copyOf(headers); - } + public HttpRequestHeaders(Map headers) { + this.headers = headers; + } + public Map asMap() { + return ImmutableMap.copyOf(headers); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/jms/AbstractJmsMessageSenderProvider.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/jms/AbstractJmsMessageSenderProvider.java index 70a0cc45fd..04279d7765 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/jms/AbstractJmsMessageSenderProvider.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/jms/AbstractJmsMessageSenderProvider.java @@ -3,6 +3,11 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; +import java.net.URI; +import java.util.concurrent.ExecutionException; +import javax.jms.ConnectionFactory; +import javax.jms.JMSContext; +import javax.jms.Message; import pl.allegro.tech.hermes.api.EndpointAddress; import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.common.exception.InternalProcessingException; @@ -12,63 +17,56 @@ import pl.allegro.tech.hermes.consumers.consumer.trace.MetadataAppender; import pl.allegro.tech.hermes.consumers.uri.UriUtils; -import java.net.URI; -import java.util.concurrent.ExecutionException; -import javax.jms.ConnectionFactory; -import javax.jms.JMSContext; -import javax.jms.Message; - public abstract class AbstractJmsMessageSenderProvider implements JmsMessageSenderProvider { - protected final LoadingCache connectionFactoryCache; - protected final MetadataAppender metadataAppender; + protected final LoadingCache connectionFactoryCache; + protected final MetadataAppender metadataAppender; - public AbstractJmsMessageSenderProvider(MetadataAppender metadataAppender) { - this.connectionFactoryCache = CacheBuilder.newBuilder().build(new ConnectionFactoryLoader()); - this.metadataAppender = metadataAppender; - } + public AbstractJmsMessageSenderProvider(MetadataAppender metadataAppender) { + this.connectionFactoryCache = CacheBuilder.newBuilder().build(new ConnectionFactoryLoader()); + this.metadataAppender = metadataAppender; + } - @Override - public MessageSender create(Subscription subscription, ResilientMessageSender resilientMessageSender) { - EndpointAddress endpoint = subscription.getEndpoint(); - URI uri = endpoint.getUri(); - ConnectionFactory connectionFactory = getConnectionFactory(uri); - JMSContext jmsContext = connectionFactory.createContext( - endpoint.getUsername(), - endpoint.getPassword() - ); + @Override + public MessageSender create( + Subscription subscription, ResilientMessageSender resilientMessageSender) { + EndpointAddress endpoint = subscription.getEndpoint(); + URI uri = endpoint.getUri(); + ConnectionFactory connectionFactory = getConnectionFactory(uri); + JMSContext jmsContext = + connectionFactory.createContext(endpoint.getUsername(), endpoint.getPassword()); - JmsMessageSender jmsMessageSender = new JmsMessageSender(jmsContext, extractTopicName(uri), metadataAppender); - return new SingleRecipientMessageSenderAdapter(jmsMessageSender, resilientMessageSender); - } + JmsMessageSender jmsMessageSender = + new JmsMessageSender(jmsContext, extractTopicName(uri), metadataAppender); + return new SingleRecipientMessageSenderAdapter(jmsMessageSender, resilientMessageSender); + } - @Override - public void start() throws Exception { - } + @Override + public void start() throws Exception {} - @Override - public void stop() throws Exception { - connectionFactoryCache.invalidateAll(); - } + @Override + public void stop() throws Exception { + connectionFactoryCache.invalidateAll(); + } - private ConnectionFactory getConnectionFactory(URI serverUri) { - try { - return connectionFactoryCache.get(serverUri); - } catch (ExecutionException e) { - throw new InternalProcessingException( - String.format("Unable to create connection factory for url %s", serverUri), e); - } + private ConnectionFactory getConnectionFactory(URI serverUri) { + try { + return connectionFactoryCache.get(serverUri); + } catch (ExecutionException e) { + throw new InternalProcessingException( + String.format("Unable to create connection factory for url %s", serverUri), e); } + } - private String extractTopicName(URI endpointURI) { - return UriUtils.extractContextFromUri(endpointURI).replaceFirst("/", ""); - } + private String extractTopicName(URI endpointURI) { + return UriUtils.extractContextFromUri(endpointURI).replaceFirst("/", ""); + } - protected class ConnectionFactoryLoader extends CacheLoader { + protected class ConnectionFactoryLoader extends CacheLoader { - @Override - public ConnectionFactory load(URI serverUri) throws Exception { - return createConnectionFactory(serverUri); - } + @Override + public ConnectionFactory load(URI serverUri) throws Exception { + return createConnectionFactory(serverUri); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/jms/JmsHornetQMessageSenderProvider.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/jms/JmsHornetQMessageSenderProvider.java index 2c5d707da4..75c4737c5a 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/jms/JmsHornetQMessageSenderProvider.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/jms/JmsHornetQMessageSenderProvider.java @@ -1,6 +1,11 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.jms; import com.google.common.collect.ImmutableSet; +import java.net.URI; +import java.util.HashMap; +import java.util.Set; +import javax.jms.ConnectionFactory; +import javax.jms.Message; import org.hornetq.api.core.TransportConfiguration; import org.hornetq.api.jms.HornetQJMSClient; import org.hornetq.api.jms.JMSFactoryType; @@ -8,33 +13,29 @@ import pl.allegro.tech.hermes.consumers.consumer.trace.MetadataAppender; import pl.allegro.tech.hermes.consumers.uri.UriUtils; -import java.net.URI; -import java.util.HashMap; -import java.util.Set; -import javax.jms.ConnectionFactory; -import javax.jms.Message; - public class JmsHornetQMessageSenderProvider extends AbstractJmsMessageSenderProvider { - private static final Set SUPPORTED_PROTOCOLS = ImmutableSet.of("jms"); + private static final Set SUPPORTED_PROTOCOLS = ImmutableSet.of("jms"); - public JmsHornetQMessageSenderProvider(MetadataAppender metadataAppender) { - super(metadataAppender); - } + public JmsHornetQMessageSenderProvider(MetadataAppender metadataAppender) { + super(metadataAppender); + } - @Override - public ConnectionFactory createConnectionFactory(URI serverUri) { - HashMap props = new HashMap<>(); - props.put("host", UriUtils.extractHostFromUri(serverUri)); - Integer port = UriUtils.extractPortFromUri(serverUri); - if (port != null) { - props.put("port", port); - } - TransportConfiguration transportConfiguration = new TransportConfiguration(NettyConnectorFactory.class.getName(), props); - return HornetQJMSClient.createConnectionFactoryWithoutHA(JMSFactoryType.CF, transportConfiguration); + @Override + public ConnectionFactory createConnectionFactory(URI serverUri) { + HashMap props = new HashMap<>(); + props.put("host", UriUtils.extractHostFromUri(serverUri)); + Integer port = UriUtils.extractPortFromUri(serverUri); + if (port != null) { + props.put("port", port); } + TransportConfiguration transportConfiguration = + new TransportConfiguration(NettyConnectorFactory.class.getName(), props); + return HornetQJMSClient.createConnectionFactoryWithoutHA( + JMSFactoryType.CF, transportConfiguration); + } - @Override - public Set getSupportedProtocols() { - return SUPPORTED_PROTOCOLS; - } + @Override + public Set getSupportedProtocols() { + return SUPPORTED_PROTOCOLS; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/jms/JmsMessageSender.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/jms/JmsMessageSender.java index 995fe555bf..1886d183b6 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/jms/JmsMessageSender.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/jms/JmsMessageSender.java @@ -1,11 +1,9 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.jms; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import pl.allegro.tech.hermes.consumers.consumer.Message; -import pl.allegro.tech.hermes.consumers.consumer.sender.CompletableFutureAwareMessageSender; -import pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult; -import pl.allegro.tech.hermes.consumers.consumer.trace.MetadataAppender; +import static pl.allegro.tech.hermes.common.http.MessageMetadataHeaders.MESSAGE_ID; +import static pl.allegro.tech.hermes.common.http.MessageMetadataHeaders.TOPIC_NAME; +import static pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult.failedResult; +import static pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult.succeededResult; import java.util.concurrent.CompletableFuture; import javax.jms.BytesMessage; @@ -13,59 +11,66 @@ import javax.jms.JMSContext; import javax.jms.JMSException; import javax.jms.JMSRuntimeException; - -import static pl.allegro.tech.hermes.common.http.MessageMetadataHeaders.MESSAGE_ID; -import static pl.allegro.tech.hermes.common.http.MessageMetadataHeaders.TOPIC_NAME; -import static pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult.failedResult; -import static pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult.succeededResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import pl.allegro.tech.hermes.consumers.consumer.Message; +import pl.allegro.tech.hermes.consumers.consumer.sender.CompletableFutureAwareMessageSender; +import pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult; +import pl.allegro.tech.hermes.consumers.consumer.trace.MetadataAppender; public class JmsMessageSender implements CompletableFutureAwareMessageSender { - private static final Logger logger = LoggerFactory.getLogger(JmsMessageSender.class); + private static final Logger logger = LoggerFactory.getLogger(JmsMessageSender.class); - private final String topicName; - private final JMSContext jmsContext; - private final MetadataAppender metadataAppender; + private final String topicName; + private final JMSContext jmsContext; + private final MetadataAppender metadataAppender; - public JmsMessageSender(JMSContext jmsContext, String destinationTopic, MetadataAppender metadataAppender) { - this.jmsContext = jmsContext; - this.topicName = destinationTopic; - this.metadataAppender = metadataAppender; - } + public JmsMessageSender( + JMSContext jmsContext, + String destinationTopic, + MetadataAppender metadataAppender) { + this.jmsContext = jmsContext; + this.topicName = destinationTopic; + this.metadataAppender = metadataAppender; + } - @Override - public void stop() { - jmsContext.close(); - } + @Override + public void stop() { + jmsContext.close(); + } - @Override - public void send(Message msg, final CompletableFuture resultFuture) { - try { - BytesMessage message = jmsContext.createBytesMessage(); - message.writeBytes(msg.getData()); - message.setStringProperty(TOPIC_NAME.getCamelCaseName(), msg.getTopic()); - message.setStringProperty(MESSAGE_ID.getCamelCaseName(), msg.getId()); + @Override + public void send(Message msg, final CompletableFuture resultFuture) { + try { + BytesMessage message = jmsContext.createBytesMessage(); + message.writeBytes(msg.getData()); + message.setStringProperty(TOPIC_NAME.getCamelCaseName(), msg.getTopic()); + message.setStringProperty(MESSAGE_ID.getCamelCaseName(), msg.getId()); - metadataAppender.append(message, msg); + metadataAppender.append(message, msg); - CompletionListener asyncListener = new CompletionListener() { - @Override - public void onCompletion(javax.jms.Message message) { - resultFuture.complete(succeededResult()); - } + CompletionListener asyncListener = + new CompletionListener() { + @Override + public void onCompletion(javax.jms.Message message) { + resultFuture.complete(succeededResult()); + } - @Override - public void onException(javax.jms.Message message, Exception exception) { - logger.warn(String.format("Exception while sending message to topic %s", topicName), exception); - resultFuture.complete(failedResult(exception)); - } - }; - jmsContext.createProducer() - .setAsync(asyncListener) - .send(jmsContext.createTopic(topicName), message); - } catch (JMSException | JMSRuntimeException e) { - resultFuture.complete(failedResult(e)); - } + @Override + public void onException(javax.jms.Message message, Exception exception) { + logger.warn( + String.format("Exception while sending message to topic %s", topicName), + exception); + resultFuture.complete(failedResult(exception)); + } + }; + jmsContext + .createProducer() + .setAsync(asyncListener) + .send(jmsContext.createTopic(topicName), message); + } catch (JMSException | JMSRuntimeException e) { + resultFuture.complete(failedResult(e)); } - + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/jms/JmsMessageSenderProvider.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/jms/JmsMessageSenderProvider.java index bd4969f5f4..9265a89bea 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/jms/JmsMessageSenderProvider.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/jms/JmsMessageSenderProvider.java @@ -1,12 +1,10 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.jms; -import pl.allegro.tech.hermes.consumers.consumer.sender.ProtocolMessageSenderProvider; - import java.net.URI; import javax.jms.ConnectionFactory; +import pl.allegro.tech.hermes.consumers.consumer.sender.ProtocolMessageSenderProvider; public interface JmsMessageSenderProvider extends ProtocolMessageSenderProvider { - ConnectionFactory createConnectionFactory(URI serverUri); - + ConnectionFactory createConnectionFactory(URI serverUri); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/jms/JmsMetadataAppender.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/jms/JmsMetadataAppender.java index c9b73e953f..fbab75c211 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/jms/JmsMetadataAppender.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/jms/JmsMetadataAppender.java @@ -1,28 +1,27 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.jms; -import pl.allegro.tech.hermes.consumers.consumer.trace.MetadataAppender; - import java.util.Map; import javax.jms.JMSException; import javax.jms.JMSRuntimeException; import javax.jms.Message; +import pl.allegro.tech.hermes.consumers.consumer.trace.MetadataAppender; public class JmsMetadataAppender implements MetadataAppender { - private static String normalize(String key) { - return key.replaceAll("-", ""); - } + private static String normalize(String key) { + return key.replaceAll("-", ""); + } - @Override - public Message append(Message target, pl.allegro.tech.hermes.consumers.consumer.Message message) { + @Override + public Message append(Message target, pl.allegro.tech.hermes.consumers.consumer.Message message) { - try { - for (Map.Entry entry : message.getExternalMetadata().entrySet()) { - target.setStringProperty(normalize(entry.getKey()), entry.getValue()); - } - return target; - } catch (JMSException e) { - throw new JMSRuntimeException(e.getMessage(), e.getErrorCode(), e); - } + try { + for (Map.Entry entry : message.getExternalMetadata().entrySet()) { + target.setStringProperty(normalize(entry.getKey()), entry.getValue()); + } + return target; + } catch (JMSException e) { + throw new JMSRuntimeException(e.getMessage(), e.getErrorCode(), e); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/resolver/EndpointAddressResolutionException.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/resolver/EndpointAddressResolutionException.java index 6cf669a5f4..37d10152e5 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/resolver/EndpointAddressResolutionException.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/resolver/EndpointAddressResolutionException.java @@ -5,27 +5,29 @@ @SuppressWarnings("serial") public class EndpointAddressResolutionException extends Exception { - private final boolean ignoreInRateCalculation; - - public EndpointAddressResolutionException(EndpointAddress endpointAddress, Throwable cause, boolean ignoreInRateCalculation) { - super("Failed to resolve " + endpointAddress, cause); - this.ignoreInRateCalculation = ignoreInRateCalculation; - } - - public EndpointAddressResolutionException(EndpointAddress endpointAddress, Throwable cause) { - this(endpointAddress, cause, false); - } - - public EndpointAddressResolutionException(String message, Throwable cause, boolean ignoreInRateCalculation) { - super(message, cause); - this.ignoreInRateCalculation = ignoreInRateCalculation; - } - - public EndpointAddressResolutionException(String message, Throwable cause) { - this(message, cause, false); - } - - public boolean isIgnoreInRateCalculation() { - return ignoreInRateCalculation; - } + private final boolean ignoreInRateCalculation; + + public EndpointAddressResolutionException( + EndpointAddress endpointAddress, Throwable cause, boolean ignoreInRateCalculation) { + super("Failed to resolve " + endpointAddress, cause); + this.ignoreInRateCalculation = ignoreInRateCalculation; + } + + public EndpointAddressResolutionException(EndpointAddress endpointAddress, Throwable cause) { + this(endpointAddress, cause, false); + } + + public EndpointAddressResolutionException( + String message, Throwable cause, boolean ignoreInRateCalculation) { + super(message, cause); + this.ignoreInRateCalculation = ignoreInRateCalculation; + } + + public EndpointAddressResolutionException(String message, Throwable cause) { + this(message, cause, false); + } + + public boolean isIgnoreInRateCalculation() { + return ignoreInRateCalculation; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/resolver/EndpointAddressResolver.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/resolver/EndpointAddressResolver.java index c5fa1e3e98..9f201d049b 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/resolver/EndpointAddressResolver.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/resolver/EndpointAddressResolver.java @@ -1,36 +1,38 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.resolver; +import java.net.URI; +import java.util.Collections; +import java.util.List; import pl.allegro.tech.hermes.api.EndpointAddress; import pl.allegro.tech.hermes.api.EndpointAddressResolverMetadata; import pl.allegro.tech.hermes.consumers.consumer.Message; import pl.allegro.tech.hermes.consumers.consumer.batch.MessageBatch; -import java.net.URI; -import java.util.Collections; -import java.util.List; - public interface EndpointAddressResolver { - default URI resolve(EndpointAddress address, Message message, EndpointAddressResolverMetadata metadata) - throws EndpointAddressResolutionException { - return resolve(address); - } + default URI resolve( + EndpointAddress address, Message message, EndpointAddressResolverMetadata metadata) + throws EndpointAddressResolutionException { + return resolve(address); + } - default URI resolve(EndpointAddress address, MessageBatch batch, EndpointAddressResolverMetadata metadata) - throws EndpointAddressResolutionException { - return resolve(address); - } + default URI resolve( + EndpointAddress address, MessageBatch batch, EndpointAddressResolverMetadata metadata) + throws EndpointAddressResolutionException { + return resolve(address); + } - static URI resolve(EndpointAddress address) throws EndpointAddressResolutionException { - try { - return address.getUri(); - } catch (Exception ex) { - throw new EndpointAddressResolutionException(address, ex); - } + static URI resolve(EndpointAddress address) throws EndpointAddressResolutionException { + try { + return address.getUri(); + } catch (Exception ex) { + throw new EndpointAddressResolutionException(address, ex); } + } - default List resolveAll(EndpointAddress address, Message message, EndpointAddressResolverMetadata metadata) - throws EndpointAddressResolutionException { - return Collections.singletonList(resolve(address, message, metadata)); - } + default List resolveAll( + EndpointAddress address, Message message, EndpointAddressResolverMetadata metadata) + throws EndpointAddressResolutionException { + return Collections.singletonList(resolve(address, message, metadata)); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/resolver/InterpolatingEndpointAddressResolver.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/resolver/InterpolatingEndpointAddressResolver.java index d025d312f5..e5f5246959 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/resolver/InterpolatingEndpointAddressResolver.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/resolver/InterpolatingEndpointAddressResolver.java @@ -1,29 +1,28 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.resolver; +import java.net.URI; import pl.allegro.tech.hermes.api.EndpointAddress; import pl.allegro.tech.hermes.api.EndpointAddressResolverMetadata; import pl.allegro.tech.hermes.consumers.consumer.Message; import pl.allegro.tech.hermes.consumers.consumer.interpolation.InterpolationException; import pl.allegro.tech.hermes.consumers.consumer.interpolation.UriInterpolator; -import java.net.URI; - public class InterpolatingEndpointAddressResolver implements EndpointAddressResolver { - private final UriInterpolator interpolator; + private final UriInterpolator interpolator; - public InterpolatingEndpointAddressResolver(UriInterpolator interpolator) { - this.interpolator = interpolator; - } + public InterpolatingEndpointAddressResolver(UriInterpolator interpolator) { + this.interpolator = interpolator; + } - @Override - public URI resolve(EndpointAddress address, Message message, EndpointAddressResolverMetadata metadata) - throws EndpointAddressResolutionException { - try { - return interpolator.interpolate(address, message); - } catch (InterpolationException ex) { - throw new EndpointAddressResolutionException(address, ex); - } + @Override + public URI resolve( + EndpointAddress address, Message message, EndpointAddressResolverMetadata metadata) + throws EndpointAddressResolutionException { + try { + return interpolator.interpolate(address, message); + } catch (InterpolationException ex) { + throw new EndpointAddressResolutionException(address, ex); } - + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/resolver/ResolvableEndpointAddress.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/resolver/ResolvableEndpointAddress.java index 6569dccbbe..c165eaa453 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/resolver/ResolvableEndpointAddress.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/resolver/ResolvableEndpointAddress.java @@ -1,39 +1,41 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.resolver; +import java.net.URI; +import java.util.List; import pl.allegro.tech.hermes.api.EndpointAddress; import pl.allegro.tech.hermes.api.EndpointAddressResolverMetadata; import pl.allegro.tech.hermes.consumers.consumer.Message; -import java.net.URI; -import java.util.List; - public class ResolvableEndpointAddress { - private final EndpointAddress address; + private final EndpointAddress address; - private final EndpointAddressResolver resolver; + private final EndpointAddressResolver resolver; - private final EndpointAddressResolverMetadata metadata; + private final EndpointAddressResolverMetadata metadata; - public ResolvableEndpointAddress(EndpointAddress address, EndpointAddressResolver resolver, EndpointAddressResolverMetadata metadata) { - this.address = address; - this.resolver = resolver; - this.metadata = metadata; - } + public ResolvableEndpointAddress( + EndpointAddress address, + EndpointAddressResolver resolver, + EndpointAddressResolverMetadata metadata) { + this.address = address; + this.resolver = resolver; + this.metadata = metadata; + } - public URI resolveFor(Message message) throws EndpointAddressResolutionException { - return resolver.resolve(address, message, metadata); - } + public URI resolveFor(Message message) throws EndpointAddressResolutionException { + return resolver.resolve(address, message, metadata); + } - public List resolveAllFor(Message message) throws EndpointAddressResolutionException { - return resolver.resolveAll(address, message, metadata); - } + public List resolveAllFor(Message message) throws EndpointAddressResolutionException { + return resolver.resolveAll(address, message, metadata); + } - public EndpointAddress getRawAddress() { - return address; - } + public EndpointAddress getRawAddress() { + return address; + } - public String toString() { - return address.toString(); - } + public String toString() { + return address.toString(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/resolver/SimpleEndpointAddressResolver.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/resolver/SimpleEndpointAddressResolver.java index e7ecf25191..fbb486dee9 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/resolver/SimpleEndpointAddressResolver.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/resolver/SimpleEndpointAddressResolver.java @@ -1,5 +1,3 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.resolver; -public class SimpleEndpointAddressResolver implements EndpointAddressResolver { - -} +public class SimpleEndpointAddressResolver implements EndpointAddressResolver {} diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/timeout/FutureAsyncTimeout.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/timeout/FutureAsyncTimeout.java index 3cc296cb42..883813f8d0 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/timeout/FutureAsyncTimeout.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/sender/timeout/FutureAsyncTimeout.java @@ -7,32 +7,34 @@ import java.util.concurrent.TimeoutException; import java.util.function.Function; -/** - * see http://www.nurkiewicz.com/2014/12/asynchronous-timeouts-with.html - */ +/** see http://www.nurkiewicz.com/2014/12/asynchronous-timeouts-with.html */ public class FutureAsyncTimeout { - private final ScheduledExecutorService executor; - - - public FutureAsyncTimeout(ScheduledExecutorService scheduledExecutorService) { - this.executor = scheduledExecutorService; - } - - public CompletableFuture within(CompletableFuture future, Duration duration, Function exceptionMapper) { - return future.applyToEither(failAfter(duration, exceptionMapper), Function.identity()); - } - - private CompletableFuture failAfter(Duration duration, Function exceptionMapper) { - final CompletableFuture promise = new CompletableFuture<>(); - executor.schedule(() -> { - TimeoutException ex = new TimeoutException("Timeout after " + duration); - return promise.complete(exceptionMapper.apply(ex)); - }, duration.toMillis(), TimeUnit.MILLISECONDS); - return promise; - } - - public void shutdown() { - executor.shutdown(); - } + private final ScheduledExecutorService executor; + + public FutureAsyncTimeout(ScheduledExecutorService scheduledExecutorService) { + this.executor = scheduledExecutorService; + } + + public CompletableFuture within( + CompletableFuture future, Duration duration, Function exceptionMapper) { + return future.applyToEither(failAfter(duration, exceptionMapper), Function.identity()); + } + + private CompletableFuture failAfter( + Duration duration, Function exceptionMapper) { + final CompletableFuture promise = new CompletableFuture<>(); + executor.schedule( + () -> { + TimeoutException ex = new TimeoutException("Timeout after " + duration); + return promise.complete(exceptionMapper.apply(ex)); + }, + duration.toMillis(), + TimeUnit.MILLISECONDS); + return promise; + } + + public void shutdown() { + executor.shutdown(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/trace/MetadataAppender.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/trace/MetadataAppender.java index f94f644d0c..00c9ec1380 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/trace/MetadataAppender.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/consumer/trace/MetadataAppender.java @@ -4,5 +4,5 @@ public interface MetadataAppender { - T append(T target, Message message); + T append(T target, Message message); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/health/Checks.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/health/Checks.java index 6ea6a97bb4..19fe5ed5e2 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/health/Checks.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/health/Checks.java @@ -1,6 +1,6 @@ package pl.allegro.tech.hermes.consumers.health; public final class Checks { - public static final String SUBSCRIPTIONS = "subscriptions"; - public static final String SUBSCRIPTIONS_COUNT = "subscriptionsCount"; + public static final String SUBSCRIPTIONS = "subscriptions"; + public static final String SUBSCRIPTIONS_COUNT = "subscriptionsCount"; } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/health/ConsumerMonitor.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/health/ConsumerMonitor.java index 557b468d7f..03aa7e1dfb 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/health/ConsumerMonitor.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/health/ConsumerMonitor.java @@ -6,17 +6,17 @@ public class ConsumerMonitor { - private Map> checks = new ConcurrentHashMap<>(); + private Map> checks = new ConcurrentHashMap<>(); - public void register(String key, Supplier check) { - checks.putIfAbsent(key, check); - } + public void register(String key, Supplier check) { + checks.putIfAbsent(key, check); + } - public Object check(String key) { - try { - return checks.getOrDefault(key, () -> "Unavailable").get(); - } catch (Exception ex) { - return "Failed to evaluate check, " + ex.getMessage(); - } + public Object check(String key) { + try { + return checks.getOrDefault(key, () -> "Unavailable").get(); + } catch (Exception ex) { + return "Failed to evaluate check, " + ex.getMessage(); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/message/undelivered/UndeliveredMessageLogPersister.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/message/undelivered/UndeliveredMessageLogPersister.java index 8dd1f6310a..911ffb3a38 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/message/undelivered/UndeliveredMessageLogPersister.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/message/undelivered/UndeliveredMessageLogPersister.java @@ -1,33 +1,36 @@ package pl.allegro.tech.hermes.consumers.message.undelivered; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import pl.allegro.tech.hermes.common.message.undelivered.UndeliveredMessageLog; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.time.Duration; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; +import pl.allegro.tech.hermes.common.message.undelivered.UndeliveredMessageLog; public class UndeliveredMessageLogPersister { - private final Duration period; - private final UndeliveredMessageLog undeliveredMessageLog; - private final ScheduledExecutorService scheduledExecutorService; - - public UndeliveredMessageLogPersister(UndeliveredMessageLog undeliveredMessageLog, Duration undeliveredMessageLogPersistPeriod) { - this.undeliveredMessageLog = undeliveredMessageLog; - this.period = undeliveredMessageLogPersistPeriod; - this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( - new ThreadFactoryBuilder().setNameFormat("undelivered-message-log-persister-%d").build()); - } - - public void start() { - scheduledExecutorService.scheduleAtFixedRate(undeliveredMessageLog::persist, period.toMillis(), period.toMillis(), MILLISECONDS); - } - - public void shutdown() { - scheduledExecutorService.shutdown(); - } - + private final Duration period; + private final UndeliveredMessageLog undeliveredMessageLog; + private final ScheduledExecutorService scheduledExecutorService; + + public UndeliveredMessageLogPersister( + UndeliveredMessageLog undeliveredMessageLog, Duration undeliveredMessageLogPersistPeriod) { + this.undeliveredMessageLog = undeliveredMessageLog; + this.period = undeliveredMessageLogPersistPeriod; + this.scheduledExecutorService = + Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder() + .setNameFormat("undelivered-message-log-persister-%d") + .build()); + } + + public void start() { + scheduledExecutorService.scheduleAtFixedRate( + undeliveredMessageLog::persist, period.toMillis(), period.toMillis(), MILLISECONDS); + } + + public void shutdown() { + scheduledExecutorService.shutdown(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/queue/FullDrainMpscQueue.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/queue/FullDrainMpscQueue.java index 3dbb63f890..58b45c29f6 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/queue/FullDrainMpscQueue.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/queue/FullDrainMpscQueue.java @@ -1,55 +1,56 @@ package pl.allegro.tech.hermes.consumers.queue; +import static org.slf4j.LoggerFactory.getLogger; + import org.jctools.queues.MessagePassingQueue; import org.jctools.queues.MpscArrayQueue; import org.slf4j.Logger; -import static org.slf4j.LoggerFactory.getLogger; - public class FullDrainMpscQueue implements MpscQueue { - private static final Logger logger = getLogger(FullDrainMpscQueue.class); - - private final MpscArrayQueue queue; - - public FullDrainMpscQueue(int capacity) { - this.queue = new MpscArrayQueue<>(capacity); + private static final Logger logger = getLogger(FullDrainMpscQueue.class); + + private final MpscArrayQueue queue; + + public FullDrainMpscQueue(int capacity) { + this.queue = new MpscArrayQueue<>(capacity); + } + + @Override + public boolean offer(T element) { + return queue.offer(element); + } + + /** + * The {@link MpscArrayQueue#drain(MessagePassingQueue.Consumer)} method may skip items with + * allocated slots by producers (who won CAS) but were not added to the queue yet. This may happen + * to broken elements chain. See explanation here. + * + *

This is an alternative approach which waits for all items to become available by using + * {@link MpscArrayQueue#poll()} underneath (which spin-waits when getting next item). + */ + @Override + public void drain(MessagePassingQueue.Consumer consumer) { + int size = queue.size(); + for (int i = 0; i < size; i++) { + T element = queue.poll(); + if (element != null) { + consumer.accept(element); + } else { + logger.warn("Unexpected null value while draining queue [idx={}, size={}]", i, size); + break; + } } + } - @Override - public boolean offer(T element) { - return queue.offer(element); - } - - /** - *

The {@link MpscArrayQueue#drain(MessagePassingQueue.Consumer)} method may skip items with allocated slots - * by producers (who won CAS) but were not added to the queue yet. This may happen to broken elements chain. - * See explanation here.

- * - *

This is an alternative approach which waits for all items to become available - * by using {@link MpscArrayQueue#poll()} underneath (which spin-waits when getting next item).

- */ - @Override - public void drain(MessagePassingQueue.Consumer consumer) { - int size = queue.size(); - for (int i = 0; i < size; i++) { - T element = queue.poll(); - if (element != null) { - consumer.accept(element); - } else { - logger.warn("Unexpected null value while draining queue [idx={}, size={}]", i, size); - break; - } - } - } + @Override + public int size() { + return queue.size(); + } - @Override - public int size() { - return queue.size(); - } - - @Override - public int capacity() { - return queue.capacity(); - } + @Override + public int capacity() { + return queue.capacity(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/queue/MonitoredMpscQueue.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/queue/MonitoredMpscQueue.java index fafc33af83..5c1b7e38a1 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/queue/MonitoredMpscQueue.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/queue/MonitoredMpscQueue.java @@ -8,43 +8,46 @@ public class MonitoredMpscQueue implements MpscQueue { - private static final Logger logger = LoggerFactory.getLogger(MonitoredMpscQueue.class); - - private final MpscQueue queue; - - private final String name; - - private final HermesCounter failuresCounter; - - public MonitoredMpscQueue(MpscQueue queue, MetricsFacade metrics, String name) { - this.queue = queue; - this.name = name; - metrics.consumer().registerQueueUtilizationGauge(queue, name, q -> (double) q.size() / q.capacity()); - this.failuresCounter = metrics.consumer().queueFailuresCounter(name); - } - - @Override - public boolean offer(T element) { - boolean accepted = queue.offer(element); - if (!accepted) { - failuresCounter.increment(); - logger.error("[Queue: {}] Unable to add item: queue is full. Offered item: {}", name, element); - } - return accepted; - } - - @Override - public void drain(MessagePassingQueue.Consumer consumer) { - queue.drain(consumer); - } - - @Override - public int size() { - return queue.size(); - } - - @Override - public int capacity() { - return queue.capacity(); + private static final Logger logger = LoggerFactory.getLogger(MonitoredMpscQueue.class); + + private final MpscQueue queue; + + private final String name; + + private final HermesCounter failuresCounter; + + public MonitoredMpscQueue(MpscQueue queue, MetricsFacade metrics, String name) { + this.queue = queue; + this.name = name; + metrics + .consumer() + .registerQueueUtilizationGauge(queue, name, q -> (double) q.size() / q.capacity()); + this.failuresCounter = metrics.consumer().queueFailuresCounter(name); + } + + @Override + public boolean offer(T element) { + boolean accepted = queue.offer(element); + if (!accepted) { + failuresCounter.increment(); + logger.error( + "[Queue: {}] Unable to add item: queue is full. Offered item: {}", name, element); } + return accepted; + } + + @Override + public void drain(MessagePassingQueue.Consumer consumer) { + queue.drain(consumer); + } + + @Override + public int size() { + return queue.size(); + } + + @Override + public int capacity() { + return queue.capacity(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/queue/MpscQueue.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/queue/MpscQueue.java index c9fe254a53..36febba296 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/queue/MpscQueue.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/queue/MpscQueue.java @@ -4,11 +4,11 @@ public interface MpscQueue { - boolean offer(T element); + boolean offer(T element); - void drain(MessagePassingQueue.Consumer consumer); + void drain(MessagePassingQueue.Consumer consumer); - int size(); + int size(); - int capacity(); + int capacity(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/queue/WaitFreeDrainMpscQueue.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/queue/WaitFreeDrainMpscQueue.java index 88cc5a5f57..c346f5edc1 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/queue/WaitFreeDrainMpscQueue.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/queue/WaitFreeDrainMpscQueue.java @@ -5,29 +5,29 @@ public class WaitFreeDrainMpscQueue implements MpscQueue { - private final MpscArrayQueue queue; - - public WaitFreeDrainMpscQueue(int capacity) { - this.queue = new MpscArrayQueue<>(capacity); - } - - @Override - public boolean offer(T element) { - return queue.offer(element); - } - - @Override - public void drain(MessagePassingQueue.Consumer consumer) { - queue.drain(consumer); - } - - @Override - public int size() { - return queue.size(); - } - - @Override - public int capacity() { - return queue.capacity(); - } + private final MpscArrayQueue queue; + + public WaitFreeDrainMpscQueue(int capacity) { + this.queue = new MpscArrayQueue<>(capacity); + } + + @Override + public boolean offer(T element) { + return queue.offer(element); + } + + @Override + public void drain(MessagePassingQueue.Consumer consumer) { + queue.drain(consumer); + } + + @Override + public int size() { + return queue.size(); + } + + @Override + public int capacity() { + return queue.capacity(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/registry/ConsumerNodesRegistry.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/registry/ConsumerNodesRegistry.java index fead0874cc..2f8cd13e80 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/registry/ConsumerNodesRegistry.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/registry/ConsumerNodesRegistry.java @@ -1,5 +1,17 @@ package pl.allegro.tech.hermes.consumers.registry; +import static java.util.stream.Collectors.toList; +import static org.apache.commons.lang3.StringUtils.substringAfterLast; +import static org.apache.zookeeper.CreateMode.EPHEMERAL; + +import java.io.IOException; +import java.time.Clock; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCache; @@ -12,165 +24,148 @@ import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.common.exception.InternalProcessingException; -import java.io.IOException; -import java.time.Clock; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; - -import static java.util.stream.Collectors.toList; -import static org.apache.commons.lang3.StringUtils.substringAfterLast; -import static org.apache.zookeeper.CreateMode.EPHEMERAL; - public class ConsumerNodesRegistry extends PathChildrenCache implements PathChildrenCacheListener { - private static final Logger logger = LoggerFactory.getLogger(ConsumerNodesRegistry.class); - - private final CuratorFramework curatorClient; - private final ConsumerNodesRegistryPaths registryPaths; - private final String consumerNodeId; - private final LeaderLatch leaderLatch; - private final Map consumersLastSeen = new ConcurrentHashMap<>(); - private final long deathOfConsumerAfterMillis; - private final Clock clock; - - public ConsumerNodesRegistry( - CuratorFramework curatorClient, - ExecutorService executorService, - ConsumerNodesRegistryPaths registryPaths, - String consumerNodeId, - long deathOfConsumerAfterSeconds, - Clock clock - ) { - super(curatorClient, registryPaths.nodesPath(), true, false, executorService); - - this.curatorClient = curatorClient; - this.registryPaths = registryPaths; - this.consumerNodeId = consumerNodeId; - this.clock = clock; - this.leaderLatch = new LeaderLatch(curatorClient, registryPaths.leaderPath(), consumerNodeId); - this.deathOfConsumerAfterMillis = TimeUnit.SECONDS.toMillis(deathOfConsumerAfterSeconds); - } - - @Override - public void start() throws Exception { - getListenable().addListener(this); - super.start(StartMode.POST_INITIALIZED_EVENT); - leaderLatch.start(); - } - - public void stop() throws IOException { - leaderLatch.close(); - close(); - } - - @Override - public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception { - switch (event.getType()) { - case INITIALIZED: - case CONNECTION_RECONNECTED: - if (!isRegistered(consumerNodeId)) { - registerConsumerNode(); - } - break; - default: - // noop - break; + private static final Logger logger = LoggerFactory.getLogger(ConsumerNodesRegistry.class); + + private final CuratorFramework curatorClient; + private final ConsumerNodesRegistryPaths registryPaths; + private final String consumerNodeId; + private final LeaderLatch leaderLatch; + private final Map consumersLastSeen = new ConcurrentHashMap<>(); + private final long deathOfConsumerAfterMillis; + private final Clock clock; + + public ConsumerNodesRegistry( + CuratorFramework curatorClient, + ExecutorService executorService, + ConsumerNodesRegistryPaths registryPaths, + String consumerNodeId, + long deathOfConsumerAfterSeconds, + Clock clock) { + super(curatorClient, registryPaths.nodesPath(), true, false, executorService); + + this.curatorClient = curatorClient; + this.registryPaths = registryPaths; + this.consumerNodeId = consumerNodeId; + this.clock = clock; + this.leaderLatch = new LeaderLatch(curatorClient, registryPaths.leaderPath(), consumerNodeId); + this.deathOfConsumerAfterMillis = TimeUnit.SECONDS.toMillis(deathOfConsumerAfterSeconds); + } + + @Override + public void start() throws Exception { + getListenable().addListener(this); + super.start(StartMode.POST_INITIALIZED_EVENT); + leaderLatch.start(); + } + + public void stop() throws IOException { + leaderLatch.close(); + close(); + } + + @Override + public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception { + switch (event.getType()) { + case INITIALIZED: + case CONNECTION_RECONNECTED: + if (!isRegistered(consumerNodeId)) { + registerConsumerNode(); } + break; + default: + // noop + break; } + } - public boolean isRegistered(String consumerNodeId) { - try { - return curatorClient.checkExists().forPath(registryPaths.nodePath(consumerNodeId)) != null; - } catch (Exception e) { - throw new InternalProcessingException(e); - } - } - - public boolean isLeader() { - return ensureRegistered() && leaderLatch.hasLeadership(); - } - - public List listConsumerNodes() { - return new ArrayList<>(consumersLastSeen.keySet()); - } - - public synchronized void refresh() { - logger.info("Refreshing current consumers registry"); - - long currentTime = clock.millis(); - List currentNodes = readCurrentNodes(); - List validNodes = currentNodes.stream() - .filter(StringUtils::isNotBlank) - .toList(); - if (currentNodes.size() != validNodes.size()) { - logger.warn("Found {} invalid consumer nodes.", currentNodes.size() - validNodes.size()); - } - validNodes.forEach(node -> consumersLastSeen.put(node, currentTime)); - - List deadConsumers = findDeadConsumers(currentTime); - if (!deadConsumers.isEmpty()) { - logger.info("Considering following consumers dead: {}", deadConsumers); - } - deadConsumers.forEach(consumersLastSeen::remove); + public boolean isRegistered(String consumerNodeId) { + try { + return curatorClient.checkExists().forPath(registryPaths.nodePath(consumerNodeId)) != null; + } catch (Exception e) { + throw new InternalProcessingException(e); } + } - private boolean ensureRegistered() { - if (curatorClient.getZookeeperClient().isConnected()) { - if (!isRegistered(consumerNodeId)) { - registerConsumerNode(); - } - return true; - } - return false; - } + public boolean isLeader() { + return ensureRegistered() && leaderLatch.hasLeadership(); + } - private void registerConsumerNode() { - try { - String nodePath = registryPaths.nodePath(consumerNodeId); - if (curatorClient.checkExists().forPath(nodePath) == null) { - curatorClient.create().creatingParentsIfNeeded() - .withMode(EPHEMERAL).forPath(nodePath); - logger.info("Registered in consumer nodes registry as {}", consumerNodeId); - } - } catch (NodeExistsException e) { - // Ignore as it is a race condition between threads trying to register the consumer node. - } catch (Exception e) { - throw new InternalProcessingException(e); - } - refresh(); - } + public List listConsumerNodes() { + return new ArrayList<>(consumersLastSeen.keySet()); + } - private List findDeadConsumers(long currentTime) { - long tooOld = currentTime - deathOfConsumerAfterMillis; - return consumersLastSeen.entrySet().stream() - .filter(entry -> { - long lastSeen = entry.getValue(); - return lastSeen < tooOld; - }) - .map(Map.Entry::getKey) - .collect(toList()); - } + public synchronized void refresh() { + logger.info("Refreshing current consumers registry"); - private List readCurrentNodes() { - return getCurrentData().stream() - .map(data -> substringAfterLast(data.getPath(), "/")) - .collect(toList()); + long currentTime = clock.millis(); + List currentNodes = readCurrentNodes(); + List validNodes = currentNodes.stream().filter(StringUtils::isNotBlank).toList(); + if (currentNodes.size() != validNodes.size()) { + logger.warn("Found {} invalid consumer nodes.", currentNodes.size() - validNodes.size()); } + validNodes.forEach(node -> consumersLastSeen.put(node, currentTime)); - public String getConsumerId() { - return consumerNodeId; + List deadConsumers = findDeadConsumers(currentTime); + if (!deadConsumers.isEmpty()) { + logger.info("Considering following consumers dead: {}", deadConsumers); } - - public void addLeaderLatchListener(LeaderLatchListener listener) { - leaderLatch.addListener(listener); + deadConsumers.forEach(consumersLastSeen::remove); + } + + private boolean ensureRegistered() { + if (curatorClient.getZookeeperClient().isConnected()) { + if (!isRegistered(consumerNodeId)) { + registerConsumerNode(); + } + return true; } - - public void removeLeaderLatchListener(LeaderLatchListener listener) { - leaderLatch.removeListener(listener); + return false; + } + + private void registerConsumerNode() { + try { + String nodePath = registryPaths.nodePath(consumerNodeId); + if (curatorClient.checkExists().forPath(nodePath) == null) { + curatorClient.create().creatingParentsIfNeeded().withMode(EPHEMERAL).forPath(nodePath); + logger.info("Registered in consumer nodes registry as {}", consumerNodeId); + } + } catch (NodeExistsException e) { + // Ignore as it is a race condition between threads trying to register the consumer node. + } catch (Exception e) { + throw new InternalProcessingException(e); } + refresh(); + } + + private List findDeadConsumers(long currentTime) { + long tooOld = currentTime - deathOfConsumerAfterMillis; + return consumersLastSeen.entrySet().stream() + .filter( + entry -> { + long lastSeen = entry.getValue(); + return lastSeen < tooOld; + }) + .map(Map.Entry::getKey) + .collect(toList()); + } + + private List readCurrentNodes() { + return getCurrentData().stream() + .map(data -> substringAfterLast(data.getPath(), "/")) + .collect(toList()); + } + + public String getConsumerId() { + return consumerNodeId; + } + + public void addLeaderLatchListener(LeaderLatchListener listener) { + leaderLatch.addListener(listener); + } + + public void removeLeaderLatchListener(LeaderLatchListener listener) { + leaderLatch.removeListener(listener); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/registry/ConsumerNodesRegistryPaths.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/registry/ConsumerNodesRegistryPaths.java index 0f51bf0881..02cdcc1056 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/registry/ConsumerNodesRegistryPaths.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/registry/ConsumerNodesRegistryPaths.java @@ -4,23 +4,24 @@ public class ConsumerNodesRegistryPaths { - private final ZookeeperPaths zookeeperPaths; - private final String registryPath; + private final ZookeeperPaths zookeeperPaths; + private final String registryPath; - public ConsumerNodesRegistryPaths(ZookeeperPaths zookeeperPaths, String clusterName) { - this.zookeeperPaths = zookeeperPaths; - this.registryPath = zookeeperPaths.join(zookeeperPaths.basePath(), "consumers-registry", clusterName); - } + public ConsumerNodesRegistryPaths(ZookeeperPaths zookeeperPaths, String clusterName) { + this.zookeeperPaths = zookeeperPaths; + this.registryPath = + zookeeperPaths.join(zookeeperPaths.basePath(), "consumers-registry", clusterName); + } - public String leaderPath() { - return zookeeperPaths.join(registryPath, "leader"); - } + public String leaderPath() { + return zookeeperPaths.join(registryPath, "leader"); + } - public String nodePath(String nodeId) { - return zookeeperPaths.join(nodesPath(), nodeId); - } + public String nodePath(String nodeId) { + return zookeeperPaths.join(nodesPath(), nodeId); + } - public String nodesPath() { - return zookeeperPaths.join(registryPath, "nodes"); - } + public String nodesPath() { + return zookeeperPaths.join(registryPath, "nodes"); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/server/ConsumerHttpServer.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/server/ConsumerHttpServer.java index a6af139fc8..722ecbc711 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/server/ConsumerHttpServer.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/server/ConsumerHttpServer.java @@ -1,70 +1,77 @@ package pl.allegro.tech.hermes.consumers.server; +import static jakarta.ws.rs.core.Response.Status.OK; +import static pl.allegro.tech.hermes.consumers.health.Checks.SUBSCRIPTIONS; +import static pl.allegro.tech.hermes.consumers.health.Checks.SUBSCRIPTIONS_COUNT; + import com.fasterxml.jackson.databind.ObjectMapper; import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpServer; import io.micrometer.prometheus.PrometheusMeterRegistry; -import pl.allegro.tech.hermes.consumers.health.ConsumerMonitor; - import java.io.IOException; import java.io.OutputStream; import java.net.InetSocketAddress; import java.util.List; - -import static jakarta.ws.rs.core.Response.Status.OK; -import static pl.allegro.tech.hermes.consumers.health.Checks.SUBSCRIPTIONS; -import static pl.allegro.tech.hermes.consumers.health.Checks.SUBSCRIPTIONS_COUNT; +import pl.allegro.tech.hermes.consumers.health.ConsumerMonitor; public class ConsumerHttpServer { - private final HttpServer server; + private final HttpServer server; - private static final String STATUS_UP = "{\"status\": \"UP\"}"; + private static final String STATUS_UP = "{\"status\": \"UP\"}"; - public ConsumerHttpServer(int healthCheckPort, ConsumerMonitor monitor, ObjectMapper mapper, - PrometheusMeterRegistry meterRegistry) throws IOException { - server = createServer(healthCheckPort); - server.createContext("/status/health", - (exchange) -> respondWithJson(exchange, STATUS_UP)); - server.createContext("/status/subscriptions", - (exchange) -> respondWithObject(exchange, mapper, monitor.check(SUBSCRIPTIONS))); - server.createContext("/status/subscriptionsCount", - (exchange) -> respondWithObject(exchange, mapper, monitor.check(SUBSCRIPTIONS_COUNT))); - server.createContext("/status/prometheus", - (exchange) -> respondWithString(exchange, meterRegistry.scrape())); - } + public ConsumerHttpServer( + int healthCheckPort, + ConsumerMonitor monitor, + ObjectMapper mapper, + PrometheusMeterRegistry meterRegistry) + throws IOException { + server = createServer(healthCheckPort); + server.createContext("/status/health", (exchange) -> respondWithJson(exchange, STATUS_UP)); + server.createContext( + "/status/subscriptions", + (exchange) -> respondWithObject(exchange, mapper, monitor.check(SUBSCRIPTIONS))); + server.createContext( + "/status/subscriptionsCount", + (exchange) -> respondWithObject(exchange, mapper, monitor.check(SUBSCRIPTIONS_COUNT))); + server.createContext( + "/status/prometheus", (exchange) -> respondWithString(exchange, meterRegistry.scrape())); + } - private HttpServer createServer(int port) throws IOException { - HttpServer httpServer = HttpServer.create(new InetSocketAddress(port), 0); - httpServer.setExecutor(null); - return httpServer; - } + private HttpServer createServer(int port) throws IOException { + HttpServer httpServer = HttpServer.create(new InetSocketAddress(port), 0); + httpServer.setExecutor(null); + return httpServer; + } - private static void respondWithObject(HttpExchange httpExchange, ObjectMapper mapper, Object response) throws IOException { - respondWithJson(httpExchange, mapper.writeValueAsString(response)); - } + private static void respondWithObject( + HttpExchange httpExchange, ObjectMapper mapper, Object response) throws IOException { + respondWithJson(httpExchange, mapper.writeValueAsString(response)); + } - private static void respondWithJson(HttpExchange httpExchange, String response) throws IOException { - httpExchange.getResponseHeaders().put("Content-Type", List.of("application/json")); - respondWithString(httpExchange, response); - } + private static void respondWithJson(HttpExchange httpExchange, String response) + throws IOException { + httpExchange.getResponseHeaders().put("Content-Type", List.of("application/json")); + respondWithString(httpExchange, response); + } - private static void respondWithString(HttpExchange httpExchange, String response) throws IOException { - httpExchange.sendResponseHeaders(OK.getStatusCode(), response.length()); - OutputStream os = httpExchange.getResponseBody(); - os.write(response.getBytes()); - os.close(); - } + private static void respondWithString(HttpExchange httpExchange, String response) + throws IOException { + httpExchange.sendResponseHeaders(OK.getStatusCode(), response.length()); + OutputStream os = httpExchange.getResponseBody(); + os.write(response.getBytes()); + os.close(); + } - public void start() { - server.start(); - } + public void start() { + server.start(); + } - public void stop() { - server.stop(0); - } + public void stop() { + server.stop(0); + } - public int getPort() { - return server.getAddress().getPort(); - } + public int getPort() { + return server.getAddress().getPort(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/cache/NotificationsBasedSubscriptionCache.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/cache/NotificationsBasedSubscriptionCache.java index 1bad30c416..5dc00dbb7e 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/cache/NotificationsBasedSubscriptionCache.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/cache/NotificationsBasedSubscriptionCache.java @@ -1,5 +1,9 @@ package pl.allegro.tech.hermes.consumers.subscription.cache; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.api.TopicName; @@ -9,74 +13,72 @@ import pl.allegro.tech.hermes.domain.subscription.SubscriptionRepository; import pl.allegro.tech.hermes.domain.topic.TopicRepository; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; - -public class NotificationsBasedSubscriptionCache implements SubscriptionsCache, SubscriptionCallback { +public class NotificationsBasedSubscriptionCache + implements SubscriptionsCache, SubscriptionCallback { - private final Map subscriptions = new ConcurrentHashMap<>(); + private final Map subscriptions = new ConcurrentHashMap<>(); - private final GroupRepository groupRepository; + private final GroupRepository groupRepository; - private final TopicRepository topicRepository; + private final TopicRepository topicRepository; - private final SubscriptionRepository subscriptionRepository; + private final SubscriptionRepository subscriptionRepository; - public NotificationsBasedSubscriptionCache(InternalNotificationsBus notificationsBus, - GroupRepository groupRepository, - TopicRepository topicRepository, - SubscriptionRepository subscriptionRepository) { - notificationsBus.registerSubscriptionCallback(this); - this.groupRepository = groupRepository; - this.topicRepository = topicRepository; - this.subscriptionRepository = subscriptionRepository; - } + public NotificationsBasedSubscriptionCache( + InternalNotificationsBus notificationsBus, + GroupRepository groupRepository, + TopicRepository topicRepository, + SubscriptionRepository subscriptionRepository) { + notificationsBus.registerSubscriptionCallback(this); + this.groupRepository = groupRepository; + this.topicRepository = topicRepository; + this.subscriptionRepository = subscriptionRepository; + } - @Override - public void onSubscriptionCreated(Subscription subscription) { - this.subscriptions.put(subscription.getQualifiedName(), subscription); - } + @Override + public void onSubscriptionCreated(Subscription subscription) { + this.subscriptions.put(subscription.getQualifiedName(), subscription); + } - @Override - public void onSubscriptionRemoved(Subscription subscription) { - this.subscriptions.remove(subscription.getQualifiedName(), subscription); - } + @Override + public void onSubscriptionRemoved(Subscription subscription) { + this.subscriptions.remove(subscription.getQualifiedName(), subscription); + } - @Override - public void onSubscriptionChanged(Subscription subscription) { - this.subscriptions.put(subscription.getQualifiedName(), subscription); - } + @Override + public void onSubscriptionChanged(Subscription subscription) { + this.subscriptions.put(subscription.getQualifiedName(), subscription); + } - @Override - public Subscription getSubscription(SubscriptionName subscriptionName) { - return subscriptions.get(subscriptionName); - } + @Override + public Subscription getSubscription(SubscriptionName subscriptionName) { + return subscriptions.get(subscriptionName); + } - @Override - public List subscriptionsOfTopic(TopicName topicName) { - return subscriptions.values().stream() - .filter(s -> s.getTopicName().equals(topicName)) - .collect(Collectors.toList()); - } + @Override + public List subscriptionsOfTopic(TopicName topicName) { + return subscriptions.values().stream() + .filter(s -> s.getTopicName().equals(topicName)) + .collect(Collectors.toList()); + } - @Override - public List listActiveSubscriptionNames() { - return subscriptions.values().stream() - .filter(Subscription::isActive) - .map(Subscription::getQualifiedName) - .collect(Collectors.toList()); - } + @Override + public List listActiveSubscriptionNames() { + return subscriptions.values().stream() + .filter(Subscription::isActive) + .map(Subscription::getQualifiedName) + .collect(Collectors.toList()); + } - @Override - public void start() { - for (String groupName : groupRepository.listGroupNames()) { - for (String topicName : topicRepository.listTopicNames(groupName)) { - for (Subscription subscription : subscriptionRepository.listSubscriptions(new TopicName(groupName, topicName))) { - subscriptions.put(subscription.getQualifiedName(), subscription); - } - } + @Override + public void start() { + for (String groupName : groupRepository.listGroupNames()) { + for (String topicName : topicRepository.listTopicNames(groupName)) { + for (Subscription subscription : + subscriptionRepository.listSubscriptions(new TopicName(groupName, topicName))) { + subscriptions.put(subscription.getQualifiedName(), subscription); } + } } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/cache/SubscriptionsCache.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/cache/SubscriptionsCache.java index 0e54e63502..27ade341fe 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/cache/SubscriptionsCache.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/cache/SubscriptionsCache.java @@ -1,18 +1,17 @@ package pl.allegro.tech.hermes.consumers.subscription.cache; +import java.util.List; import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.api.TopicName; -import java.util.List; - public interface SubscriptionsCache { - Subscription getSubscription(SubscriptionName subscriptionName); + Subscription getSubscription(SubscriptionName subscriptionName); - List subscriptionsOfTopic(TopicName topicName); + List subscriptionsOfTopic(TopicName topicName); - List listActiveSubscriptionNames(); + List listActiveSubscriptionNames(); - void start(); + void start(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/id/NotificationAwareSubscriptionIdsCache.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/id/NotificationAwareSubscriptionIdsCache.java index f3838d10d3..80b96e9376 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/id/NotificationAwareSubscriptionIdsCache.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/id/NotificationAwareSubscriptionIdsCache.java @@ -4,6 +4,10 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.cache.RemovalListener; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.Subscription; @@ -12,95 +16,103 @@ import pl.allegro.tech.hermes.domain.notifications.InternalNotificationsBus; import pl.allegro.tech.hermes.domain.notifications.SubscriptionCallback; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; - -public class NotificationAwareSubscriptionIdsCache implements SubscriptionIds, SubscriptionCallback { - - private static final Logger logger = LoggerFactory.getLogger(NotificationAwareSubscriptionIdsCache.class); - - private final SubscriptionsCache subscriptionsCache; - private final SubscriptionIdProvider subscriptionIdProvider; - private final Map nameToIdMap = new ConcurrentHashMap<>(); - private final Map valueToIdMap = new ConcurrentHashMap<>(); - private final Cache nameToIdMapOfRemoved; - private final Cache valueToIdMapOfRemoved; - - public NotificationAwareSubscriptionIdsCache(InternalNotificationsBus notificationsBus, - SubscriptionsCache subscriptionsCache, - SubscriptionIdProvider subscriptionIdProvider, - long removedSubscriptionsExpireAfterAccessSeconds, - Ticker ticker) { - this.subscriptionsCache = subscriptionsCache; - this.subscriptionIdProvider = subscriptionIdProvider; - - this.nameToIdMapOfRemoved = createExpiringCache( - removedSubscriptionsExpireAfterAccessSeconds, ticker, - notification -> logger.info("Removing expired subscription {} id from name->id cache", - notification.getKey().getQualifiedName()) - ); - - this.valueToIdMapOfRemoved = createExpiringCache( - removedSubscriptionsExpireAfterAccessSeconds, ticker, - notification -> logger.info("Removing expired subscription {} id from value->id cache", - notification.getValue().getSubscriptionName().getQualifiedName()) - ); - - notificationsBus.registerSubscriptionCallback(this); - } - - private Cache createExpiringCache(long expireAfterSeconds, Ticker ticker, RemovalListener removalListener) { - CacheBuilder cacheBuilder = CacheBuilder.newBuilder() - .expireAfterAccess(expireAfterSeconds, TimeUnit.SECONDS) - .ticker(ticker) - .removalListener(removalListener); - return cacheBuilder.build(); - } - - @Override - public void start() { - subscriptionsCache.listActiveSubscriptionNames() - .forEach(this::putSubscriptionId); - } - - private void putSubscriptionId(SubscriptionName name) { - SubscriptionId id = subscriptionIdProvider.getSubscriptionId(name); - nameToIdMap.put(name, id); - valueToIdMap.put(id.getValue(), id); - } - - @Override - public Optional getSubscriptionId(SubscriptionName subscriptionName) { - return Optional.ofNullable(Optional.ofNullable(nameToIdMap.get(subscriptionName)) - .orElseGet(() -> nameToIdMapOfRemoved.getIfPresent(subscriptionName))); - } - - @Override - public Optional getSubscriptionId(long id) { - return Optional.ofNullable(Optional.ofNullable(valueToIdMap.get(id)) - .orElseGet(() -> valueToIdMapOfRemoved.getIfPresent(id))); - } - - @Override - public void onSubscriptionCreated(Subscription subscription) { - putSubscriptionId(subscription.getQualifiedName()); - } - - @Override - public void onSubscriptionChanged(Subscription subscription) { - putSubscriptionId(subscription.getQualifiedName()); - } - - @Override - public void onSubscriptionRemoved(Subscription subscription) { - Optional.ofNullable(nameToIdMap.remove(subscription.getQualifiedName())) - .ifPresent(id -> { - valueToIdMap.remove(id.getValue()); - - nameToIdMapOfRemoved.put(subscription.getQualifiedName(), id); - valueToIdMapOfRemoved.put(id.getValue(), id); - }); - } +public class NotificationAwareSubscriptionIdsCache + implements SubscriptionIds, SubscriptionCallback { + + private static final Logger logger = + LoggerFactory.getLogger(NotificationAwareSubscriptionIdsCache.class); + + private final SubscriptionsCache subscriptionsCache; + private final SubscriptionIdProvider subscriptionIdProvider; + private final Map nameToIdMap = new ConcurrentHashMap<>(); + private final Map valueToIdMap = new ConcurrentHashMap<>(); + private final Cache nameToIdMapOfRemoved; + private final Cache valueToIdMapOfRemoved; + + public NotificationAwareSubscriptionIdsCache( + InternalNotificationsBus notificationsBus, + SubscriptionsCache subscriptionsCache, + SubscriptionIdProvider subscriptionIdProvider, + long removedSubscriptionsExpireAfterAccessSeconds, + Ticker ticker) { + this.subscriptionsCache = subscriptionsCache; + this.subscriptionIdProvider = subscriptionIdProvider; + + this.nameToIdMapOfRemoved = + createExpiringCache( + removedSubscriptionsExpireAfterAccessSeconds, + ticker, + notification -> + logger.info( + "Removing expired subscription {} id from name->id cache", + notification.getKey().getQualifiedName())); + + this.valueToIdMapOfRemoved = + createExpiringCache( + removedSubscriptionsExpireAfterAccessSeconds, + ticker, + notification -> + logger.info( + "Removing expired subscription {} id from value->id cache", + notification.getValue().getSubscriptionName().getQualifiedName())); + + notificationsBus.registerSubscriptionCallback(this); + } + + private Cache createExpiringCache( + long expireAfterSeconds, Ticker ticker, RemovalListener removalListener) { + CacheBuilder cacheBuilder = + CacheBuilder.newBuilder() + .expireAfterAccess(expireAfterSeconds, TimeUnit.SECONDS) + .ticker(ticker) + .removalListener(removalListener); + return cacheBuilder.build(); + } + + @Override + public void start() { + subscriptionsCache.listActiveSubscriptionNames().forEach(this::putSubscriptionId); + } + + private void putSubscriptionId(SubscriptionName name) { + SubscriptionId id = subscriptionIdProvider.getSubscriptionId(name); + nameToIdMap.put(name, id); + valueToIdMap.put(id.getValue(), id); + } + + @Override + public Optional getSubscriptionId(SubscriptionName subscriptionName) { + return Optional.ofNullable( + Optional.ofNullable(nameToIdMap.get(subscriptionName)) + .orElseGet(() -> nameToIdMapOfRemoved.getIfPresent(subscriptionName))); + } + + @Override + public Optional getSubscriptionId(long id) { + return Optional.ofNullable( + Optional.ofNullable(valueToIdMap.get(id)) + .orElseGet(() -> valueToIdMapOfRemoved.getIfPresent(id))); + } + + @Override + public void onSubscriptionCreated(Subscription subscription) { + putSubscriptionId(subscription.getQualifiedName()); + } + + @Override + public void onSubscriptionChanged(Subscription subscription) { + putSubscriptionId(subscription.getQualifiedName()); + } + + @Override + public void onSubscriptionRemoved(Subscription subscription) { + Optional.ofNullable(nameToIdMap.remove(subscription.getQualifiedName())) + .ifPresent( + id -> { + valueToIdMap.remove(id.getValue()); + + nameToIdMapOfRemoved.put(subscription.getQualifiedName(), id); + valueToIdMapOfRemoved.put(id.getValue(), id); + }); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/id/SubscriptionId.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/id/SubscriptionId.java index c5c6184c24..4d0763afd1 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/id/SubscriptionId.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/id/SubscriptionId.java @@ -1,46 +1,45 @@ package pl.allegro.tech.hermes.consumers.subscription.id; -import pl.allegro.tech.hermes.api.SubscriptionName; - import java.util.Objects; +import pl.allegro.tech.hermes.api.SubscriptionName; public class SubscriptionId { - private final long value; + private final long value; - private final SubscriptionName subscriptionName; + private final SubscriptionName subscriptionName; - private SubscriptionId(SubscriptionName subscriptionName, long value) { - this.value = value; - this.subscriptionName = subscriptionName; - } + private SubscriptionId(SubscriptionName subscriptionName, long value) { + this.value = value; + this.subscriptionName = subscriptionName; + } - public static SubscriptionId from(SubscriptionName subscriptionName, long value) { - return new SubscriptionId(subscriptionName, value); - } + public static SubscriptionId from(SubscriptionName subscriptionName, long value) { + return new SubscriptionId(subscriptionName, value); + } - public long getValue() { - return value; - } + public long getValue() { + return value; + } - public SubscriptionName getSubscriptionName() { - return subscriptionName; - } + public SubscriptionName getSubscriptionName() { + return subscriptionName; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - SubscriptionId that = (SubscriptionId) o; - return value == that.value; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(value); + if (o == null || getClass() != o.getClass()) { + return false; } + SubscriptionId that = (SubscriptionId) o; + return value == that.value; + } + + @Override + public int hashCode() { + return Objects.hash(value); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/id/SubscriptionIdProvider.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/id/SubscriptionIdProvider.java index 991b76b12e..23a4f71f10 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/id/SubscriptionIdProvider.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/id/SubscriptionIdProvider.java @@ -4,5 +4,5 @@ public interface SubscriptionIdProvider { - SubscriptionId getSubscriptionId(SubscriptionName name); + SubscriptionId getSubscriptionId(SubscriptionName name); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/id/SubscriptionIds.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/id/SubscriptionIds.java index 91671e76b2..4c5498baaf 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/id/SubscriptionIds.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/id/SubscriptionIds.java @@ -1,14 +1,13 @@ package pl.allegro.tech.hermes.consumers.subscription.id; -import pl.allegro.tech.hermes.api.SubscriptionName; - import java.util.Optional; +import pl.allegro.tech.hermes.api.SubscriptionName; public interface SubscriptionIds { - Optional getSubscriptionId(SubscriptionName subscriptionName); + Optional getSubscriptionId(SubscriptionName subscriptionName); - Optional getSubscriptionId(long id); + Optional getSubscriptionId(long id); - void start(); + void start(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/id/ZookeeperSubscriptionIdProvider.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/id/ZookeeperSubscriptionIdProvider.java index bded66c883..23774950af 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/id/ZookeeperSubscriptionIdProvider.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/subscription/id/ZookeeperSubscriptionIdProvider.java @@ -1,39 +1,45 @@ package pl.allegro.tech.hermes.consumers.subscription.id; +import java.util.Optional; import org.apache.curator.framework.CuratorFramework; import org.apache.zookeeper.data.Stat; import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.common.exception.InternalProcessingException; import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; -import java.util.Optional; - public class ZookeeperSubscriptionIdProvider implements SubscriptionIdProvider { - private final CuratorFramework curatorFramework; - private final ZookeeperPaths zookeeperPaths; + private final CuratorFramework curatorFramework; + private final ZookeeperPaths zookeeperPaths; - public ZookeeperSubscriptionIdProvider(CuratorFramework curatorFramework, ZookeeperPaths zookeeperPaths) { - this.curatorFramework = curatorFramework; - this.zookeeperPaths = zookeeperPaths; - } + public ZookeeperSubscriptionIdProvider( + CuratorFramework curatorFramework, ZookeeperPaths zookeeperPaths) { + this.curatorFramework = curatorFramework; + this.zookeeperPaths = zookeeperPaths; + } - @Override - public SubscriptionId getSubscriptionId(SubscriptionName name) { - return Optional.ofNullable(getZnodeStat(name)) - .map(Stat::getCzxid) - .map(czxid -> SubscriptionId.from(name, czxid)) - .orElseThrow(() -> new IllegalStateException( - String.format("Cannot get czxid of subscription %s as it doesn't exist", name.getQualifiedName()))); - } + @Override + public SubscriptionId getSubscriptionId(SubscriptionName name) { + return Optional.ofNullable(getZnodeStat(name)) + .map(Stat::getCzxid) + .map(czxid -> SubscriptionId.from(name, czxid)) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot get czxid of subscription %s as it doesn't exist", + name.getQualifiedName()))); + } - private Stat getZnodeStat(SubscriptionName name) { - String path = zookeeperPaths.subscriptionPath(name.getTopicName(), name.getName()); - try { - return curatorFramework.checkExists().forPath(path); - } catch (Exception e) { - throw new InternalProcessingException(String.format("Could not check existence of subscription %s node", - name.getQualifiedName()), e); - } + private Stat getZnodeStat(SubscriptionName name) { + String path = zookeeperPaths.subscriptionPath(name.getTopicName(), name.getName()); + try { + return curatorFramework.checkExists().forPath(path); + } catch (Exception e) { + throw new InternalProcessingException( + String.format( + "Could not check existence of subscription %s node", name.getQualifiedName()), + e); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/ConsumerFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/ConsumerFactory.java index f0108ccd3b..8ae3000766 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/ConsumerFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/ConsumerFactory.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.consumers.supervisor; +import java.time.Clock; +import java.time.Duration; import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.common.message.wrapper.CompositeMessageContentWrapper; @@ -23,105 +25,108 @@ import pl.allegro.tech.hermes.domain.topic.TopicRepository; import pl.allegro.tech.hermes.tracker.consumers.Trackers; -import java.time.Clock; -import java.time.Duration; - public class ConsumerFactory { - private final ConsumerRateLimitSupervisor consumerRateLimitSupervisor; - private final OutputRateCalculatorFactory outputRateCalculatorFactory; - private final ReceiverFactory messageReceiverFactory; - private final MetricsFacade metrics; - private final CommonConsumerParameters commonConsumerParameters; - private final Trackers trackers; - private final ConsumerMessageSenderFactory consumerMessageSenderFactory; - private final TopicRepository topicRepository; - private final MessageConverterResolver messageConverterResolver; - private final MessageBatchFactory batchFactory; - private final CompositeMessageContentWrapper compositeMessageContentWrapper; - private final MessageBatchSenderFactory batchSenderFactory; - private final ConsumerAuthorizationHandler consumerAuthorizationHandler; - private final Clock clock; - private final SubscriptionLoadRecordersRegistry subscriptionLoadRecordersRegistry; - private final ConsumerPartitionAssignmentState consumerPartitionAssignmentState; - private final Duration commitPeriod; - private final int offsetQueueSize; + private final ConsumerRateLimitSupervisor consumerRateLimitSupervisor; + private final OutputRateCalculatorFactory outputRateCalculatorFactory; + private final ReceiverFactory messageReceiverFactory; + private final MetricsFacade metrics; + private final CommonConsumerParameters commonConsumerParameters; + private final Trackers trackers; + private final ConsumerMessageSenderFactory consumerMessageSenderFactory; + private final TopicRepository topicRepository; + private final MessageConverterResolver messageConverterResolver; + private final MessageBatchFactory batchFactory; + private final CompositeMessageContentWrapper compositeMessageContentWrapper; + private final MessageBatchSenderFactory batchSenderFactory; + private final ConsumerAuthorizationHandler consumerAuthorizationHandler; + private final Clock clock; + private final SubscriptionLoadRecordersRegistry subscriptionLoadRecordersRegistry; + private final ConsumerPartitionAssignmentState consumerPartitionAssignmentState; + private final Duration commitPeriod; + private final int offsetQueueSize; - public ConsumerFactory(ReceiverFactory messageReceiverFactory, - MetricsFacade metrics, - CommonConsumerParameters commonConsumerParameters, - ConsumerRateLimitSupervisor consumerRateLimitSupervisor, - OutputRateCalculatorFactory outputRateCalculatorFactory, - Trackers trackers, - ConsumerMessageSenderFactory consumerMessageSenderFactory, - TopicRepository topicRepository, - MessageConverterResolver messageConverterResolver, - MessageBatchFactory byteBufferMessageBatchFactory, - CompositeMessageContentWrapper compositeMessageContentWrapper, - MessageBatchSenderFactory batchSenderFactory, - ConsumerAuthorizationHandler consumerAuthorizationHandler, - Clock clock, - SubscriptionLoadRecordersRegistry subscriptionLoadRecordersRegistry, - ConsumerPartitionAssignmentState consumerPartitionAssignmentState, - Duration commitPeriod, - int offsetQueueSize) { - this.messageReceiverFactory = messageReceiverFactory; - this.metrics = metrics; - this.commonConsumerParameters = commonConsumerParameters; - this.consumerRateLimitSupervisor = consumerRateLimitSupervisor; - this.outputRateCalculatorFactory = outputRateCalculatorFactory; - this.trackers = trackers; - this.consumerMessageSenderFactory = consumerMessageSenderFactory; - this.topicRepository = topicRepository; - this.messageConverterResolver = messageConverterResolver; - this.batchFactory = byteBufferMessageBatchFactory; - this.compositeMessageContentWrapper = compositeMessageContentWrapper; - this.batchSenderFactory = batchSenderFactory; - this.consumerAuthorizationHandler = consumerAuthorizationHandler; - this.clock = clock; - this.subscriptionLoadRecordersRegistry = subscriptionLoadRecordersRegistry; - this.consumerPartitionAssignmentState = consumerPartitionAssignmentState; - this.commitPeriod = commitPeriod; - this.offsetQueueSize = offsetQueueSize; - } + public ConsumerFactory( + ReceiverFactory messageReceiverFactory, + MetricsFacade metrics, + CommonConsumerParameters commonConsumerParameters, + ConsumerRateLimitSupervisor consumerRateLimitSupervisor, + OutputRateCalculatorFactory outputRateCalculatorFactory, + Trackers trackers, + ConsumerMessageSenderFactory consumerMessageSenderFactory, + TopicRepository topicRepository, + MessageConverterResolver messageConverterResolver, + MessageBatchFactory byteBufferMessageBatchFactory, + CompositeMessageContentWrapper compositeMessageContentWrapper, + MessageBatchSenderFactory batchSenderFactory, + ConsumerAuthorizationHandler consumerAuthorizationHandler, + Clock clock, + SubscriptionLoadRecordersRegistry subscriptionLoadRecordersRegistry, + ConsumerPartitionAssignmentState consumerPartitionAssignmentState, + Duration commitPeriod, + int offsetQueueSize) { + this.messageReceiverFactory = messageReceiverFactory; + this.metrics = metrics; + this.commonConsumerParameters = commonConsumerParameters; + this.consumerRateLimitSupervisor = consumerRateLimitSupervisor; + this.outputRateCalculatorFactory = outputRateCalculatorFactory; + this.trackers = trackers; + this.consumerMessageSenderFactory = consumerMessageSenderFactory; + this.topicRepository = topicRepository; + this.messageConverterResolver = messageConverterResolver; + this.batchFactory = byteBufferMessageBatchFactory; + this.compositeMessageContentWrapper = compositeMessageContentWrapper; + this.batchSenderFactory = batchSenderFactory; + this.consumerAuthorizationHandler = consumerAuthorizationHandler; + this.clock = clock; + this.subscriptionLoadRecordersRegistry = subscriptionLoadRecordersRegistry; + this.consumerPartitionAssignmentState = consumerPartitionAssignmentState; + this.commitPeriod = commitPeriod; + this.offsetQueueSize = offsetQueueSize; + } - public Consumer createConsumer(Subscription subscription) { - Topic topic = topicRepository.getTopicDetails(subscription.getTopicName()); - SubscriptionLoadRecorder loadRecorder = subscriptionLoadRecordersRegistry.register(subscription.getQualifiedName()); - if (subscription.isBatchSubscription()) { - return new BatchConsumer(messageReceiverFactory, - batchSenderFactory.create(subscription), - batchFactory, - messageConverterResolver, - compositeMessageContentWrapper, - metrics, - trackers, - subscription, - topic, - commonConsumerParameters.isUseTopicMessageSizeEnabled(), - loadRecorder, - commitPeriod - ); - } else { - SerialConsumerRateLimiter consumerRateLimiter = new SerialConsumerRateLimiter(subscription, - outputRateCalculatorFactory, metrics, consumerRateLimitSupervisor, clock); + public Consumer createConsumer(Subscription subscription) { + Topic topic = topicRepository.getTopicDetails(subscription.getTopicName()); + SubscriptionLoadRecorder loadRecorder = + subscriptionLoadRecordersRegistry.register(subscription.getQualifiedName()); + if (subscription.isBatchSubscription()) { + return new BatchConsumer( + messageReceiverFactory, + batchSenderFactory.create(subscription), + batchFactory, + messageConverterResolver, + compositeMessageContentWrapper, + metrics, + trackers, + subscription, + topic, + commonConsumerParameters.isUseTopicMessageSizeEnabled(), + loadRecorder, + commitPeriod); + } else { + SerialConsumerRateLimiter consumerRateLimiter = + new SerialConsumerRateLimiter( + subscription, + outputRateCalculatorFactory, + metrics, + consumerRateLimitSupervisor, + clock); - return new SerialConsumer( - messageReceiverFactory, - metrics, - subscription, - consumerRateLimiter, - consumerMessageSenderFactory, - trackers, - messageConverterResolver, - topic, - commonConsumerParameters, - consumerAuthorizationHandler, - loadRecorder, - consumerPartitionAssignmentState, - commitPeriod, - offsetQueueSize - ); - } + return new SerialConsumer( + messageReceiverFactory, + metrics, + subscription, + consumerRateLimiter, + consumerMessageSenderFactory, + trackers, + messageConverterResolver, + topic, + commonConsumerParameters, + consumerAuthorizationHandler, + loadRecorder, + consumerPartitionAssignmentState, + commitPeriod, + offsetQueueSize); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/ConsumerHolder.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/ConsumerHolder.java index c0182bd104..e9496fa7a6 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/ConsumerHolder.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/ConsumerHolder.java @@ -3,38 +3,37 @@ import com.google.common.collect.HashBasedTable; import com.google.common.collect.Lists; import com.google.common.collect.Table; -import pl.allegro.tech.hermes.api.TopicName; -import pl.allegro.tech.hermes.consumers.consumer.Consumer; - import java.util.Iterator; import java.util.Optional; +import pl.allegro.tech.hermes.api.TopicName; +import pl.allegro.tech.hermes.consumers.consumer.Consumer; public class ConsumerHolder implements Iterable { - private Table consumers = HashBasedTable.create(); + private Table consumers = HashBasedTable.create(); - public synchronized void add(TopicName topicName, String subscriptionName, Consumer consumer) { - consumers.put(topicName, subscriptionName, consumer); - } + public synchronized void add(TopicName topicName, String subscriptionName, Consumer consumer) { + consumers.put(topicName, subscriptionName, consumer); + } - public synchronized void remove(TopicName topicName, String subscriptionName) { - consumers.remove(topicName, subscriptionName); - } + public synchronized void remove(TopicName topicName, String subscriptionName) { + consumers.remove(topicName, subscriptionName); + } - public synchronized Optional get(TopicName topicName, String subscriptionName) { - return Optional.ofNullable(consumers.get(topicName, subscriptionName)); - } + public synchronized Optional get(TopicName topicName, String subscriptionName) { + return Optional.ofNullable(consumers.get(topicName, subscriptionName)); + } - public synchronized boolean contains(TopicName topicName, String subscriptionName) { - return consumers.contains(topicName, subscriptionName); - } + public synchronized boolean contains(TopicName topicName, String subscriptionName) { + return consumers.contains(topicName, subscriptionName); + } - @Override - public synchronized Iterator iterator() { - return Lists.newArrayList(consumers.values()).iterator(); - } + @Override + public synchronized Iterator iterator() { + return Lists.newArrayList(consumers.values()).iterator(); + } - public synchronized void clear() { - consumers.clear(); - } + public synchronized void clear() { + consumers.clear(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/ConsumersExecutorService.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/ConsumersExecutorService.java index d44c145bce..6a9eef256e 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/ConsumersExecutorService.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/ConsumersExecutorService.java @@ -1,43 +1,46 @@ package pl.allegro.tech.hermes.consumers.supervisor; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import pl.allegro.tech.hermes.common.metric.MetricsFacade; -import pl.allegro.tech.hermes.consumers.supervisor.process.ConsumerProcess; - import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import pl.allegro.tech.hermes.common.metric.MetricsFacade; +import pl.allegro.tech.hermes.consumers.supervisor.process.ConsumerProcess; public class ConsumersExecutorService { - private static final Logger logger = LoggerFactory.getLogger(ConsumersExecutorService.class); - private final ThreadPoolExecutor executor; + private static final Logger logger = LoggerFactory.getLogger(ConsumersExecutorService.class); + private final ThreadPoolExecutor executor; - public ConsumersExecutorService(int poolSize, MetricsFacade metrics) { - ThreadFactory threadFactory = new ThreadFactoryBuilder() + public ConsumersExecutorService(int poolSize, MetricsFacade metrics) { + ThreadFactory threadFactory = + new ThreadFactoryBuilder() .setNameFormat("Consumer-%d") - .setUncaughtExceptionHandler((t, e) -> logger.error("Exception from consumer with name {}", t.getName(), e)).build(); - - executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(poolSize, threadFactory); - - metrics.consumer().registerConsumerProcessesThreadsGauge(executor, ThreadPoolExecutor::getActiveCount); - } - - public Future execute(ConsumerProcess consumer) { - return executor.submit(consumer); - } - - public void shutdown() { - executor.shutdownNow(); - try { - executor.awaitTermination(1, TimeUnit.MINUTES); - } catch (InterruptedException e) { - logger.error("Termination of consumers executor service interrupted.", e); - } + .setUncaughtExceptionHandler( + (t, e) -> logger.error("Exception from consumer with name {}", t.getName(), e)) + .build(); + + executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(poolSize, threadFactory); + + metrics + .consumer() + .registerConsumerProcessesThreadsGauge(executor, ThreadPoolExecutor::getActiveCount); + } + + public Future execute(ConsumerProcess consumer) { + return executor.submit(consumer); + } + + public void shutdown() { + executor.shutdownNow(); + try { + executor.awaitTermination(1, TimeUnit.MINUTES); + } catch (InterruptedException e) { + logger.error("Termination of consumers executor service interrupted.", e); } - + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/ConsumersSupervisor.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/ConsumersSupervisor.java index aa533f3001..5a3409dd60 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/ConsumersSupervisor.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/ConsumersSupervisor.java @@ -1,26 +1,25 @@ package pl.allegro.tech.hermes.consumers.supervisor; +import java.util.Set; import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.api.Topic; -import java.util.Set; - public interface ConsumersSupervisor { - void assignConsumerForSubscription(Subscription subscription); + void assignConsumerForSubscription(Subscription subscription); - void deleteConsumerForSubscriptionName(SubscriptionName subscription); + void deleteConsumerForSubscriptionName(SubscriptionName subscription); - void updateSubscription(Subscription subscription); + void updateSubscription(Subscription subscription); - void updateTopic(Subscription subscription, Topic topic); + void updateTopic(Subscription subscription, Topic topic); - void shutdown() throws InterruptedException; + void shutdown() throws InterruptedException; - void retransmit(SubscriptionName subscription) throws Exception; + void retransmit(SubscriptionName subscription) throws Exception; - Set runningConsumers(); + Set runningConsumers(); - void start() throws Exception; + void start() throws Exception; } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/NonblockingConsumersSupervisor.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/NonblockingConsumersSupervisor.java index 0911f05fea..d628050a0d 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/NonblockingConsumersSupervisor.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/NonblockingConsumersSupervisor.java @@ -1,6 +1,18 @@ package pl.allegro.tech.hermes.consumers.supervisor; +import static pl.allegro.tech.hermes.api.Subscription.State.ACTIVE; +import static pl.allegro.tech.hermes.api.Subscription.State.PENDING; +import static pl.allegro.tech.hermes.consumers.health.Checks.SUBSCRIPTIONS; +import static pl.allegro.tech.hermes.consumers.health.Checks.SUBSCRIPTIONS_COUNT; + import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.time.Clock; +import java.time.Duration; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.Subscription; @@ -17,118 +29,121 @@ import pl.allegro.tech.hermes.consumers.supervisor.process.Signal; import pl.allegro.tech.hermes.domain.subscription.SubscriptionRepository; -import java.time.Clock; -import java.time.Duration; -import java.util.Set; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; - -import static pl.allegro.tech.hermes.api.Subscription.State.ACTIVE; -import static pl.allegro.tech.hermes.api.Subscription.State.PENDING; -import static pl.allegro.tech.hermes.consumers.health.Checks.SUBSCRIPTIONS; -import static pl.allegro.tech.hermes.consumers.health.Checks.SUBSCRIPTIONS_COUNT; - public class NonblockingConsumersSupervisor implements ConsumersSupervisor { - private static final Logger logger = LoggerFactory.getLogger(NonblockingConsumersSupervisor.class); - - private final ConsumerProcessSupervisor backgroundProcess; - private final UndeliveredMessageLogPersister undeliveredMessageLogPersister; - private final Duration backgroundSupervisorInterval; - private final SubscriptionRepository subscriptionRepository; - - private final ScheduledExecutorService scheduledExecutor; - - public NonblockingConsumersSupervisor(CommonConsumerParameters commonConsumerParameters, - ConsumersExecutorService executor, - ConsumerFactory consumerFactory, - ConsumerPartitionAssignmentState consumerPartitionAssignmentState, - Retransmitter retransmitter, - UndeliveredMessageLogPersister undeliveredMessageLogPersister, - SubscriptionRepository subscriptionRepository, - MetricsFacade metrics, - ConsumerMonitor monitor, - Clock clock) { - this.undeliveredMessageLogPersister = undeliveredMessageLogPersister; - this.subscriptionRepository = subscriptionRepository; - this.backgroundSupervisorInterval = commonConsumerParameters.getBackgroundSupervisor().getInterval(); - this.backgroundProcess = new ConsumerProcessSupervisor(executor, clock, metrics, - new ConsumerProcessFactory( - retransmitter, - consumerFactory, - commonConsumerParameters.getBackgroundSupervisor().getUnhealthyAfter(), - clock), - commonConsumerParameters.getSignalProcessingQueueSize(), - commonConsumerParameters.getBackgroundSupervisor().getKillAfter()); - this.scheduledExecutor = createExecutorForSupervision(); - monitor.register(SUBSCRIPTIONS, backgroundProcess::runningSubscriptionsStatus); - monitor.register(SUBSCRIPTIONS_COUNT, backgroundProcess::countRunningProcesses); - } - - private ScheduledExecutorService createExecutorForSupervision() { - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setNameFormat("NonblockingConsumersSupervisor-%d") - .setUncaughtExceptionHandler((t, e) -> logger.error("Exception from supervisor with name {}", t.getName(), e)).build(); - return Executors.newSingleThreadScheduledExecutor(threadFactory); - } - - @Override - public void assignConsumerForSubscription(Subscription subscription) { - try { - Signal start = Signal.of(Signal.SignalType.START, subscription.getQualifiedName(), subscription); - - backgroundProcess.accept(start); - - if (subscription.getState() == PENDING) { - subscriptionRepository.updateSubscriptionState(subscription.getTopicName(), subscription.getName(), ACTIVE); - } - } catch (RuntimeException e) { - logger.error("Error during assigning subscription {} to consumer", subscription.getQualifiedName(), e); - } - } - - @Override - public void deleteConsumerForSubscriptionName(SubscriptionName subscription) { - Signal stop = Signal.of(Signal.SignalType.STOP, subscription); - logger.info("Deleting consumer for {}. {}", subscription, stop.getLogWithIdAndType()); - backgroundProcess.accept(stop); - } - - @Override - public void updateTopic(Subscription subscription, Topic topic) { - backgroundProcess.accept(Signal.of(Signal.SignalType.UPDATE_TOPIC, subscription.getQualifiedName(), topic)); - } - - @Override - public void updateSubscription(Subscription subscription) { - backgroundProcess.accept(Signal.of(Signal.SignalType.UPDATE_SUBSCRIPTION, subscription.getQualifiedName(), subscription)); - } - - @Override - public void retransmit(SubscriptionName subscription) { - backgroundProcess.accept(Signal.of(Signal.SignalType.RETRANSMIT, subscription)); - } - - @Override - public Set runningConsumers() { - return backgroundProcess.existingConsumers(); - } - - @Override - public void start() { - scheduledExecutor.scheduleAtFixedRate( - backgroundProcess, - backgroundSupervisorInterval.toMillis(), - backgroundSupervisorInterval.toMillis(), - TimeUnit.MILLISECONDS); - undeliveredMessageLogPersister.start(); - } - - @Override - public void shutdown() { - backgroundProcess.shutdown(); - scheduledExecutor.shutdown(); - undeliveredMessageLogPersister.shutdown(); + private static final Logger logger = + LoggerFactory.getLogger(NonblockingConsumersSupervisor.class); + + private final ConsumerProcessSupervisor backgroundProcess; + private final UndeliveredMessageLogPersister undeliveredMessageLogPersister; + private final Duration backgroundSupervisorInterval; + private final SubscriptionRepository subscriptionRepository; + + private final ScheduledExecutorService scheduledExecutor; + + public NonblockingConsumersSupervisor( + CommonConsumerParameters commonConsumerParameters, + ConsumersExecutorService executor, + ConsumerFactory consumerFactory, + ConsumerPartitionAssignmentState consumerPartitionAssignmentState, + Retransmitter retransmitter, + UndeliveredMessageLogPersister undeliveredMessageLogPersister, + SubscriptionRepository subscriptionRepository, + MetricsFacade metrics, + ConsumerMonitor monitor, + Clock clock) { + this.undeliveredMessageLogPersister = undeliveredMessageLogPersister; + this.subscriptionRepository = subscriptionRepository; + this.backgroundSupervisorInterval = + commonConsumerParameters.getBackgroundSupervisor().getInterval(); + this.backgroundProcess = + new ConsumerProcessSupervisor( + executor, + clock, + metrics, + new ConsumerProcessFactory( + retransmitter, + consumerFactory, + commonConsumerParameters.getBackgroundSupervisor().getUnhealthyAfter(), + clock), + commonConsumerParameters.getSignalProcessingQueueSize(), + commonConsumerParameters.getBackgroundSupervisor().getKillAfter()); + this.scheduledExecutor = createExecutorForSupervision(); + monitor.register(SUBSCRIPTIONS, backgroundProcess::runningSubscriptionsStatus); + monitor.register(SUBSCRIPTIONS_COUNT, backgroundProcess::countRunningProcesses); + } + + private ScheduledExecutorService createExecutorForSupervision() { + ThreadFactory threadFactory = + new ThreadFactoryBuilder() + .setNameFormat("NonblockingConsumersSupervisor-%d") + .setUncaughtExceptionHandler( + (t, e) -> logger.error("Exception from supervisor with name {}", t.getName(), e)) + .build(); + return Executors.newSingleThreadScheduledExecutor(threadFactory); + } + + @Override + public void assignConsumerForSubscription(Subscription subscription) { + try { + Signal start = + Signal.of(Signal.SignalType.START, subscription.getQualifiedName(), subscription); + + backgroundProcess.accept(start); + + if (subscription.getState() == PENDING) { + subscriptionRepository.updateSubscriptionState( + subscription.getTopicName(), subscription.getName(), ACTIVE); + } + } catch (RuntimeException e) { + logger.error( + "Error during assigning subscription {} to consumer", subscription.getQualifiedName(), e); } + } + + @Override + public void deleteConsumerForSubscriptionName(SubscriptionName subscription) { + Signal stop = Signal.of(Signal.SignalType.STOP, subscription); + logger.info("Deleting consumer for {}. {}", subscription, stop.getLogWithIdAndType()); + backgroundProcess.accept(stop); + } + + @Override + public void updateTopic(Subscription subscription, Topic topic) { + backgroundProcess.accept( + Signal.of(Signal.SignalType.UPDATE_TOPIC, subscription.getQualifiedName(), topic)); + } + + @Override + public void updateSubscription(Subscription subscription) { + backgroundProcess.accept( + Signal.of( + Signal.SignalType.UPDATE_SUBSCRIPTION, subscription.getQualifiedName(), subscription)); + } + + @Override + public void retransmit(SubscriptionName subscription) { + backgroundProcess.accept(Signal.of(Signal.SignalType.RETRANSMIT, subscription)); + } + + @Override + public Set runningConsumers() { + return backgroundProcess.existingConsumers(); + } + + @Override + public void start() { + scheduledExecutor.scheduleAtFixedRate( + backgroundProcess, + backgroundSupervisorInterval.toMillis(), + backgroundSupervisorInterval.toMillis(), + TimeUnit.MILLISECONDS); + undeliveredMessageLogPersister.start(); + } + + @Override + public void shutdown() { + backgroundProcess.shutdown(); + scheduledExecutor.shutdown(); + undeliveredMessageLogPersister.shutdown(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/SupervisorParameters.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/SupervisorParameters.java index cd4cf48cbb..a5a75bc7c1 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/SupervisorParameters.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/SupervisorParameters.java @@ -4,9 +4,9 @@ public interface SupervisorParameters { - Duration getInterval(); + Duration getInterval(); - Duration getUnhealthyAfter(); + Duration getUnhealthyAfter(); - Duration getKillAfter(); + Duration getKillAfter(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/monitor/ConsumersRuntimeMonitor.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/monitor/ConsumersRuntimeMonitor.java index 5416ffc344..1bef4fe15f 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/monitor/ConsumersRuntimeMonitor.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/monitor/ConsumersRuntimeMonitor.java @@ -2,6 +2,12 @@ import com.google.common.collect.Sets; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.time.Duration; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.SubscriptionName; @@ -10,140 +16,140 @@ import pl.allegro.tech.hermes.consumers.supervisor.ConsumersSupervisor; import pl.allegro.tech.hermes.consumers.supervisor.workload.WorkloadSupervisor; -import java.time.Duration; -import java.util.Set; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; - public class ConsumersRuntimeMonitor implements Runnable { - private static final Logger logger = LoggerFactory.getLogger(ConsumersRuntimeMonitor.class); + private static final Logger logger = LoggerFactory.getLogger(ConsumersRuntimeMonitor.class); - private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor( - new ThreadFactoryBuilder().setNameFormat("consumer-monitor-%d").build() - ); + private final ScheduledExecutorService executor = + Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder().setNameFormat("consumer-monitor-%d").build()); - private final Duration scanInterval; + private final Duration scanInterval; - private final ConsumersSupervisor consumerSupervisor; + private final ConsumersSupervisor consumerSupervisor; - private final WorkloadSupervisor workloadSupervisor; + private final WorkloadSupervisor workloadSupervisor; - private final SubscriptionsCache subscriptionsCache; + private final SubscriptionsCache subscriptionsCache; - private final MonitorMetrics monitorMetrics = new MonitorMetrics(); + private final MonitorMetrics monitorMetrics = new MonitorMetrics(); - private ScheduledFuture monitoringTask; + private ScheduledFuture monitoringTask; - public ConsumersRuntimeMonitor(ConsumersSupervisor consumerSupervisor, - WorkloadSupervisor workloadSupervisor, - MetricsFacade metrics, - SubscriptionsCache subscriptionsCache, - Duration scanInterval) { - this.consumerSupervisor = consumerSupervisor; - this.workloadSupervisor = workloadSupervisor; - this.subscriptionsCache = subscriptionsCache; - this.scanInterval = scanInterval; + public ConsumersRuntimeMonitor( + ConsumersSupervisor consumerSupervisor, + WorkloadSupervisor workloadSupervisor, + MetricsFacade metrics, + SubscriptionsCache subscriptionsCache, + Duration scanInterval) { + this.consumerSupervisor = consumerSupervisor; + this.workloadSupervisor = workloadSupervisor; + this.subscriptionsCache = subscriptionsCache; + this.scanInterval = scanInterval; - metrics.workload().registerRunningSubscriptionsGauge(monitorMetrics, mm -> mm.running); - metrics.workload().registerAssignedSubscriptionsGauge(monitorMetrics, mm -> mm.assigned); - metrics.workload().registerMissingSubscriptionsGauge(monitorMetrics, mm -> mm.missing); - metrics.workload().registerOversubscribedGauge(monitorMetrics, mm -> mm.oversubscribed); - } + metrics.workload().registerRunningSubscriptionsGauge(monitorMetrics, mm -> mm.running); + metrics.workload().registerAssignedSubscriptionsGauge(monitorMetrics, mm -> mm.assigned); + metrics.workload().registerMissingSubscriptionsGauge(monitorMetrics, mm -> mm.missing); + metrics.workload().registerOversubscribedGauge(monitorMetrics, mm -> mm.oversubscribed); + } - @Override - public void run() { - try { - Set assigned = workloadSupervisor.assignedSubscriptions(); - Set running = consumerSupervisor.runningConsumers(); - Set missing = missing(assigned, running); - Set oversubscribed = oversubscribed(assigned, running); - - log(assigned, running, missing, oversubscribed); - updateMetrics(assigned, running, missing, oversubscribed); - - ensureCorrectness(missing, oversubscribed); - } catch (Exception exception) { - logger.error("Could not check correctness of assignments", exception); - } - } + @Override + public void run() { + try { + Set assigned = workloadSupervisor.assignedSubscriptions(); + Set running = consumerSupervisor.runningConsumers(); + Set missing = missing(assigned, running); + Set oversubscribed = oversubscribed(assigned, running); - public void start() { - this.monitoringTask = executor.scheduleWithFixedDelay(this, scanInterval.toSeconds(), scanInterval.toSeconds(), TimeUnit.SECONDS); - } + log(assigned, running, missing, oversubscribed); + updateMetrics(assigned, running, missing, oversubscribed); - public void shutdown() throws InterruptedException { - try { - monitoringTask.cancel(false); - executor.shutdown(); - executor.awaitTermination(1, TimeUnit.MINUTES); - } catch (InterruptedException exception) { - logger.warn("Got exception when stopping consumers runtime monitor", exception); - } + ensureCorrectness(missing, oversubscribed); + } catch (Exception exception) { + logger.error("Could not check correctness of assignments", exception); } - - private void ensureCorrectness(Set missing, Set oversubscribed) { - if (!missing.isEmpty() || !oversubscribed.isEmpty()) { - logger.info("Fixing runtime. Creating {} and killing {} consumers", missing.size(), oversubscribed.size()); - } - missing.stream() - .map(subscriptionsCache::getSubscription) - .forEach(consumerSupervisor::assignConsumerForSubscription); - oversubscribed.forEach(consumerSupervisor::deleteConsumerForSubscriptionName); + } + + public void start() { + this.monitoringTask = + executor.scheduleWithFixedDelay( + this, scanInterval.toSeconds(), scanInterval.toSeconds(), TimeUnit.SECONDS); + } + + public void shutdown() throws InterruptedException { + try { + monitoringTask.cancel(false); + executor.shutdown(); + executor.awaitTermination(1, TimeUnit.MINUTES); + } catch (InterruptedException exception) { + logger.warn("Got exception when stopping consumers runtime monitor", exception); } - - private void log(Set assigned, - Set running, - Set missing, - Set oversubscribed) { - for (SubscriptionName subscriptionName : missing) { - logger.warn("Missing consumer process for subscription: {}", subscriptionName); - } - - for (SubscriptionName subscriptionName : oversubscribed) { - logger.warn("Unwanted consumer process for subscription: {}", subscriptionName); - } - - logger.info( - "Subscriptions assigned: {}, existing subscriptions: {}, missing: {}, oversubscribed: {}", - assigned.size(), - running.size(), - missing.size(), - oversubscribed.size() - ); + } + + private void ensureCorrectness( + Set missing, Set oversubscribed) { + if (!missing.isEmpty() || !oversubscribed.isEmpty()) { + logger.info( + "Fixing runtime. Creating {} and killing {} consumers", + missing.size(), + oversubscribed.size()); } - - private void updateMetrics(Set assigned, - Set running, - Set missing, - Set oversubscribed) { - monitorMetrics.assigned = assigned.size(); - monitorMetrics.running = running.size(); - monitorMetrics.missing = missing.size(); - monitorMetrics.oversubscribed = oversubscribed.size(); - } - - private Set missing(Set assignedSubscriptions, - Set runningSubscriptions) { - return Sets.difference(assignedSubscriptions, runningSubscriptions).immutableCopy(); + missing.stream() + .map(subscriptionsCache::getSubscription) + .forEach(consumerSupervisor::assignConsumerForSubscription); + oversubscribed.forEach(consumerSupervisor::deleteConsumerForSubscriptionName); + } + + private void log( + Set assigned, + Set running, + Set missing, + Set oversubscribed) { + for (SubscriptionName subscriptionName : missing) { + logger.warn("Missing consumer process for subscription: {}", subscriptionName); } - private Set oversubscribed(Set assignedSubscriptions, - Set runningSubscriptions) { - return Sets.difference(runningSubscriptions, assignedSubscriptions).immutableCopy(); + for (SubscriptionName subscriptionName : oversubscribed) { + logger.warn("Unwanted consumer process for subscription: {}", subscriptionName); } - private static class MonitorMetrics { - - volatile int assigned; - - volatile int running; - - volatile int missing; - - volatile int oversubscribed; - - } + logger.info( + "Subscriptions assigned: {}, existing subscriptions: {}, missing: {}, oversubscribed: {}", + assigned.size(), + running.size(), + missing.size(), + oversubscribed.size()); + } + + private void updateMetrics( + Set assigned, + Set running, + Set missing, + Set oversubscribed) { + monitorMetrics.assigned = assigned.size(); + monitorMetrics.running = running.size(); + monitorMetrics.missing = missing.size(); + monitorMetrics.oversubscribed = oversubscribed.size(); + } + + private Set missing( + Set assignedSubscriptions, Set runningSubscriptions) { + return Sets.difference(assignedSubscriptions, runningSubscriptions).immutableCopy(); + } + + private Set oversubscribed( + Set assignedSubscriptions, Set runningSubscriptions) { + return Sets.difference(runningSubscriptions, assignedSubscriptions).immutableCopy(); + } + + private static class MonitorMetrics { + + volatile int assigned; + + volatile int running; + + volatile int missing; + + volatile int oversubscribed; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/monitor/DifferenceCalculator.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/monitor/DifferenceCalculator.java index 405f1b7311..df2e843501 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/monitor/DifferenceCalculator.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/monitor/DifferenceCalculator.java @@ -1,16 +1,13 @@ package pl.allegro.tech.hermes.consumers.supervisor.monitor; import com.google.common.collect.Sets; - import java.util.Set; class DifferenceCalculator { - static SetDifference calculate(Set collection1, Set collection2) { - return new SetDifference<>( - Sets.difference(collection2, collection1).immutableCopy(), - Sets.difference(collection1, collection2).immutableCopy() - ); - } - + static SetDifference calculate(Set collection1, Set collection2) { + return new SetDifference<>( + Sets.difference(collection2, collection1).immutableCopy(), + Sets.difference(collection1, collection2).immutableCopy()); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/monitor/SetDifference.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/monitor/SetDifference.java index 9726206986..06ec54d30c 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/monitor/SetDifference.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/monitor/SetDifference.java @@ -6,32 +6,29 @@ class SetDifference { - private final Set added; - - private final Set removed; - - SetDifference(Set added, Set removed) { - this.added = Collections.unmodifiableSet(new HashSet<>(added)); - this.removed = Collections.unmodifiableSet(new HashSet<>(removed)); - } - - Set added() { - return added; - } - - Set removed() { - return removed; - } - - boolean containsChanges() { - return !added.isEmpty() || !removed.isEmpty(); - } - - @Override - public String toString() { - return "SetDifference{" - + "added=" + added - + ", removed=" + removed - + '}'; - } + private final Set added; + + private final Set removed; + + SetDifference(Set added, Set removed) { + this.added = Collections.unmodifiableSet(new HashSet<>(added)); + this.removed = Collections.unmodifiableSet(new HashSet<>(removed)); + } + + Set added() { + return added; + } + + Set removed() { + return removed; + } + + boolean containsChanges() { + return !added.isEmpty() || !removed.isEmpty(); + } + + @Override + public String toString() { + return "SetDifference{" + "added=" + added + ", removed=" + removed + '}'; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/ConsumerProcess.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/ConsumerProcess.java index c9d5983646..821fbf7340 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/ConsumerProcess.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/ConsumerProcess.java @@ -1,6 +1,13 @@ package pl.allegro.tech.hermes.consumers.supervisor.process; +import static pl.allegro.tech.hermes.consumers.supervisor.process.Signal.SignalType.START; + import com.google.common.collect.ImmutableMap; +import java.time.Clock; +import java.time.Duration; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; import org.jctools.queues.SpscArrayQueue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -8,209 +15,221 @@ import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.consumers.consumer.Consumer; -import java.time.Clock; -import java.time.Duration; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; - -import static pl.allegro.tech.hermes.consumers.supervisor.process.Signal.SignalType.START; - public class ConsumerProcess implements Runnable { - private static final Logger logger = LoggerFactory.getLogger(ConsumerProcess.class); - - private final SpscArrayQueue signals = new SpscArrayQueue<>(100); - - private final Clock clock; - - private final Consumer consumer; - - private final Retransmitter retransmitter; - - private final java.util.function.Consumer onConsumerStopped; - - private final Duration unhealthyAfter; - - private volatile boolean running = true; - - private volatile long healthcheckRefreshTime; - - private final Map signalTimesheet = new ConcurrentHashMap<>(); - - public ConsumerProcess( - Signal startSignal, - Consumer consumer, - Retransmitter retransmitter, - Clock clock, - Duration unhealthyAfter, - java.util.function.Consumer onConsumerStopped) { - this.consumer = consumer; - this.retransmitter = retransmitter; - this.onConsumerStopped = onConsumerStopped; - this.clock = clock; - this.healthcheckRefreshTime = clock.millis(); - this.unhealthyAfter = unhealthyAfter; - - addSignal(startSignal); - } - - @Override - public void run() { - try { - Thread.currentThread().setName("consumer-" + getSubscriptionName()); - - while (running && !Thread.currentThread().isInterrupted()) { - consumer.consume(this::processSignals); - } - } catch (Throwable throwable) { - logger.error("Consumer process of subscription {} failed", getSubscriptionName(), throwable); - } finally { - logger.info("Releasing consumer process thread of subscription {}", getSubscriptionName()); - refreshHealthcheck(); - try { - stop(); - } catch (Exception exceptionWhileStopping) { - logger.error("An error occurred while stopping consumer process of subscription {}", - getSubscriptionName(), exceptionWhileStopping); - } finally { - onConsumerStopped.accept(getSubscriptionName()); - Thread.currentThread().setName("consumer-released-thread"); - } - } - } - - public ConsumerProcess accept(Signal signal) { - addSignal(signal); - return this; - } - - public boolean isHealthy() { - return unhealthyAfter.toMillis() > lastSeen(); - } - - public long lastSeen() { - return clock.millis() - healthcheckRefreshTime; - } - - public long healthcheckRefreshTime() { - return healthcheckRefreshTime; - } - - private void processSignals() { - refreshHealthcheck(); - signals.drain(this::process); - refreshHealthcheck(); - } - - private void refreshHealthcheck() { - this.healthcheckRefreshTime = clock.millis(); - } - - private void process(Signal signal) { - try { - switch (signal.getType()) { - case START: - start(signal); - break; - case STOP: - logger.info("Stopping main loop for consumer {}. {}", signal.getTarget(), signal.getLogWithIdAndType()); - this.running = false; - break; - case RETRANSMIT: - retransmit(signal); - break; - case UPDATE_SUBSCRIPTION: - consumer.updateSubscription(signal.getPayload()); - break; - case UPDATE_TOPIC: - consumer.updateTopic(signal.getPayload()); - break; - default: - logger.warn("Unhandled signal found {}", signal); - break; - } - signalTimesheet.put(signal.getType(), clock.millis()); - } catch (Exception ex) { - logger.error("Failed to process signal {}", signal, ex); - } - } - - private void start(Signal signal) { - long startTime = clock.millis(); - logger.info("Starting consumer for subscription {}. {}", - getSubscriptionName(), signal.getLogWithIdAndType()); - - consumer.initialize(); - - long initializationTime = clock.millis(); - logger.info("Started consumer for subscription {} in {}ms. {}", - getSubscriptionName(), initializationTime - startTime, signal.getLogWithIdAndType()); - signalTimesheet.put(START, initializationTime); - } - - private void stop() { - long startTime = clock.millis(); - logger.info("Stopping consumer for subscription {}", getSubscriptionName()); - - consumer.tearDown(); - - logger.info("Stopped consumer for subscription {} in {}ms", getSubscriptionName(), clock.millis() - startTime); - } - - private void retransmit(Signal signal) { - long startTime = clock.millis(); - logger.info("Starting retransmission for consumer of subscription {}. {}", - getSubscriptionName(), signal.getLogWithIdAndType()); - try { - retransmitter.reloadOffsets(getSubscriptionName(), consumer); - logger.info("Done retransmission for consumer of subscription {} in {}ms", - getSubscriptionName(), clock.millis() - startTime); - } catch (Exception ex) { - logger.error("Failed retransmission for consumer of subscription {} in {}ms", - getSubscriptionName(), clock.millis() - startTime, ex); - } - } - - @Override - public String toString() { - return "ConsumerProcess{" - + "subscriptionName=" + getSubscriptionName() - + '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ConsumerProcess that = (ConsumerProcess) o; - return Objects.equals(getSubscriptionName(), that.getSubscriptionName()); - } - - @Override - public int hashCode() { - return Objects.hash(getSubscriptionName()); - } - - public Subscription getSubscription() { - return consumer.getSubscription(); - } - - public Map getSignalTimesheet() { - return ImmutableMap.copyOf(signalTimesheet); - } - - SubscriptionName getSubscriptionName() { - return getSubscription().getQualifiedName(); - } - - private void addSignal(Signal signal) { - if (!this.signals.add(signal)) { - logger.error("[Queue: consumerProcessSignals] Unable to add item: queue is full. Offered item: {}", signal); - } - } + private static final Logger logger = LoggerFactory.getLogger(ConsumerProcess.class); + + private final SpscArrayQueue signals = new SpscArrayQueue<>(100); + + private final Clock clock; + + private final Consumer consumer; + + private final Retransmitter retransmitter; + + private final java.util.function.Consumer onConsumerStopped; + + private final Duration unhealthyAfter; + + private volatile boolean running = true; + + private volatile long healthcheckRefreshTime; + + private final Map signalTimesheet = new ConcurrentHashMap<>(); + + public ConsumerProcess( + Signal startSignal, + Consumer consumer, + Retransmitter retransmitter, + Clock clock, + Duration unhealthyAfter, + java.util.function.Consumer onConsumerStopped) { + this.consumer = consumer; + this.retransmitter = retransmitter; + this.onConsumerStopped = onConsumerStopped; + this.clock = clock; + this.healthcheckRefreshTime = clock.millis(); + this.unhealthyAfter = unhealthyAfter; + + addSignal(startSignal); + } + + @Override + public void run() { + try { + Thread.currentThread().setName("consumer-" + getSubscriptionName()); + + while (running && !Thread.currentThread().isInterrupted()) { + consumer.consume(this::processSignals); + } + } catch (Throwable throwable) { + logger.error("Consumer process of subscription {} failed", getSubscriptionName(), throwable); + } finally { + logger.info("Releasing consumer process thread of subscription {}", getSubscriptionName()); + refreshHealthcheck(); + try { + stop(); + } catch (Exception exceptionWhileStopping) { + logger.error( + "An error occurred while stopping consumer process of subscription {}", + getSubscriptionName(), + exceptionWhileStopping); + } finally { + onConsumerStopped.accept(getSubscriptionName()); + Thread.currentThread().setName("consumer-released-thread"); + } + } + } + + public ConsumerProcess accept(Signal signal) { + addSignal(signal); + return this; + } + + public boolean isHealthy() { + return unhealthyAfter.toMillis() > lastSeen(); + } + + public long lastSeen() { + return clock.millis() - healthcheckRefreshTime; + } + + public long healthcheckRefreshTime() { + return healthcheckRefreshTime; + } + + private void processSignals() { + refreshHealthcheck(); + signals.drain(this::process); + refreshHealthcheck(); + } + + private void refreshHealthcheck() { + this.healthcheckRefreshTime = clock.millis(); + } + + private void process(Signal signal) { + try { + switch (signal.getType()) { + case START: + start(signal); + break; + case STOP: + logger.info( + "Stopping main loop for consumer {}. {}", + signal.getTarget(), + signal.getLogWithIdAndType()); + this.running = false; + break; + case RETRANSMIT: + retransmit(signal); + break; + case UPDATE_SUBSCRIPTION: + consumer.updateSubscription(signal.getPayload()); + break; + case UPDATE_TOPIC: + consumer.updateTopic(signal.getPayload()); + break; + default: + logger.warn("Unhandled signal found {}", signal); + break; + } + signalTimesheet.put(signal.getType(), clock.millis()); + } catch (Exception ex) { + logger.error("Failed to process signal {}", signal, ex); + } + } + + private void start(Signal signal) { + long startTime = clock.millis(); + logger.info( + "Starting consumer for subscription {}. {}", + getSubscriptionName(), + signal.getLogWithIdAndType()); + + consumer.initialize(); + + long initializationTime = clock.millis(); + logger.info( + "Started consumer for subscription {} in {}ms. {}", + getSubscriptionName(), + initializationTime - startTime, + signal.getLogWithIdAndType()); + signalTimesheet.put(START, initializationTime); + } + + private void stop() { + long startTime = clock.millis(); + logger.info("Stopping consumer for subscription {}", getSubscriptionName()); + + consumer.tearDown(); + + logger.info( + "Stopped consumer for subscription {} in {}ms", + getSubscriptionName(), + clock.millis() - startTime); + } + + private void retransmit(Signal signal) { + long startTime = clock.millis(); + logger.info( + "Starting retransmission for consumer of subscription {}. {}", + getSubscriptionName(), + signal.getLogWithIdAndType()); + try { + retransmitter.reloadOffsets(getSubscriptionName(), consumer); + logger.info( + "Done retransmission for consumer of subscription {} in {}ms", + getSubscriptionName(), + clock.millis() - startTime); + } catch (Exception ex) { + logger.error( + "Failed retransmission for consumer of subscription {} in {}ms", + getSubscriptionName(), + clock.millis() - startTime, + ex); + } + } + + @Override + public String toString() { + return "ConsumerProcess{" + "subscriptionName=" + getSubscriptionName() + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ConsumerProcess that = (ConsumerProcess) o; + return Objects.equals(getSubscriptionName(), that.getSubscriptionName()); + } + + @Override + public int hashCode() { + return Objects.hash(getSubscriptionName()); + } + + public Subscription getSubscription() { + return consumer.getSubscription(); + } + + public Map getSignalTimesheet() { + return ImmutableMap.copyOf(signalTimesheet); + } + + SubscriptionName getSubscriptionName() { + return getSubscription().getQualifiedName(); + } + + private void addSignal(Signal signal) { + if (!this.signals.add(signal)) { + logger.error( + "[Queue: consumerProcessSignals] Unable to add item: queue is full. Offered item: {}", + signal); + } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/ConsumerProcessFactory.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/ConsumerProcessFactory.java index 0657629307..1b155821a6 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/ConsumerProcessFactory.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/ConsumerProcessFactory.java @@ -1,39 +1,41 @@ package pl.allegro.tech.hermes.consumers.supervisor.process; +import java.time.Clock; +import java.time.Duration; import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.consumers.supervisor.ConsumerFactory; -import java.time.Clock; -import java.time.Duration; - public class ConsumerProcessFactory implements ConsumerProcessSupplier { - private final Retransmitter retransmitter; - private final Duration unhealthyAfter; - private final Clock clock; - private final ConsumerFactory consumerFactory; + private final Retransmitter retransmitter; + private final Duration unhealthyAfter; + private final Clock clock; + private final ConsumerFactory consumerFactory; - public ConsumerProcessFactory(Retransmitter retransmitter, - ConsumerFactory consumerFactory, - Duration unhealthyAfter, - Clock clock) { - this.retransmitter = retransmitter; - this.consumerFactory = consumerFactory; - this.unhealthyAfter = unhealthyAfter; - this.clock = clock; - } + public ConsumerProcessFactory( + Retransmitter retransmitter, + ConsumerFactory consumerFactory, + Duration unhealthyAfter, + Clock clock) { + this.retransmitter = retransmitter; + this.consumerFactory = consumerFactory; + this.unhealthyAfter = unhealthyAfter; + this.clock = clock; + } - @Override - public ConsumerProcess createProcess(Subscription subscription, - Signal startSignal, - java.util.function.Consumer onConsumerStopped) { + @Override + public ConsumerProcess createProcess( + Subscription subscription, + Signal startSignal, + java.util.function.Consumer onConsumerStopped) { - return new ConsumerProcess(startSignal, - consumerFactory.createConsumer(subscription), - retransmitter, - clock, - unhealthyAfter, - onConsumerStopped); - } -} \ No newline at end of file + return new ConsumerProcess( + startSignal, + consumerFactory.createConsumer(subscription), + retransmitter, + clock, + unhealthyAfter, + onConsumerStopped); + } +} diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/ConsumerProcessKiller.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/ConsumerProcessKiller.java index 19d4a9043f..f461f5477f 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/ConsumerProcessKiller.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/ConsumerProcessKiller.java @@ -1,51 +1,49 @@ package pl.allegro.tech.hermes.consumers.supervisor.process; +import java.time.Clock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.SubscriptionName; -import java.time.Clock; - class ConsumerProcessKiller { - private static final Logger logger = LoggerFactory.getLogger(ConsumerProcessKiller.class); - - private final long killAfterMs; - private final Clock clock; - private final RunningConsumerProcesses dyingConsumerProcesses; - - ConsumerProcessKiller(long killAfterMs, Clock clock) { - this.clock = clock; - this.killAfterMs = killAfterMs; - this.dyingConsumerProcesses = new RunningConsumerProcesses(clock); - } - - int countDying() { - return dyingConsumerProcesses.count(); - } - - void killAllDying() { - dyingConsumerProcesses - .stream() - .filter(RunningConsumerProcess::shouldBeCanceledNow) - .forEach(RunningConsumerProcess::cancel); - } - - void kill(RunningConsumerProcess process) { - observe(process).cancel(); - } - - boolean isDying(SubscriptionName subscriptionName) { - return dyingConsumerProcesses.hasProcess(subscriptionName); - } - - void cleanup(SubscriptionName subscriptionName) { - logger.info("Removing consumer process for subscription {}", subscriptionName); - dyingConsumerProcesses.remove(subscriptionName); - } - - RunningConsumerProcess observe(RunningConsumerProcess process) { - RunningConsumerProcess observed = process.copyWithTimeOfDeath(clock.millis() + killAfterMs); - dyingConsumerProcesses.add(observed); - return observed; - } -} \ No newline at end of file + private static final Logger logger = LoggerFactory.getLogger(ConsumerProcessKiller.class); + + private final long killAfterMs; + private final Clock clock; + private final RunningConsumerProcesses dyingConsumerProcesses; + + ConsumerProcessKiller(long killAfterMs, Clock clock) { + this.clock = clock; + this.killAfterMs = killAfterMs; + this.dyingConsumerProcesses = new RunningConsumerProcesses(clock); + } + + int countDying() { + return dyingConsumerProcesses.count(); + } + + void killAllDying() { + dyingConsumerProcesses.stream() + .filter(RunningConsumerProcess::shouldBeCanceledNow) + .forEach(RunningConsumerProcess::cancel); + } + + void kill(RunningConsumerProcess process) { + observe(process).cancel(); + } + + boolean isDying(SubscriptionName subscriptionName) { + return dyingConsumerProcesses.hasProcess(subscriptionName); + } + + void cleanup(SubscriptionName subscriptionName) { + logger.info("Removing consumer process for subscription {}", subscriptionName); + dyingConsumerProcesses.remove(subscriptionName); + } + + RunningConsumerProcess observe(RunningConsumerProcess process) { + RunningConsumerProcess observed = process.copyWithTimeOfDeath(clock.millis() + killAfterMs); + dyingConsumerProcesses.add(observed); + return observed; + } +} diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/ConsumerProcessSupervisor.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/ConsumerProcessSupervisor.java index 458ba8f0c0..ec6595ce3e 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/ConsumerProcessSupervisor.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/ConsumerProcessSupervisor.java @@ -1,5 +1,16 @@ package pl.allegro.tech.hermes.consumers.supervisor.process; +import static pl.allegro.tech.hermes.consumers.supervisor.process.Signal.SignalType.START; +import static pl.allegro.tech.hermes.consumers.supervisor.process.Signal.SignalType.STOP; + +import java.time.Clock; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Future; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.Subscription; @@ -11,232 +22,267 @@ import pl.allegro.tech.hermes.consumers.supervisor.ConsumersExecutorService; import pl.allegro.tech.hermes.metrics.HermesCounter; -import java.time.Clock; -import java.time.Duration; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.Future; - -import static pl.allegro.tech.hermes.consumers.supervisor.process.Signal.SignalType.START; -import static pl.allegro.tech.hermes.consumers.supervisor.process.Signal.SignalType.STOP; - public class ConsumerProcessSupervisor implements Runnable { - private static final Logger logger = LoggerFactory.getLogger(ConsumerProcessSupervisor.class); - - private final MpscQueue taskQueue; - - private final RunningConsumerProcesses runningConsumerProcesses; - - private final ConsumerProcessKiller processKiller; - - private final ConsumersExecutorService executor; - - private final Clock clock; - - private final MetricsFacade metrics; - - private final SignalsFilter signalsFilter; - - private final ConsumerProcessSupplier processFactory; - - private final Map processedSignalsCounters = new HashMap<>(); - - private final Map droppedSignalsCounters = new HashMap<>(); - - public ConsumerProcessSupervisor(ConsumersExecutorService executor, - Clock clock, - MetricsFacade metrics, - ConsumerProcessSupplier processFactory, - int signalQueueSize, - Duration backgroundSupervisorKillAfter) { - this.executor = executor; - this.clock = clock; - this.metrics = metrics; - this.taskQueue = new MonitoredMpscQueue<>(new WaitFreeDrainMpscQueue<>(signalQueueSize), metrics, "signalQueue"); - this.signalsFilter = new SignalsFilter(taskQueue, clock); - this.runningConsumerProcesses = new RunningConsumerProcesses(clock); - this.processKiller = new ConsumerProcessKiller(backgroundSupervisorKillAfter.toMillis(), clock); - this.processFactory = processFactory; - - metrics.consumer().registerRunningConsumerProcessesGauge(runningConsumerProcesses, RunningConsumerProcesses::count); - metrics.consumer().registerDyingConsumerProcessesGauge(processKiller, ConsumerProcessKiller::countDying); + private static final Logger logger = LoggerFactory.getLogger(ConsumerProcessSupervisor.class); + + private final MpscQueue taskQueue; + + private final RunningConsumerProcesses runningConsumerProcesses; + + private final ConsumerProcessKiller processKiller; + + private final ConsumersExecutorService executor; + + private final Clock clock; + + private final MetricsFacade metrics; + + private final SignalsFilter signalsFilter; + + private final ConsumerProcessSupplier processFactory; + + private final Map processedSignalsCounters = new HashMap<>(); + + private final Map droppedSignalsCounters = new HashMap<>(); + + public ConsumerProcessSupervisor( + ConsumersExecutorService executor, + Clock clock, + MetricsFacade metrics, + ConsumerProcessSupplier processFactory, + int signalQueueSize, + Duration backgroundSupervisorKillAfter) { + this.executor = executor; + this.clock = clock; + this.metrics = metrics; + this.taskQueue = + new MonitoredMpscQueue<>( + new WaitFreeDrainMpscQueue<>(signalQueueSize), metrics, "signalQueue"); + this.signalsFilter = new SignalsFilter(taskQueue, clock); + this.runningConsumerProcesses = new RunningConsumerProcesses(clock); + this.processKiller = new ConsumerProcessKiller(backgroundSupervisorKillAfter.toMillis(), clock); + this.processFactory = processFactory; + + metrics + .consumer() + .registerRunningConsumerProcessesGauge( + runningConsumerProcesses, RunningConsumerProcesses::count); + metrics + .consumer() + .registerDyingConsumerProcessesGauge(processKiller, ConsumerProcessKiller::countDying); + } + + public ConsumerProcessSupervisor accept(Signal signal) { + taskQueue.offer(signal); + return this; + } + + public Set existingConsumers() { + return runningConsumerProcesses.existingConsumers(); + } + + @Override + public void run() { + logger.debug("Starting process supervisor loop"); + final long currentTime = clock.millis(); + + restartUnhealthy(); + processKiller.killAllDying(); + + List signalsToProcess = new ArrayList<>(); + taskQueue.drain(signalsToProcess::add); + signalsFilter.filterSignals(signalsToProcess).forEach(this::tryToProcessSignal); + + logger.debug( + "Process supervisor loop took {} ms to check all consumers", clock.millis() - currentTime); + } + + public void shutdown() { + runningConsumerProcesses.stream() + .forEach( + p -> + p.getConsumerProcess() + .accept(Signal.of(STOP, p.getConsumerProcess().getSubscriptionName()))); + + processKiller.killAllDying(); + executor.shutdown(); + } + + public List runningSubscriptionsStatus() { + return runningConsumerProcesses.listRunningSubscriptions(); + } + + public Integer countRunningProcesses() { + return runningConsumerProcesses.count(); + } + + private void restartUnhealthy() { + runningConsumerProcesses.stream() + .filter(process -> !process.getConsumerProcess().isHealthy()) + .toList() + .forEach( + process -> { + logger.info( + "Lost contact with consumer {} (last seen {}ms ago). Attempting to kill this process and spawn new one.", + process.getConsumerProcess(), + process.getConsumerProcess().lastSeen()); + processKiller.kill(process); + runningConsumerProcesses.remove(process); + taskQueue.offer( + Signal.of( + START, + process.getSubscription().getQualifiedName(), + process.getSubscription())); + }); + } + + private void tryToProcessSignal(Signal signal) { + try { + processSignal(signal); + } catch (Exception exception) { + logger.error("Supervisor failed to process signal {}", signal, exception); } - - public ConsumerProcessSupervisor accept(Signal signal) { - taskQueue.offer(signal); - return this; + } + + private void processSignal(Signal signal) { + logger.debug("Processing signal: {}", signal); + processedSignalsCounters + .computeIfAbsent( + signal.getType().name(), name -> metrics.consumer().processedSignalsCounter(name)) + .increment(); + + switch (signal.getType()) { + case START: + start(signal); + break; + case UPDATE_SUBSCRIPTION: + updateSubscription(signal); + break; + case UPDATE_TOPIC: + case RETRANSMIT: + forRunningConsumerProcess( + signal, runningProcess -> runningProcess.getConsumerProcess().accept(signal)); + break; + case STOP: + stop(signal); + break; + default: + logger.warn("Unknown signal {}", signal); + break; } + } - public Set existingConsumers() { - return runningConsumerProcesses.existingConsumers(); + private void updateSubscription(Signal signal) { + if (runningConsumerProcesses.hasProcess(signal.getTarget())) { + stopOrUpdateConsumer(signal); + } else { + drop(signal); } - - @Override - public void run() { - logger.debug("Starting process supervisor loop"); - final long currentTime = clock.millis(); - - restartUnhealthy(); - processKiller.killAllDying(); - - List signalsToProcess = new ArrayList<>(); - taskQueue.drain(signalsToProcess::add); - signalsFilter.filterSignals(signalsToProcess).forEach(this::tryToProcessSignal); - - logger.debug("Process supervisor loop took {} ms to check all consumers", clock.millis() - currentTime); + } + + private void stopOrUpdateConsumer(Signal signal) { + Subscription signalSubscription = signal.getPayload(); + if (!deliveryTypesEqual(signalSubscription)) { + logger.info( + "Stopping subscription: {} because of delivery type update", + signalSubscription.getQualifiedName()); + stop(Signal.of(STOP, signal.getTarget())); + } else { + forRunningConsumerProcess( + signal, runningProcess -> runningProcess.getConsumerProcess().accept(signal)); } - - public void shutdown() { - runningConsumerProcesses.stream() - .forEach(p -> p.getConsumerProcess().accept(Signal.of(STOP, p.getConsumerProcess().getSubscriptionName()))); - - processKiller.killAllDying(); - executor.shutdown(); - } - - public List runningSubscriptionsStatus() { - return runningConsumerProcesses.listRunningSubscriptions(); - } - - public Integer countRunningProcesses() { - return runningConsumerProcesses.count(); - } - - private void restartUnhealthy() { - runningConsumerProcesses.stream() - .filter(process -> !process.getConsumerProcess().isHealthy()) - .toList() - .forEach(process -> { - logger.info("Lost contact with consumer {} (last seen {}ms ago). Attempting to kill this process and spawn new one.", - process.getConsumerProcess(), process.getConsumerProcess().lastSeen()); - processKiller.kill(process); - runningConsumerProcesses.remove(process); - taskQueue.offer(Signal.of(START, process.getSubscription().getQualifiedName(), process.getSubscription())); - }); - } - - private void tryToProcessSignal(Signal signal) { - try { - processSignal(signal); - } catch (Exception exception) { - logger.error("Supervisor failed to process signal {}", signal, exception); - } - } - - private void processSignal(Signal signal) { - logger.debug("Processing signal: {}", signal); - processedSignalsCounters.computeIfAbsent( - signal.getType().name(), - name -> metrics.consumer().processedSignalsCounter(name) - ).increment(); - - switch (signal.getType()) { - case START: - start(signal); - break; - case UPDATE_SUBSCRIPTION: - updateSubscription(signal); - break; - case UPDATE_TOPIC: - case RETRANSMIT: - forRunningConsumerProcess(signal, runningProcess -> runningProcess.getConsumerProcess().accept(signal)); - break; - case STOP: - stop(signal); - break; - default: - logger.warn("Unknown signal {}", signal); - break; - } - } - - private void updateSubscription(Signal signal) { - if (runningConsumerProcesses.hasProcess(signal.getTarget())) { - stopOrUpdateConsumer(signal); - } else { - drop(signal); - } - } - - private void stopOrUpdateConsumer(Signal signal) { - Subscription signalSubscription = signal.getPayload(); - if (!deliveryTypesEqual(signalSubscription)) { - logger.info("Stopping subscription: {} because of delivery type update", signalSubscription.getQualifiedName()); - stop(Signal.of(STOP, signal.getTarget())); - } else { - forRunningConsumerProcess(signal, runningProcess -> runningProcess.getConsumerProcess().accept(signal)); - } - } - - private boolean deliveryTypesEqual(Subscription signalSubscription) { - return signalSubscription.getDeliveryType() == runningConsumerProcesses.getProcess(signalSubscription.getQualifiedName()) - .getSubscription() - .getDeliveryType(); - } - - private void stop(Signal signal) { - forRunningConsumerProcess(signal, runningProcess -> { - processKiller.observe(runningProcess); - runningConsumerProcesses.remove(runningProcess); - runningProcess.getConsumerProcess().accept(signal); + } + + private boolean deliveryTypesEqual(Subscription signalSubscription) { + return signalSubscription.getDeliveryType() + == runningConsumerProcesses + .getProcess(signalSubscription.getQualifiedName()) + .getSubscription() + .getDeliveryType(); + } + + private void stop(Signal signal) { + forRunningConsumerProcess( + signal, + runningProcess -> { + processKiller.observe(runningProcess); + runningConsumerProcesses.remove(runningProcess); + runningProcess.getConsumerProcess().accept(signal); }); + } + + private void forRunningConsumerProcess( + Signal signal, java.util.function.Consumer consumerProcessConsumer) { + if (runningConsumerProcesses.hasProcess(signal.getTarget())) { + consumerProcessConsumer.accept(runningConsumerProcesses.getProcess(signal.getTarget())); + } else { + drop(signal); } - - private void forRunningConsumerProcess(Signal signal, java.util.function.Consumer consumerProcessConsumer) { - if (runningConsumerProcesses.hasProcess(signal.getTarget())) { - consumerProcessConsumer.accept(runningConsumerProcesses.getProcess(signal.getTarget())); - } else { - drop(signal); - } - } - - private void drop(Signal signal) { - droppedSignalsCounters.computeIfAbsent( - signal.getType().name(), - name -> metrics.consumer().droppedSignalsCounter(name) - ).increment(); - logger.warn("Dropping signal {} as running target consumer process does not exist.", signal); - } - - private void start(Signal start) { - Subscription subscription = getSubscriptionFromPayload(start); - - if (!hasProcess(start.getTarget())) { - try { - logger.info("Creating consumer for {}", subscription.getQualifiedName()); - ConsumerProcess process = processFactory.createProcess(subscription, start, processKiller::cleanup); - logger.info("Created consumer for {}. {}", subscription.getQualifiedName(), start.getLogWithIdAndType()); - - logger.info("Starting consumer process for subscription {}. {}", start.getTarget(), start.getLogWithIdAndType()); - Future future = executor.execute(process); - logger.info("Consumer for {} was added for execution. {}", subscription.getQualifiedName(), start.getLogWithIdAndType()); - - runningConsumerProcesses.add(process, future); - logger.info("Started consumer process for subscription {}. {}", start.getTarget(), start.getLogWithIdAndType()); - } catch (Exception ex) { - logger.error("Failed to create consumer for subscription {}", subscription.getQualifiedName(), ex); - } - } else if (processKiller.isDying(start.getTarget())) { - logger.info("Consumer process for {} is already dying, startup deferred.", subscription.getQualifiedName()); - accept(start); - } else { - logger.info("Abort consumer process start: process for subscription {} is already running. {}", - start.getTarget(), start.getLogWithIdAndType()); - } + } + + private void drop(Signal signal) { + droppedSignalsCounters + .computeIfAbsent( + signal.getType().name(), name -> metrics.consumer().droppedSignalsCounter(name)) + .increment(); + logger.warn("Dropping signal {} as running target consumer process does not exist.", signal); + } + + private void start(Signal start) { + Subscription subscription = getSubscriptionFromPayload(start); + + if (!hasProcess(start.getTarget())) { + try { + logger.info("Creating consumer for {}", subscription.getQualifiedName()); + ConsumerProcess process = + processFactory.createProcess(subscription, start, processKiller::cleanup); + logger.info( + "Created consumer for {}. {}", + subscription.getQualifiedName(), + start.getLogWithIdAndType()); + + logger.info( + "Starting consumer process for subscription {}. {}", + start.getTarget(), + start.getLogWithIdAndType()); + Future future = executor.execute(process); + logger.info( + "Consumer for {} was added for execution. {}", + subscription.getQualifiedName(), + start.getLogWithIdAndType()); + + runningConsumerProcesses.add(process, future); + logger.info( + "Started consumer process for subscription {}. {}", + start.getTarget(), + start.getLogWithIdAndType()); + } catch (Exception ex) { + logger.error( + "Failed to create consumer for subscription {}", subscription.getQualifiedName(), ex); + } + } else if (processKiller.isDying(start.getTarget())) { + logger.info( + "Consumer process for {} is already dying, startup deferred.", + subscription.getQualifiedName()); + accept(start); + } else { + logger.info( + "Abort consumer process start: process for subscription {} is already running. {}", + start.getTarget(), + start.getLogWithIdAndType()); } + } - private boolean hasProcess(SubscriptionName subscriptionName) { - return runningConsumerProcesses.hasProcess(subscriptionName) || processKiller.isDying(subscriptionName); - } + private boolean hasProcess(SubscriptionName subscriptionName) { + return runningConsumerProcesses.hasProcess(subscriptionName) + || processKiller.isDying(subscriptionName); + } - private Subscription getSubscriptionFromPayload(Signal startSignal) { - if (!(startSignal.getPayload() instanceof Subscription)) { - throw new IllegalArgumentException("Signal's payload has to be Subscription type"); - } - return startSignal.getPayload(); + private Subscription getSubscriptionFromPayload(Signal startSignal) { + if (!(startSignal.getPayload() instanceof Subscription)) { + throw new IllegalArgumentException("Signal's payload has to be Subscription type"); } + return startSignal.getPayload(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/ConsumerProcessSupplier.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/ConsumerProcessSupplier.java index d9b8627d12..e478500eeb 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/ConsumerProcessSupplier.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/ConsumerProcessSupplier.java @@ -4,7 +4,8 @@ import pl.allegro.tech.hermes.api.SubscriptionName; public interface ConsumerProcessSupplier { - ConsumerProcess createProcess(Subscription subscription, - Signal startSignal, - java.util.function.Consumer onConsumerStopped); -} \ No newline at end of file + ConsumerProcess createProcess( + Subscription subscription, + Signal startSignal, + java.util.function.Consumer onConsumerStopped); +} diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/Retransmitter.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/Retransmitter.java index f78e2939ff..3ccd141f89 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/Retransmitter.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/Retransmitter.java @@ -11,50 +11,55 @@ public class Retransmitter { - private static final Logger logger = LoggerFactory.getLogger(Retransmitter.class); + private static final Logger logger = LoggerFactory.getLogger(Retransmitter.class); - private final SubscriptionOffsetChangeIndicator subscriptionOffsetChangeIndicator; - private final String brokersClusterName; + private final SubscriptionOffsetChangeIndicator subscriptionOffsetChangeIndicator; + private final String brokersClusterName; - public Retransmitter(SubscriptionOffsetChangeIndicator subscriptionOffsetChangeIndicator, - String kafkaClusterName) { - this.subscriptionOffsetChangeIndicator = subscriptionOffsetChangeIndicator; - this.brokersClusterName = kafkaClusterName; - } + public Retransmitter( + SubscriptionOffsetChangeIndicator subscriptionOffsetChangeIndicator, + String kafkaClusterName) { + this.subscriptionOffsetChangeIndicator = subscriptionOffsetChangeIndicator; + this.brokersClusterName = kafkaClusterName; + } + + public void reloadOffsets(SubscriptionName subscriptionName, Consumer consumer) { + logger.info("Reloading offsets for {}", subscriptionName); + try { + PartitionOffsets offsets = + subscriptionOffsetChangeIndicator.getSubscriptionOffsets( + subscriptionName.getTopicName(), subscriptionName.getName(), brokersClusterName); - public void reloadOffsets(SubscriptionName subscriptionName, Consumer consumer) { - logger.info("Reloading offsets for {}", subscriptionName); - try { - PartitionOffsets offsets = subscriptionOffsetChangeIndicator.getSubscriptionOffsets( - subscriptionName.getTopicName(), subscriptionName.getName(), brokersClusterName); - - for (PartitionOffset partitionOffset : offsets) { - if (moveOffset(subscriptionName, consumer, partitionOffset)) { - subscriptionOffsetChangeIndicator.removeOffset( - subscriptionName.getTopicName(), - subscriptionName.getName(), - brokersClusterName, - partitionOffset.getTopic(), - partitionOffset.getPartition() - ); - logger.info("Removed offset indicator for subscription={} and partition={}", - subscriptionName, partitionOffset.getPartition()); - } - } - } catch (Exception ex) { - throw new RetransmissionException(ex); + for (PartitionOffset partitionOffset : offsets) { + if (moveOffset(subscriptionName, consumer, partitionOffset)) { + subscriptionOffsetChangeIndicator.removeOffset( + subscriptionName.getTopicName(), + subscriptionName.getName(), + brokersClusterName, + partitionOffset.getTopic(), + partitionOffset.getPartition()); + logger.info( + "Removed offset indicator for subscription={} and partition={}", + subscriptionName, + partitionOffset.getPartition()); } + } + } catch (Exception ex) { + throw new RetransmissionException(ex); } + } - private boolean moveOffset(SubscriptionName subscriptionName, - Consumer consumer, - PartitionOffset partitionOffset) { - try { - return consumer.moveOffset(partitionOffset); - } catch (IllegalStateException ex) { - logger.warn("Cannot move offset for subscription={} and partition={} , possibly owned by different node", - subscriptionName, partitionOffset.getPartition(), ex); - return false; - } + private boolean moveOffset( + SubscriptionName subscriptionName, Consumer consumer, PartitionOffset partitionOffset) { + try { + return consumer.moveOffset(partitionOffset); + } catch (IllegalStateException ex) { + logger.warn( + "Cannot move offset for subscription={} and partition={} , possibly owned by different node", + subscriptionName, + partitionOffset.getPartition(), + ex); + return false; } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/RunningConsumerProcess.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/RunningConsumerProcess.java index fbf80fb4f9..b7ca2b27b3 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/RunningConsumerProcess.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/RunningConsumerProcess.java @@ -1,59 +1,64 @@ package pl.allegro.tech.hermes.consumers.supervisor.process; +import java.time.Clock; +import java.util.Optional; +import java.util.concurrent.Future; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.api.SubscriptionName; -import java.time.Clock; -import java.util.Optional; -import java.util.concurrent.Future; - class RunningConsumerProcess { - private static final Logger logger = LoggerFactory.getLogger(RunningConsumerProcess.class); - - private ConsumerProcess process; - private Future executionHandle; - private Clock clock; - private Optional timeOfDeath = Optional.empty(); - - RunningConsumerProcess(ConsumerProcess process, Future executionHandle, Clock clock) { - this.process = process; - this.executionHandle = executionHandle; - this.clock = clock; - } - - private RunningConsumerProcess(ConsumerProcess process, Future executionHandle, long killTime, Clock clock) { - this(process, executionHandle, clock); - this.timeOfDeath = Optional.of(killTime); - } - - void cancel() { - timeOfDeath.ifPresent(timeOfDeath -> { - SubscriptionName subscriptionName = process.getSubscriptionName(); - logger.info("Canceling consumer process for subscription {}. It should have been killed {}ms ago.", - subscriptionName, clock.millis() - timeOfDeath); - if (executionHandle.cancel(true)) { - logger.info("Canceled consumer process for subscription {}", subscriptionName); - } else { - logger.error("Failed to cancel consumer process for {}.", subscriptionName); - } + private static final Logger logger = LoggerFactory.getLogger(RunningConsumerProcess.class); + + private ConsumerProcess process; + private Future executionHandle; + private Clock clock; + private Optional timeOfDeath = Optional.empty(); + + RunningConsumerProcess(ConsumerProcess process, Future executionHandle, Clock clock) { + this.process = process; + this.executionHandle = executionHandle; + this.clock = clock; + } + + private RunningConsumerProcess( + ConsumerProcess process, Future executionHandle, long killTime, Clock clock) { + this(process, executionHandle, clock); + this.timeOfDeath = Optional.of(killTime); + } + + void cancel() { + timeOfDeath.ifPresent( + timeOfDeath -> { + SubscriptionName subscriptionName = process.getSubscriptionName(); + logger.info( + "Canceling consumer process for subscription {}. It should have been killed {}ms ago.", + subscriptionName, + clock.millis() - timeOfDeath); + if (executionHandle.cancel(true)) { + logger.info("Canceled consumer process for subscription {}", subscriptionName); + } else { + logger.error("Failed to cancel consumer process for {}.", subscriptionName); + } }); - } + } - boolean shouldBeCanceledNow() { - return timeOfDeath.map(time -> time < clock.millis() && !executionHandle.isDone()).orElse(false); - } + boolean shouldBeCanceledNow() { + return timeOfDeath + .map(time -> time < clock.millis() && !executionHandle.isDone()) + .orElse(false); + } - RunningConsumerProcess copyWithTimeOfDeath(long timeOfDeath) { - return new RunningConsumerProcess(process, executionHandle, timeOfDeath, clock); - } + RunningConsumerProcess copyWithTimeOfDeath(long timeOfDeath) { + return new RunningConsumerProcess(process, executionHandle, timeOfDeath, clock); + } - ConsumerProcess getConsumerProcess() { - return process; - } + ConsumerProcess getConsumerProcess() { + return process; + } - public Subscription getSubscription() { - return getConsumerProcess().getSubscription(); - } + public Subscription getSubscription() { + return getConsumerProcess().getSubscription(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/RunningConsumerProcesses.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/RunningConsumerProcesses.java index a836ed2bae..9669d12b23 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/RunningConsumerProcesses.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/RunningConsumerProcesses.java @@ -1,8 +1,8 @@ package pl.allegro.tech.hermes.consumers.supervisor.process; -import com.google.common.collect.ImmutableSet; -import pl.allegro.tech.hermes.api.SubscriptionName; +import static java.util.stream.Collectors.toList; +import com.google.common.collect.ImmutableSet; import java.time.Clock; import java.util.List; import java.util.Map; @@ -10,64 +10,69 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Future; import java.util.stream.Stream; - -import static java.util.stream.Collectors.toList; +import pl.allegro.tech.hermes.api.SubscriptionName; class RunningConsumerProcesses { - private final Map processes = new ConcurrentHashMap<>(); - private final Clock clock; - - RunningConsumerProcesses(Clock clock) { - this.clock = clock; - } - - void add(ConsumerProcess process, Future executionHandle) { - this.processes.put(process.getSubscription().getQualifiedName(), new RunningConsumerProcess(process, executionHandle, clock)); - } - - void add(RunningConsumerProcess process) { - this.processes.put(process.getConsumerProcess().getSubscriptionName(), process); - } - - void remove(SubscriptionName subscriptionName) { - processes.remove(subscriptionName); - } - - public void remove(ConsumerProcess consumerProcess) { - remove(consumerProcess.getSubscriptionName()); - } - - void remove(RunningConsumerProcess runningProcess) { - remove(runningProcess.getConsumerProcess().getSubscriptionName()); - } - - RunningConsumerProcess getProcess(SubscriptionName subscriptionName) { - return processes.get(subscriptionName); - } - - boolean hasProcess(SubscriptionName subscriptionName) { - return processes.containsKey(subscriptionName); - } - - Stream stream() { - return processes.values().stream(); - } - - Set existingConsumers() { - return ImmutableSet.copyOf(processes.keySet()); - } - - List listRunningSubscriptions() { - return processes.entrySet().stream() - .map(entry -> new RunningSubscriptionStatus( - entry.getKey().getQualifiedName(), - entry.getValue().getConsumerProcess().getSignalTimesheet())) - .sorted((s1, s2) -> String.CASE_INSENSITIVE_ORDER.compare(s1.getQualifiedName(), s2.getQualifiedName())) - .collect(toList()); - } - - Integer count() { - return processes.size(); - } -} \ No newline at end of file + private final Map processes = new ConcurrentHashMap<>(); + private final Clock clock; + + RunningConsumerProcesses(Clock clock) { + this.clock = clock; + } + + void add(ConsumerProcess process, Future executionHandle) { + this.processes.put( + process.getSubscription().getQualifiedName(), + new RunningConsumerProcess(process, executionHandle, clock)); + } + + void add(RunningConsumerProcess process) { + this.processes.put(process.getConsumerProcess().getSubscriptionName(), process); + } + + void remove(SubscriptionName subscriptionName) { + processes.remove(subscriptionName); + } + + public void remove(ConsumerProcess consumerProcess) { + remove(consumerProcess.getSubscriptionName()); + } + + void remove(RunningConsumerProcess runningProcess) { + remove(runningProcess.getConsumerProcess().getSubscriptionName()); + } + + RunningConsumerProcess getProcess(SubscriptionName subscriptionName) { + return processes.get(subscriptionName); + } + + boolean hasProcess(SubscriptionName subscriptionName) { + return processes.containsKey(subscriptionName); + } + + Stream stream() { + return processes.values().stream(); + } + + Set existingConsumers() { + return ImmutableSet.copyOf(processes.keySet()); + } + + List listRunningSubscriptions() { + return processes.entrySet().stream() + .map( + entry -> + new RunningSubscriptionStatus( + entry.getKey().getQualifiedName(), + entry.getValue().getConsumerProcess().getSignalTimesheet())) + .sorted( + (s1, s2) -> + String.CASE_INSENSITIVE_ORDER.compare(s1.getQualifiedName(), s2.getQualifiedName())) + .collect(toList()); + } + + Integer count() { + return processes.size(); + } +} diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/RunningSubscriptionStatus.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/RunningSubscriptionStatus.java index 2c289cc71d..840214865b 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/RunningSubscriptionStatus.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/RunningSubscriptionStatus.java @@ -1,25 +1,25 @@ package pl.allegro.tech.hermes.consumers.supervisor.process; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.Map; public class RunningSubscriptionStatus { - private final String qualifiedName; - private final Map signalTimesheet; + private final String qualifiedName; + private final Map signalTimesheet; - public RunningSubscriptionStatus(@JsonProperty("qualifiedName") String qualifiedName, - @JsonProperty("signals") Map signalTimesheet) { + public RunningSubscriptionStatus( + @JsonProperty("qualifiedName") String qualifiedName, + @JsonProperty("signals") Map signalTimesheet) { - this.qualifiedName = qualifiedName; - this.signalTimesheet = signalTimesheet; - } + this.qualifiedName = qualifiedName; + this.signalTimesheet = signalTimesheet; + } - public String getQualifiedName() { - return qualifiedName; - } + public String getQualifiedName() { + return qualifiedName; + } - public Map getSignalTimesheet() { - return signalTimesheet; - } + public Map getSignalTimesheet() { + return signalTimesheet; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/Signal.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/Signal.java index 57a17b7bc0..fa5066d9b3 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/Signal.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/Signal.java @@ -1,91 +1,100 @@ package pl.allegro.tech.hermes.consumers.supervisor.process; -import pl.allegro.tech.hermes.api.SubscriptionName; - import java.util.Objects; import java.util.concurrent.atomic.AtomicLong; +import pl.allegro.tech.hermes.api.SubscriptionName; public class Signal { - private final SignalType type; - - private final SubscriptionName target; - - private final Object payload; - - private final long executeAfterTimestamp; - private final long id; - - public enum SignalType { - START, STOP, RETRANSMIT, UPDATE_SUBSCRIPTION, UPDATE_TOPIC - } - - private static final AtomicLong SIGNALS_COUNTER = new AtomicLong(); - - private Signal(SignalType type, SubscriptionName target, Object payload, long executeAfterTimestamp, long id) { - this.type = type; - this.target = target; - this.payload = payload; - this.executeAfterTimestamp = executeAfterTimestamp; - this.id = id; - } - - public static Signal of(SignalType type, SubscriptionName target) { - return of(type, target, null); - } - - public static Signal of(SignalType type, SubscriptionName target, Object payload) { - return of(type, target, payload, -1); - } - - public static Signal of(SignalType type, SubscriptionName target, Object payload, long executeAfterTimestamp) { - return new Signal(type, target, payload, executeAfterTimestamp, SIGNALS_COUNTER.incrementAndGet()); + private final SignalType type; + + private final SubscriptionName target; + + private final Object payload; + + private final long executeAfterTimestamp; + private final long id; + + public enum SignalType { + START, + STOP, + RETRANSMIT, + UPDATE_SUBSCRIPTION, + UPDATE_TOPIC + } + + private static final AtomicLong SIGNALS_COUNTER = new AtomicLong(); + + private Signal( + SignalType type, + SubscriptionName target, + Object payload, + long executeAfterTimestamp, + long id) { + this.type = type; + this.target = target; + this.payload = payload; + this.executeAfterTimestamp = executeAfterTimestamp; + this.id = id; + } + + public static Signal of(SignalType type, SubscriptionName target) { + return of(type, target, null); + } + + public static Signal of(SignalType type, SubscriptionName target, Object payload) { + return of(type, target, payload, -1); + } + + public static Signal of( + SignalType type, SubscriptionName target, Object payload, long executeAfterTimestamp) { + return new Signal( + type, target, payload, executeAfterTimestamp, SIGNALS_COUNTER.incrementAndGet()); + } + + Signal createChild(SignalType type) { + return new Signal(type, target, payload, executeAfterTimestamp, id); + } + + SignalType getType() { + return type; + } + + SubscriptionName getTarget() { + return target; + } + + boolean canExecuteNow(long currentTimestamp) { + return currentTimestamp >= executeAfterTimestamp; + } + + @SuppressWarnings("unchecked") + T getPayload() { + return (T) payload; + } + + @Override + public String toString() { + return "Signal(" + id + ", " + type + ", " + target + ")"; + } + + public String getLogWithIdAndType() { + return "[Signal(" + id + ", " + type + ")]"; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - Signal createChild(SignalType type) { - return new Signal(type, target, payload, executeAfterTimestamp, id); - } - - SignalType getType() { - return type; - } - - SubscriptionName getTarget() { - return target; - } - - boolean canExecuteNow(long currentTimestamp) { - return currentTimestamp >= executeAfterTimestamp; - } - - @SuppressWarnings("unchecked") - T getPayload() { - return (T) payload; - } - - @Override - public String toString() { - return "Signal(" + id + ", " + type + ", " + target + ")"; - } - - public String getLogWithIdAndType() { - return "[Signal(" + id + ", " + type + ")]"; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof Signal signal)) { - return false; - } - return type == signal.type - && Objects.equals(target, signal.target); + if (!(o instanceof Signal signal)) { + return false; } + return type == signal.type && Objects.equals(target, signal.target); + } - @Override - public int hashCode() { - return Objects.hash(type, target); - } + @Override + public int hashCode() { + return Objects.hash(type, target); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/SignalsFilter.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/SignalsFilter.java index ff2b56f963..a79d0cc4cb 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/SignalsFilter.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/process/SignalsFilter.java @@ -1,62 +1,62 @@ package pl.allegro.tech.hermes.consumers.supervisor.process; import com.google.common.collect.ImmutableMap; -import pl.allegro.tech.hermes.consumers.queue.MpscQueue; -import pl.allegro.tech.hermes.consumers.supervisor.process.Signal.SignalType; - import java.time.Clock; import java.util.ArrayList; import java.util.List; import java.util.Map; +import pl.allegro.tech.hermes.consumers.queue.MpscQueue; +import pl.allegro.tech.hermes.consumers.supervisor.process.Signal.SignalType; class SignalsFilter { - private static final Map MERGEABLE_SIGNALS = ImmutableMap.builder() - .put(SignalType.START, SignalType.STOP) - .put(SignalType.STOP, SignalType.START) - .build(); + private static final Map MERGEABLE_SIGNALS = + ImmutableMap.builder() + .put(SignalType.START, SignalType.STOP) + .put(SignalType.STOP, SignalType.START) + .build(); - private final Clock clock; + private final Clock clock; - private final MpscQueue taskQueue; + private final MpscQueue taskQueue; - SignalsFilter(MpscQueue taskQueue, Clock clock) { - this.taskQueue = taskQueue; - this.clock = clock; - } + SignalsFilter(MpscQueue taskQueue, Clock clock) { + this.taskQueue = taskQueue; + this.clock = clock; + } - List filterSignals(List signals) { - List filteredSignals = new ArrayList<>(signals.size()); - - for (Signal signal : signals) { - boolean merged = merge(filteredSignals, signal); - if (!merged) { - if (signal.canExecuteNow(clock.millis())) { - addWithoutDuplicationMergeableSignals(filteredSignals, signal); - } else { - taskQueue.offer(signal); - } - } - } + List filterSignals(List signals) { + List filteredSignals = new ArrayList<>(signals.size()); - return filteredSignals; - } - - private void addWithoutDuplicationMergeableSignals(List filteredSignals, Signal signal) { - if (MERGEABLE_SIGNALS.containsKey(signal.getType())) { - if (!filteredSignals.contains(signal)) { - filteredSignals.add(signal); - } + for (Signal signal : signals) { + boolean merged = merge(filteredSignals, signal); + if (!merged) { + if (signal.canExecuteNow(clock.millis())) { + addWithoutDuplicationMergeableSignals(filteredSignals, signal); } else { - filteredSignals.add(signal); + taskQueue.offer(signal); } + } } - private boolean merge(List filteredSignals, Signal signal) { - SignalType signalTypeToMerge = MERGEABLE_SIGNALS.get(signal.getType()); - if (signalTypeToMerge != null) { - return filteredSignals.remove(signal.createChild(signalTypeToMerge)); - } - return false; + return filteredSignals; + } + + private void addWithoutDuplicationMergeableSignals(List filteredSignals, Signal signal) { + if (MERGEABLE_SIGNALS.containsKey(signal.getType())) { + if (!filteredSignals.contains(signal)) { + filteredSignals.add(signal); + } + } else { + filteredSignals.add(signal); + } + } + + private boolean merge(List filteredSignals, Signal signal) { + SignalType signalTypeToMerge = MERGEABLE_SIGNALS.get(signal.getType()); + if (signalTypeToMerge != null) { + return filteredSignals.remove(signal.createChild(signalTypeToMerge)); } + return false; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/BalancingJob.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/BalancingJob.java index b0cdbdb274..cf794b7c51 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/BalancingJob.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/BalancingJob.java @@ -1,5 +1,6 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.SubscriptionName; @@ -10,144 +11,159 @@ import pl.allegro.tech.hermes.domain.workload.constraints.WorkloadConstraintsRepository; import pl.allegro.tech.hermes.metrics.HermesTimerContext; -import java.util.List; - class BalancingJob implements Runnable { - private static final Logger logger = LoggerFactory.getLogger(BalancingJob.class); - - private final ConsumerNodesRegistry consumersRegistry; - private final WorkBalancingParameters workBalancingParameters; - private final SubscriptionsCache subscriptionsCache; - private final ClusterAssignmentCache clusterAssignmentCache; - private final ConsumerAssignmentRegistry consumerAssignmentRegistry; - private final WorkBalancer workBalancer; - private final MetricsFacade metrics; - private final String kafkaCluster; - private final WorkloadConstraintsRepository workloadConstraintsRepository; - private final BalancingListener balancingListener; - private final BalancingJobMetrics balancingMetrics = new BalancingJobMetrics(); - - BalancingJob(ConsumerNodesRegistry consumersRegistry, - WorkBalancingParameters workBalancingParameters, - SubscriptionsCache subscriptionsCache, - ClusterAssignmentCache clusterAssignmentCache, - ConsumerAssignmentRegistry consumerAssignmentRegistry, - WorkBalancer workBalancer, - MetricsFacade metrics, - String kafkaCluster, - WorkloadConstraintsRepository workloadConstraintsRepository, - BalancingListener balancingListener) { - this.consumersRegistry = consumersRegistry; - this.workBalancingParameters = workBalancingParameters; - this.subscriptionsCache = subscriptionsCache; - this.clusterAssignmentCache = clusterAssignmentCache; - this.consumerAssignmentRegistry = consumerAssignmentRegistry; - this.workBalancer = workBalancer; - this.metrics = metrics; - this.kafkaCluster = kafkaCluster; - this.workloadConstraintsRepository = workloadConstraintsRepository; - this.balancingListener = balancingListener; - metrics.workload().registerAllAssignmentsGauge(balancingMetrics, kafkaCluster, bm -> bm.allAssignments); - metrics.workload().registerMissingResourcesGauge(balancingMetrics, kafkaCluster, bm -> bm.missingResources); - metrics.workload().registerDeletedAssignmentsGauge(balancingMetrics, kafkaCluster, bm -> bm.deletedAssignments); - metrics.workload().registerCreatedAssignmentsGauge(balancingMetrics, kafkaCluster, bm -> bm.createdAssignments); - } - - @Override - public void run() { - try { - consumersRegistry.refresh(); - if (consumersRegistry.isLeader()) { - try (HermesTimerContext ignored = metrics.workload().rebalanceDurationTimer(kafkaCluster).time()) { - logger.info("Initializing workload balance."); - clusterAssignmentCache.refresh(); - - SubscriptionAssignmentView initialState = clusterAssignmentCache.createSnapshot(); - List activeConsumers = consumersRegistry.listConsumerNodes(); - List activeSubscriptions = subscriptionsCache.listActiveSubscriptionNames(); - - balancingListener.onBeforeBalancing(activeConsumers); - - WorkBalancingResult work = workBalancer.balance( - activeSubscriptions, - activeConsumers, - initialState, - prepareWorkloadConstraints(activeConsumers) - ); - - if (consumersRegistry.isLeader()) { - logger.info("Applying workload balance changes"); - WorkDistributionChanges changes = calculateWorkDistributionChanges(initialState, work); - applyWorkloadChanges(changes, work); - logger.info("Finished workload balance"); - - clusterAssignmentCache.refresh(); // refresh cache with just stored data - - balancingListener.onAfterBalancing(changes); - - updateMetrics(work, changes); - } else { - logger.info("Lost leadership before applying changes"); - } - } - } else { - balancingMetrics.reset(); - balancingListener.onBalancingSkipped(); - } - } catch (Exception e) { - logger.error("Caught exception when running balancing job", e); + private static final Logger logger = LoggerFactory.getLogger(BalancingJob.class); + + private final ConsumerNodesRegistry consumersRegistry; + private final WorkBalancingParameters workBalancingParameters; + private final SubscriptionsCache subscriptionsCache; + private final ClusterAssignmentCache clusterAssignmentCache; + private final ConsumerAssignmentRegistry consumerAssignmentRegistry; + private final WorkBalancer workBalancer; + private final MetricsFacade metrics; + private final String kafkaCluster; + private final WorkloadConstraintsRepository workloadConstraintsRepository; + private final BalancingListener balancingListener; + private final BalancingJobMetrics balancingMetrics = new BalancingJobMetrics(); + + BalancingJob( + ConsumerNodesRegistry consumersRegistry, + WorkBalancingParameters workBalancingParameters, + SubscriptionsCache subscriptionsCache, + ClusterAssignmentCache clusterAssignmentCache, + ConsumerAssignmentRegistry consumerAssignmentRegistry, + WorkBalancer workBalancer, + MetricsFacade metrics, + String kafkaCluster, + WorkloadConstraintsRepository workloadConstraintsRepository, + BalancingListener balancingListener) { + this.consumersRegistry = consumersRegistry; + this.workBalancingParameters = workBalancingParameters; + this.subscriptionsCache = subscriptionsCache; + this.clusterAssignmentCache = clusterAssignmentCache; + this.consumerAssignmentRegistry = consumerAssignmentRegistry; + this.workBalancer = workBalancer; + this.metrics = metrics; + this.kafkaCluster = kafkaCluster; + this.workloadConstraintsRepository = workloadConstraintsRepository; + this.balancingListener = balancingListener; + metrics + .workload() + .registerAllAssignmentsGauge(balancingMetrics, kafkaCluster, bm -> bm.allAssignments); + metrics + .workload() + .registerMissingResourcesGauge(balancingMetrics, kafkaCluster, bm -> bm.missingResources); + metrics + .workload() + .registerDeletedAssignmentsGauge( + balancingMetrics, kafkaCluster, bm -> bm.deletedAssignments); + metrics + .workload() + .registerCreatedAssignmentsGauge( + balancingMetrics, kafkaCluster, bm -> bm.createdAssignments); + } + + @Override + public void run() { + try { + consumersRegistry.refresh(); + if (consumersRegistry.isLeader()) { + try (HermesTimerContext ignored = + metrics.workload().rebalanceDurationTimer(kafkaCluster).time()) { + logger.info("Initializing workload balance."); + clusterAssignmentCache.refresh(); + + SubscriptionAssignmentView initialState = clusterAssignmentCache.createSnapshot(); + List activeConsumers = consumersRegistry.listConsumerNodes(); + List activeSubscriptions = + subscriptionsCache.listActiveSubscriptionNames(); + + balancingListener.onBeforeBalancing(activeConsumers); + + WorkBalancingResult work = + workBalancer.balance( + activeSubscriptions, + activeConsumers, + initialState, + prepareWorkloadConstraints(activeConsumers)); + + if (consumersRegistry.isLeader()) { + logger.info("Applying workload balance changes"); + WorkDistributionChanges changes = calculateWorkDistributionChanges(initialState, work); + applyWorkloadChanges(changes, work); + logger.info("Finished workload balance"); + + clusterAssignmentCache.refresh(); // refresh cache with just stored data + + balancingListener.onAfterBalancing(changes); + + updateMetrics(work, changes); + } else { + logger.info("Lost leadership before applying changes"); + } } + } else { + balancingMetrics.reset(); + balancingListener.onBalancingSkipped(); + } + } catch (Exception e) { + logger.error("Caught exception when running balancing job", e); } - - private WorkloadConstraints prepareWorkloadConstraints(List activeConsumers) { - ConsumersWorkloadConstraints constraints = workloadConstraintsRepository.getConsumersWorkloadConstraints(); - return WorkloadConstraints.builder() - .withActiveConsumers(activeConsumers.size()) - .withConsumersPerSubscription(workBalancingParameters.getConsumersPerSubscription()) - .withMaxSubscriptionsPerConsumer(workBalancingParameters.getMaxSubscriptionsPerConsumer()) - .withSubscriptionConstraints(constraints.getSubscriptionConstraints()) - .withTopicConstraints(constraints.getTopicConstraints()) - .build(); + } + + private WorkloadConstraints prepareWorkloadConstraints(List activeConsumers) { + ConsumersWorkloadConstraints constraints = + workloadConstraintsRepository.getConsumersWorkloadConstraints(); + return WorkloadConstraints.builder() + .withActiveConsumers(activeConsumers.size()) + .withConsumersPerSubscription(workBalancingParameters.getConsumersPerSubscription()) + .withMaxSubscriptionsPerConsumer(workBalancingParameters.getMaxSubscriptionsPerConsumer()) + .withSubscriptionConstraints(constraints.getSubscriptionConstraints()) + .withTopicConstraints(constraints.getTopicConstraints()) + .build(); + } + + private WorkDistributionChanges calculateWorkDistributionChanges( + SubscriptionAssignmentView initialState, WorkBalancingResult workBalancingResult) { + SubscriptionAssignmentView balancedState = workBalancingResult.getAssignmentsView(); + SubscriptionAssignmentView deletions = initialState.deletions(balancedState); + SubscriptionAssignmentView additions = initialState.additions(balancedState); + return new WorkDistributionChanges(deletions, additions); + } + + private void applyWorkloadChanges( + WorkDistributionChanges changes, WorkBalancingResult workBalancingResult) { + SubscriptionAssignmentView balancedState = workBalancingResult.getAssignmentsView(); + for (String consumerId : changes.getModifiedConsumerNodes()) { + consumerAssignmentRegistry.updateAssignments( + consumerId, balancedState.getSubscriptionsForConsumerNode(consumerId)); } + } - private WorkDistributionChanges calculateWorkDistributionChanges(SubscriptionAssignmentView initialState, - WorkBalancingResult workBalancingResult) { - SubscriptionAssignmentView balancedState = workBalancingResult.getAssignmentsView(); - SubscriptionAssignmentView deletions = initialState.deletions(balancedState); - SubscriptionAssignmentView additions = initialState.additions(balancedState); - return new WorkDistributionChanges(deletions, additions); - } + private void updateMetrics(WorkBalancingResult balancingResult, WorkDistributionChanges changes) { + this.balancingMetrics.allAssignments = + balancingResult.getAssignmentsView().getAllAssignments().size(); + this.balancingMetrics.missingResources = balancingResult.getMissingResources(); + this.balancingMetrics.createdAssignments = changes.getCreatedAssignmentsCount(); + this.balancingMetrics.deletedAssignments = changes.getDeletedAssignmentsCount(); + } - private void applyWorkloadChanges(WorkDistributionChanges changes, WorkBalancingResult workBalancingResult) { - SubscriptionAssignmentView balancedState = workBalancingResult.getAssignmentsView(); - for (String consumerId : changes.getModifiedConsumerNodes()) { - consumerAssignmentRegistry.updateAssignments(consumerId, balancedState.getSubscriptionsForConsumerNode(consumerId)); - } - } - - private void updateMetrics(WorkBalancingResult balancingResult, WorkDistributionChanges changes) { - this.balancingMetrics.allAssignments = balancingResult.getAssignmentsView().getAllAssignments().size(); - this.balancingMetrics.missingResources = balancingResult.getMissingResources(); - this.balancingMetrics.createdAssignments = changes.getCreatedAssignmentsCount(); - this.balancingMetrics.deletedAssignments = changes.getDeletedAssignmentsCount(); - } - - private static class BalancingJobMetrics { + private static class BalancingJobMetrics { - volatile int allAssignments; + volatile int allAssignments; - volatile int missingResources; + volatile int missingResources; - volatile int deletedAssignments; + volatile int deletedAssignments; - volatile int createdAssignments; + volatile int createdAssignments; - void reset() { - this.allAssignments = 0; - this.missingResources = 0; - this.deletedAssignments = 0; - this.createdAssignments = 0; - } + void reset() { + this.allAssignments = 0; + this.missingResources = 0; + this.deletedAssignments = 0; + this.createdAssignments = 0; } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/BalancingListener.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/BalancingListener.java index 5d8aa9a3f7..41a2fea0fc 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/BalancingListener.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/BalancingListener.java @@ -4,9 +4,9 @@ public interface BalancingListener { - void onBeforeBalancing(List activeConsumers); + void onBeforeBalancing(List activeConsumers); - void onAfterBalancing(WorkDistributionChanges changes); + void onAfterBalancing(WorkDistributionChanges changes); - void onBalancingSkipped(); + void onBalancingSkipped(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ClusterAssignmentCache.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ClusterAssignmentCache.java index 5d78913a6d..b3641ecfb3 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ClusterAssignmentCache.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ClusterAssignmentCache.java @@ -1,11 +1,6 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload; -import org.apache.curator.framework.CuratorFramework; -import org.slf4j.Logger; -import pl.allegro.tech.hermes.api.SubscriptionName; -import pl.allegro.tech.hermes.consumers.registry.ConsumerNodesRegistry; -import pl.allegro.tech.hermes.consumers.subscription.id.SubscriptionIds; -import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; +import static org.slf4j.LoggerFactory.getLogger; import java.util.Collections; import java.util.HashMap; @@ -15,117 +10,128 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; - -import static org.slf4j.LoggerFactory.getLogger; +import org.apache.curator.framework.CuratorFramework; +import org.slf4j.Logger; +import pl.allegro.tech.hermes.api.SubscriptionName; +import pl.allegro.tech.hermes.consumers.registry.ConsumerNodesRegistry; +import pl.allegro.tech.hermes.consumers.subscription.id.SubscriptionIds; +import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; public class ClusterAssignmentCache { - private static final Logger logger = getLogger(ClusterAssignmentCache.class); - - private final ConsumerNodesRegistry consumerNodesRegistry; - private final WorkloadRegistryPaths paths; - private final ConsumerWorkloadDecoder consumerWorkloadDecoder; - private final String consumersPath; - private final ZookeeperOperations zookeeper; - - private final Map> consumerSubscriptions = new ConcurrentHashMap<>(); - private final Set assignments = Collections.newSetFromMap(new ConcurrentHashMap<>()); - - public ClusterAssignmentCache(CuratorFramework curator, - String clusterName, - ZookeeperPaths zookeeperPaths, - SubscriptionIds subscriptionIds, - ConsumerNodesRegistry consumerNodesRegistry) { - this.zookeeper = new ZookeeperOperations(curator); - this.consumerNodesRegistry = consumerNodesRegistry; - this.paths = new WorkloadRegistryPaths(zookeeperPaths, clusterName); - this.consumerWorkloadDecoder = new ConsumerWorkloadDecoder(subscriptionIds); - this.consumersPath = paths.consumersWorkloadCurrentClusterRuntimeBinaryPath(); - } - - public void refresh() { - logger.info("Refreshing consumer workload assignments"); - List currentlyAssignedConsumers = getWorkloadConsumers(); - List activeConsumers = consumerNodesRegistry.listConsumerNodes(); - - assignments.clear(); - consumerSubscriptions.clear(); - - for (String consumer : currentlyAssignedConsumers) { - if (activeConsumers.contains(consumer)) { - Set subscriptions = readConsumerWorkload(consumer); - consumerSubscriptions.put(consumer, subscriptions); - subscriptions.forEach(subscription -> { - assignments.add(new SubscriptionAssignment(consumer, subscription)); - }); - } else { - logger.info("Deleting consumer {} from workload", consumer); - deleteConsumerWorkloadNode(consumer); - } - } - } - - private Set readConsumerWorkload(String consumer) { - Optional nodeData = zookeeper.getNodeData(paths.consumerWorkloadPath(consumer)); - if (nodeData.isPresent()) { - byte[] data = nodeData.get(); - return consumerWorkloadDecoder.decode(data); - } else { - logger.info("No workload data available for consumer {}", consumer); - return Collections.emptySet(); - } + private static final Logger logger = getLogger(ClusterAssignmentCache.class); + + private final ConsumerNodesRegistry consumerNodesRegistry; + private final WorkloadRegistryPaths paths; + private final ConsumerWorkloadDecoder consumerWorkloadDecoder; + private final String consumersPath; + private final ZookeeperOperations zookeeper; + + private final Map> consumerSubscriptions = + new ConcurrentHashMap<>(); + private final Set assignments = + Collections.newSetFromMap(new ConcurrentHashMap<>()); + + public ClusterAssignmentCache( + CuratorFramework curator, + String clusterName, + ZookeeperPaths zookeeperPaths, + SubscriptionIds subscriptionIds, + ConsumerNodesRegistry consumerNodesRegistry) { + this.zookeeper = new ZookeeperOperations(curator); + this.consumerNodesRegistry = consumerNodesRegistry; + this.paths = new WorkloadRegistryPaths(zookeeperPaths, clusterName); + this.consumerWorkloadDecoder = new ConsumerWorkloadDecoder(subscriptionIds); + this.consumersPath = paths.consumersWorkloadCurrentClusterRuntimeBinaryPath(); + } + + public void refresh() { + logger.info("Refreshing consumer workload assignments"); + List currentlyAssignedConsumers = getWorkloadConsumers(); + List activeConsumers = consumerNodesRegistry.listConsumerNodes(); + + assignments.clear(); + consumerSubscriptions.clear(); + + for (String consumer : currentlyAssignedConsumers) { + if (activeConsumers.contains(consumer)) { + Set subscriptions = readConsumerWorkload(consumer); + consumerSubscriptions.put(consumer, subscriptions); + subscriptions.forEach( + subscription -> { + assignments.add(new SubscriptionAssignment(consumer, subscription)); + }); + } else { + logger.info("Deleting consumer {} from workload", consumer); + deleteConsumerWorkloadNode(consumer); + } } - - private void deleteConsumerWorkloadNode(String consumer) { - String path = paths.consumerWorkloadPath(consumer); - try { - zookeeper.deleteNode(path); - } catch (Exception e) { - logger.warn("Could not delete consumer workload node at {}", path, e); - } + } + + private Set readConsumerWorkload(String consumer) { + Optional nodeData = zookeeper.getNodeData(paths.consumerWorkloadPath(consumer)); + if (nodeData.isPresent()) { + byte[] data = nodeData.get(); + return consumerWorkloadDecoder.decode(data); + } else { + logger.info("No workload data available for consumer {}", consumer); + return Collections.emptySet(); } - - private List getWorkloadConsumers() { - try { - if (zookeeper.exists(consumersPath)) { - return zookeeper.getNodeChildren(consumersPath); - } - } catch (Exception e) { - logger.warn("Could not get workload consumer nodes list", e); - } - return Collections.emptyList(); + } + + private void deleteConsumerWorkloadNode(String consumer) { + String path = paths.consumerWorkloadPath(consumer); + try { + zookeeper.deleteNode(path); + } catch (Exception e) { + logger.warn("Could not delete consumer workload node at {}", path, e); } - - public SubscriptionAssignmentView createSnapshot() { - return SubscriptionAssignmentView.of(assignments); + } + + private List getWorkloadConsumers() { + try { + if (zookeeper.exists(consumersPath)) { + return zookeeper.getNodeChildren(consumersPath); + } + } catch (Exception e) { + logger.warn("Could not get workload consumer nodes list", e); } - - public boolean isAssignedTo(String consumerId, SubscriptionName subscription) { - Set subscriptions = consumerSubscriptions.getOrDefault(consumerId, Collections.emptySet()); - return subscriptions.contains(subscription); - } - - public Map> getSubscriptionConsumers() { - Map> result = new HashMap<>(); - consumerSubscriptions.forEach((consumer, subscriptions) -> { - subscriptions.forEach(subscription -> { + return Collections.emptyList(); + } + + public SubscriptionAssignmentView createSnapshot() { + return SubscriptionAssignmentView.of(assignments); + } + + public boolean isAssignedTo(String consumerId, SubscriptionName subscription) { + Set subscriptions = + consumerSubscriptions.getOrDefault(consumerId, Collections.emptySet()); + return subscriptions.contains(subscription); + } + + public Map> getSubscriptionConsumers() { + Map> result = new HashMap<>(); + consumerSubscriptions.forEach( + (consumer, subscriptions) -> { + subscriptions.forEach( + subscription -> { if (result.containsKey(subscription)) { - result.get(subscription).add(consumer); + result.get(subscription).add(consumer); } else { - HashSet consumers = new HashSet<>(); - consumers.add(consumer); - result.put(subscription, consumers); + HashSet consumers = new HashSet<>(); + consumers.add(consumer); + result.put(subscription, consumers); } - }); + }); }); - return result; - } + return result; + } - public Set getAssignedConsumers() { - return consumerSubscriptions.keySet(); - } + public Set getAssignedConsumers() { + return consumerSubscriptions.keySet(); + } - public Set getConsumerSubscriptions(String consumerId) { - return consumerSubscriptions.getOrDefault(consumerId, Collections.emptySet()); - } + public Set getConsumerSubscriptions(String consumerId) { + return consumerSubscriptions.getOrDefault(consumerId, Collections.emptySet()); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ConsumerAssignmentCache.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ConsumerAssignmentCache.java index 2a28adbb51..7b2ea7f5b0 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ConsumerAssignmentCache.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ConsumerAssignmentCache.java @@ -1,7 +1,13 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload; +import static org.slf4j.LoggerFactory.getLogger; + import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; +import java.io.IOException; +import java.util.Collections; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.NodeCache; @@ -11,105 +17,106 @@ import pl.allegro.tech.hermes.consumers.subscription.id.SubscriptionIds; import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; -import java.io.IOException; -import java.util.Collections; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; - -import static org.slf4j.LoggerFactory.getLogger; - public class ConsumerAssignmentCache implements NodeCacheListener { - private static final Logger logger = getLogger(ConsumerAssignmentCache.class); - - private final WorkloadRegistryPaths paths; - private final String consumerId; - private final NodeCache workloadNodeCache; - private final ConsumerWorkloadDecoder consumerWorkloadDecoder; - private final Set currentlyAssignedSubscriptions = Collections.newSetFromMap(new ConcurrentHashMap<>()); - private final Set callbacks = Collections.newSetFromMap(new ConcurrentHashMap<>()); - - public ConsumerAssignmentCache(CuratorFramework curator, - String consumerId, - String clusterName, - ZookeeperPaths zookeeperPaths, - SubscriptionIds subscriptionIds) { - this.paths = new WorkloadRegistryPaths(zookeeperPaths, clusterName); - this.consumerId = consumerId; - - String path = paths.consumerWorkloadPath(consumerId); - this.workloadNodeCache = new NodeCache(curator, path); - workloadNodeCache.getListenable().addListener(this); - - this.consumerWorkloadDecoder = new ConsumerWorkloadDecoder(subscriptionIds); - } - - public void start() throws Exception { - try { - logger.info("Starting binary workload assignment cache at {}, watching current consumer path at {}", - paths.consumersWorkloadCurrentClusterRuntimeBinaryPath(), paths.consumerWorkloadPath(consumerId)); - workloadNodeCache.start(true); - } catch (Exception e) { - throw new IllegalStateException("Could not start node cache for consumer workload", e); - } - refreshConsumerWorkload(); + private static final Logger logger = getLogger(ConsumerAssignmentCache.class); + + private final WorkloadRegistryPaths paths; + private final String consumerId; + private final NodeCache workloadNodeCache; + private final ConsumerWorkloadDecoder consumerWorkloadDecoder; + private final Set currentlyAssignedSubscriptions = + Collections.newSetFromMap(new ConcurrentHashMap<>()); + private final Set callbacks = + Collections.newSetFromMap(new ConcurrentHashMap<>()); + + public ConsumerAssignmentCache( + CuratorFramework curator, + String consumerId, + String clusterName, + ZookeeperPaths zookeeperPaths, + SubscriptionIds subscriptionIds) { + this.paths = new WorkloadRegistryPaths(zookeeperPaths, clusterName); + this.consumerId = consumerId; + + String path = paths.consumerWorkloadPath(consumerId); + this.workloadNodeCache = new NodeCache(curator, path); + workloadNodeCache.getListenable().addListener(this); + + this.consumerWorkloadDecoder = new ConsumerWorkloadDecoder(subscriptionIds); + } + + public void start() throws Exception { + try { + logger.info( + "Starting binary workload assignment cache at {}, watching current consumer path at {}", + paths.consumersWorkloadCurrentClusterRuntimeBinaryPath(), + paths.consumerWorkloadPath(consumerId)); + workloadNodeCache.start(true); + } catch (Exception e) { + throw new IllegalStateException("Could not start node cache for consumer workload", e); } - - private void refreshConsumerWorkload() { - ChildData nodeData = workloadNodeCache.getCurrentData(); - if (nodeData != null) { - byte[] data = nodeData.getData(); - Set subscriptions = consumerWorkloadDecoder.decode(data); - updateAssignedSubscriptions(subscriptions); - } else { - logger.info("No workload data available for consumer"); - } + refreshConsumerWorkload(); + } + + private void refreshConsumerWorkload() { + ChildData nodeData = workloadNodeCache.getCurrentData(); + if (nodeData != null) { + byte[] data = nodeData.getData(); + Set subscriptions = consumerWorkloadDecoder.decode(data); + updateAssignedSubscriptions(subscriptions); + } else { + logger.info("No workload data available for consumer"); } - - private void updateAssignedSubscriptions(Set targetAssignments) { - ImmutableSet assignmentDeletions = Sets.difference(currentlyAssignedSubscriptions, targetAssignments) - .immutableCopy(); - ImmutableSet assignmentsAdditions = Sets.difference(targetAssignments, currentlyAssignedSubscriptions) - .immutableCopy(); - - - assignmentDeletions.forEach(s -> logger.info("Assignment deletion for subscription {}", s.getQualifiedName())); - assignmentsAdditions.forEach(s -> logger.info("Assignment addition for subscription {}", s.getQualifiedName())); - - currentlyAssignedSubscriptions.clear(); - currentlyAssignedSubscriptions.addAll(targetAssignments); - - callbacks.forEach(callback -> { - if (!callback.watchedConsumerId().isPresent() || callback.watchedConsumerId().get().equals(consumerId)) { - assignmentDeletions.forEach(callback::onAssignmentRemoved); - assignmentsAdditions.forEach(callback::onSubscriptionAssigned); - } + } + + private void updateAssignedSubscriptions(Set targetAssignments) { + ImmutableSet assignmentDeletions = + Sets.difference(currentlyAssignedSubscriptions, targetAssignments).immutableCopy(); + ImmutableSet assignmentsAdditions = + Sets.difference(targetAssignments, currentlyAssignedSubscriptions).immutableCopy(); + + assignmentDeletions.forEach( + s -> logger.info("Assignment deletion for subscription {}", s.getQualifiedName())); + assignmentsAdditions.forEach( + s -> logger.info("Assignment addition for subscription {}", s.getQualifiedName())); + + currentlyAssignedSubscriptions.clear(); + currentlyAssignedSubscriptions.addAll(targetAssignments); + + callbacks.forEach( + callback -> { + if (!callback.watchedConsumerId().isPresent() + || callback.watchedConsumerId().get().equals(consumerId)) { + assignmentDeletions.forEach(callback::onAssignmentRemoved); + assignmentsAdditions.forEach(callback::onSubscriptionAssigned); + } }); + } + + public void stop() throws Exception { + try { + logger.info("Stopping binary workload assignment cache"); + workloadNodeCache.close(); + } catch (IOException e) { + throw new RuntimeException("Could not stop node cache for consumer workload", e); } + } - public void stop() throws Exception { - try { - logger.info("Stopping binary workload assignment cache"); - workloadNodeCache.close(); - } catch (IOException e) { - throw new RuntimeException("Could not stop node cache for consumer workload", e); - } - } + public boolean isAssignedTo(SubscriptionName subscription) { + return currentlyAssignedSubscriptions.contains(subscription); + } - public boolean isAssignedTo(SubscriptionName subscription) { - return currentlyAssignedSubscriptions.contains(subscription); - } + public void registerAssignmentCallback(SubscriptionAssignmentAware callback) { + callbacks.add(callback); + } - public void registerAssignmentCallback(SubscriptionAssignmentAware callback) { - callbacks.add(callback); - } + public Set getConsumerSubscriptions() { + return ImmutableSet.copyOf(currentlyAssignedSubscriptions); + } - public Set getConsumerSubscriptions() { - return ImmutableSet.copyOf(currentlyAssignedSubscriptions); - } - - @Override - public void nodeChanged() { - refreshConsumerWorkload(); - } + @Override + public void nodeChanged() { + refreshConsumerWorkload(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ConsumerAssignmentRegistry.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ConsumerAssignmentRegistry.java index e179e30e40..34475cb806 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ConsumerAssignmentRegistry.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ConsumerAssignmentRegistry.java @@ -1,40 +1,41 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload; +import static org.slf4j.LoggerFactory.getLogger; + +import java.util.Set; import org.apache.curator.framework.CuratorFramework; import org.slf4j.Logger; import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.consumers.subscription.id.SubscriptionIds; import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; -import java.util.Set; - -import static org.slf4j.LoggerFactory.getLogger; - public class ConsumerAssignmentRegistry { - private static final Logger logger = getLogger(ConsumerAssignmentRegistry.class); - - private final ZookeeperOperations zookeeper; - private final ConsumerWorkloadEncoder consumerAssignmentsEncoder; - private final WorkloadRegistryPaths paths; - - public ConsumerAssignmentRegistry(CuratorFramework curator, - int assignmentsEncoderBufferSize, - String clusterName, - ZookeeperPaths zookeeperPaths, - SubscriptionIds subscriptionIds) { - this.zookeeper = new ZookeeperOperations(curator); - this.consumerAssignmentsEncoder = new ConsumerWorkloadEncoder(subscriptionIds, assignmentsEncoderBufferSize); - - this.paths = new WorkloadRegistryPaths(zookeeperPaths, clusterName); - } - - public void updateAssignments(String consumerNode, Set subscriptions) { - byte[] encoded = consumerAssignmentsEncoder.encode(subscriptions); - try { - String path = paths.consumerWorkloadPath(consumerNode); - zookeeper.writeOrCreatePersistent(path, encoded); - } catch (Exception e) { - logger.error("Could not write consumer workload for {}", consumerNode); - } + private static final Logger logger = getLogger(ConsumerAssignmentRegistry.class); + + private final ZookeeperOperations zookeeper; + private final ConsumerWorkloadEncoder consumerAssignmentsEncoder; + private final WorkloadRegistryPaths paths; + + public ConsumerAssignmentRegistry( + CuratorFramework curator, + int assignmentsEncoderBufferSize, + String clusterName, + ZookeeperPaths zookeeperPaths, + SubscriptionIds subscriptionIds) { + this.zookeeper = new ZookeeperOperations(curator); + this.consumerAssignmentsEncoder = + new ConsumerWorkloadEncoder(subscriptionIds, assignmentsEncoderBufferSize); + + this.paths = new WorkloadRegistryPaths(zookeeperPaths, clusterName); + } + + public void updateAssignments(String consumerNode, Set subscriptions) { + byte[] encoded = consumerAssignmentsEncoder.encode(subscriptions); + try { + String path = paths.consumerWorkloadPath(consumerNode); + zookeeper.writeOrCreatePersistent(path, encoded); + } catch (Exception e) { + logger.error("Could not write consumer workload for {}", consumerNode); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ConsumerWorkloadDecoder.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ConsumerWorkloadDecoder.java index 59c7de54ff..87265af541 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ConsumerWorkloadDecoder.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ConsumerWorkloadDecoder.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload; +import static org.slf4j.LoggerFactory.getLogger; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; import org.agrona.concurrent.UnsafeBuffer; import org.slf4j.Logger; import pl.allegro.tech.hermes.api.SubscriptionName; @@ -8,46 +13,45 @@ import pl.allegro.tech.hermes.consumers.supervisor.workload.sbe.stubs.AssignmentsDecoder; import pl.allegro.tech.hermes.consumers.supervisor.workload.sbe.stubs.MessageHeaderDecoder; -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; +class ConsumerWorkloadDecoder { -import static org.slf4j.LoggerFactory.getLogger; + private static final Logger logger = getLogger(ConsumerWorkloadDecoder.class); -class ConsumerWorkloadDecoder { + private final SubscriptionIds subscriptionIds; - private static final Logger logger = getLogger(ConsumerWorkloadDecoder.class); + ConsumerWorkloadDecoder(SubscriptionIds subscriptionIds) { + this.subscriptionIds = subscriptionIds; + } - private final SubscriptionIds subscriptionIds; + Set decode(byte[] data) { + MessageHeaderDecoder header = new MessageHeaderDecoder(); + AssignmentsDecoder body = new AssignmentsDecoder(); - ConsumerWorkloadDecoder(SubscriptionIds subscriptionIds) { - this.subscriptionIds = subscriptionIds; - } + UnsafeBuffer buffer = new UnsafeBuffer(data); + header.wrap(buffer, 0); - Set decode(byte[] data) { - MessageHeaderDecoder header = new MessageHeaderDecoder(); - AssignmentsDecoder body = new AssignmentsDecoder(); - - UnsafeBuffer buffer = new UnsafeBuffer(data); - header.wrap(buffer, 0); - - if (header.schemaId() != AssignmentsDecoder.SCHEMA_ID || header.templateId() != AssignmentsDecoder.TEMPLATE_ID) { - logger.warn("Unable to decode assignments, schema or template id mismatch. " - + "Required by decoder: [schema id={}, template id={}], " - + "encoded in payload: [schema id={}, template id={}]", - AssignmentsDecoder.SCHEMA_ID, AssignmentsDecoder.TEMPLATE_ID, - header.schemaId(), header.templateId()); - return Collections.emptySet(); - } - body.wrap(buffer, header.encodedLength(), header.blockLength(), header.version()); - - Set subscriptions = new HashSet<>(); - for (AssignmentsDecoder.SubscriptionsDecoder subscriptionDecoder : body.subscriptions()) { - long id = subscriptionDecoder.id(); - subscriptionIds.getSubscriptionId(id) - .map(SubscriptionId::getSubscriptionName) - .ifPresent(subscriptions::add); - } - return subscriptions; + if (header.schemaId() != AssignmentsDecoder.SCHEMA_ID + || header.templateId() != AssignmentsDecoder.TEMPLATE_ID) { + logger.warn( + "Unable to decode assignments, schema or template id mismatch. " + + "Required by decoder: [schema id={}, template id={}], " + + "encoded in payload: [schema id={}, template id={}]", + AssignmentsDecoder.SCHEMA_ID, + AssignmentsDecoder.TEMPLATE_ID, + header.schemaId(), + header.templateId()); + return Collections.emptySet(); + } + body.wrap(buffer, header.encodedLength(), header.blockLength(), header.version()); + + Set subscriptions = new HashSet<>(); + for (AssignmentsDecoder.SubscriptionsDecoder subscriptionDecoder : body.subscriptions()) { + long id = subscriptionDecoder.id(); + subscriptionIds + .getSubscriptionId(id) + .map(SubscriptionId::getSubscriptionName) + .ifPresent(subscriptions::add); } + return subscriptions; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ConsumerWorkloadEncoder.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ConsumerWorkloadEncoder.java index d63e089a4b..4de977276f 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ConsumerWorkloadEncoder.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ConsumerWorkloadEncoder.java @@ -1,5 +1,9 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload; +import java.util.Collection; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; import org.agrona.ExpandableDirectByteBuffer; import org.agrona.MutableDirectBuffer; import pl.allegro.tech.hermes.api.SubscriptionName; @@ -8,40 +12,35 @@ import pl.allegro.tech.hermes.consumers.supervisor.workload.sbe.stubs.AssignmentsEncoder; import pl.allegro.tech.hermes.consumers.supervisor.workload.sbe.stubs.MessageHeaderEncoder; -import java.util.Collection; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - class ConsumerWorkloadEncoder { - private final SubscriptionIds subscriptionIds; - private final MutableDirectBuffer buffer; + private final SubscriptionIds subscriptionIds; + private final MutableDirectBuffer buffer; - ConsumerWorkloadEncoder(SubscriptionIds subscriptionIds, int bufferSize) { - this.subscriptionIds = subscriptionIds; - this.buffer = new ExpandableDirectByteBuffer(bufferSize); - } + ConsumerWorkloadEncoder(SubscriptionIds subscriptionIds, int bufferSize) { + this.subscriptionIds = subscriptionIds; + this.buffer = new ExpandableDirectByteBuffer(bufferSize); + } - byte[] encode(Collection subscriptions) { - MessageHeaderEncoder headerEncoder = new MessageHeaderEncoder(); - AssignmentsEncoder body = new AssignmentsEncoder(); + byte[] encode(Collection subscriptions) { + MessageHeaderEncoder headerEncoder = new MessageHeaderEncoder(); + AssignmentsEncoder body = new AssignmentsEncoder(); - Set ids = subscriptions.stream() - .map(this.subscriptionIds::getSubscriptionId) - .filter(Optional::isPresent) - .map(Optional::get) - .collect(Collectors.toSet()); + Set ids = + subscriptions.stream() + .map(this.subscriptionIds::getSubscriptionId) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toSet()); - AssignmentsEncoder.SubscriptionsEncoder subscriptionsEncoder = body.wrapAndApplyHeader(buffer, 0, headerEncoder) - .subscriptionsCount(ids.size()); - ids.forEach(id -> subscriptionsEncoder.next() - .id(id.getValue())); + AssignmentsEncoder.SubscriptionsEncoder subscriptionsEncoder = + body.wrapAndApplyHeader(buffer, 0, headerEncoder).subscriptionsCount(ids.size()); + ids.forEach(id -> subscriptionsEncoder.next().id(id.getValue())); - int len = headerEncoder.encodedLength() + body.encodedLength(); + int len = headerEncoder.encodedLength() + body.encodedLength(); - byte[] dst = new byte[len]; - buffer.getBytes(0, dst); - return dst; - } + byte[] dst = new byte[len]; + buffer.getBytes(0, dst); + return dst; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/NoOpBalancingListener.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/NoOpBalancingListener.java index 6c4b08f0fc..e252bb228a 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/NoOpBalancingListener.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/NoOpBalancingListener.java @@ -4,18 +4,12 @@ public class NoOpBalancingListener implements BalancingListener { - @Override - public void onBeforeBalancing(List activeConsumers) { + @Override + public void onBeforeBalancing(List activeConsumers) {} - } + @Override + public void onAfterBalancing(WorkDistributionChanges changes) {} - @Override - public void onAfterBalancing(WorkDistributionChanges changes) { - - } - - @Override - public void onBalancingSkipped() { - - } + @Override + public void onBalancingSkipped() {} } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/SubscriptionAssignment.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/SubscriptionAssignment.java index 96f8626c01..dd684180a8 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/SubscriptionAssignment.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/SubscriptionAssignment.java @@ -1,41 +1,40 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload; -import pl.allegro.tech.hermes.api.SubscriptionName; - import java.util.Objects; +import pl.allegro.tech.hermes.api.SubscriptionName; public class SubscriptionAssignment { - private final String consumerNodeId; - private final SubscriptionName subscriptionName; - - public SubscriptionAssignment(String consumerNodeId, SubscriptionName subscriptionName) { - this.consumerNodeId = consumerNodeId; - this.subscriptionName = subscriptionName; - } - - public String getConsumerNodeId() { - return consumerNodeId; + private final String consumerNodeId; + private final SubscriptionName subscriptionName; + + public SubscriptionAssignment(String consumerNodeId, SubscriptionName subscriptionName) { + this.consumerNodeId = consumerNodeId; + this.subscriptionName = subscriptionName; + } + + public String getConsumerNodeId() { + return consumerNodeId; + } + + public SubscriptionName getSubscriptionName() { + return subscriptionName; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - public SubscriptionName getSubscriptionName() { - return subscriptionName; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - SubscriptionAssignment that = (SubscriptionAssignment) o; - return Objects.equals(consumerNodeId, that.consumerNodeId) - && Objects.equals(subscriptionName, that.subscriptionName); - } - - @Override - public int hashCode() { - return Objects.hash(consumerNodeId, subscriptionName); + if (o == null || getClass() != o.getClass()) { + return false; } + SubscriptionAssignment that = (SubscriptionAssignment) o; + return Objects.equals(consumerNodeId, that.consumerNodeId) + && Objects.equals(subscriptionName, that.subscriptionName); + } + + @Override + public int hashCode() { + return Objects.hash(consumerNodeId, subscriptionName); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/SubscriptionAssignmentAware.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/SubscriptionAssignmentAware.java index a53c618075..0e069c8232 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/SubscriptionAssignmentAware.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/SubscriptionAssignmentAware.java @@ -1,13 +1,12 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload; -import pl.allegro.tech.hermes.api.SubscriptionName; - import java.util.Optional; +import pl.allegro.tech.hermes.api.SubscriptionName; public interface SubscriptionAssignmentAware { - default void onSubscriptionAssigned(SubscriptionName subscriptionName) {} + default void onSubscriptionAssigned(SubscriptionName subscriptionName) {} - default void onAssignmentRemoved(SubscriptionName subscriptionName) {} + default void onAssignmentRemoved(SubscriptionName subscriptionName) {} - Optional watchedConsumerId(); + Optional watchedConsumerId(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/SubscriptionAssignmentView.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/SubscriptionAssignmentView.java index 098aa2b677..229bafc4dc 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/SubscriptionAssignmentView.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/SubscriptionAssignmentView.java @@ -1,8 +1,9 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload; -import com.google.common.collect.Sets; -import pl.allegro.tech.hermes.api.SubscriptionName; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toSet; +import com.google.common.collect.Sets; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -11,229 +12,251 @@ import java.util.Objects; import java.util.Set; import java.util.function.BiConsumer; - -import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toSet; +import pl.allegro.tech.hermes.api.SubscriptionName; public class SubscriptionAssignmentView { - private final Map> subscriptionAssignments; - private final Map> consumerNodeAssignments; - - public SubscriptionAssignmentView(Map> view) { - this.subscriptionAssignments = setupSubscriptionAssignments(view); - this.consumerNodeAssignments = setupConsumerNodeAssignments(view); - } - - private Map> setupSubscriptionAssignments( - Map> view - ) { - Map> map = new HashMap<>(); - view.entrySet().forEach(entry -> map.put(entry.getKey(), new HashSet<>(entry.getValue()))); - return map; - } - - private Map> setupConsumerNodeAssignments(Map> view) { - Map> map = new HashMap<>(); - view.values().stream().flatMap(Set::stream).forEach(assignment -> { - if (!map.containsKey(assignment.getConsumerNodeId())) { + private final Map> subscriptionAssignments; + private final Map> consumerNodeAssignments; + + public SubscriptionAssignmentView(Map> view) { + this.subscriptionAssignments = setupSubscriptionAssignments(view); + this.consumerNodeAssignments = setupConsumerNodeAssignments(view); + } + + private Map> setupSubscriptionAssignments( + Map> view) { + Map> map = new HashMap<>(); + view.entrySet().forEach(entry -> map.put(entry.getKey(), new HashSet<>(entry.getValue()))); + return map; + } + + private Map> setupConsumerNodeAssignments( + Map> view) { + Map> map = new HashMap<>(); + view.values().stream() + .flatMap(Set::stream) + .forEach( + assignment -> { + if (!map.containsKey(assignment.getConsumerNodeId())) { map.put(assignment.getConsumerNodeId(), new HashSet<>()); - } - map.get(assignment.getConsumerNodeId()).add(assignment); - }); - return map; - } - - public Set getSubscriptions() { - return subscriptionAssignments.keySet(); - } - - public int getSubscriptionsCount() { - return subscriptionAssignments.size(); - } - - public Set getConsumerNodes() { - return consumerNodeAssignments.keySet(); - } - - public List getAllAssignments() { - return subscriptionAssignments.values().stream().flatMap(Set::stream).collect(toList()); - } - - public Set getConsumerNodesForSubscription(SubscriptionName subscriptionName) { - return getAssignmentsForSubscription(subscriptionName).stream().map(SubscriptionAssignment::getConsumerNodeId).collect(toSet()); - } - - public Set getAssignmentsForSubscription(SubscriptionName subscriptionName) { - return Collections.unmodifiableSet(subscriptionAssignments.getOrDefault(subscriptionName, Collections.emptySet())); - } - - public Set getSubscriptionsForConsumerNode(String nodeId) { - return getAssignmentsForConsumerNode(nodeId).stream().map(SubscriptionAssignment::getSubscriptionName).collect(toSet()); - } - - public Set getAssignmentsForConsumerNode(String nodeId) { - return Collections.unmodifiableSet(consumerNodeAssignments.getOrDefault(nodeId, Collections.emptySet())); - } - - private void removeSubscription(SubscriptionName subscription) { - consumerNodeAssignments.values() - .forEach(assignments -> assignments.removeIf(assignment -> assignment.getSubscriptionName().equals(subscription))); - subscriptionAssignments.remove(subscription); - } - - private void removeConsumerNode(String nodeId) { - subscriptionAssignments.values() - .forEach(assignments -> assignments.removeIf(assignment -> assignment.getConsumerNodeId().equals(nodeId))); - consumerNodeAssignments.remove(nodeId); - } - - private void addSubscription(SubscriptionName subscriptionName) { - subscriptionAssignments.putIfAbsent(subscriptionName, new HashSet<>()); - } - - private void addConsumerNode(String nodeId) { - consumerNodeAssignments.putIfAbsent(nodeId, new HashSet<>()); - } - - private void addAssignment(SubscriptionAssignment assignment) { - subscriptionAssignments.get(assignment.getSubscriptionName()).add(assignment); - if (!consumerNodeAssignments.containsKey(assignment.getConsumerNodeId())) { - addConsumerNode(assignment.getConsumerNodeId()); - } - consumerNodeAssignments.get(assignment.getConsumerNodeId()).add(assignment); - } - - private void removeAssignment(SubscriptionAssignment assignment) { - subscriptionAssignments.get(assignment.getSubscriptionName()).remove(assignment); - consumerNodeAssignments.get(assignment.getConsumerNodeId()).remove(assignment); - } - - private void transferAssignment(String from, String to, SubscriptionName subscriptionName) { - removeAssignment(new SubscriptionAssignment(from, subscriptionName)); - addAssignment(new SubscriptionAssignment(to, subscriptionName)); - } - - public SubscriptionAssignmentView deletions(SubscriptionAssignmentView target) { - return difference(this, target); - } - - public SubscriptionAssignmentView additions(SubscriptionAssignmentView target) { - return difference(target, this); - } - - private static SubscriptionAssignmentView difference(SubscriptionAssignmentView first, SubscriptionAssignmentView second) { - HashMap> result = new HashMap<>(); - for (SubscriptionName subscription : first.getSubscriptions()) { - Set assignments = first.getAssignmentsForSubscription(subscription); - if (!second.getSubscriptions().contains(subscription)) { - result.put(subscription, assignments); - } else { - Sets.SetView difference = - Sets.difference(assignments, second.getAssignmentsForSubscription(subscription)); - if (!difference.isEmpty()) { - result.put(subscription, difference); - } - } - } - return new SubscriptionAssignmentView(result); - } - - public static SubscriptionAssignmentView of(Set assignments) { - Map> snapshot = new HashMap<>(); - for (SubscriptionAssignment assignment : assignments) { - snapshot.compute(assignment.getSubscriptionName(), (k, v) -> { - v = (v == null ? new HashSet<>() : v); - v.add(assignment); - return v; + } + map.get(assignment.getConsumerNodeId()).add(assignment); }); - } - return new SubscriptionAssignmentView(snapshot); + return map; + } + + public Set getSubscriptions() { + return subscriptionAssignments.keySet(); + } + + public int getSubscriptionsCount() { + return subscriptionAssignments.size(); + } + + public Set getConsumerNodes() { + return consumerNodeAssignments.keySet(); + } + + public List getAllAssignments() { + return subscriptionAssignments.values().stream().flatMap(Set::stream).collect(toList()); + } + + public Set getConsumerNodesForSubscription(SubscriptionName subscriptionName) { + return getAssignmentsForSubscription(subscriptionName).stream() + .map(SubscriptionAssignment::getConsumerNodeId) + .collect(toSet()); + } + + public Set getAssignmentsForSubscription( + SubscriptionName subscriptionName) { + return Collections.unmodifiableSet( + subscriptionAssignments.getOrDefault(subscriptionName, Collections.emptySet())); + } + + public Set getSubscriptionsForConsumerNode(String nodeId) { + return getAssignmentsForConsumerNode(nodeId).stream() + .map(SubscriptionAssignment::getSubscriptionName) + .collect(toSet()); + } + + public Set getAssignmentsForConsumerNode(String nodeId) { + return Collections.unmodifiableSet( + consumerNodeAssignments.getOrDefault(nodeId, Collections.emptySet())); + } + + private void removeSubscription(SubscriptionName subscription) { + consumerNodeAssignments + .values() + .forEach( + assignments -> + assignments.removeIf( + assignment -> assignment.getSubscriptionName().equals(subscription))); + subscriptionAssignments.remove(subscription); + } + + private void removeConsumerNode(String nodeId) { + subscriptionAssignments + .values() + .forEach( + assignments -> + assignments.removeIf(assignment -> assignment.getConsumerNodeId().equals(nodeId))); + consumerNodeAssignments.remove(nodeId); + } + + private void addSubscription(SubscriptionName subscriptionName) { + subscriptionAssignments.putIfAbsent(subscriptionName, new HashSet<>()); + } + + private void addConsumerNode(String nodeId) { + consumerNodeAssignments.putIfAbsent(nodeId, new HashSet<>()); + } + + private void addAssignment(SubscriptionAssignment assignment) { + subscriptionAssignments.get(assignment.getSubscriptionName()).add(assignment); + if (!consumerNodeAssignments.containsKey(assignment.getConsumerNodeId())) { + addConsumerNode(assignment.getConsumerNodeId()); } - - public static SubscriptionAssignmentView copyOf(SubscriptionAssignmentView currentState) { - return new SubscriptionAssignmentView(currentState.subscriptionAssignments); + consumerNodeAssignments.get(assignment.getConsumerNodeId()).add(assignment); + } + + private void removeAssignment(SubscriptionAssignment assignment) { + subscriptionAssignments.get(assignment.getSubscriptionName()).remove(assignment); + consumerNodeAssignments.get(assignment.getConsumerNodeId()).remove(assignment); + } + + private void transferAssignment(String from, String to, SubscriptionName subscriptionName) { + removeAssignment(new SubscriptionAssignment(from, subscriptionName)); + addAssignment(new SubscriptionAssignment(to, subscriptionName)); + } + + public SubscriptionAssignmentView deletions(SubscriptionAssignmentView target) { + return difference(this, target); + } + + public SubscriptionAssignmentView additions(SubscriptionAssignmentView target) { + return difference(target, this); + } + + private static SubscriptionAssignmentView difference( + SubscriptionAssignmentView first, SubscriptionAssignmentView second) { + HashMap> result = new HashMap<>(); + for (SubscriptionName subscription : first.getSubscriptions()) { + Set assignments = first.getAssignmentsForSubscription(subscription); + if (!second.getSubscriptions().contains(subscription)) { + result.put(subscription, assignments); + } else { + Sets.SetView difference = + Sets.difference(assignments, second.getAssignmentsForSubscription(subscription)); + if (!difference.isEmpty()) { + result.put(subscription, difference); + } + } } - - public int getAssignmentsCountForSubscription(SubscriptionName subscription) { - return subscriptionAssignments.get(subscription).size(); + return new SubscriptionAssignmentView(result); + } + + public static SubscriptionAssignmentView of(Set assignments) { + Map> snapshot = new HashMap<>(); + for (SubscriptionAssignment assignment : assignments) { + snapshot.compute( + assignment.getSubscriptionName(), + (k, v) -> { + v = (v == null ? new HashSet<>() : v); + v.add(assignment); + return v; + }); } - - public int getAssignmentsCountForConsumerNode(String nodeId) { - return consumerNodeAssignments.get(nodeId).size(); - } - - @Override - public int hashCode() { - return Objects.hash(subscriptionAssignments); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - SubscriptionAssignmentView that = (SubscriptionAssignmentView) o; - return Objects.equals(subscriptionAssignments, that.subscriptionAssignments); + return new SubscriptionAssignmentView(snapshot); + } + + public static SubscriptionAssignmentView copyOf(SubscriptionAssignmentView currentState) { + return new SubscriptionAssignmentView(currentState.subscriptionAssignments); + } + + public int getAssignmentsCountForSubscription(SubscriptionName subscription) { + return subscriptionAssignments.get(subscription).size(); + } + + public int getAssignmentsCountForConsumerNode(String nodeId) { + return consumerNodeAssignments.get(nodeId).size(); + } + + @Override + public int hashCode() { + return Objects.hash(subscriptionAssignments); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - public interface Transformer { - void removeSubscription(SubscriptionName subscriptionName); - - void removeConsumerNode(String nodeId); - - void addSubscription(SubscriptionName subscriptionName); - - void addConsumerNode(String nodeId); - - void addAssignment(SubscriptionAssignment assignment); - - void removeAssignment(SubscriptionAssignment assignment); - - void transferAssignment(String from, String to, SubscriptionName subscriptionName); + if (o == null || getClass() != o.getClass()) { + return false; } - - public SubscriptionAssignmentView transform(BiConsumer consumer) { - SubscriptionAssignmentView view = SubscriptionAssignmentView.copyOf(this); - consumer.accept(view, new Transformer() { - @Override - public void removeSubscription(SubscriptionName subscriptionName) { - view.removeSubscription(subscriptionName); - } - - @Override - public void removeConsumerNode(String nodeId) { - view.removeConsumerNode(nodeId); - } - - @Override - public void addSubscription(SubscriptionName subscriptionName) { - view.addSubscription(subscriptionName); - } - - @Override - public void addConsumerNode(String nodeId) { - view.addConsumerNode(nodeId); - } - - @Override - public void addAssignment(SubscriptionAssignment assignment) { - view.addAssignment(assignment); - } - - @Override - public void removeAssignment(SubscriptionAssignment assignment) { - view.removeAssignment(assignment); - } - - @Override - public void transferAssignment(String from, String to, SubscriptionName subscriptionName) { - view.transferAssignment(from, to, subscriptionName); - } + SubscriptionAssignmentView that = (SubscriptionAssignmentView) o; + return Objects.equals(subscriptionAssignments, that.subscriptionAssignments); + } + + public interface Transformer { + void removeSubscription(SubscriptionName subscriptionName); + + void removeConsumerNode(String nodeId); + + void addSubscription(SubscriptionName subscriptionName); + + void addConsumerNode(String nodeId); + + void addAssignment(SubscriptionAssignment assignment); + + void removeAssignment(SubscriptionAssignment assignment); + + void transferAssignment(String from, String to, SubscriptionName subscriptionName); + } + + public SubscriptionAssignmentView transform( + BiConsumer consumer) { + SubscriptionAssignmentView view = SubscriptionAssignmentView.copyOf(this); + consumer.accept( + view, + new Transformer() { + @Override + public void removeSubscription(SubscriptionName subscriptionName) { + view.removeSubscription(subscriptionName); + } + + @Override + public void removeConsumerNode(String nodeId) { + view.removeConsumerNode(nodeId); + } + + @Override + public void addSubscription(SubscriptionName subscriptionName) { + view.addSubscription(subscriptionName); + } + + @Override + public void addConsumerNode(String nodeId) { + view.addConsumerNode(nodeId); + } + + @Override + public void addAssignment(SubscriptionAssignment assignment) { + view.addAssignment(assignment); + } + + @Override + public void removeAssignment(SubscriptionAssignment assignment) { + view.removeAssignment(assignment); + } + + @Override + public void transferAssignment( + String from, String to, SubscriptionName subscriptionName) { + view.transferAssignment(from, to, subscriptionName); + } }); - return copyOf(view); - } + return copyOf(view); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkBalancer.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkBalancer.java index b8a73af77b..5e8555c0e1 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkBalancer.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkBalancer.java @@ -1,13 +1,13 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload; -import pl.allegro.tech.hermes.api.SubscriptionName; - import java.util.List; +import pl.allegro.tech.hermes.api.SubscriptionName; public interface WorkBalancer { - WorkBalancingResult balance(List subscriptions, - List activeConsumerNodes, - SubscriptionAssignmentView currentState, - WorkloadConstraints constraints); + WorkBalancingResult balance( + List subscriptions, + List activeConsumerNodes, + SubscriptionAssignmentView currentState, + WorkloadConstraints constraints); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkBalancingParameters.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkBalancingParameters.java index 0e42c7ab4e..0186a5b779 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkBalancingParameters.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkBalancingParameters.java @@ -4,11 +4,11 @@ public interface WorkBalancingParameters { - Duration getRebalanceInterval(); + Duration getRebalanceInterval(); - int getConsumersPerSubscription(); + int getConsumersPerSubscription(); - int getMaxSubscriptionsPerConsumer(); + int getMaxSubscriptionsPerConsumer(); - boolean isAutoRebalance(); + boolean isAutoRebalance(); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkBalancingResult.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkBalancingResult.java index 563bdf2c6a..5753192f37 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkBalancingResult.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkBalancingResult.java @@ -2,19 +2,19 @@ public class WorkBalancingResult { - private final SubscriptionAssignmentView state; - private final int missingResources; + private final SubscriptionAssignmentView state; + private final int missingResources; - public WorkBalancingResult(SubscriptionAssignmentView balancedState, int missingResources) { - this.state = balancedState; - this.missingResources = missingResources; - } + public WorkBalancingResult(SubscriptionAssignmentView balancedState, int missingResources) { + this.state = balancedState; + this.missingResources = missingResources; + } - public SubscriptionAssignmentView getAssignmentsView() { - return state; - } + public SubscriptionAssignmentView getAssignmentsView() { + return state; + } - public int getMissingResources() { - return missingResources; - } + public int getMissingResources() { + return missingResources; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkDistributionChanges.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkDistributionChanges.java index b0f90b57bf..ea28327d46 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkDistributionChanges.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkDistributionChanges.java @@ -1,44 +1,41 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload; -import com.google.common.collect.Sets; -import pl.allegro.tech.hermes.api.SubscriptionName; +import static java.util.stream.Collectors.toSet; +import com.google.common.collect.Sets; import java.util.Set; import java.util.stream.Stream; - -import static java.util.stream.Collectors.toSet; +import pl.allegro.tech.hermes.api.SubscriptionName; public class WorkDistributionChanges { - private final SubscriptionAssignmentView deletions; - private final SubscriptionAssignmentView additions; - private final Set modifiedConsumerNodes; - - WorkDistributionChanges(SubscriptionAssignmentView deletions, SubscriptionAssignmentView additions) { - this.deletions = deletions; - this.additions = additions; - this.modifiedConsumerNodes = Sets.union( - deletions.getConsumerNodes(), - additions.getConsumerNodes() - ); - } - - int getDeletedAssignmentsCount() { - return deletions.getAllAssignments().size(); - } - - int getCreatedAssignmentsCount() { - return additions.getAllAssignments().size(); - } - - Set getModifiedConsumerNodes() { - return modifiedConsumerNodes; - } - - public Set getRebalancedSubscriptions() { - return Stream.concat( - additions.getSubscriptions().stream(), - deletions.getSubscriptions().stream() - ).collect(toSet()); - } + private final SubscriptionAssignmentView deletions; + private final SubscriptionAssignmentView additions; + private final Set modifiedConsumerNodes; + + WorkDistributionChanges( + SubscriptionAssignmentView deletions, SubscriptionAssignmentView additions) { + this.deletions = deletions; + this.additions = additions; + this.modifiedConsumerNodes = + Sets.union(deletions.getConsumerNodes(), additions.getConsumerNodes()); + } + + int getDeletedAssignmentsCount() { + return deletions.getAllAssignments().size(); + } + + int getCreatedAssignmentsCount() { + return additions.getAllAssignments().size(); + } + + Set getModifiedConsumerNodes() { + return modifiedConsumerNodes; + } + + public Set getRebalancedSubscriptions() { + return Stream.concat( + additions.getSubscriptions().stream(), deletions.getSubscriptions().stream()) + .collect(toSet()); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkloadConstraints.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkloadConstraints.java index d34b1ae046..f0a82bf9ea 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkloadConstraints.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkloadConstraints.java @@ -1,93 +1,94 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload; -import pl.allegro.tech.hermes.api.Constraints; -import pl.allegro.tech.hermes.api.SubscriptionName; -import pl.allegro.tech.hermes.api.TopicName; +import static java.util.Collections.emptyMap; import java.util.HashMap; import java.util.Map; - -import static java.util.Collections.emptyMap; +import pl.allegro.tech.hermes.api.Constraints; +import pl.allegro.tech.hermes.api.SubscriptionName; +import pl.allegro.tech.hermes.api.TopicName; public class WorkloadConstraints { - private final int activeConsumerCount; - private final int consumersPerSubscription; - private final int maxSubscriptionsPerConsumer; - private final Map subscriptionConstraints; - private final Map topicConstraints; - - private WorkloadConstraints(int activeConsumerCount, - int consumersPerSubscription, - int maxSubscriptionsPerConsumer, - Map subscriptionConstraints, - Map topicConstraints) { - this.activeConsumerCount = activeConsumerCount; - this.consumersPerSubscription = consumersPerSubscription; - this.maxSubscriptionsPerConsumer = maxSubscriptionsPerConsumer; - this.subscriptionConstraints = subscriptionConstraints != null ? subscriptionConstraints : emptyMap(); - this.topicConstraints = topicConstraints != null ? topicConstraints : emptyMap(); + private final int activeConsumerCount; + private final int consumersPerSubscription; + private final int maxSubscriptionsPerConsumer; + private final Map subscriptionConstraints; + private final Map topicConstraints; + + private WorkloadConstraints( + int activeConsumerCount, + int consumersPerSubscription, + int maxSubscriptionsPerConsumer, + Map subscriptionConstraints, + Map topicConstraints) { + this.activeConsumerCount = activeConsumerCount; + this.consumersPerSubscription = consumersPerSubscription; + this.maxSubscriptionsPerConsumer = maxSubscriptionsPerConsumer; + this.subscriptionConstraints = + subscriptionConstraints != null ? subscriptionConstraints : emptyMap(); + this.topicConstraints = topicConstraints != null ? topicConstraints : emptyMap(); + } + + public int getConsumerCount(SubscriptionName subscriptionName) { + Constraints requiredConsumers = subscriptionConstraints.get(subscriptionName); + if (requiredConsumers == null) { + requiredConsumers = topicConstraints.get(subscriptionName.getTopicName()); } - - public int getConsumerCount(SubscriptionName subscriptionName) { - Constraints requiredConsumers = subscriptionConstraints.get(subscriptionName); - if (requiredConsumers == null) { - requiredConsumers = topicConstraints.get(subscriptionName.getTopicName()); - } - if (requiredConsumers != null && requiredConsumers.getConsumersNumber() > 0) { - return Math.min(requiredConsumers.getConsumersNumber(), activeConsumerCount); - } - return Math.min(consumersPerSubscription, activeConsumerCount); + if (requiredConsumers != null && requiredConsumers.getConsumersNumber() > 0) { + return Math.min(requiredConsumers.getConsumersNumber(), activeConsumerCount); } - - public int getMaxSubscriptionsPerConsumer() { - return maxSubscriptionsPerConsumer; + return Math.min(consumersPerSubscription, activeConsumerCount); + } + + public int getMaxSubscriptionsPerConsumer() { + return maxSubscriptionsPerConsumer; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private int activeConsumerCount; + private int consumersPerSubscription; + private int maxSubscriptionsPerConsumer; + private Map subscriptionConstraints = new HashMap<>(); + private Map topicConstraints = new HashMap<>(); + + public Builder withActiveConsumers(int activeConsumerCount) { + this.activeConsumerCount = activeConsumerCount; + return this; } - public static Builder builder() { - return new Builder(); + public Builder withConsumersPerSubscription(int consumersPerSubscription) { + this.consumersPerSubscription = consumersPerSubscription; + return this; } - public static class Builder { - private int activeConsumerCount; - private int consumersPerSubscription; - private int maxSubscriptionsPerConsumer; - private Map subscriptionConstraints = new HashMap<>(); - private Map topicConstraints = new HashMap<>(); - - public Builder withActiveConsumers(int activeConsumerCount) { - this.activeConsumerCount = activeConsumerCount; - return this; - } - - public Builder withConsumersPerSubscription(int consumersPerSubscription) { - this.consumersPerSubscription = consumersPerSubscription; - return this; - } - - public Builder withMaxSubscriptionsPerConsumer(int maxSubscriptionsPerConsumer) { - this.maxSubscriptionsPerConsumer = maxSubscriptionsPerConsumer; - return this; - } + public Builder withMaxSubscriptionsPerConsumer(int maxSubscriptionsPerConsumer) { + this.maxSubscriptionsPerConsumer = maxSubscriptionsPerConsumer; + return this; + } - public Builder withSubscriptionConstraints(Map subscriptionConstraints) { - this.subscriptionConstraints = subscriptionConstraints; - return this; - } + public Builder withSubscriptionConstraints( + Map subscriptionConstraints) { + this.subscriptionConstraints = subscriptionConstraints; + return this; + } - public Builder withTopicConstraints(Map topicConstraints) { - this.topicConstraints = topicConstraints; - return this; - } + public Builder withTopicConstraints(Map topicConstraints) { + this.topicConstraints = topicConstraints; + return this; + } - public WorkloadConstraints build() { - return new WorkloadConstraints( - activeConsumerCount, - consumersPerSubscription, - maxSubscriptionsPerConsumer, - subscriptionConstraints, - topicConstraints - ); - } + public WorkloadConstraints build() { + return new WorkloadConstraints( + activeConsumerCount, + consumersPerSubscription, + maxSubscriptionsPerConsumer, + subscriptionConstraints, + topicConstraints); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkloadRegistryPaths.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkloadRegistryPaths.java index d562ae7ef4..beed7e2828 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkloadRegistryPaths.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkloadRegistryPaths.java @@ -1,26 +1,27 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload; -import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; - import static pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths.CONSUMERS_WORKLOAD_PATH; +import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; + class WorkloadRegistryPaths { - private static final String WORKLOAD_RUNTIME_PATH = "runtime-bin"; + private static final String WORKLOAD_RUNTIME_PATH = "runtime-bin"; - private final ZookeeperPaths zookeeperPaths; - private final String clusterName; + private final ZookeeperPaths zookeeperPaths; + private final String clusterName; - WorkloadRegistryPaths(ZookeeperPaths zookeeperPaths, String clusterName) { - this.zookeeperPaths = zookeeperPaths; - this.clusterName = clusterName; - } + WorkloadRegistryPaths(ZookeeperPaths zookeeperPaths, String clusterName) { + this.zookeeperPaths = zookeeperPaths; + this.clusterName = clusterName; + } - String consumerWorkloadPath(String consumerId) { - return zookeeperPaths.join(consumersWorkloadCurrentClusterRuntimeBinaryPath(), consumerId); - } + String consumerWorkloadPath(String consumerId) { + return zookeeperPaths.join(consumersWorkloadCurrentClusterRuntimeBinaryPath(), consumerId); + } - String consumersWorkloadCurrentClusterRuntimeBinaryPath() { - return zookeeperPaths.join(zookeeperPaths.basePath(), CONSUMERS_WORKLOAD_PATH, clusterName, WORKLOAD_RUNTIME_PATH); - } + String consumersWorkloadCurrentClusterRuntimeBinaryPath() { + return zookeeperPaths.join( + zookeeperPaths.basePath(), CONSUMERS_WORKLOAD_PATH, clusterName, WORKLOAD_RUNTIME_PATH); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkloadSupervisor.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkloadSupervisor.java index 1e85e5b351..6556872cc2 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkloadSupervisor.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkloadSupervisor.java @@ -1,6 +1,15 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.Subscription; @@ -17,162 +26,158 @@ import pl.allegro.tech.hermes.domain.notifications.TopicCallback; import pl.allegro.tech.hermes.domain.workload.constraints.WorkloadConstraintsRepository; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; - -public class WorkloadSupervisor implements SubscriptionCallback, TopicCallback, SubscriptionAssignmentAware, AdminOperationsCallback { - - private static final Logger logger = LoggerFactory.getLogger(WorkloadSupervisor.class); - - private final ConsumersSupervisor supervisor; - private final InternalNotificationsBus notificationsBus; - private final SubscriptionsCache subscriptionsCache; - private final ConsumerAssignmentCache assignmentCache; - private final ConsumerNodesRegistry consumersRegistry; - private final BalancingJob balancingJob; - private final ZookeeperAdminCache adminCache; - private final WorkBalancingParameters workBalancingParameters; - private final ExecutorService assignmentExecutor; - private final ScheduledExecutorService rebalanceScheduler; - - public WorkloadSupervisor(ConsumersSupervisor supervisor, - InternalNotificationsBus notificationsBus, - SubscriptionsCache subscriptionsCache, - ConsumerAssignmentCache assignmentCache, - ConsumerAssignmentRegistry consumerAssignmentRegistry, - ClusterAssignmentCache clusterAssignmentCache, - ConsumerNodesRegistry consumersRegistry, - ZookeeperAdminCache adminCache, - ExecutorService assignmentExecutor, - WorkBalancingParameters workBalancingParameters, - String kafkaClusterName, - MetricsFacade metrics, - WorkloadConstraintsRepository workloadConstraintsRepository, - WorkBalancer workBalancer, - BalancingListener balancingListener) { - this.supervisor = supervisor; - this.notificationsBus = notificationsBus; - this.subscriptionsCache = subscriptionsCache; - this.assignmentCache = assignmentCache; - this.consumersRegistry = consumersRegistry; - this.adminCache = adminCache; - this.assignmentExecutor = assignmentExecutor; - this.workBalancingParameters = workBalancingParameters; - this.balancingJob = new BalancingJob( - consumersRegistry, - workBalancingParameters, - subscriptionsCache, - clusterAssignmentCache, - consumerAssignmentRegistry, - workBalancer, - metrics, - kafkaClusterName, - workloadConstraintsRepository, - balancingListener - ); - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setNameFormat("balancing-executor-%d").build(); - this.rebalanceScheduler = Executors.newSingleThreadScheduledExecutor(threadFactory); - } - - @Override - public void onSubscriptionAssigned(SubscriptionName subscriptionName) { - Subscription subscription = subscriptionsCache.getSubscription(subscriptionName); - logger.info("Scheduling assignment consumer for {}", subscription.getQualifiedName()); - assignmentExecutor.execute(() -> { - logger.info("Assigning consumer for {}", subscription.getQualifiedName()); - supervisor.assignConsumerForSubscription(subscription); - logger.info("Consumer assigned for {}", subscription.getQualifiedName()); +public class WorkloadSupervisor + implements SubscriptionCallback, + TopicCallback, + SubscriptionAssignmentAware, + AdminOperationsCallback { + + private static final Logger logger = LoggerFactory.getLogger(WorkloadSupervisor.class); + + private final ConsumersSupervisor supervisor; + private final InternalNotificationsBus notificationsBus; + private final SubscriptionsCache subscriptionsCache; + private final ConsumerAssignmentCache assignmentCache; + private final ConsumerNodesRegistry consumersRegistry; + private final BalancingJob balancingJob; + private final ZookeeperAdminCache adminCache; + private final WorkBalancingParameters workBalancingParameters; + private final ExecutorService assignmentExecutor; + private final ScheduledExecutorService rebalanceScheduler; + + public WorkloadSupervisor( + ConsumersSupervisor supervisor, + InternalNotificationsBus notificationsBus, + SubscriptionsCache subscriptionsCache, + ConsumerAssignmentCache assignmentCache, + ConsumerAssignmentRegistry consumerAssignmentRegistry, + ClusterAssignmentCache clusterAssignmentCache, + ConsumerNodesRegistry consumersRegistry, + ZookeeperAdminCache adminCache, + ExecutorService assignmentExecutor, + WorkBalancingParameters workBalancingParameters, + String kafkaClusterName, + MetricsFacade metrics, + WorkloadConstraintsRepository workloadConstraintsRepository, + WorkBalancer workBalancer, + BalancingListener balancingListener) { + this.supervisor = supervisor; + this.notificationsBus = notificationsBus; + this.subscriptionsCache = subscriptionsCache; + this.assignmentCache = assignmentCache; + this.consumersRegistry = consumersRegistry; + this.adminCache = adminCache; + this.assignmentExecutor = assignmentExecutor; + this.workBalancingParameters = workBalancingParameters; + this.balancingJob = + new BalancingJob( + consumersRegistry, + workBalancingParameters, + subscriptionsCache, + clusterAssignmentCache, + consumerAssignmentRegistry, + workBalancer, + metrics, + kafkaClusterName, + workloadConstraintsRepository, + balancingListener); + ThreadFactory threadFactory = + new ThreadFactoryBuilder().setNameFormat("balancing-executor-%d").build(); + this.rebalanceScheduler = Executors.newSingleThreadScheduledExecutor(threadFactory); + } + + @Override + public void onSubscriptionAssigned(SubscriptionName subscriptionName) { + Subscription subscription = subscriptionsCache.getSubscription(subscriptionName); + logger.info("Scheduling assignment consumer for {}", subscription.getQualifiedName()); + assignmentExecutor.execute( + () -> { + logger.info("Assigning consumer for {}", subscription.getQualifiedName()); + supervisor.assignConsumerForSubscription(subscription); + logger.info("Consumer assigned for {}", subscription.getQualifiedName()); }); - } - - @Override - public void onAssignmentRemoved(SubscriptionName subscription) { - logger.info("Scheduling assignment removal consumer for {}", subscription.getQualifiedName()); - assignmentExecutor.execute(() -> { - logger.info("Removing assignment from consumer for {}", subscription.getQualifiedName()); - supervisor.deleteConsumerForSubscriptionName(subscription); - logger.info("Consumer removed for {}", subscription.getName()); + } + + @Override + public void onAssignmentRemoved(SubscriptionName subscription) { + logger.info("Scheduling assignment removal consumer for {}", subscription.getQualifiedName()); + assignmentExecutor.execute( + () -> { + logger.info("Removing assignment from consumer for {}", subscription.getQualifiedName()); + supervisor.deleteConsumerForSubscriptionName(subscription); + logger.info("Consumer removed for {}", subscription.getName()); }); - } - - @Override - public void onSubscriptionChanged(Subscription subscription) { - if (assignmentCache.isAssignedTo(subscription.getQualifiedName())) { - logger.info("Updating subscription {}", subscription.getName()); - supervisor.updateSubscription(subscription); - } - } - - @Override - public void onTopicChanged(Topic topic) { - for (Subscription subscription : subscriptionsCache.subscriptionsOfTopic(topic.getName())) { - if (assignmentCache.isAssignedTo(subscription.getQualifiedName())) { - supervisor.updateTopic(subscription, topic); - } - } - } + } - public void start() throws Exception { - final long startTime = System.currentTimeMillis(); - - adminCache.start(); - adminCache.addCallback(this); - - notificationsBus.registerSubscriptionCallback(this); - notificationsBus.registerTopicCallback(this); - assignmentCache.registerAssignmentCallback(this); - - supervisor.start(); - if (workBalancingParameters.isAutoRebalance()) { - rebalanceScheduler.scheduleWithFixedDelay( - balancingJob, - workBalancingParameters.getRebalanceInterval().toMillis(), - workBalancingParameters.getRebalanceInterval().toMillis(), - MILLISECONDS - ); - } else { - logger.info("Automatic workload rebalancing is disabled."); - } - - logger.info("Consumer boot complete in {} ms.", System.currentTimeMillis() - startTime); + @Override + public void onSubscriptionChanged(Subscription subscription) { + if (assignmentCache.isAssignedTo(subscription.getQualifiedName())) { + logger.info("Updating subscription {}", subscription.getName()); + supervisor.updateSubscription(subscription); } - - public Set assignedSubscriptions() { - return assignmentCache.getConsumerSubscriptions(); + } + + @Override + public void onTopicChanged(Topic topic) { + for (Subscription subscription : subscriptionsCache.subscriptionsOfTopic(topic.getName())) { + if (assignmentCache.isAssignedTo(subscription.getQualifiedName())) { + supervisor.updateTopic(subscription, topic); + } } - - public void shutdown() throws Exception { - rebalanceScheduler.shutdown(); - rebalanceScheduler.awaitTermination(1, TimeUnit.MINUTES); - supervisor.shutdown(); - } - - @Override - public Optional watchedConsumerId() { - return Optional.of(consumersRegistry.getConsumerId()); - } - - public String consumerId() { - return consumersRegistry.getConsumerId(); - } - - public boolean isLeader() { - return consumersRegistry.isLeader(); + } + + public void start() throws Exception { + final long startTime = System.currentTimeMillis(); + + adminCache.start(); + adminCache.addCallback(this); + + notificationsBus.registerSubscriptionCallback(this); + notificationsBus.registerTopicCallback(this); + assignmentCache.registerAssignmentCallback(this); + + supervisor.start(); + if (workBalancingParameters.isAutoRebalance()) { + rebalanceScheduler.scheduleWithFixedDelay( + balancingJob, + workBalancingParameters.getRebalanceInterval().toMillis(), + workBalancingParameters.getRebalanceInterval().toMillis(), + MILLISECONDS); + } else { + logger.info("Automatic workload rebalancing is disabled."); } - @Override - public void onRetransmissionStarts(SubscriptionName subscription) throws Exception { - if (assignmentCache.isAssignedTo(subscription)) { - logger.info("Triggering retransmission for subscription {}", subscription); - supervisor.retransmit(subscription); - } + logger.info("Consumer boot complete in {} ms.", System.currentTimeMillis() - startTime); + } + + public Set assignedSubscriptions() { + return assignmentCache.getConsumerSubscriptions(); + } + + public void shutdown() throws Exception { + rebalanceScheduler.shutdown(); + rebalanceScheduler.awaitTermination(1, TimeUnit.MINUTES); + supervisor.shutdown(); + } + + @Override + public Optional watchedConsumerId() { + return Optional.of(consumersRegistry.getConsumerId()); + } + + public String consumerId() { + return consumersRegistry.getConsumerId(); + } + + public boolean isLeader() { + return consumersRegistry.isLeader(); + } + + @Override + public void onRetransmissionStarts(SubscriptionName subscription) throws Exception { + if (assignmentCache.isAssignedTo(subscription)) { + logger.info("Triggering retransmission for subscription {}", subscription); + supervisor.retransmit(subscription); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ZookeeperOperations.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ZookeeperOperations.java index b2743ee15d..3701572966 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ZookeeperOperations.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ZookeeperOperations.java @@ -1,69 +1,71 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload; +import java.util.List; +import java.util.Optional; import org.apache.curator.framework.CuratorFramework; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import pl.allegro.tech.hermes.common.exception.InternalProcessingException; -import java.util.List; -import java.util.Optional; - class ZookeeperOperations { - private final CuratorFramework curator; + private final CuratorFramework curator; - ZookeeperOperations(CuratorFramework curator) { - this.curator = curator; - } + ZookeeperOperations(CuratorFramework curator) { + this.curator = curator; + } - void writeOrCreatePersistent(String path, byte[] serializedData) throws Exception { - try { - curator.setData().forPath(path, serializedData); - } catch (KeeperException.NoNodeException e) { - try { - curator.create().creatingParentContainersIfNeeded() - .withMode(CreateMode.PERSISTENT) - .forPath(path, serializedData); - } catch (KeeperException.NodeExistsException ex) { - // ignore - } - } + void writeOrCreatePersistent(String path, byte[] serializedData) throws Exception { + try { + curator.setData().forPath(path, serializedData); + } catch (KeeperException.NoNodeException e) { + try { + curator + .create() + .creatingParentContainersIfNeeded() + .withMode(CreateMode.PERSISTENT) + .forPath(path, serializedData); + } catch (KeeperException.NodeExistsException ex) { + // ignore + } } + } - Optional getNodeData(String path) { - try { - if (curator.checkExists().forPath(path) != null) { - return Optional.of(curator.getData().forPath(path)); - } - } catch (Exception e) { - throw new InternalProcessingException(String.format("Could not read node data on path %s", path), e); - } - return Optional.empty(); + Optional getNodeData(String path) { + try { + if (curator.checkExists().forPath(path) != null) { + return Optional.of(curator.getData().forPath(path)); + } + } catch (Exception e) { + throw new InternalProcessingException( + String.format("Could not read node data on path %s", path), e); } + return Optional.empty(); + } - void deleteNode(String path) { - try { - if (curator.checkExists().forPath(path) != null) { - curator.delete().forPath(path); - } - } catch (Exception e) { - throw new InternalProcessingException("Could not delete node " + path, e); - } + void deleteNode(String path) { + try { + if (curator.checkExists().forPath(path) != null) { + curator.delete().forPath(path); + } + } catch (Exception e) { + throw new InternalProcessingException("Could not delete node " + path, e); } + } - List getNodeChildren(String path) { - try { - return curator.getChildren().forPath(path); - } catch (Exception e) { - throw new InternalProcessingException("Could not get children of node " + path, e); - } + List getNodeChildren(String path) { + try { + return curator.getChildren().forPath(path); + } catch (Exception e) { + throw new InternalProcessingException("Could not get children of node " + path, e); } + } - boolean exists(String path) { - try { - return curator.checkExists().forPath(path) != null; - } catch (Exception e) { - throw new InternalProcessingException("Unable to check existence of node " + path, e); - } + boolean exists(String path) { + try { + return curator.checkExists().forPath(path) != null; + } catch (Exception e) { + throw new InternalProcessingException("Unable to check existence of node " + path, e); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/selective/AvailableWork.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/selective/AvailableWork.java index 6e411e668a..d77969beab 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/selective/AvailableWork.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/selective/AvailableWork.java @@ -1,11 +1,8 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload.selective; -import com.google.common.collect.Sets; -import pl.allegro.tech.hermes.api.SubscriptionName; -import pl.allegro.tech.hermes.consumers.supervisor.workload.SubscriptionAssignment; -import pl.allegro.tech.hermes.consumers.supervisor.workload.SubscriptionAssignmentView; -import pl.allegro.tech.hermes.consumers.supervisor.workload.WorkloadConstraints; +import static java.util.stream.Collectors.toSet; +import com.google.common.collect.Sets; import java.util.Comparator; import java.util.Optional; import java.util.Set; @@ -13,59 +10,73 @@ import java.util.function.Consumer; import java.util.stream.Stream; import java.util.stream.StreamSupport; - -import static java.util.stream.Collectors.toSet; +import pl.allegro.tech.hermes.api.SubscriptionName; +import pl.allegro.tech.hermes.consumers.supervisor.workload.SubscriptionAssignment; +import pl.allegro.tech.hermes.consumers.supervisor.workload.SubscriptionAssignmentView; +import pl.allegro.tech.hermes.consumers.supervisor.workload.WorkloadConstraints; class AvailableWork extends Spliterators.AbstractSpliterator { - private final SubscriptionAssignmentView state; - private final WorkloadConstraints constraints; + private final SubscriptionAssignmentView state; + private final WorkloadConstraints constraints; - private AvailableWork(SubscriptionAssignmentView state, WorkloadConstraints constraints) { - super(Long.MAX_VALUE, 0); - this.state = state; - this.constraints = constraints; - } + private AvailableWork(SubscriptionAssignmentView state, WorkloadConstraints constraints) { + super(Long.MAX_VALUE, 0); + this.state = state; + this.constraints = constraints; + } - @Override - public boolean tryAdvance(Consumer action) { - Set availableConsumers = availableConsumerNodes(state); - if (!availableConsumers.isEmpty()) { - Optional subscriptionAssignment = getNextSubscription(state, availableConsumers) - .map(subscription -> getNextSubscriptionAssignment(state, availableConsumers, subscription)); - if (subscriptionAssignment.isPresent()) { - action.accept(subscriptionAssignment.get()); - return true; - } - } - return false; + @Override + public boolean tryAdvance(Consumer action) { + Set availableConsumers = availableConsumerNodes(state); + if (!availableConsumers.isEmpty()) { + Optional subscriptionAssignment = + getNextSubscription(state, availableConsumers) + .map( + subscription -> + getNextSubscriptionAssignment(state, availableConsumers, subscription)); + if (subscriptionAssignment.isPresent()) { + action.accept(subscriptionAssignment.get()); + return true; + } } + return false; + } - private Optional getNextSubscription(SubscriptionAssignmentView state, Set availableConsumerNodes) { - return state.getSubscriptions().stream() - .filter(s -> state.getAssignmentsCountForSubscription(s) < constraints.getConsumerCount(s)) - .filter(s -> !Sets.difference(availableConsumerNodes, state.getConsumerNodesForSubscription(s)).isEmpty()) - .min(Comparator.comparingInt(state::getAssignmentsCountForSubscription)); - } + private Optional getNextSubscription( + SubscriptionAssignmentView state, Set availableConsumerNodes) { + return state.getSubscriptions().stream() + .filter(s -> state.getAssignmentsCountForSubscription(s) < constraints.getConsumerCount(s)) + .filter( + s -> + !Sets.difference(availableConsumerNodes, state.getConsumerNodesForSubscription(s)) + .isEmpty()) + .min(Comparator.comparingInt(state::getAssignmentsCountForSubscription)); + } - private SubscriptionAssignment getNextSubscriptionAssignment(SubscriptionAssignmentView state, - Set availableConsumerNodes, - SubscriptionName subscriptionName) { - return availableConsumerNodes.stream() - .filter(s -> !state.getSubscriptionsForConsumerNode(s).contains(subscriptionName)) - .min(Comparator.comparingInt(state::getAssignmentsCountForConsumerNode)) - .map(s -> new SubscriptionAssignment(s, subscriptionName)) - .get(); - } + private SubscriptionAssignment getNextSubscriptionAssignment( + SubscriptionAssignmentView state, + Set availableConsumerNodes, + SubscriptionName subscriptionName) { + return availableConsumerNodes.stream() + .filter(s -> !state.getSubscriptionsForConsumerNode(s).contains(subscriptionName)) + .min(Comparator.comparingInt(state::getAssignmentsCountForConsumerNode)) + .map(s -> new SubscriptionAssignment(s, subscriptionName)) + .get(); + } - private Set availableConsumerNodes(SubscriptionAssignmentView state) { - return state.getConsumerNodes().stream() - .filter(s -> state.getAssignmentsCountForConsumerNode(s) < constraints.getMaxSubscriptionsPerConsumer()) - .filter(s -> state.getAssignmentsCountForConsumerNode(s) < state.getSubscriptionsCount()) - .collect(toSet()); - } + private Set availableConsumerNodes(SubscriptionAssignmentView state) { + return state.getConsumerNodes().stream() + .filter( + s -> + state.getAssignmentsCountForConsumerNode(s) + < constraints.getMaxSubscriptionsPerConsumer()) + .filter(s -> state.getAssignmentsCountForConsumerNode(s) < state.getSubscriptionsCount()) + .collect(toSet()); + } - static Stream stream(SubscriptionAssignmentView state, WorkloadConstraints constraints) { - AvailableWork work = new AvailableWork(state, constraints); - return StreamSupport.stream(work, false); - } + static Stream stream( + SubscriptionAssignmentView state, WorkloadConstraints constraints) { + AvailableWork work = new AvailableWork(state, constraints); + return StreamSupport.stream(work, false); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/selective/SelectiveWorkBalancer.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/selective/SelectiveWorkBalancer.java index 964efc40ce..ca71507a89 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/selective/SelectiveWorkBalancer.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/selective/SelectiveWorkBalancer.java @@ -1,5 +1,12 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload.selective; +import static java.util.Comparator.comparingInt; +import static java.util.stream.Collectors.toList; + +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.SubscriptionName; @@ -9,171 +16,197 @@ import pl.allegro.tech.hermes.consumers.supervisor.workload.WorkBalancingResult; import pl.allegro.tech.hermes.consumers.supervisor.workload.WorkloadConstraints; -import java.util.Iterator; -import java.util.List; -import java.util.Optional; -import java.util.stream.Stream; - -import static java.util.Comparator.comparingInt; -import static java.util.stream.Collectors.toList; - public class SelectiveWorkBalancer implements WorkBalancer { - private static final Logger logger = LoggerFactory.getLogger(SelectiveWorkBalancer.class); - - @Override - public WorkBalancingResult balance(List subscriptions, - List activeConsumerNodes, - SubscriptionAssignmentView currentState, - WorkloadConstraints constraints) { - - List removedSubscriptions = findRemovedSubscriptions(currentState, subscriptions); - List inactiveConsumers = findInactiveConsumers(currentState, activeConsumerNodes); - List newSubscriptions = findNewSubscriptions(currentState, subscriptions); - List newConsumers = findNewConsumers(currentState, activeConsumerNodes); - - SubscriptionAssignmentView balancedState = - balance(currentState, removedSubscriptions, inactiveConsumers, newSubscriptions, newConsumers, constraints); - - log(subscriptions, activeConsumerNodes, currentState, balancedState); - - return new WorkBalancingResult( - balancedState, - countMissingResources(subscriptions, balancedState, constraints) - ); - } - - private SubscriptionAssignmentView balance(SubscriptionAssignmentView currentState, - List removedSubscriptions, - List inactiveConsumers, - List newSubscriptions, - List newConsumers, - WorkloadConstraints constraints) { - return currentState.transform((state, transformer) -> { - removedSubscriptions.forEach(transformer::removeSubscription); - inactiveConsumers.forEach(transformer::removeConsumerNode); - newSubscriptions.forEach(transformer::addSubscription); - newConsumers.forEach(transformer::addConsumerNode); - minimizeWorkload(state, transformer, constraints); - AvailableWork.stream(state, constraints) - .forEach(transformer::addAssignment); - equalizeWorkload(state, transformer); + private static final Logger logger = LoggerFactory.getLogger(SelectiveWorkBalancer.class); + + @Override + public WorkBalancingResult balance( + List subscriptions, + List activeConsumerNodes, + SubscriptionAssignmentView currentState, + WorkloadConstraints constraints) { + + List removedSubscriptions = + findRemovedSubscriptions(currentState, subscriptions); + List inactiveConsumers = findInactiveConsumers(currentState, activeConsumerNodes); + List newSubscriptions = findNewSubscriptions(currentState, subscriptions); + List newConsumers = findNewConsumers(currentState, activeConsumerNodes); + + SubscriptionAssignmentView balancedState = + balance( + currentState, + removedSubscriptions, + inactiveConsumers, + newSubscriptions, + newConsumers, + constraints); + + log(subscriptions, activeConsumerNodes, currentState, balancedState); + + return new WorkBalancingResult( + balancedState, countMissingResources(subscriptions, balancedState, constraints)); + } + + private SubscriptionAssignmentView balance( + SubscriptionAssignmentView currentState, + List removedSubscriptions, + List inactiveConsumers, + List newSubscriptions, + List newConsumers, + WorkloadConstraints constraints) { + return currentState.transform( + (state, transformer) -> { + removedSubscriptions.forEach(transformer::removeSubscription); + inactiveConsumers.forEach(transformer::removeConsumerNode); + newSubscriptions.forEach(transformer::addSubscription); + newConsumers.forEach(transformer::addConsumerNode); + minimizeWorkload(state, transformer, constraints); + AvailableWork.stream(state, constraints).forEach(transformer::addAssignment); + equalizeWorkload(state, transformer); }); + } + + private void minimizeWorkload( + SubscriptionAssignmentView state, + SubscriptionAssignmentView.Transformer transformer, + WorkloadConstraints workloadConstraints) { + state.getSubscriptions().stream() + .flatMap( + subscriptionName -> + findRedundantAssignments(state, subscriptionName, workloadConstraints)) + .forEach(transformer::removeAssignment); + } + + private Stream findRedundantAssignments( + SubscriptionAssignmentView state, + SubscriptionName subscriptionName, + WorkloadConstraints constraints) { + final int assignedConsumers = state.getAssignmentsCountForSubscription(subscriptionName); + final int requiredConsumers = constraints.getConsumerCount(subscriptionName); + int redundantConsumers = assignedConsumers - requiredConsumers; + if (redundantConsumers > 0) { + Stream.Builder redundant = Stream.builder(); + Iterator iterator = + state.getAssignmentsForSubscription(subscriptionName).iterator(); + while (redundantConsumers > 0 && iterator.hasNext()) { + SubscriptionAssignment assignment = iterator.next(); + redundant.add(assignment); + redundantConsumers--; + } + return redundant.build(); } - - private void minimizeWorkload(SubscriptionAssignmentView state, - SubscriptionAssignmentView.Transformer transformer, - WorkloadConstraints workloadConstraints) { - state.getSubscriptions() - .stream() - .flatMap(subscriptionName -> findRedundantAssignments(state, subscriptionName, workloadConstraints)) - .forEach(transformer::removeAssignment); - } - - private Stream findRedundantAssignments(SubscriptionAssignmentView state, - SubscriptionName subscriptionName, - WorkloadConstraints constraints) { - final int assignedConsumers = state.getAssignmentsCountForSubscription(subscriptionName); - final int requiredConsumers = constraints.getConsumerCount(subscriptionName); - int redundantConsumers = assignedConsumers - requiredConsumers; - if (redundantConsumers > 0) { - Stream.Builder redundant = Stream.builder(); - Iterator iterator = state.getAssignmentsForSubscription(subscriptionName).iterator(); - while (redundantConsumers > 0 && iterator.hasNext()) { - SubscriptionAssignment assignment = iterator.next(); - redundant.add(assignment); - redundantConsumers--; - } - return redundant.build(); - } - return Stream.empty(); - } - - private int countMissingResources(List subscriptions, - SubscriptionAssignmentView state, - WorkloadConstraints constraints) { - return subscriptions.stream() - .mapToInt(s -> { - int requiredConsumers = constraints.getConsumerCount(s); - int subscriptionAssignments = state.getAssignmentsCountForSubscription(s); - int missing = requiredConsumers - subscriptionAssignments; - if (missing != 0) { - logger.info("Subscription {} has {} != {} (default) assignments", - s, subscriptionAssignments, requiredConsumers); - } - return missing; - }) - .sum(); - } - - private void equalizeWorkload(SubscriptionAssignmentView state, - SubscriptionAssignmentView.Transformer transformer) { - if (state.getSubscriptionsCount() > 1 && !state.getConsumerNodes().isEmpty()) { - boolean transferred; - do { - transferred = false; - - String maxLoaded = maxLoadedConsumerNode(state); - String minLoaded = minLoadedConsumerNode(state); - int maxLoad = state.getAssignmentsCountForConsumerNode(maxLoaded); - int minLoad = state.getAssignmentsCountForConsumerNode(minLoaded); - - while (maxLoad > minLoad + 1) { - Optional subscription = getSubscriptionForTransfer(state, maxLoaded, minLoaded); - if (subscription.isPresent()) { - transformer.transferAssignment(maxLoaded, minLoaded, subscription.get()); - transferred = true; - } else { - break; - } - maxLoad--; - minLoad++; - } - } while (transferred); + return Stream.empty(); + } + + private int countMissingResources( + List subscriptions, + SubscriptionAssignmentView state, + WorkloadConstraints constraints) { + return subscriptions.stream() + .mapToInt( + s -> { + int requiredConsumers = constraints.getConsumerCount(s); + int subscriptionAssignments = state.getAssignmentsCountForSubscription(s); + int missing = requiredConsumers - subscriptionAssignments; + if (missing != 0) { + logger.info( + "Subscription {} has {} != {} (default) assignments", + s, + subscriptionAssignments, + requiredConsumers); + } + return missing; + }) + .sum(); + } + + private void equalizeWorkload( + SubscriptionAssignmentView state, SubscriptionAssignmentView.Transformer transformer) { + if (state.getSubscriptionsCount() > 1 && !state.getConsumerNodes().isEmpty()) { + boolean transferred; + do { + transferred = false; + + String maxLoaded = maxLoadedConsumerNode(state); + String minLoaded = minLoadedConsumerNode(state); + int maxLoad = state.getAssignmentsCountForConsumerNode(maxLoaded); + int minLoad = state.getAssignmentsCountForConsumerNode(minLoaded); + + while (maxLoad > minLoad + 1) { + Optional subscription = + getSubscriptionForTransfer(state, maxLoaded, minLoaded); + if (subscription.isPresent()) { + transformer.transferAssignment(maxLoaded, minLoaded, subscription.get()); + transferred = true; + } else { + break; + } + maxLoad--; + minLoad++; } + } while (transferred); } - - private String maxLoadedConsumerNode(SubscriptionAssignmentView state) { - return state.getConsumerNodes().stream().max(comparingInt(state::getAssignmentsCountForConsumerNode)).get(); - } - - private String minLoadedConsumerNode(SubscriptionAssignmentView state) { - return state.getConsumerNodes().stream().min(comparingInt(state::getAssignmentsCountForConsumerNode)).get(); - } - - private Optional getSubscriptionForTransfer(SubscriptionAssignmentView state, - String maxLoaded, - String minLoaded) { - return state.getSubscriptionsForConsumerNode(maxLoaded).stream() - .filter(s -> !state.getConsumerNodesForSubscription(s).contains(minLoaded)) - .findAny(); - } - - private List findRemovedSubscriptions(SubscriptionAssignmentView state, - List subscriptions) { - return state.getSubscriptions().stream().filter(s -> !subscriptions.contains(s)).collect(toList()); - } - - private List findInactiveConsumers(SubscriptionAssignmentView state, List activeConsumers) { - return state.getConsumerNodes().stream().filter(c -> !activeConsumers.contains(c)).collect(toList()); - } - - private List findNewSubscriptions(SubscriptionAssignmentView state, - List subscriptions) { - return subscriptions.stream().filter(s -> !state.getSubscriptions().contains(s)).collect(toList()); - } - - private List findNewConsumers(SubscriptionAssignmentView state, List activeConsumers) { - return activeConsumers.stream().filter(c -> !state.getConsumerNodes().contains(c)).collect(toList()); - } - - private void log(List subscriptions, List activeConsumerNodes, - SubscriptionAssignmentView currentState, SubscriptionAssignmentView balancedState) { - logger.info("Balancing {} subscriptions across {} nodes with previous {} assignments" - + " produced {} assignments", - subscriptions.size(), - activeConsumerNodes.size(), - currentState.getAllAssignments().size(), - balancedState.getAllAssignments().size()); - } + } + + private String maxLoadedConsumerNode(SubscriptionAssignmentView state) { + return state.getConsumerNodes().stream() + .max(comparingInt(state::getAssignmentsCountForConsumerNode)) + .get(); + } + + private String minLoadedConsumerNode(SubscriptionAssignmentView state) { + return state.getConsumerNodes().stream() + .min(comparingInt(state::getAssignmentsCountForConsumerNode)) + .get(); + } + + private Optional getSubscriptionForTransfer( + SubscriptionAssignmentView state, String maxLoaded, String minLoaded) { + return state.getSubscriptionsForConsumerNode(maxLoaded).stream() + .filter(s -> !state.getConsumerNodesForSubscription(s).contains(minLoaded)) + .findAny(); + } + + private List findRemovedSubscriptions( + SubscriptionAssignmentView state, List subscriptions) { + return state.getSubscriptions().stream() + .filter(s -> !subscriptions.contains(s)) + .collect(toList()); + } + + private List findInactiveConsumers( + SubscriptionAssignmentView state, List activeConsumers) { + return state.getConsumerNodes().stream() + .filter(c -> !activeConsumers.contains(c)) + .collect(toList()); + } + + private List findNewSubscriptions( + SubscriptionAssignmentView state, List subscriptions) { + return subscriptions.stream() + .filter(s -> !state.getSubscriptions().contains(s)) + .collect(toList()); + } + + private List findNewConsumers( + SubscriptionAssignmentView state, List activeConsumers) { + return activeConsumers.stream() + .filter(c -> !state.getConsumerNodes().contains(c)) + .collect(toList()); + } + + private void log( + List subscriptions, + List activeConsumerNodes, + SubscriptionAssignmentView currentState, + SubscriptionAssignmentView balancedState) { + logger.info( + "Balancing {} subscriptions across {} nodes with previous {} assignments" + + " produced {} assignments", + subscriptions.size(), + activeConsumerNodes.size(), + currentState.getAllAssignments().size(), + balancedState.getAllAssignments().size()); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/AvgTargetWeightCalculator.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/AvgTargetWeightCalculator.java index 3372d146c1..4e4d83f019 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/AvgTargetWeightCalculator.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/AvgTargetWeightCalculator.java @@ -1,33 +1,31 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload.weighted; +import static java.util.stream.Collectors.toMap; + import java.util.Collection; import java.util.Map; -import static java.util.stream.Collectors.toMap; - public class AvgTargetWeightCalculator implements TargetWeightCalculator { - private final WeightedWorkloadMetricsReporter metrics; + private final WeightedWorkloadMetricsReporter metrics; - public AvgTargetWeightCalculator(WeightedWorkloadMetricsReporter metrics) { - this.metrics = metrics; - } + public AvgTargetWeightCalculator(WeightedWorkloadMetricsReporter metrics) { + this.metrics = metrics; + } - @Override - public Map calculate(Collection consumers) { - if (consumers.isEmpty()) { - return Map.of(); - } - - metrics.reportCurrentWeights(consumers); - Weight sum = consumers.stream() - .map(ConsumerNode::getWeight) - .reduce(Weight.ZERO, Weight::add); - Weight average = sum.divide(consumers.size()); - - Map newWeights = consumers.stream() - .collect(toMap(ConsumerNode::getConsumerId, ignore -> average)); - metrics.reportProposedWeights(newWeights); - return newWeights; + @Override + public Map calculate(Collection consumers) { + if (consumers.isEmpty()) { + return Map.of(); } + + metrics.reportCurrentWeights(consumers); + Weight sum = consumers.stream().map(ConsumerNode::getWeight).reduce(Weight.ZERO, Weight::add); + Weight average = sum.divide(consumers.size()); + + Map newWeights = + consumers.stream().collect(toMap(ConsumerNode::getConsumerId, ignore -> average)); + metrics.reportProposedWeights(newWeights); + return newWeights; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ConsumerNode.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ConsumerNode.java index 875a63188b..9340186787 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ConsumerNode.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ConsumerNode.java @@ -1,90 +1,90 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload.weighted; +import static java.util.Comparator.comparing; + import java.util.Comparator; import java.util.HashSet; import java.util.Objects; import java.util.Set; -import static java.util.Comparator.comparing; - class ConsumerNode { - static Comparator LIGHTEST_CONSUMER_FIRST = comparing(ConsumerNode::getWeight); - - private final String consumerId; - private final ConsumerNodeLoad initialLoad; - private final int maxSubscriptionsPerConsumer; - private final Set tasks = new HashSet<>(); - private Weight weight = Weight.ZERO; - - ConsumerNode(String consumerId, ConsumerNodeLoad initialLoad, int maxSubscriptionsPerConsumer) { - this.consumerId = consumerId; - this.initialLoad = initialLoad; - this.maxSubscriptionsPerConsumer = maxSubscriptionsPerConsumer; - } - - String getConsumerId() { - return consumerId; - } - - void assign(ConsumerTask task) { - tasks.add(task); - weight = weight.add(task.getWeight()); - } - - void unassign(ConsumerTask task) { - tasks.remove(task); - weight = weight.subtract(task.getWeight()); - } - - void swap(ConsumerTask moveOut, ConsumerTask moveIn) { - unassign(moveOut); - assign(moveIn); - } - - Set getAssignedTasks() { - return tasks; + static Comparator LIGHTEST_CONSUMER_FIRST = comparing(ConsumerNode::getWeight); + + private final String consumerId; + private final ConsumerNodeLoad initialLoad; + private final int maxSubscriptionsPerConsumer; + private final Set tasks = new HashSet<>(); + private Weight weight = Weight.ZERO; + + ConsumerNode(String consumerId, ConsumerNodeLoad initialLoad, int maxSubscriptionsPerConsumer) { + this.consumerId = consumerId; + this.initialLoad = initialLoad; + this.maxSubscriptionsPerConsumer = maxSubscriptionsPerConsumer; + } + + String getConsumerId() { + return consumerId; + } + + void assign(ConsumerTask task) { + tasks.add(task); + weight = weight.add(task.getWeight()); + } + + void unassign(ConsumerTask task) { + tasks.remove(task); + weight = weight.subtract(task.getWeight()); + } + + void swap(ConsumerTask moveOut, ConsumerTask moveIn) { + unassign(moveOut); + assign(moveIn); + } + + Set getAssignedTasks() { + return tasks; + } + + int getAssignedTaskCount() { + return tasks.size(); + } + + boolean isNotAssigned(ConsumerTask consumerTask) { + return !tasks.contains(consumerTask); + } + + boolean isFull() { + return tasks.size() >= maxSubscriptionsPerConsumer; + } + + Weight getWeight() { + return weight; + } + + ConsumerNodeLoad getInitialLoad() { + return initialLoad; + } + + @Override + public String toString() { + return consumerId; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - int getAssignedTaskCount() { - return tasks.size(); - } - - boolean isNotAssigned(ConsumerTask consumerTask) { - return !tasks.contains(consumerTask); - } - - boolean isFull() { - return tasks.size() >= maxSubscriptionsPerConsumer; - } - - Weight getWeight() { - return weight; - } - - ConsumerNodeLoad getInitialLoad() { - return initialLoad; - } - - @Override - public String toString() { - return consumerId; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ConsumerNode that = (ConsumerNode) o; - return Objects.equals(consumerId, that.consumerId); - } - - @Override - public int hashCode() { - return Objects.hash(consumerId); + if (o == null || getClass() != o.getClass()) { + return false; } + ConsumerNode that = (ConsumerNode) o; + return Objects.equals(consumerId, that.consumerId); + } + + @Override + public int hashCode() { + return Objects.hash(consumerId); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ConsumerNodeLoad.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ConsumerNodeLoad.java index 4c6000b515..24087e19be 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ConsumerNodeLoad.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ConsumerNodeLoad.java @@ -1,36 +1,36 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload.weighted; -import pl.allegro.tech.hermes.api.SubscriptionName; - import java.util.Map; +import pl.allegro.tech.hermes.api.SubscriptionName; class ConsumerNodeLoad { - static final ConsumerNodeLoad UNDEFINED = new ConsumerNodeLoad(-1d, Map.of()); + static final ConsumerNodeLoad UNDEFINED = new ConsumerNodeLoad(-1d, Map.of()); - private final double cpuUtilization; - private final Map loadPerSubscription; + private final double cpuUtilization; + private final Map loadPerSubscription; - ConsumerNodeLoad(double cpuUtilization, Map loadPerSubscription) { - this.cpuUtilization = cpuUtilization; - this.loadPerSubscription = loadPerSubscription; - } + ConsumerNodeLoad( + double cpuUtilization, Map loadPerSubscription) { + this.cpuUtilization = cpuUtilization; + this.loadPerSubscription = loadPerSubscription; + } - Map getLoadPerSubscription() { - return loadPerSubscription; - } + Map getLoadPerSubscription() { + return loadPerSubscription; + } - double getCpuUtilization() { - return cpuUtilization; - } + double getCpuUtilization() { + return cpuUtilization; + } - double sumOperationsPerSecond() { - return loadPerSubscription.values().stream() - .mapToDouble(SubscriptionLoad::getOperationsPerSecond) - .sum(); - } + double sumOperationsPerSecond() { + return loadPerSubscription.values().stream() + .mapToDouble(SubscriptionLoad::getOperationsPerSecond) + .sum(); + } - boolean isDefined() { - return cpuUtilization != UNDEFINED.getCpuUtilization(); - } + boolean isDefined() { + return cpuUtilization != UNDEFINED.getCpuUtilization(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ConsumerNodeLoadDecoder.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ConsumerNodeLoadDecoder.java index c9dee485ae..63b8a7dddf 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ConsumerNodeLoadDecoder.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ConsumerNodeLoadDecoder.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload.weighted; +import static org.slf4j.LoggerFactory.getLogger; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; import org.agrona.concurrent.UnsafeBuffer; import org.slf4j.Logger; import pl.allegro.tech.hermes.api.SubscriptionName; @@ -9,54 +14,53 @@ import pl.allegro.tech.hermes.consumers.supervisor.workload.sbe.stubs.ConsumerLoadDecoder.SubscriptionsDecoder; import pl.allegro.tech.hermes.consumers.supervisor.workload.sbe.stubs.MessageHeaderDecoder; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -import static org.slf4j.LoggerFactory.getLogger; - class ConsumerNodeLoadDecoder { - private static final Logger logger = getLogger(ConsumerNodeLoadDecoder.class); + private static final Logger logger = getLogger(ConsumerNodeLoadDecoder.class); - private final SubscriptionIds subscriptionIds; + private final SubscriptionIds subscriptionIds; - ConsumerNodeLoadDecoder(SubscriptionIds subscriptionIds) { - this.subscriptionIds = subscriptionIds; - } + ConsumerNodeLoadDecoder(SubscriptionIds subscriptionIds) { + this.subscriptionIds = subscriptionIds; + } - ConsumerNodeLoad decode(byte[] bytes) { - MessageHeaderDecoder header = new MessageHeaderDecoder(); - ConsumerLoadDecoder body = new ConsumerLoadDecoder(); + ConsumerNodeLoad decode(byte[] bytes) { + MessageHeaderDecoder header = new MessageHeaderDecoder(); + ConsumerLoadDecoder body = new ConsumerLoadDecoder(); - UnsafeBuffer buffer = new UnsafeBuffer(bytes); - header.wrap(buffer, 0); + UnsafeBuffer buffer = new UnsafeBuffer(bytes); + header.wrap(buffer, 0); - if (header.schemaId() != ConsumerLoadDecoder.SCHEMA_ID || header.templateId() != ConsumerLoadDecoder.TEMPLATE_ID) { - logger.warn("Unable to decode consumer node load, schema or template id mismatch. " - + "Required by decoder: [schema id={}, template id={}], " - + "encoded in payload: [schema id={}, template id={}]", - ConsumerLoadDecoder.SCHEMA_ID, ConsumerLoadDecoder.TEMPLATE_ID, - header.schemaId(), header.templateId()); - return ConsumerNodeLoad.UNDEFINED; - } + if (header.schemaId() != ConsumerLoadDecoder.SCHEMA_ID + || header.templateId() != ConsumerLoadDecoder.TEMPLATE_ID) { + logger.warn( + "Unable to decode consumer node load, schema or template id mismatch. " + + "Required by decoder: [schema id={}, template id={}], " + + "encoded in payload: [schema id={}, template id={}]", + ConsumerLoadDecoder.SCHEMA_ID, + ConsumerLoadDecoder.TEMPLATE_ID, + header.schemaId(), + header.templateId()); + return ConsumerNodeLoad.UNDEFINED; + } - body.wrap(buffer, header.encodedLength(), header.blockLength(), header.version()); + body.wrap(buffer, header.encodedLength(), header.blockLength(), header.version()); - return new ConsumerNodeLoad(body.cpuUtilization(), decodeSubscriptionLoads(body)); - } + return new ConsumerNodeLoad(body.cpuUtilization(), decodeSubscriptionLoads(body)); + } - private Map decodeSubscriptionLoads(ConsumerLoadDecoder body) { - Map subscriptionLoads = new HashMap<>(); - for (SubscriptionsDecoder loadPerSubscriptionDecoder : body.subscriptions()) { - long id = loadPerSubscriptionDecoder.id(); - Optional subscriptionId = subscriptionIds.getSubscriptionId(id); - if (subscriptionId.isPresent()) { - double operationsPerSecond = loadPerSubscriptionDecoder.operationsPerSecond(); - SubscriptionLoad load = new SubscriptionLoad(operationsPerSecond); - subscriptionLoads.put(subscriptionId.get().getSubscriptionName(), load); - } - } - return subscriptionLoads; + private Map decodeSubscriptionLoads( + ConsumerLoadDecoder body) { + Map subscriptionLoads = new HashMap<>(); + for (SubscriptionsDecoder loadPerSubscriptionDecoder : body.subscriptions()) { + long id = loadPerSubscriptionDecoder.id(); + Optional subscriptionId = subscriptionIds.getSubscriptionId(id); + if (subscriptionId.isPresent()) { + double operationsPerSecond = loadPerSubscriptionDecoder.operationsPerSecond(); + SubscriptionLoad load = new SubscriptionLoad(operationsPerSecond); + subscriptionLoads.put(subscriptionId.get().getSubscriptionName(), load); + } } + return subscriptionLoads; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ConsumerNodeLoadEncoder.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ConsumerNodeLoadEncoder.java index 83c20cc474..a2a83f3f0a 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ConsumerNodeLoadEncoder.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ConsumerNodeLoadEncoder.java @@ -1,5 +1,8 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload.weighted; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; import org.agrona.ExpandableDirectByteBuffer; import org.agrona.MutableDirectBuffer; import pl.allegro.tech.hermes.api.SubscriptionName; @@ -9,52 +12,51 @@ import pl.allegro.tech.hermes.consumers.supervisor.workload.sbe.stubs.ConsumerLoadEncoder.SubscriptionsEncoder; import pl.allegro.tech.hermes.consumers.supervisor.workload.sbe.stubs.MessageHeaderEncoder; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - class ConsumerNodeLoadEncoder { - private final SubscriptionIds subscriptionIds; - private final MutableDirectBuffer buffer; - - ConsumerNodeLoadEncoder(SubscriptionIds subscriptionIds, int bufferSize) { - this.subscriptionIds = subscriptionIds; - this.buffer = new ExpandableDirectByteBuffer(bufferSize); + private final SubscriptionIds subscriptionIds; + private final MutableDirectBuffer buffer; + + ConsumerNodeLoadEncoder(SubscriptionIds subscriptionIds, int bufferSize) { + this.subscriptionIds = subscriptionIds; + this.buffer = new ExpandableDirectByteBuffer(bufferSize); + } + + byte[] encode(ConsumerNodeLoad consumerNodeLoad) { + Map subscriptionLoads = + mapToSubscriptionIds(consumerNodeLoad); + + MessageHeaderEncoder headerEncoder = new MessageHeaderEncoder(); + ConsumerLoadEncoder body = + new ConsumerLoadEncoder().wrapAndApplyHeader(buffer, 0, headerEncoder); + + SubscriptionsEncoder loadPerSubscriptionEncoder = + body.cpuUtilization(consumerNodeLoad.getCpuUtilization()) + .subscriptionsCount(subscriptionLoads.size()); + + for (Map.Entry entry : subscriptionLoads.entrySet()) { + SubscriptionId subscriptionId = entry.getKey(); + SubscriptionLoad load = entry.getValue(); + loadPerSubscriptionEncoder + .next() + .id(subscriptionId.getValue()) + .operationsPerSecond(load.getOperationsPerSecond()); } - byte[] encode(ConsumerNodeLoad consumerNodeLoad) { - Map subscriptionLoads = mapToSubscriptionIds(consumerNodeLoad); - - MessageHeaderEncoder headerEncoder = new MessageHeaderEncoder(); - ConsumerLoadEncoder body = new ConsumerLoadEncoder() - .wrapAndApplyHeader(buffer, 0, headerEncoder); - - SubscriptionsEncoder loadPerSubscriptionEncoder = body - .cpuUtilization(consumerNodeLoad.getCpuUtilization()) - .subscriptionsCount(subscriptionLoads.size()); + int len = headerEncoder.encodedLength() + body.encodedLength(); - for (Map.Entry entry : subscriptionLoads.entrySet()) { - SubscriptionId subscriptionId = entry.getKey(); - SubscriptionLoad load = entry.getValue(); - loadPerSubscriptionEncoder.next() - .id(subscriptionId.getValue()) - .operationsPerSecond(load.getOperationsPerSecond()); - } - - int len = headerEncoder.encodedLength() + body.encodedLength(); - - byte[] dst = new byte[len]; - buffer.getBytes(0, dst); - return dst; - } + byte[] dst = new byte[len]; + buffer.getBytes(0, dst); + return dst; + } - private Map mapToSubscriptionIds(ConsumerNodeLoad metrics) { - Map subscriptionLoads = new HashMap<>(); - for (Map.Entry entry : metrics.getLoadPerSubscription().entrySet()) { - Optional subscriptionId = subscriptionIds.getSubscriptionId(entry.getKey()); - subscriptionId.ifPresent(id -> subscriptionLoads.put(id, entry.getValue())); - } - return subscriptionLoads; + private Map mapToSubscriptionIds(ConsumerNodeLoad metrics) { + Map subscriptionLoads = new HashMap<>(); + for (Map.Entry entry : + metrics.getLoadPerSubscription().entrySet()) { + Optional subscriptionId = subscriptionIds.getSubscriptionId(entry.getKey()); + subscriptionId.ifPresent(id -> subscriptionLoads.put(id, entry.getValue())); } + return subscriptionLoads; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ConsumerNodeLoadRegistry.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ConsumerNodeLoadRegistry.java index 66ab51c634..3e64ce60fa 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ConsumerNodeLoadRegistry.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ConsumerNodeLoadRegistry.java @@ -4,9 +4,9 @@ public interface ConsumerNodeLoadRegistry extends SubscriptionLoadRecordersRegistry { - void start(); + void start(); - void stop(); + void stop(); - ConsumerNodeLoad get(String consumerId); + ConsumerNodeLoad get(String consumerId); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ConsumerTask.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ConsumerTask.java index a231f76fd6..78ac8f90ad 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ConsumerTask.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ConsumerTask.java @@ -1,53 +1,53 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload.weighted; -import pl.allegro.tech.hermes.api.SubscriptionName; +import static java.util.Comparator.comparing; import java.time.Instant; import java.util.Comparator; import java.util.Objects; - -import static java.util.Comparator.comparing; +import pl.allegro.tech.hermes.api.SubscriptionName; class ConsumerTask { - static Comparator HEAVIEST_TASK_FIRST = comparing(ConsumerTask::getWeight).reversed(); + static Comparator HEAVIEST_TASK_FIRST = + comparing(ConsumerTask::getWeight).reversed(); - private final SubscriptionName subscriptionName; - private final Instant lastRebalanceTimestamp; - private final Weight weight; + private final SubscriptionName subscriptionName; + private final Instant lastRebalanceTimestamp; + private final Weight weight; - ConsumerTask(SubscriptionName subscriptionName, SubscriptionProfile subscriptionProfile) { - this.subscriptionName = subscriptionName; - this.lastRebalanceTimestamp = subscriptionProfile.getLastRebalanceTimestamp(); - this.weight = subscriptionProfile.getWeight(); - } + ConsumerTask(SubscriptionName subscriptionName, SubscriptionProfile subscriptionProfile) { + this.subscriptionName = subscriptionName; + this.lastRebalanceTimestamp = subscriptionProfile.getLastRebalanceTimestamp(); + this.weight = subscriptionProfile.getWeight(); + } - SubscriptionName getSubscriptionName() { - return subscriptionName; - } + SubscriptionName getSubscriptionName() { + return subscriptionName; + } - Weight getWeight() { - return weight; - } + Weight getWeight() { + return weight; + } - Instant getLastRebalanceTimestamp() { - return lastRebalanceTimestamp; - } + Instant getLastRebalanceTimestamp() { + return lastRebalanceTimestamp; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ConsumerTask that = (ConsumerTask) o; - return Objects.equals(subscriptionName, that.subscriptionName); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(subscriptionName); + if (o == null || getClass() != o.getClass()) { + return false; } + ConsumerTask that = (ConsumerTask) o; + return Objects.equals(subscriptionName, that.subscriptionName); + } + + @Override + public int hashCode() { + return Objects.hash(subscriptionName); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/CurrentLoadProvider.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/CurrentLoadProvider.java index e23674e2b9..b5b5bd8f7f 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/CurrentLoadProvider.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/CurrentLoadProvider.java @@ -5,28 +5,28 @@ public class CurrentLoadProvider { - private final Map consumerLoads = new ConcurrentHashMap<>(); - private volatile SubscriptionProfiles profiles = SubscriptionProfiles.EMPTY; - - SubscriptionProfiles getProfiles() { - return profiles; - } - - ConsumerNodeLoad getConsumerNodeLoad(String consumerId) { - return consumerLoads.getOrDefault(consumerId, ConsumerNodeLoad.UNDEFINED); - } - - void updateConsumerNodeLoads(Map newConsumerLoads) { - consumerLoads.clear(); - consumerLoads.putAll(newConsumerLoads); - } - - void updateProfiles(SubscriptionProfiles newProfiles) { - profiles = newProfiles; - } - - void clear() { - consumerLoads.clear(); - profiles = SubscriptionProfiles.EMPTY; - } + private final Map consumerLoads = new ConcurrentHashMap<>(); + private volatile SubscriptionProfiles profiles = SubscriptionProfiles.EMPTY; + + SubscriptionProfiles getProfiles() { + return profiles; + } + + ConsumerNodeLoad getConsumerNodeLoad(String consumerId) { + return consumerLoads.getOrDefault(consumerId, ConsumerNodeLoad.UNDEFINED); + } + + void updateConsumerNodeLoads(Map newConsumerLoads) { + consumerLoads.clear(); + consumerLoads.putAll(newConsumerLoads); + } + + void updateProfiles(SubscriptionProfiles newProfiles) { + profiles = newProfiles; + } + + void clear() { + consumerLoads.clear(); + profiles = SubscriptionProfiles.EMPTY; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ExponentiallyWeightedMovingAverage.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ExponentiallyWeightedMovingAverage.java index 4c2782059c..3d43e0168b 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ExponentiallyWeightedMovingAverage.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ExponentiallyWeightedMovingAverage.java @@ -6,27 +6,27 @@ class ExponentiallyWeightedMovingAverage { - private final Duration windowSize; + private final Duration windowSize; - private Instant previousUpdateTimestamp; - private double currentAverage = 0d; + private Instant previousUpdateTimestamp; + private double currentAverage = 0d; - ExponentiallyWeightedMovingAverage(Duration windowSize) { - this.windowSize = windowSize; - } + ExponentiallyWeightedMovingAverage(Duration windowSize) { + this.windowSize = windowSize; + } - double update(double sample, Instant now) { - if (previousUpdateTimestamp == null) { - currentAverage = sample; - } else { - // This calculation is done in the same way as the Linux load average is calculated. - // See: https://www.helpsystems.com/resources/guides/unix-load-average-part-1-how-it-works - Duration elapsed = Duration.between(previousUpdateTimestamp, now); - long elapsedMillis = Math.max(TimeUnit.MILLISECONDS.convert(elapsed), 0); - double alpha = 1.0 - Math.exp(-1.0 * ((double) elapsedMillis / windowSize.toMillis())); - currentAverage = (sample * alpha) + (currentAverage * (1.0 - alpha)); - } - previousUpdateTimestamp = now; - return currentAverage; + double update(double sample, Instant now) { + if (previousUpdateTimestamp == null) { + currentAverage = sample; + } else { + // This calculation is done in the same way as the Linux load average is calculated. + // See: https://www.helpsystems.com/resources/guides/unix-load-average-part-1-how-it-works + Duration elapsed = Duration.between(previousUpdateTimestamp, now); + long elapsedMillis = Math.max(TimeUnit.MILLISECONDS.convert(elapsed), 0); + double alpha = 1.0 - Math.exp(-1.0 * ((double) elapsedMillis / windowSize.toMillis())); + currentAverage = (sample * alpha) + (currentAverage * (1.0 - alpha)); } + previousUpdateTimestamp = now; + return currentAverage; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/NoOpConsumerNodeLoadRegistry.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/NoOpConsumerNodeLoadRegistry.java index c2d4dc04d7..41eea86a47 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/NoOpConsumerNodeLoadRegistry.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/NoOpConsumerNodeLoadRegistry.java @@ -5,43 +5,34 @@ public class NoOpConsumerNodeLoadRegistry implements ConsumerNodeLoadRegistry { - private static final NoOpSubscriptionLoadRecorder SUBSCRIPTION_LOAD_RECORDER = new NoOpSubscriptionLoadRecorder(); + private static final NoOpSubscriptionLoadRecorder SUBSCRIPTION_LOAD_RECORDER = + new NoOpSubscriptionLoadRecorder(); - @Override - public void start() { + @Override + public void start() {} - } + @Override + public void stop() {} - @Override - public void stop() { + @Override + public ConsumerNodeLoad get(String consumerId) { + return ConsumerNodeLoad.UNDEFINED; + } - } + @Override + public SubscriptionLoadRecorder register(SubscriptionName subscriptionName) { + return SUBSCRIPTION_LOAD_RECORDER; + } - @Override - public ConsumerNodeLoad get(String consumerId) { - return ConsumerNodeLoad.UNDEFINED; - } + private static class NoOpSubscriptionLoadRecorder implements SubscriptionLoadRecorder { @Override - public SubscriptionLoadRecorder register(SubscriptionName subscriptionName) { - return SUBSCRIPTION_LOAD_RECORDER; - } - - private static class NoOpSubscriptionLoadRecorder implements SubscriptionLoadRecorder { - - @Override - public void initialize() { + public void initialize() {} - } - - @Override - public void recordSingleOperation() { - - } - - @Override - public void shutdown() { + @Override + public void recordSingleOperation() {} - } - } + @Override + public void shutdown() {} + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ScoringTargetWeightCalculator.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ScoringTargetWeightCalculator.java index 8ba1210f50..2c92fb6cf2 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ScoringTargetWeightCalculator.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ScoringTargetWeightCalculator.java @@ -1,5 +1,9 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload.weighted; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toMap; +import static java.util.stream.Collectors.toSet; + import java.time.Clock; import java.time.Duration; import java.util.Collection; @@ -8,135 +12,129 @@ import java.util.Map; import java.util.Set; -import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toMap; -import static java.util.stream.Collectors.toSet; - public class ScoringTargetWeightCalculator implements TargetWeightCalculator { - private static final double MIN_SCORE = 0.01d; - private static final double MAX_SCORE = 1.0d; - - private final WeightedWorkloadMetricsReporter metrics; - private final Clock clock; - private final Duration scoringWindowSize; - private final double scoringGain; - private final Map scores = new HashMap<>(); - - public ScoringTargetWeightCalculator(WeightedWorkloadMetricsReporter metrics, - Clock clock, - Duration scoringWindowSize, - double scoringGain) { - this.metrics = metrics; - this.clock = clock; - this.scoringWindowSize = scoringWindowSize; - this.scoringGain = scoringGain; - } - - @Override - public Map calculate(Collection consumers) { - removeScoresForInactiveConsumers(consumers); - - metrics.reportCurrentWeights(consumers); - Map loadPerConsumer = mapConsumerIdToLoad(consumers); - double targetCpuUtilization = calculateTargetCpuUtilization(loadPerConsumer); - Map currentScores = calculateCurrentScores(loadPerConsumer); - - Map newScores = new HashMap<>(); - for (Map.Entry entry : currentScores.entrySet()) { - String consumerId = entry.getKey(); - double cpuUtilization = loadPerConsumer.get(consumerId).getCpuUtilization(); - double error = targetCpuUtilization - cpuUtilization; - double currentScore = entry.getValue(); - double newScore = calculateNewScore(consumerId, currentScore, error); - newScores.put(consumerId, newScore); - metrics.reportCurrentScore(consumerId, currentScore); - metrics.reportProposedScore(consumerId, newScore); - metrics.reportScoringError(consumerId, error); - } - - Map newWeights = calculateWeights(consumers, newScores); - metrics.reportProposedWeights(newWeights); - return newWeights; - } - - private void removeScoresForInactiveConsumers(Collection consumers) { - Set consumerIds = consumers.stream().map(ConsumerNode::getConsumerId).collect(toSet()); - scores.entrySet().removeIf(e -> !consumerIds.contains(e.getKey())); - } - - private Map mapConsumerIdToLoad(Collection consumers) { - return consumers.stream() - .collect(toMap(ConsumerNode::getConsumerId, ConsumerNode::getInitialLoad)); + private static final double MIN_SCORE = 0.01d; + private static final double MAX_SCORE = 1.0d; + + private final WeightedWorkloadMetricsReporter metrics; + private final Clock clock; + private final Duration scoringWindowSize; + private final double scoringGain; + private final Map scores = new HashMap<>(); + + public ScoringTargetWeightCalculator( + WeightedWorkloadMetricsReporter metrics, + Clock clock, + Duration scoringWindowSize, + double scoringGain) { + this.metrics = metrics; + this.clock = clock; + this.scoringWindowSize = scoringWindowSize; + this.scoringGain = scoringGain; + } + + @Override + public Map calculate(Collection consumers) { + removeScoresForInactiveConsumers(consumers); + + metrics.reportCurrentWeights(consumers); + Map loadPerConsumer = mapConsumerIdToLoad(consumers); + double targetCpuUtilization = calculateTargetCpuUtilization(loadPerConsumer); + Map currentScores = calculateCurrentScores(loadPerConsumer); + + Map newScores = new HashMap<>(); + for (Map.Entry entry : currentScores.entrySet()) { + String consumerId = entry.getKey(); + double cpuUtilization = loadPerConsumer.get(consumerId).getCpuUtilization(); + double error = targetCpuUtilization - cpuUtilization; + double currentScore = entry.getValue(); + double newScore = calculateNewScore(consumerId, currentScore, error); + newScores.put(consumerId, newScore); + metrics.reportCurrentScore(consumerId, currentScore); + metrics.reportProposedScore(consumerId, newScore); + metrics.reportScoringError(consumerId, error); } - private double calculateTargetCpuUtilization(Map loadPerConsumer) { - return loadPerConsumer.values().stream() - .filter(ConsumerNodeLoad::isDefined) - .mapToDouble(ConsumerNodeLoad::getCpuUtilization) - .average() - .orElse(0d); + Map newWeights = calculateWeights(consumers, newScores); + metrics.reportProposedWeights(newWeights); + return newWeights; + } + + private void removeScoresForInactiveConsumers(Collection consumers) { + Set consumerIds = consumers.stream().map(ConsumerNode::getConsumerId).collect(toSet()); + scores.entrySet().removeIf(e -> !consumerIds.contains(e.getKey())); + } + + private Map mapConsumerIdToLoad(Collection consumers) { + return consumers.stream() + .collect(toMap(ConsumerNode::getConsumerId, ConsumerNode::getInitialLoad)); + } + + private double calculateTargetCpuUtilization(Map loadPerConsumer) { + return loadPerConsumer.values().stream() + .filter(ConsumerNodeLoad::isDefined) + .mapToDouble(ConsumerNodeLoad::getCpuUtilization) + .average() + .orElse(0d); + } + + private Map calculateCurrentScores( + Map loadPerConsumer) { + Map opsPerConsumer = + loadPerConsumer.entrySet().stream() + .filter(e -> e.getValue().isDefined()) + .collect(toMap(Map.Entry::getKey, e -> e.getValue().sumOperationsPerSecond())); + double opsSum = opsPerConsumer.values().stream().mapToDouble(ops -> ops).sum(); + return opsPerConsumer.entrySet().stream() + .collect(toMap(Map.Entry::getKey, e -> calculateCurrentScore(e.getValue(), opsSum))); + } + + private double calculateCurrentScore(double ops, double opsSum) { + if (opsSum > 0) { + return ops / opsSum; } - - private Map calculateCurrentScores(Map loadPerConsumer) { - Map opsPerConsumer = loadPerConsumer.entrySet().stream() - .filter(e -> e.getValue().isDefined()) - .collect(toMap(Map.Entry::getKey, e -> e.getValue().sumOperationsPerSecond())); - double opsSum = opsPerConsumer.values().stream() - .mapToDouble(ops -> ops) - .sum(); - return opsPerConsumer.entrySet().stream() - .collect(toMap(Map.Entry::getKey, e -> calculateCurrentScore(e.getValue(), opsSum))); - } - - private double calculateCurrentScore(double ops, double opsSum) { - if (opsSum > 0) { - return ops / opsSum; - } - return 0; + return 0; + } + + private double calculateNewScore(String consumerId, double currentScore, double error) { + double rawScore = currentScore + scoringGain * error; + ExponentiallyWeightedMovingAverage average = + scores.computeIfAbsent( + consumerId, ignore -> new ExponentiallyWeightedMovingAverage(scoringWindowSize)); + double avg = average.update(rawScore, clock.instant()); + return ensureScoreRanges(avg); + } + + private double ensureScoreRanges(double score) { + return Math.max(Math.min(score, MAX_SCORE), MIN_SCORE); + } + + private Map calculateWeights( + Collection consumers, Map newScores) { + Weight sum = consumers.stream().map(ConsumerNode::getWeight).reduce(Weight.ZERO, Weight::add); + Weight avgWeight = calculateAvgWeight(sum, consumers.size()); + List consumersWithoutScore = + consumers.stream() + .filter(consumerNode -> !newScores.containsKey(consumerNode.getConsumerId())) + .collect(toList()); + Map newWeights = new HashMap<>(); + for (ConsumerNode consumerNode : consumersWithoutScore) { + newWeights.put(consumerNode.getConsumerId(), avgWeight); + sum = sum.subtract(avgWeight); } - - private double calculateNewScore(String consumerId, double currentScore, double error) { - double rawScore = currentScore + scoringGain * error; - ExponentiallyWeightedMovingAverage average = scores.computeIfAbsent( - consumerId, - ignore -> new ExponentiallyWeightedMovingAverage(scoringWindowSize) - ); - double avg = average.update(rawScore, clock.instant()); - return ensureScoreRanges(avg); - } - - private double ensureScoreRanges(double score) { - return Math.max(Math.min(score, MAX_SCORE), MIN_SCORE); - } - - private Map calculateWeights(Collection consumers, Map newScores) { - Weight sum = consumers.stream() - .map(ConsumerNode::getWeight) - .reduce(Weight.ZERO, Weight::add); - Weight avgWeight = calculateAvgWeight(sum, consumers.size()); - List consumersWithoutScore = consumers.stream() - .filter(consumerNode -> !newScores.containsKey(consumerNode.getConsumerId())) - .collect(toList()); - Map newWeights = new HashMap<>(); - for (ConsumerNode consumerNode : consumersWithoutScore) { - newWeights.put(consumerNode.getConsumerId(), avgWeight); - sum = sum.subtract(avgWeight); - } - double newScoresSum = newScores.values().stream() - .mapToDouble(score -> score) - .sum(); - for (Map.Entry entry : newScores.entrySet()) { - Weight weight = sum.multiply(entry.getValue() / newScoresSum); - newWeights.put(entry.getKey(), weight); - } - return newWeights; + double newScoresSum = newScores.values().stream().mapToDouble(score -> score).sum(); + for (Map.Entry entry : newScores.entrySet()) { + Weight weight = sum.multiply(entry.getValue() / newScoresSum); + newWeights.put(entry.getKey(), weight); } + return newWeights; + } - private Weight calculateAvgWeight(Weight sum, int consumerCount) { - if (consumerCount == 0) { - return Weight.ZERO; - } - return sum.divide(consumerCount); + private Weight calculateAvgWeight(Weight sum, int consumerCount) { + if (consumerCount == 0) { + return Weight.ZERO; } + return sum.divide(consumerCount); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionLoad.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionLoad.java index b1eaaffc30..92dea02ca0 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionLoad.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionLoad.java @@ -4,30 +4,30 @@ class SubscriptionLoad { - private final double operationsPerSecond; + private final double operationsPerSecond; - SubscriptionLoad(double operationsPerSecond) { - this.operationsPerSecond = operationsPerSecond; - } + SubscriptionLoad(double operationsPerSecond) { + this.operationsPerSecond = operationsPerSecond; + } - double getOperationsPerSecond() { - return operationsPerSecond; - } + double getOperationsPerSecond() { + return operationsPerSecond; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - SubscriptionLoad that = (SubscriptionLoad) o; - return Double.compare(that.operationsPerSecond, operationsPerSecond) == 0; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(operationsPerSecond); + if (o == null || getClass() != o.getClass()) { + return false; } + SubscriptionLoad that = (SubscriptionLoad) o; + return Double.compare(that.operationsPerSecond, operationsPerSecond) == 0; + } + + @Override + public int hashCode() { + return Objects.hash(operationsPerSecond); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionProfile.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionProfile.java index ddcb1ae53a..2b0bacf9a0 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionProfile.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionProfile.java @@ -4,21 +4,21 @@ class SubscriptionProfile { - static SubscriptionProfile UNDEFINED = new SubscriptionProfile(Instant.MIN, Weight.ZERO); + static SubscriptionProfile UNDEFINED = new SubscriptionProfile(Instant.MIN, Weight.ZERO); - private final Instant lastRebalanceTimestamp; - private final Weight weight; + private final Instant lastRebalanceTimestamp; + private final Weight weight; - SubscriptionProfile(Instant lastRebalanceTimestamp, Weight weight) { - this.lastRebalanceTimestamp = lastRebalanceTimestamp; - this.weight = weight; - } + SubscriptionProfile(Instant lastRebalanceTimestamp, Weight weight) { + this.lastRebalanceTimestamp = lastRebalanceTimestamp; + this.weight = weight; + } - Weight getWeight() { - return weight; - } + Weight getWeight() { + return weight; + } - Instant getLastRebalanceTimestamp() { - return lastRebalanceTimestamp; - } + Instant getLastRebalanceTimestamp() { + return lastRebalanceTimestamp; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionProfileRegistry.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionProfileRegistry.java index a33a6121d6..d63d2a0def 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionProfileRegistry.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionProfileRegistry.java @@ -2,7 +2,7 @@ public interface SubscriptionProfileRegistry { - SubscriptionProfiles fetch(); + SubscriptionProfiles fetch(); - void persist(SubscriptionProfiles profiles); + void persist(SubscriptionProfiles profiles); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionProfiles.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionProfiles.java index 32f205485b..7838b9fbad 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionProfiles.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionProfiles.java @@ -1,36 +1,36 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload.weighted; -import pl.allegro.tech.hermes.api.SubscriptionName; - import java.time.Instant; import java.util.Map; import java.util.Set; +import pl.allegro.tech.hermes.api.SubscriptionName; class SubscriptionProfiles { - static final SubscriptionProfiles EMPTY = new SubscriptionProfiles(Map.of(), Instant.MIN); + static final SubscriptionProfiles EMPTY = new SubscriptionProfiles(Map.of(), Instant.MIN); - private final Map profiles; - private final Instant updateTimestamp; + private final Map profiles; + private final Instant updateTimestamp; - SubscriptionProfiles(Map profiles, Instant updateTimestamp) { - this.profiles = profiles; - this.updateTimestamp = updateTimestamp; - } + SubscriptionProfiles( + Map profiles, Instant updateTimestamp) { + this.profiles = profiles; + this.updateTimestamp = updateTimestamp; + } - Instant getUpdateTimestamp() { - return updateTimestamp; - } + Instant getUpdateTimestamp() { + return updateTimestamp; + } - Set getSubscriptions() { - return profiles.keySet(); - } + Set getSubscriptions() { + return profiles.keySet(); + } - SubscriptionProfile getProfile(SubscriptionName subscriptionName) { - return profiles.getOrDefault(subscriptionName, SubscriptionProfile.UNDEFINED); - } + SubscriptionProfile getProfile(SubscriptionName subscriptionName) { + return profiles.getOrDefault(subscriptionName, SubscriptionProfile.UNDEFINED); + } - Map getProfiles() { - return profiles; - } + Map getProfiles() { + return profiles; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionProfilesCalculator.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionProfilesCalculator.java index 2e8c12bacf..4a839abcfe 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionProfilesCalculator.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionProfilesCalculator.java @@ -1,60 +1,66 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload.weighted; -import pl.allegro.tech.hermes.api.SubscriptionName; - import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.Collection; import java.util.HashMap; import java.util.Map; +import pl.allegro.tech.hermes.api.SubscriptionName; class SubscriptionProfilesCalculator { - private final Clock clock; - private final Duration weightWindowSize; + private final Clock clock; + private final Duration weightWindowSize; - SubscriptionProfilesCalculator(Clock clock, Duration weightWindowSize) { - this.clock = clock; - this.weightWindowSize = weightWindowSize; - } + SubscriptionProfilesCalculator(Clock clock, Duration weightWindowSize) { + this.clock = clock; + this.weightWindowSize = weightWindowSize; + } - SubscriptionProfiles calculate(Collection consumerLoads, SubscriptionProfiles previousProfiles) { - Map currentWeights = calculateCurrentWeights(consumerLoads); - Map newProfiles = new HashMap<>(); - Instant now = clock.instant(); - for (Map.Entry entry : currentWeights.entrySet()) { - SubscriptionName subscriptionName = entry.getKey(); - Weight currentWeight = entry.getValue(); - SubscriptionProfile newProfile = applyCurrentWeight(previousProfiles, subscriptionName, currentWeight, now); - newProfiles.put(subscriptionName, newProfile); - } - return new SubscriptionProfiles(newProfiles, now); + SubscriptionProfiles calculate( + Collection consumerLoads, SubscriptionProfiles previousProfiles) { + Map currentWeights = calculateCurrentWeights(consumerLoads); + Map newProfiles = new HashMap<>(); + Instant now = clock.instant(); + for (Map.Entry entry : currentWeights.entrySet()) { + SubscriptionName subscriptionName = entry.getKey(); + Weight currentWeight = entry.getValue(); + SubscriptionProfile newProfile = + applyCurrentWeight(previousProfiles, subscriptionName, currentWeight, now); + newProfiles.put(subscriptionName, newProfile); } + return new SubscriptionProfiles(newProfiles, now); + } - private Map calculateCurrentWeights(Collection consumerLoads) { - Map currentWeights = new HashMap<>(); - for (ConsumerNodeLoad consumerLoad : consumerLoads) { - for (Map.Entry entry : consumerLoad.getLoadPerSubscription().entrySet()) { - SubscriptionName subscriptionName = entry.getKey(); - Weight currentConsumerWeight = new Weight(entry.getValue().getOperationsPerSecond()); - Weight currentMaxWeight = currentWeights.computeIfAbsent(subscriptionName, subscription -> Weight.ZERO); - Weight newMaxWeight = Weight.max(currentMaxWeight, currentConsumerWeight); - currentWeights.put(subscriptionName, newMaxWeight); - } - } - return currentWeights; + private Map calculateCurrentWeights( + Collection consumerLoads) { + Map currentWeights = new HashMap<>(); + for (ConsumerNodeLoad consumerLoad : consumerLoads) { + for (Map.Entry entry : + consumerLoad.getLoadPerSubscription().entrySet()) { + SubscriptionName subscriptionName = entry.getKey(); + Weight currentConsumerWeight = new Weight(entry.getValue().getOperationsPerSecond()); + Weight currentMaxWeight = + currentWeights.computeIfAbsent(subscriptionName, subscription -> Weight.ZERO); + Weight newMaxWeight = Weight.max(currentMaxWeight, currentConsumerWeight); + currentWeights.put(subscriptionName, newMaxWeight); + } } + return currentWeights; + } - private SubscriptionProfile applyCurrentWeight(SubscriptionProfiles previousProfiles, - SubscriptionName subscriptionName, - Weight currentWeight, - Instant now) { - SubscriptionProfile previousProfile = previousProfiles.getProfile(subscriptionName); - Weight previousWeight = previousProfile.getWeight(); - ExponentiallyWeightedMovingAverage average = new ExponentiallyWeightedMovingAverage(weightWindowSize); - average.update(previousWeight.getOperationsPerSecond(), previousProfiles.getUpdateTimestamp()); - double opsAvg = average.update(currentWeight.getOperationsPerSecond(), now); - return new SubscriptionProfile(previousProfile.getLastRebalanceTimestamp(), new Weight(opsAvg)); - } + private SubscriptionProfile applyCurrentWeight( + SubscriptionProfiles previousProfiles, + SubscriptionName subscriptionName, + Weight currentWeight, + Instant now) { + SubscriptionProfile previousProfile = previousProfiles.getProfile(subscriptionName); + Weight previousWeight = previousProfile.getWeight(); + ExponentiallyWeightedMovingAverage average = + new ExponentiallyWeightedMovingAverage(weightWindowSize); + average.update(previousWeight.getOperationsPerSecond(), previousProfiles.getUpdateTimestamp()); + double opsAvg = average.update(currentWeight.getOperationsPerSecond(), now); + return new SubscriptionProfile(previousProfile.getLastRebalanceTimestamp(), new Weight(opsAvg)); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionProfilesDecoder.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionProfilesDecoder.java index e89d820222..5dc9fa755b 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionProfilesDecoder.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionProfilesDecoder.java @@ -1,5 +1,11 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload.weighted; +import static org.slf4j.LoggerFactory.getLogger; + +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; import org.agrona.concurrent.UnsafeBuffer; import org.slf4j.Logger; import pl.allegro.tech.hermes.api.SubscriptionName; @@ -9,67 +15,62 @@ import pl.allegro.tech.hermes.consumers.supervisor.workload.sbe.stubs.ProfilesDecoder; import pl.allegro.tech.hermes.consumers.supervisor.workload.sbe.stubs.ProfilesDecoder.SubscriptionsDecoder; -import java.time.Instant; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -import static org.slf4j.LoggerFactory.getLogger; - class SubscriptionProfilesDecoder { - private static final Logger logger = getLogger(SubscriptionProfilesDecoder.class); + private static final Logger logger = getLogger(SubscriptionProfilesDecoder.class); - private final SubscriptionIds subscriptionIds; + private final SubscriptionIds subscriptionIds; - SubscriptionProfilesDecoder(SubscriptionIds subscriptionIds) { - this.subscriptionIds = subscriptionIds; - } + SubscriptionProfilesDecoder(SubscriptionIds subscriptionIds) { + this.subscriptionIds = subscriptionIds; + } - SubscriptionProfiles decode(byte[] bytes) { - MessageHeaderDecoder header = new MessageHeaderDecoder(); - ProfilesDecoder body = new ProfilesDecoder(); + SubscriptionProfiles decode(byte[] bytes) { + MessageHeaderDecoder header = new MessageHeaderDecoder(); + ProfilesDecoder body = new ProfilesDecoder(); - UnsafeBuffer buffer = new UnsafeBuffer(bytes); - header.wrap(buffer, 0); + UnsafeBuffer buffer = new UnsafeBuffer(bytes); + header.wrap(buffer, 0); - if (header.schemaId() != ProfilesDecoder.SCHEMA_ID || header.templateId() != ProfilesDecoder.TEMPLATE_ID) { - logger.warn("Unable to decode subscription profiles, schema or template id mismatch. " - + "Required by decoder: [schema id={}, template id={}], " - + "encoded in payload: [schema id={}, template id={}]", - ProfilesDecoder.SCHEMA_ID, ProfilesDecoder.TEMPLATE_ID, - header.schemaId(), header.templateId()); - return SubscriptionProfiles.EMPTY; - } + if (header.schemaId() != ProfilesDecoder.SCHEMA_ID + || header.templateId() != ProfilesDecoder.TEMPLATE_ID) { + logger.warn( + "Unable to decode subscription profiles, schema or template id mismatch. " + + "Required by decoder: [schema id={}, template id={}], " + + "encoded in payload: [schema id={}, template id={}]", + ProfilesDecoder.SCHEMA_ID, + ProfilesDecoder.TEMPLATE_ID, + header.schemaId(), + header.templateId()); + return SubscriptionProfiles.EMPTY; + } - body.wrap(buffer, header.encodedLength(), header.blockLength(), header.version()); + body.wrap(buffer, header.encodedLength(), header.blockLength(), header.version()); - return decodeSubscriptionProfiles(body); - } + return decodeSubscriptionProfiles(body); + } - private SubscriptionProfiles decodeSubscriptionProfiles(ProfilesDecoder body) { - Instant updateTimestamp = toInstant(body.updateTimestamp()); - Map subscriptionProfiles = new HashMap<>(); - for (SubscriptionsDecoder subscriptionsDecoder : body.subscriptions()) { - long id = subscriptionsDecoder.id(); - Optional subscriptionId = subscriptionIds.getSubscriptionId(id); - if (subscriptionId.isPresent()) { - double operationsPerSecond = subscriptionsDecoder.operationsPerSecond(); - Instant lastRebalance = toInstant(subscriptionsDecoder.lastRebalanceTimestamp()); - SubscriptionProfile profile = new SubscriptionProfile( - lastRebalance, - new Weight(operationsPerSecond) - ); - subscriptionProfiles.put(subscriptionId.get().getSubscriptionName(), profile); - } - } - return new SubscriptionProfiles(subscriptionProfiles, updateTimestamp); + private SubscriptionProfiles decodeSubscriptionProfiles(ProfilesDecoder body) { + Instant updateTimestamp = toInstant(body.updateTimestamp()); + Map subscriptionProfiles = new HashMap<>(); + for (SubscriptionsDecoder subscriptionsDecoder : body.subscriptions()) { + long id = subscriptionsDecoder.id(); + Optional subscriptionId = subscriptionIds.getSubscriptionId(id); + if (subscriptionId.isPresent()) { + double operationsPerSecond = subscriptionsDecoder.operationsPerSecond(); + Instant lastRebalance = toInstant(subscriptionsDecoder.lastRebalanceTimestamp()); + SubscriptionProfile profile = + new SubscriptionProfile(lastRebalance, new Weight(operationsPerSecond)); + subscriptionProfiles.put(subscriptionId.get().getSubscriptionName(), profile); + } } + return new SubscriptionProfiles(subscriptionProfiles, updateTimestamp); + } - private Instant toInstant(long millis) { - if (millis < 0) { - return Instant.MIN; - } - return Instant.ofEpochMilli(millis); + private Instant toInstant(long millis) { + if (millis < 0) { + return Instant.MIN; } + return Instant.ofEpochMilli(millis); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionProfilesEncoder.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionProfilesEncoder.java index 63dafb023b..a89afc0c58 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionProfilesEncoder.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/SubscriptionProfilesEncoder.java @@ -1,5 +1,9 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload.weighted; +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; import org.agrona.ExpandableDirectByteBuffer; import org.agrona.MutableDirectBuffer; import pl.allegro.tech.hermes.api.SubscriptionName; @@ -8,61 +12,58 @@ import pl.allegro.tech.hermes.consumers.supervisor.workload.sbe.stubs.MessageHeaderEncoder; import pl.allegro.tech.hermes.consumers.supervisor.workload.sbe.stubs.ProfilesEncoder; -import java.time.Instant; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - class SubscriptionProfilesEncoder { - private final SubscriptionIds subscriptionIds; - private final MutableDirectBuffer buffer; + private final SubscriptionIds subscriptionIds; + private final MutableDirectBuffer buffer; - SubscriptionProfilesEncoder(SubscriptionIds subscriptionIds, int bufferSize) { - this.subscriptionIds = subscriptionIds; - this.buffer = new ExpandableDirectByteBuffer(bufferSize); - } + SubscriptionProfilesEncoder(SubscriptionIds subscriptionIds, int bufferSize) { + this.subscriptionIds = subscriptionIds; + this.buffer = new ExpandableDirectByteBuffer(bufferSize); + } - byte[] encode(SubscriptionProfiles profiles) { - Map subscriptionProfiles = mapToSubscriptionIds(profiles); + byte[] encode(SubscriptionProfiles profiles) { + Map subscriptionProfiles = mapToSubscriptionIds(profiles); - MessageHeaderEncoder headerEncoder = new MessageHeaderEncoder(); - ProfilesEncoder body = new ProfilesEncoder() - .wrapAndApplyHeader(buffer, 0, headerEncoder); + MessageHeaderEncoder headerEncoder = new MessageHeaderEncoder(); + ProfilesEncoder body = new ProfilesEncoder().wrapAndApplyHeader(buffer, 0, headerEncoder); - ProfilesEncoder.SubscriptionsEncoder subscriptionsEncoder = body - .updateTimestamp(toMillis(profiles.getUpdateTimestamp())) - .subscriptionsCount(subscriptionProfiles.size()); + ProfilesEncoder.SubscriptionsEncoder subscriptionsEncoder = + body.updateTimestamp(toMillis(profiles.getUpdateTimestamp())) + .subscriptionsCount(subscriptionProfiles.size()); - for (Map.Entry entry : subscriptionProfiles.entrySet()) { - SubscriptionId subscriptionId = entry.getKey(); - SubscriptionProfile profile = entry.getValue(); - subscriptionsEncoder.next() - .id(subscriptionId.getValue()) - .operationsPerSecond(profile.getWeight().getOperationsPerSecond()) - .lastRebalanceTimestamp(toMillis(profile.getLastRebalanceTimestamp())); - } + for (Map.Entry entry : subscriptionProfiles.entrySet()) { + SubscriptionId subscriptionId = entry.getKey(); + SubscriptionProfile profile = entry.getValue(); + subscriptionsEncoder + .next() + .id(subscriptionId.getValue()) + .operationsPerSecond(profile.getWeight().getOperationsPerSecond()) + .lastRebalanceTimestamp(toMillis(profile.getLastRebalanceTimestamp())); + } - int len = headerEncoder.encodedLength() + body.encodedLength(); + int len = headerEncoder.encodedLength() + body.encodedLength(); - byte[] dst = new byte[len]; - buffer.getBytes(0, dst); - return dst; - } + byte[] dst = new byte[len]; + buffer.getBytes(0, dst); + return dst; + } - private Map mapToSubscriptionIds(SubscriptionProfiles profiles) { - Map subscriptionProfiles = new HashMap<>(); - for (SubscriptionName subscriptionName : profiles.getSubscriptions()) { - Optional subscriptionId = subscriptionIds.getSubscriptionId(subscriptionName); - subscriptionId.ifPresent(id -> subscriptionProfiles.put(id, profiles.getProfile(subscriptionName))); - } - return subscriptionProfiles; + private Map mapToSubscriptionIds( + SubscriptionProfiles profiles) { + Map subscriptionProfiles = new HashMap<>(); + for (SubscriptionName subscriptionName : profiles.getSubscriptions()) { + Optional subscriptionId = subscriptionIds.getSubscriptionId(subscriptionName); + subscriptionId.ifPresent( + id -> subscriptionProfiles.put(id, profiles.getProfile(subscriptionName))); } + return subscriptionProfiles; + } - private long toMillis(Instant timestamp) { - if (timestamp == Instant.MIN) { - return -1; - } - return timestamp.toEpochMilli(); + private long toMillis(Instant timestamp) { + if (timestamp == Instant.MIN) { + return -1; } + return timestamp.toEpochMilli(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/TargetWeightCalculator.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/TargetWeightCalculator.java index a8ea650fab..10e68c67b4 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/TargetWeightCalculator.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/TargetWeightCalculator.java @@ -5,5 +5,5 @@ public interface TargetWeightCalculator { - Map calculate(Collection consumers); + Map calculate(Collection consumers); } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/Weight.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/Weight.java index 809f98a678..b2e787ce63 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/Weight.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/Weight.java @@ -1,93 +1,96 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload.weighted; -import java.util.Objects; - import static com.google.common.base.Preconditions.checkArgument; -class Weight implements Comparable { - - static final Weight ZERO = new Weight(0d); - - private final double operationsPerSecond; - - Weight(double operationsPerSecond) { - checkArgument(operationsPerSecond >= 0d, "operationsPerSecond should be greater than or equal to zero"); - this.operationsPerSecond = operationsPerSecond; - } - - static double calculatePercentageChange(Weight initialWeight, Weight finalWeight) { - checkArgument(initialWeight.operationsPerSecond > 0d, "initialWeight.operationsPerSecond should be greater than zero"); - double delta = Math.abs(finalWeight.operationsPerSecond - initialWeight.operationsPerSecond); - return (delta / initialWeight.operationsPerSecond) * 100; - } - - static Weight max(Weight first, Weight second) { - if (first.operationsPerSecond > second.operationsPerSecond) { - return first; - } - return second; - } - - Weight add(Weight addend) { - return new Weight(operationsPerSecond + addend.operationsPerSecond); - } - - Weight subtract(Weight subtrahend) { - return new Weight(Math.max(operationsPerSecond - subtrahend.operationsPerSecond, 0d)); - } - - Weight multiply(double multiplicand) { - return new Weight(operationsPerSecond * multiplicand); - } - - Weight divide(int divisor) { - return new Weight(operationsPerSecond / divisor); - } - - boolean isGreaterThan(Weight other) { - return operationsPerSecond > other.operationsPerSecond; - } - - boolean isGreaterThanOrEqualTo(Weight other) { - return isGreaterThan(other) || isEqualTo(other); - } - - boolean isLessThan(Weight other) { - return operationsPerSecond < other.operationsPerSecond; - } - - boolean isEqualTo(Weight other) { - return Double.compare(operationsPerSecond, other.operationsPerSecond) == 0; - } - - double getOperationsPerSecond() { - return operationsPerSecond; - } - - @Override - public int compareTo(Weight o) { - return Double.compare(operationsPerSecond, o.operationsPerSecond); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Weight weight = (Weight) o; - return Double.compare(weight.operationsPerSecond, operationsPerSecond) == 0; - } +import java.util.Objects; - @Override - public int hashCode() { - return Objects.hash(operationsPerSecond); - } +class Weight implements Comparable { - @Override - public String toString() { - return "(ops=" + operationsPerSecond + ")"; - } + static final Weight ZERO = new Weight(0d); + + private final double operationsPerSecond; + + Weight(double operationsPerSecond) { + checkArgument( + operationsPerSecond >= 0d, "operationsPerSecond should be greater than or equal to zero"); + this.operationsPerSecond = operationsPerSecond; + } + + static double calculatePercentageChange(Weight initialWeight, Weight finalWeight) { + checkArgument( + initialWeight.operationsPerSecond > 0d, + "initialWeight.operationsPerSecond should be greater than zero"); + double delta = Math.abs(finalWeight.operationsPerSecond - initialWeight.operationsPerSecond); + return (delta / initialWeight.operationsPerSecond) * 100; + } + + static Weight max(Weight first, Weight second) { + if (first.operationsPerSecond > second.operationsPerSecond) { + return first; + } + return second; + } + + Weight add(Weight addend) { + return new Weight(operationsPerSecond + addend.operationsPerSecond); + } + + Weight subtract(Weight subtrahend) { + return new Weight(Math.max(operationsPerSecond - subtrahend.operationsPerSecond, 0d)); + } + + Weight multiply(double multiplicand) { + return new Weight(operationsPerSecond * multiplicand); + } + + Weight divide(int divisor) { + return new Weight(operationsPerSecond / divisor); + } + + boolean isGreaterThan(Weight other) { + return operationsPerSecond > other.operationsPerSecond; + } + + boolean isGreaterThanOrEqualTo(Weight other) { + return isGreaterThan(other) || isEqualTo(other); + } + + boolean isLessThan(Weight other) { + return operationsPerSecond < other.operationsPerSecond; + } + + boolean isEqualTo(Weight other) { + return Double.compare(operationsPerSecond, other.operationsPerSecond) == 0; + } + + double getOperationsPerSecond() { + return operationsPerSecond; + } + + @Override + public int compareTo(Weight o) { + return Double.compare(operationsPerSecond, o.operationsPerSecond); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Weight weight = (Weight) o; + return Double.compare(weight.operationsPerSecond, operationsPerSecond) == 0; + } + + @Override + public int hashCode() { + return Objects.hash(operationsPerSecond); + } + + @Override + public String toString() { + return "(ops=" + operationsPerSecond + ")"; + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/WeightedWorkBalancer.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/WeightedWorkBalancer.java index 47aec5f7cc..73e34a72d7 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/WeightedWorkBalancer.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/WeightedWorkBalancer.java @@ -1,13 +1,10 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload.weighted; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import pl.allegro.tech.hermes.api.SubscriptionName; -import pl.allegro.tech.hermes.consumers.supervisor.workload.SubscriptionAssignment; -import pl.allegro.tech.hermes.consumers.supervisor.workload.SubscriptionAssignmentView; -import pl.allegro.tech.hermes.consumers.supervisor.workload.WorkBalancer; -import pl.allegro.tech.hermes.consumers.supervisor.workload.WorkBalancingResult; -import pl.allegro.tech.hermes.consumers.supervisor.workload.WorkloadConstraints; +import static java.util.stream.Collectors.toCollection; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toMap; +import static pl.allegro.tech.hermes.consumers.supervisor.workload.weighted.ConsumerNode.LIGHTEST_CONSUMER_FIRST; +import static pl.allegro.tech.hermes.consumers.supervisor.workload.weighted.ConsumerTask.HEAVIEST_TASK_FIRST; import java.time.Clock; import java.time.Duration; @@ -26,393 +23,435 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.IntStream; - -import static java.util.stream.Collectors.toCollection; -import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toMap; -import static pl.allegro.tech.hermes.consumers.supervisor.workload.weighted.ConsumerNode.LIGHTEST_CONSUMER_FIRST; -import static pl.allegro.tech.hermes.consumers.supervisor.workload.weighted.ConsumerTask.HEAVIEST_TASK_FIRST; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import pl.allegro.tech.hermes.api.SubscriptionName; +import pl.allegro.tech.hermes.consumers.supervisor.workload.SubscriptionAssignment; +import pl.allegro.tech.hermes.consumers.supervisor.workload.SubscriptionAssignmentView; +import pl.allegro.tech.hermes.consumers.supervisor.workload.WorkBalancer; +import pl.allegro.tech.hermes.consumers.supervisor.workload.WorkBalancingResult; +import pl.allegro.tech.hermes.consumers.supervisor.workload.WorkloadConstraints; public class WeightedWorkBalancer implements WorkBalancer { - private static final Logger logger = LoggerFactory.getLogger(WeightedWorkBalancer.class); - - private final Clock clock; - private final Duration stabilizationWindowSize; - private final double minSignificantChangePercent; - private final CurrentLoadProvider currentLoadProvider; - private final TargetWeightCalculator targetWeightCalculator; - - public WeightedWorkBalancer(Clock clock, - Duration stabilizationWindowSize, - double minSignificantChangePercent, - CurrentLoadProvider currentLoadProvider, - TargetWeightCalculator targetWeightCalculator) { - this.clock = clock; - this.stabilizationWindowSize = stabilizationWindowSize; - this.minSignificantChangePercent = minSignificantChangePercent; - this.currentLoadProvider = currentLoadProvider; - this.targetWeightCalculator = targetWeightCalculator; + private static final Logger logger = LoggerFactory.getLogger(WeightedWorkBalancer.class); + + private final Clock clock; + private final Duration stabilizationWindowSize; + private final double minSignificantChangePercent; + private final CurrentLoadProvider currentLoadProvider; + private final TargetWeightCalculator targetWeightCalculator; + + public WeightedWorkBalancer( + Clock clock, + Duration stabilizationWindowSize, + double minSignificantChangePercent, + CurrentLoadProvider currentLoadProvider, + TargetWeightCalculator targetWeightCalculator) { + this.clock = clock; + this.stabilizationWindowSize = stabilizationWindowSize; + this.minSignificantChangePercent = minSignificantChangePercent; + this.currentLoadProvider = currentLoadProvider; + this.targetWeightCalculator = targetWeightCalculator; + } + + @Override + public WorkBalancingResult balance( + List subscriptions, + List activeConsumerNodes, + SubscriptionAssignmentView currentState, + WorkloadConstraints constraints) { + AssignmentPlan initialPlan = + restoreValidAssignments(subscriptions, activeConsumerNodes, currentState, constraints); + AssignmentPlan planWithAllPossibleAssignments = tryAssignUnassignedTasks(initialPlan); + AssignmentPlan finalPlan = rebalance(planWithAllPossibleAssignments); + return finalPlan.toWorkBalancingResult(); + } + + private AssignmentPlan restoreValidAssignments( + List subscriptions, + List activeConsumerNodes, + SubscriptionAssignmentView currentState, + WorkloadConstraints constraints) { + Map consumers = createConsumers(activeConsumerNodes, constraints); + List unassignedTasks = new ArrayList<>(); + SubscriptionProfiles profiles = currentLoadProvider.getProfiles(); + for (SubscriptionName subscriptionName : subscriptions) { + SubscriptionProfile subscriptionProfile = profiles.getProfile(subscriptionName); + Queue consumerTasks = + createConsumerTasks(subscriptionName, subscriptionProfile, constraints); + Set consumerNodesForSubscription = + currentState.getConsumerNodesForSubscription(subscriptionName); + for (String consumerId : consumerNodesForSubscription) { + ConsumerNode consumerNode = consumers.get(consumerId); + if (consumerNode != null && !consumerTasks.isEmpty()) { + ConsumerTask consumerTask = consumerTasks.poll(); + consumerNode.assign(consumerTask); + } + } + unassignedTasks.addAll(consumerTasks); } - - @Override - public WorkBalancingResult balance(List subscriptions, - List activeConsumerNodes, - SubscriptionAssignmentView currentState, - WorkloadConstraints constraints) { - AssignmentPlan initialPlan = restoreValidAssignments(subscriptions, activeConsumerNodes, currentState, constraints); - AssignmentPlan planWithAllPossibleAssignments = tryAssignUnassignedTasks(initialPlan); - AssignmentPlan finalPlan = rebalance(planWithAllPossibleAssignments); - return finalPlan.toWorkBalancingResult(); - } - - private AssignmentPlan restoreValidAssignments(List subscriptions, - List activeConsumerNodes, - SubscriptionAssignmentView currentState, - WorkloadConstraints constraints) { - Map consumers = createConsumers(activeConsumerNodes, constraints); - List unassignedTasks = new ArrayList<>(); - SubscriptionProfiles profiles = currentLoadProvider.getProfiles(); - for (SubscriptionName subscriptionName : subscriptions) { - SubscriptionProfile subscriptionProfile = profiles.getProfile(subscriptionName); - Queue consumerTasks = createConsumerTasks(subscriptionName, subscriptionProfile, constraints); - Set consumerNodesForSubscription = currentState.getConsumerNodesForSubscription(subscriptionName); - for (String consumerId : consumerNodesForSubscription) { - ConsumerNode consumerNode = consumers.get(consumerId); - if (consumerNode != null && !consumerTasks.isEmpty()) { - ConsumerTask consumerTask = consumerTasks.poll(); - consumerNode.assign(consumerTask); - } - } - unassignedTasks.addAll(consumerTasks); - } - return new AssignmentPlan(unassignedTasks, consumers.values()); - } - - private Map createConsumers(List activeConsumerNodes, WorkloadConstraints constraints) { - return activeConsumerNodes.stream() - .map(consumerId -> { - ConsumerNodeLoad consumerNodeLoad = currentLoadProvider.getConsumerNodeLoad(consumerId); - return new ConsumerNode(consumerId, consumerNodeLoad, constraints.getMaxSubscriptionsPerConsumer()); - }) - .collect(toMap(ConsumerNode::getConsumerId, Function.identity())); - } - - private Queue createConsumerTasks(SubscriptionName subscriptionName, - SubscriptionProfile subscriptionProfile, - WorkloadConstraints constraints) { - int consumerCount = constraints.getConsumerCount(subscriptionName); - return IntStream.range(0, consumerCount) - .mapToObj(ignore -> new ConsumerTask(subscriptionName, subscriptionProfile)) - .collect(toCollection(ArrayDeque::new)); - } - - private AssignmentPlan tryAssignUnassignedTasks(AssignmentPlan assignmentPlan) { - PriorityQueue tasksToAssign = new PriorityQueue<>(HEAVIEST_TASK_FIRST); - tasksToAssign.addAll(assignmentPlan.getUnassignedTasks()); - List unassignedTasks = new ArrayList<>(); - while (!tasksToAssign.isEmpty()) { - ConsumerTask consumerTask = tasksToAssign.poll(); - Optional candidate = selectConsumerNode(consumerTask, assignmentPlan.getConsumers()); - if (candidate.isPresent()) { - candidate.get().assign(consumerTask); - } else { - unassignedTasks.add(consumerTask); - } - } - return new AssignmentPlan(unassignedTasks, assignmentPlan.getConsumers()); - } - - private Optional selectConsumerNode(ConsumerTask consumerTask, Collection consumers) { - return consumers.stream() - .filter(consumerNode -> consumerNode.isNotAssigned(consumerTask)) - .filter(consumerNode -> !consumerNode.isFull()) - .min(LIGHTEST_CONSUMER_FIRST); - } - - private AssignmentPlan rebalance(AssignmentPlan plan) { - Collection consumers = plan.getConsumers(); - TargetConsumerLoad targetLoad = calculateTargetConsumerLoad(consumers); - List overloadedConsumers = consumers.stream() - .filter(consumerNode -> isOverloaded(consumerNode, targetLoad)) - .sorted(new MostOverloadedConsumerFirst(targetLoad)) - .collect(toList()); - for (ConsumerNode overloaded : overloadedConsumers) { - List candidates = consumers.stream() - .filter(consumer -> !consumer.equals(overloaded)) - .collect(toList()); - for (ConsumerNode candidate : candidates) { - tryMoveOutTasks(overloaded, candidate, targetLoad); - trySwapTasks(overloaded, candidate, targetLoad); - if (isBalanced(overloaded, targetLoad)) { - break; - } - } - } - return new AssignmentPlan(plan.getUnassignedTasks(), consumers); + return new AssignmentPlan(unassignedTasks, consumers.values()); + } + + private Map createConsumers( + List activeConsumerNodes, WorkloadConstraints constraints) { + return activeConsumerNodes.stream() + .map( + consumerId -> { + ConsumerNodeLoad consumerNodeLoad = + currentLoadProvider.getConsumerNodeLoad(consumerId); + return new ConsumerNode( + consumerId, consumerNodeLoad, constraints.getMaxSubscriptionsPerConsumer()); + }) + .collect(toMap(ConsumerNode::getConsumerId, Function.identity())); + } + + private Queue createConsumerTasks( + SubscriptionName subscriptionName, + SubscriptionProfile subscriptionProfile, + WorkloadConstraints constraints) { + int consumerCount = constraints.getConsumerCount(subscriptionName); + return IntStream.range(0, consumerCount) + .mapToObj(ignore -> new ConsumerTask(subscriptionName, subscriptionProfile)) + .collect(toCollection(ArrayDeque::new)); + } + + private AssignmentPlan tryAssignUnassignedTasks(AssignmentPlan assignmentPlan) { + PriorityQueue tasksToAssign = new PriorityQueue<>(HEAVIEST_TASK_FIRST); + tasksToAssign.addAll(assignmentPlan.getUnassignedTasks()); + List unassignedTasks = new ArrayList<>(); + while (!tasksToAssign.isEmpty()) { + ConsumerTask consumerTask = tasksToAssign.poll(); + Optional candidate = + selectConsumerNode(consumerTask, assignmentPlan.getConsumers()); + if (candidate.isPresent()) { + candidate.get().assign(consumerTask); + } else { + unassignedTasks.add(consumerTask); + } } - - private TargetConsumerLoad calculateTargetConsumerLoad(Collection consumers) { - int totalNumberOfTasks = consumers.stream().mapToInt(ConsumerNode::getAssignedTaskCount).sum(); - int consumerCount = consumers.size(); - int maxNumberOfTasksPerConsumer = consumerCount == 0 ? 0 : divideWithRoundingUp(totalNumberOfTasks, consumerCount); - Map targetWeights = targetWeightCalculator.calculate(consumers); - return new TargetConsumerLoad(targetWeights, maxNumberOfTasksPerConsumer); + return new AssignmentPlan(unassignedTasks, assignmentPlan.getConsumers()); + } + + private Optional selectConsumerNode( + ConsumerTask consumerTask, Collection consumers) { + return consumers.stream() + .filter(consumerNode -> consumerNode.isNotAssigned(consumerTask)) + .filter(consumerNode -> !consumerNode.isFull()) + .min(LIGHTEST_CONSUMER_FIRST); + } + + private AssignmentPlan rebalance(AssignmentPlan plan) { + Collection consumers = plan.getConsumers(); + TargetConsumerLoad targetLoad = calculateTargetConsumerLoad(consumers); + List overloadedConsumers = + consumers.stream() + .filter(consumerNode -> isOverloaded(consumerNode, targetLoad)) + .sorted(new MostOverloadedConsumerFirst(targetLoad)) + .collect(toList()); + for (ConsumerNode overloaded : overloadedConsumers) { + List candidates = + consumers.stream().filter(consumer -> !consumer.equals(overloaded)).collect(toList()); + for (ConsumerNode candidate : candidates) { + tryMoveOutTasks(overloaded, candidate, targetLoad); + trySwapTasks(overloaded, candidate, targetLoad); + if (isBalanced(overloaded, targetLoad)) { + break; + } + } } - - private int divideWithRoundingUp(int dividend, int divisor) { - return (dividend / divisor) + (dividend % divisor > 0 ? 1 : 0); + return new AssignmentPlan(plan.getUnassignedTasks(), consumers); + } + + private TargetConsumerLoad calculateTargetConsumerLoad(Collection consumers) { + int totalNumberOfTasks = consumers.stream().mapToInt(ConsumerNode::getAssignedTaskCount).sum(); + int consumerCount = consumers.size(); + int maxNumberOfTasksPerConsumer = + consumerCount == 0 ? 0 : divideWithRoundingUp(totalNumberOfTasks, consumerCount); + Map targetWeights = targetWeightCalculator.calculate(consumers); + return new TargetConsumerLoad(targetWeights, maxNumberOfTasksPerConsumer); + } + + private int divideWithRoundingUp(int dividend, int divisor) { + return (dividend / divisor) + (dividend % divisor > 0 ? 1 : 0); + } + + private void tryMoveOutTasks( + ConsumerNode overloaded, ConsumerNode candidate, TargetConsumerLoad targetLoad) { + List candidatesToMoveOut = + overloaded.getAssignedTasks().stream().filter(candidate::isNotAssigned).collect(toList()); + ListIterator consumerTaskIterator = candidatesToMoveOut.listIterator(); + while (consumerTaskIterator.hasNext() + && hasTooManyTasks(overloaded, targetLoad) + && !hasTooManyTasks(candidate, targetLoad)) { + ConsumerTask taskFromOverloaded = consumerTaskIterator.next(); + MoveOutProposal proposal = new MoveOutProposal(overloaded, candidate, taskFromOverloaded); + if (isProfitable(proposal, targetLoad)) { + overloaded.unassign(taskFromOverloaded); + candidate.assign(taskFromOverloaded); + } } - - private void tryMoveOutTasks(ConsumerNode overloaded, ConsumerNode candidate, TargetConsumerLoad targetLoad) { - List candidatesToMoveOut = overloaded.getAssignedTasks().stream() - .filter(candidate::isNotAssigned) - .collect(toList()); - ListIterator consumerTaskIterator = candidatesToMoveOut.listIterator(); - while (consumerTaskIterator.hasNext() && hasTooManyTasks(overloaded, targetLoad) && !hasTooManyTasks(candidate, targetLoad)) { - ConsumerTask taskFromOverloaded = consumerTaskIterator.next(); - MoveOutProposal proposal = new MoveOutProposal(overloaded, candidate, taskFromOverloaded); - if (isProfitable(proposal, targetLoad)) { - overloaded.unassign(taskFromOverloaded); - candidate.assign(taskFromOverloaded); - } - } + } + + private boolean hasTooManyTasks( + ConsumerNode consumerNode, TargetConsumerLoad targetConsumerLoad) { + return consumerNode.getAssignedTaskCount() >= targetConsumerLoad.getNumberOfTasks(); + } + + private boolean isProfitable(MoveOutProposal proposal, TargetConsumerLoad targetLoad) { + Weight targetWeight = targetLoad.getWeightForConsumer(proposal.getCandidateId()); + if (targetWeight.isGreaterThanOrEqualTo(proposal.getFinalCandidateWeight())) { + logger.debug("MoveOut proposal will be applied:\n{}", proposal); + return true; } - - private boolean hasTooManyTasks(ConsumerNode consumerNode, TargetConsumerLoad targetConsumerLoad) { - return consumerNode.getAssignedTaskCount() >= targetConsumerLoad.getNumberOfTasks(); + return false; + } + + private boolean isProfitable(SwapProposal proposal, TargetConsumerLoad targetLoad) { + Weight initialOverloadedWeight = proposal.getOverloadedWeight(); + Weight finalOverloadedWeight = proposal.getFinalOverloadedWeight(); + Weight finalCandidateWeight = proposal.getFinalCandidateWeight(); + Weight overloadedTargetWeight = targetLoad.getWeightForConsumer(proposal.getOverloadedId()); + Weight candidateTargetWeight = targetLoad.getWeightForConsumer(proposal.getCandidateId()); + + if (initialOverloadedWeight.isLessThan(overloadedTargetWeight)) { + return false; } - - private boolean isProfitable(MoveOutProposal proposal, TargetConsumerLoad targetLoad) { - Weight targetWeight = targetLoad.getWeightForConsumer(proposal.getCandidateId()); - if (targetWeight.isGreaterThanOrEqualTo(proposal.getFinalCandidateWeight())) { - logger.debug("MoveOut proposal will be applied:\n{}", proposal); - return true; - } - return false; + if (finalCandidateWeight.isGreaterThan(candidateTargetWeight)) { + return false; } - - private boolean isProfitable(SwapProposal proposal, TargetConsumerLoad targetLoad) { - Weight initialOverloadedWeight = proposal.getOverloadedWeight(); - Weight finalOverloadedWeight = proposal.getFinalOverloadedWeight(); - Weight finalCandidateWeight = proposal.getFinalCandidateWeight(); - Weight overloadedTargetWeight = targetLoad.getWeightForConsumer(proposal.getOverloadedId()); - Weight candidateTargetWeight = targetLoad.getWeightForConsumer(proposal.getCandidateId()); - - if (initialOverloadedWeight.isLessThan(overloadedTargetWeight)) { - return false; - } - if (finalCandidateWeight.isGreaterThan(candidateTargetWeight)) { - return false; - } - if (finalOverloadedWeight.isGreaterThan(initialOverloadedWeight)) { - return false; - } - if (initialOverloadedWeight.isEqualTo(Weight.ZERO)) { - return false; - } - double percentageChange = Weight.calculatePercentageChange(initialOverloadedWeight, finalOverloadedWeight); - if (percentageChange >= minSignificantChangePercent) { - logger.debug("Swap proposal will be applied:\n{}", proposal); - return true; - } - return false; - } - - private void trySwapTasks(ConsumerNode overloaded, ConsumerNode candidate, TargetConsumerLoad targetLoad) { - List tasksFromOverloaded = findTasksForMovingOut(overloaded, candidate); - List tasksFromCandidate = findTasksForMovingOut(candidate, overloaded); - for (ConsumerTask taskFromOverloaded : tasksFromOverloaded) { - ListIterator tasksFromCandidateIterator = tasksFromCandidate.listIterator(); - while (tasksFromCandidateIterator.hasNext()) { - ConsumerTask taskFromCandidate = tasksFromCandidateIterator.next(); - SwapProposal proposal = new SwapProposal(overloaded, candidate, taskFromOverloaded, taskFromCandidate); - if (isProfitable(proposal, targetLoad)) { - overloaded.swap(taskFromOverloaded, taskFromCandidate); - candidate.swap(taskFromCandidate, taskFromOverloaded); - tasksFromCandidateIterator.remove(); - } - if (isBalanced(overloaded, targetLoad)) { - return; - } - } - } + if (finalOverloadedWeight.isGreaterThan(initialOverloadedWeight)) { + return false; } - - private List findTasksForMovingOut(ConsumerNode source, ConsumerNode destination) { - return source.getAssignedTasks().stream() - .filter(destination::isNotAssigned) - .filter(this::isStable) - .sorted(HEAVIEST_TASK_FIRST) - .collect(toList()); + if (initialOverloadedWeight.isEqualTo(Weight.ZERO)) { + return false; } - - private boolean isStable(ConsumerTask task) { - return clock.instant().isAfter(task.getLastRebalanceTimestamp().plus(stabilizationWindowSize)); + double percentageChange = + Weight.calculatePercentageChange(initialOverloadedWeight, finalOverloadedWeight); + if (percentageChange >= minSignificantChangePercent) { + logger.debug("Swap proposal will be applied:\n{}", proposal); + return true; } - - private boolean isOverloaded(ConsumerNode consumerNode, TargetConsumerLoad targetLoad) { - Weight targetWeight = targetLoad.getWeightForConsumer(consumerNode.getConsumerId()); - return consumerNode.getWeight().isGreaterThan(targetWeight) - || consumerNode.getAssignedTaskCount() > targetLoad.getNumberOfTasks(); + return false; + } + + private void trySwapTasks( + ConsumerNode overloaded, ConsumerNode candidate, TargetConsumerLoad targetLoad) { + List tasksFromOverloaded = findTasksForMovingOut(overloaded, candidate); + List tasksFromCandidate = findTasksForMovingOut(candidate, overloaded); + for (ConsumerTask taskFromOverloaded : tasksFromOverloaded) { + ListIterator tasksFromCandidateIterator = tasksFromCandidate.listIterator(); + while (tasksFromCandidateIterator.hasNext()) { + ConsumerTask taskFromCandidate = tasksFromCandidateIterator.next(); + SwapProposal proposal = + new SwapProposal(overloaded, candidate, taskFromOverloaded, taskFromCandidate); + if (isProfitable(proposal, targetLoad)) { + overloaded.swap(taskFromOverloaded, taskFromCandidate); + candidate.swap(taskFromCandidate, taskFromOverloaded); + tasksFromCandidateIterator.remove(); + } + if (isBalanced(overloaded, targetLoad)) { + return; + } + } } - - private boolean isBalanced(ConsumerNode consumerNode, TargetConsumerLoad targetLoad) { - Weight targetWeight = targetLoad.getWeightForConsumer(consumerNode.getConsumerId()); - return consumerNode.getWeight().isEqualTo(targetWeight) - && consumerNode.getAssignedTaskCount() <= targetLoad.getNumberOfTasks(); + } + + private List findTasksForMovingOut(ConsumerNode source, ConsumerNode destination) { + return source.getAssignedTasks().stream() + .filter(destination::isNotAssigned) + .filter(this::isStable) + .sorted(HEAVIEST_TASK_FIRST) + .collect(toList()); + } + + private boolean isStable(ConsumerTask task) { + return clock.instant().isAfter(task.getLastRebalanceTimestamp().plus(stabilizationWindowSize)); + } + + private boolean isOverloaded(ConsumerNode consumerNode, TargetConsumerLoad targetLoad) { + Weight targetWeight = targetLoad.getWeightForConsumer(consumerNode.getConsumerId()); + return consumerNode.getWeight().isGreaterThan(targetWeight) + || consumerNode.getAssignedTaskCount() > targetLoad.getNumberOfTasks(); + } + + private boolean isBalanced(ConsumerNode consumerNode, TargetConsumerLoad targetLoad) { + Weight targetWeight = targetLoad.getWeightForConsumer(consumerNode.getConsumerId()); + return consumerNode.getWeight().isEqualTo(targetWeight) + && consumerNode.getAssignedTaskCount() <= targetLoad.getNumberOfTasks(); + } + + private static class AssignmentPlan { + + private final List unassignedTasks; + private final Collection consumers; + + AssignmentPlan(List unassignedTasks, Collection consumers) { + this.unassignedTasks = unassignedTasks; + this.consumers = consumers; } - private static class AssignmentPlan { - - private final List unassignedTasks; - private final Collection consumers; - - AssignmentPlan(List unassignedTasks, Collection consumers) { - this.unassignedTasks = unassignedTasks; - this.consumers = consumers; - } - - Collection getConsumers() { - return consumers; - } - - List getUnassignedTasks() { - return unassignedTasks; - } + Collection getConsumers() { + return consumers; + } - WorkBalancingResult toWorkBalancingResult() { - Map> targetView = new HashMap<>(); - for (ConsumerNode consumer : consumers) { - for (ConsumerTask consumerTask : consumer.getAssignedTasks()) { - SubscriptionName subscriptionName = consumerTask.getSubscriptionName(); - Set assignments = targetView.computeIfAbsent(subscriptionName, ignore -> new HashSet<>()); - assignments.add(new SubscriptionAssignment(consumer.getConsumerId(), subscriptionName)); - } - } - return new WorkBalancingResult(new SubscriptionAssignmentView(targetView), unassignedTasks.size()); - } + List getUnassignedTasks() { + return unassignedTasks; } - private static class MoveOutProposal { + WorkBalancingResult toWorkBalancingResult() { + Map> targetView = new HashMap<>(); + for (ConsumerNode consumer : consumers) { + for (ConsumerTask consumerTask : consumer.getAssignedTasks()) { + SubscriptionName subscriptionName = consumerTask.getSubscriptionName(); + Set assignments = + targetView.computeIfAbsent(subscriptionName, ignore -> new HashSet<>()); + assignments.add(new SubscriptionAssignment(consumer.getConsumerId(), subscriptionName)); + } + } + return new WorkBalancingResult( + new SubscriptionAssignmentView(targetView), unassignedTasks.size()); + } + } - private final ConsumerNode overloaded; - private final ConsumerNode candidate; - private final Weight finalOverloadedWeight; - private final Weight finalCandidateWeight; + private static class MoveOutProposal { - MoveOutProposal(ConsumerNode overloaded, ConsumerNode candidate, ConsumerTask taskFromOverloaded) { - this.overloaded = overloaded; - this.candidate = candidate; - this.finalOverloadedWeight = overloaded.getWeight() - .subtract(taskFromOverloaded.getWeight()); - this.finalCandidateWeight = candidate.getWeight() - .add(taskFromOverloaded.getWeight()); - } + private final ConsumerNode overloaded; + private final ConsumerNode candidate; + private final Weight finalOverloadedWeight; + private final Weight finalCandidateWeight; - Weight getFinalCandidateWeight() { - return finalCandidateWeight; - } + MoveOutProposal( + ConsumerNode overloaded, ConsumerNode candidate, ConsumerTask taskFromOverloaded) { + this.overloaded = overloaded; + this.candidate = candidate; + this.finalOverloadedWeight = overloaded.getWeight().subtract(taskFromOverloaded.getWeight()); + this.finalCandidateWeight = candidate.getWeight().add(taskFromOverloaded.getWeight()); + } - String getCandidateId() { - return candidate.getConsumerId(); - } + Weight getFinalCandidateWeight() { + return finalCandidateWeight; + } - @Override - public String toString() { - return toString(overloaded, finalOverloadedWeight) + "\n" + toString(candidate, finalCandidateWeight); - } + String getCandidateId() { + return candidate.getConsumerId(); + } - private String toString(ConsumerNode consumerNode, Weight newWeight) { - return consumerNode + " (old weight = " + consumerNode.getWeight() + ", new weight = " + newWeight + ")"; - } + @Override + public String toString() { + return toString(overloaded, finalOverloadedWeight) + + "\n" + + toString(candidate, finalCandidateWeight); } - private static class SwapProposal { - - private final ConsumerNode overloaded; - private final ConsumerNode candidate; - private final Weight finalOverloadedWeight; - private final Weight finalCandidateWeight; - - SwapProposal(ConsumerNode overloaded, - ConsumerNode candidate, - ConsumerTask taskFromOverloaded, - ConsumerTask taskFromCandidate) { - this.overloaded = overloaded; - this.candidate = candidate; - this.finalOverloadedWeight = overloaded.getWeight() - .add(taskFromCandidate.getWeight()) - .subtract(taskFromOverloaded.getWeight()); - this.finalCandidateWeight = candidate.getWeight() - .add(taskFromOverloaded.getWeight()) - .subtract(taskFromCandidate.getWeight()); - } + private String toString(ConsumerNode consumerNode, Weight newWeight) { + return consumerNode + + " (old weight = " + + consumerNode.getWeight() + + ", new weight = " + + newWeight + + ")"; + } + } + + private static class SwapProposal { + + private final ConsumerNode overloaded; + private final ConsumerNode candidate; + private final Weight finalOverloadedWeight; + private final Weight finalCandidateWeight; + + SwapProposal( + ConsumerNode overloaded, + ConsumerNode candidate, + ConsumerTask taskFromOverloaded, + ConsumerTask taskFromCandidate) { + this.overloaded = overloaded; + this.candidate = candidate; + this.finalOverloadedWeight = + overloaded + .getWeight() + .add(taskFromCandidate.getWeight()) + .subtract(taskFromOverloaded.getWeight()); + this.finalCandidateWeight = + candidate + .getWeight() + .add(taskFromOverloaded.getWeight()) + .subtract(taskFromCandidate.getWeight()); + } - Weight getOverloadedWeight() { - return overloaded.getWeight(); - } + Weight getOverloadedWeight() { + return overloaded.getWeight(); + } - Weight getFinalOverloadedWeight() { - return finalOverloadedWeight; - } + Weight getFinalOverloadedWeight() { + return finalOverloadedWeight; + } - Weight getFinalCandidateWeight() { - return finalCandidateWeight; - } + Weight getFinalCandidateWeight() { + return finalCandidateWeight; + } - String getCandidateId() { - return candidate.getConsumerId(); - } + String getCandidateId() { + return candidate.getConsumerId(); + } - String getOverloadedId() { - return overloaded.getConsumerId(); - } + String getOverloadedId() { + return overloaded.getConsumerId(); + } - @Override - public String toString() { - return toString(overloaded, finalOverloadedWeight) + "\n" + toString(candidate, finalCandidateWeight); - } + @Override + public String toString() { + return toString(overloaded, finalOverloadedWeight) + + "\n" + + toString(candidate, finalCandidateWeight); + } - private String toString(ConsumerNode consumerNode, Weight newWeight) { - return consumerNode + " (old weight = " + consumerNode.getWeight() + ", new weight = " + newWeight + ")"; - } + private String toString(ConsumerNode consumerNode, Weight newWeight) { + return consumerNode + + " (old weight = " + + consumerNode.getWeight() + + ", new weight = " + + newWeight + + ")"; } + } - private static class TargetConsumerLoad { + private static class TargetConsumerLoad { - private final Map weights; - private final int numberOfTasks; + private final Map weights; + private final int numberOfTasks; - TargetConsumerLoad(Map weights, int numberOfTasks) { - this.weights = weights; - this.numberOfTasks = numberOfTasks; - } + TargetConsumerLoad(Map weights, int numberOfTasks) { + this.weights = weights; + this.numberOfTasks = numberOfTasks; + } - Weight getWeightForConsumer(String consumerId) { - return weights.get(consumerId); - } + Weight getWeightForConsumer(String consumerId) { + return weights.get(consumerId); + } - int getNumberOfTasks() { - return numberOfTasks; - } + int getNumberOfTasks() { + return numberOfTasks; } + } - private static class MostOverloadedConsumerFirst implements Comparator { + private static class MostOverloadedConsumerFirst implements Comparator { - private final TargetConsumerLoad targetLoad; + private final TargetConsumerLoad targetLoad; - MostOverloadedConsumerFirst(TargetConsumerLoad targetLoad) { - this.targetLoad = targetLoad; - } + MostOverloadedConsumerFirst(TargetConsumerLoad targetLoad) { + this.targetLoad = targetLoad; + } - @Override - public int compare(ConsumerNode first, ConsumerNode second) { - Weight firstTargetLoad = targetLoad.getWeightForConsumer(first.getConsumerId()); - Weight secondTargetLoad = targetLoad.getWeightForConsumer(second.getConsumerId()); - Weight firstError = first.getWeight().subtract(firstTargetLoad); - Weight secondError = second.getWeight().subtract(secondTargetLoad); - return secondError.compareTo(firstError); - } + @Override + public int compare(ConsumerNode first, ConsumerNode second) { + Weight firstTargetLoad = targetLoad.getWeightForConsumer(first.getConsumerId()); + Weight secondTargetLoad = targetLoad.getWeightForConsumer(second.getConsumerId()); + Weight firstError = first.getWeight().subtract(firstTargetLoad); + Weight secondError = second.getWeight().subtract(secondTargetLoad); + return secondError.compareTo(firstError); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/WeightedWorkBalancingListener.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/WeightedWorkBalancingListener.java index 3cf6c6f745..933cb64924 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/WeightedWorkBalancingListener.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/WeightedWorkBalancingListener.java @@ -1,8 +1,6 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload.weighted; -import pl.allegro.tech.hermes.api.SubscriptionName; -import pl.allegro.tech.hermes.consumers.supervisor.workload.BalancingListener; -import pl.allegro.tech.hermes.consumers.supervisor.workload.WorkDistributionChanges; +import static java.util.stream.Collectors.toMap; import java.time.Clock; import java.time.Duration; @@ -13,73 +11,82 @@ import java.util.Map; import java.util.Set; import java.util.function.Function; - -import static java.util.stream.Collectors.toMap; +import pl.allegro.tech.hermes.api.SubscriptionName; +import pl.allegro.tech.hermes.consumers.supervisor.workload.BalancingListener; +import pl.allegro.tech.hermes.consumers.supervisor.workload.WorkDistributionChanges; public class WeightedWorkBalancingListener implements BalancingListener { - private final ConsumerNodeLoadRegistry consumerNodeLoadRegistry; - private final SubscriptionProfileRegistry subscriptionProfileRegistry; - private final CurrentLoadProvider currentLoadProvider; - private final WeightedWorkloadMetricsReporter weightedWorkloadMetrics; - private final SubscriptionProfilesCalculator subscriptionProfilesCalculator; - private final Clock clock; + private final ConsumerNodeLoadRegistry consumerNodeLoadRegistry; + private final SubscriptionProfileRegistry subscriptionProfileRegistry; + private final CurrentLoadProvider currentLoadProvider; + private final WeightedWorkloadMetricsReporter weightedWorkloadMetrics; + private final SubscriptionProfilesCalculator subscriptionProfilesCalculator; + private final Clock clock; - public WeightedWorkBalancingListener(ConsumerNodeLoadRegistry consumerNodeLoadRegistry, - SubscriptionProfileRegistry subscriptionProfileRegistry, - CurrentLoadProvider currentLoadProvider, - WeightedWorkloadMetricsReporter weightedWorkloadMetrics, - Clock clock, - Duration weightWindowSize) { - this.consumerNodeLoadRegistry = consumerNodeLoadRegistry; - this.subscriptionProfileRegistry = subscriptionProfileRegistry; - this.currentLoadProvider = currentLoadProvider; - this.weightedWorkloadMetrics = weightedWorkloadMetrics; - this.subscriptionProfilesCalculator = new SubscriptionProfilesCalculator(clock, weightWindowSize); - this.clock = clock; - } + public WeightedWorkBalancingListener( + ConsumerNodeLoadRegistry consumerNodeLoadRegistry, + SubscriptionProfileRegistry subscriptionProfileRegistry, + CurrentLoadProvider currentLoadProvider, + WeightedWorkloadMetricsReporter weightedWorkloadMetrics, + Clock clock, + Duration weightWindowSize) { + this.consumerNodeLoadRegistry = consumerNodeLoadRegistry; + this.subscriptionProfileRegistry = subscriptionProfileRegistry; + this.currentLoadProvider = currentLoadProvider; + this.weightedWorkloadMetrics = weightedWorkloadMetrics; + this.subscriptionProfilesCalculator = + new SubscriptionProfilesCalculator(clock, weightWindowSize); + this.clock = clock; + } - @Override - public void onBeforeBalancing(List activeConsumers) { - weightedWorkloadMetrics.unregisterMetricsForConsumersOtherThan(new HashSet<>(activeConsumers)); - Map newConsumerLoads = fetchConsumerNodeLoads(activeConsumers); - currentLoadProvider.updateConsumerNodeLoads(newConsumerLoads); - SubscriptionProfiles currentProfiles = recalculateSubscriptionProfiles(newConsumerLoads.values()); - currentLoadProvider.updateProfiles(currentProfiles); - } + @Override + public void onBeforeBalancing(List activeConsumers) { + weightedWorkloadMetrics.unregisterMetricsForConsumersOtherThan(new HashSet<>(activeConsumers)); + Map newConsumerLoads = fetchConsumerNodeLoads(activeConsumers); + currentLoadProvider.updateConsumerNodeLoads(newConsumerLoads); + SubscriptionProfiles currentProfiles = + recalculateSubscriptionProfiles(newConsumerLoads.values()); + currentLoadProvider.updateProfiles(currentProfiles); + } - private Map fetchConsumerNodeLoads(List activeConsumers) { - return activeConsumers.stream() - .collect(toMap(Function.identity(), consumerNodeLoadRegistry::get)); - } + private Map fetchConsumerNodeLoads(List activeConsumers) { + return activeConsumers.stream() + .collect(toMap(Function.identity(), consumerNodeLoadRegistry::get)); + } - private SubscriptionProfiles recalculateSubscriptionProfiles(Collection consumerNodeLoads) { - SubscriptionProfiles previousProfiles = subscriptionProfileRegistry.fetch(); - return subscriptionProfilesCalculator.calculate(consumerNodeLoads, previousProfiles); - } + private SubscriptionProfiles recalculateSubscriptionProfiles( + Collection consumerNodeLoads) { + SubscriptionProfiles previousProfiles = subscriptionProfileRegistry.fetch(); + return subscriptionProfilesCalculator.calculate(consumerNodeLoads, previousProfiles); + } - @Override - public void onAfterBalancing(WorkDistributionChanges changes) { - applyRebalanceTimestampToSubscriptionProfiles(changes.getRebalancedSubscriptions()); - } + @Override + public void onAfterBalancing(WorkDistributionChanges changes) { + applyRebalanceTimestampToSubscriptionProfiles(changes.getRebalancedSubscriptions()); + } - private void applyRebalanceTimestampToSubscriptionProfiles(Set rebalancedSubscriptions) { - SubscriptionProfiles currentProfiles = currentLoadProvider.getProfiles(); - Map profilePerSubscription = new HashMap<>(currentProfiles.getProfiles()); - for (SubscriptionName subscriptionName : rebalancedSubscriptions) { - SubscriptionProfile profile = profilePerSubscription.get(subscriptionName); - if (profile != null) { - profilePerSubscription.put(subscriptionName, new SubscriptionProfile(clock.instant(), profile.getWeight())); - } - } - SubscriptionProfiles finalProfiles = new SubscriptionProfiles(profilePerSubscription, currentProfiles.getUpdateTimestamp()); - subscriptionProfileRegistry.persist(finalProfiles); - currentLoadProvider.updateProfiles(finalProfiles); + private void applyRebalanceTimestampToSubscriptionProfiles( + Set rebalancedSubscriptions) { + SubscriptionProfiles currentProfiles = currentLoadProvider.getProfiles(); + Map profilePerSubscription = + new HashMap<>(currentProfiles.getProfiles()); + for (SubscriptionName subscriptionName : rebalancedSubscriptions) { + SubscriptionProfile profile = profilePerSubscription.get(subscriptionName); + if (profile != null) { + profilePerSubscription.put( + subscriptionName, new SubscriptionProfile(clock.instant(), profile.getWeight())); + } } + SubscriptionProfiles finalProfiles = + new SubscriptionProfiles(profilePerSubscription, currentProfiles.getUpdateTimestamp()); + subscriptionProfileRegistry.persist(finalProfiles); + currentLoadProvider.updateProfiles(finalProfiles); + } - @Override - public void onBalancingSkipped() { - weightedWorkloadMetrics.unregisterLeaderMetrics(); - currentLoadProvider.clear(); - } + @Override + public void onBalancingSkipped() { + weightedWorkloadMetrics.unregisterLeaderMetrics(); + currentLoadProvider.clear(); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/WeightedWorkloadMetricsReporter.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/WeightedWorkloadMetricsReporter.java index a5138ded40..acfdd792d3 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/WeightedWorkloadMetricsReporter.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/WeightedWorkloadMetricsReporter.java @@ -1,92 +1,94 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload.weighted; +import static java.util.Collections.emptySet; +import static java.util.stream.Collectors.toSet; + import com.google.common.collect.Iterables; import com.google.common.collect.Sets; -import pl.allegro.tech.hermes.common.metric.MetricsFacade; -import pl.allegro.tech.hermes.common.metric.WorkloadMetrics; - import java.util.Collection; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; - -import static java.util.Collections.emptySet; -import static java.util.stream.Collectors.toSet; +import pl.allegro.tech.hermes.common.metric.MetricsFacade; +import pl.allegro.tech.hermes.common.metric.WorkloadMetrics; public class WeightedWorkloadMetricsReporter { - private final WorkloadMetrics metrics; - private final Map currentWeights = new ConcurrentHashMap<>(); - private final Map proposedWeights = new ConcurrentHashMap<>(); - private final Map currentScores = new ConcurrentHashMap<>(); - private final Map proposedScores = new ConcurrentHashMap<>(); - private final Map scoringErrors = new ConcurrentHashMap<>(); + private final WorkloadMetrics metrics; + private final Map currentWeights = new ConcurrentHashMap<>(); + private final Map proposedWeights = new ConcurrentHashMap<>(); + private final Map currentScores = new ConcurrentHashMap<>(); + private final Map proposedScores = new ConcurrentHashMap<>(); + private final Map scoringErrors = new ConcurrentHashMap<>(); - public WeightedWorkloadMetricsReporter(MetricsFacade metrics) { - this.metrics = metrics.workload(); - } + public WeightedWorkloadMetricsReporter(MetricsFacade metrics) { + this.metrics = metrics.workload(); + } - void reportCurrentScore(String consumerId, double score) { - if (!currentScores.containsKey(consumerId)) { - metrics.registerCurrentScoreGauge(consumerId, currentScores, scores -> scores.getOrDefault(consumerId, 0d)); - } - currentScores.put(consumerId, score); + void reportCurrentScore(String consumerId, double score) { + if (!currentScores.containsKey(consumerId)) { + metrics.registerCurrentScoreGauge( + consumerId, currentScores, scores -> scores.getOrDefault(consumerId, 0d)); } + currentScores.put(consumerId, score); + } - void reportProposedScore(String consumerId, double score) { - if (!currentScores.containsKey(consumerId)) { - metrics.registerProposedErrorGauge(consumerId, proposedScores, scores -> scores.getOrDefault(consumerId, 0d)); - } - proposedScores.put(consumerId, score); + void reportProposedScore(String consumerId, double score) { + if (!currentScores.containsKey(consumerId)) { + metrics.registerProposedErrorGauge( + consumerId, proposedScores, scores -> scores.getOrDefault(consumerId, 0d)); } + proposedScores.put(consumerId, score); + } - void reportScoringError(String consumerId, double error) { - if (!scoringErrors.containsKey(consumerId)) { - metrics.registerScoringErrorGauge(consumerId, scoringErrors, errors -> errors.getOrDefault(consumerId, 0d)); - } - scoringErrors.put(consumerId, error); + void reportScoringError(String consumerId, double error) { + if (!scoringErrors.containsKey(consumerId)) { + metrics.registerScoringErrorGauge( + consumerId, scoringErrors, errors -> errors.getOrDefault(consumerId, 0d)); } + scoringErrors.put(consumerId, error); + } - void reportCurrentWeights(Collection consumers) { - for (ConsumerNode consumerNode : consumers) { - String consumerId = consumerNode.getConsumerId(); - if (!currentWeights.containsKey(consumerId)) { - metrics.registerCurrentWeightGauge(consumerId, currentWeights, weights -> weights.getOrDefault(consumerId, 0d)); - } - currentWeights.put(consumerId, consumerNode.getWeight().getOperationsPerSecond()); - } + void reportCurrentWeights(Collection consumers) { + for (ConsumerNode consumerNode : consumers) { + String consumerId = consumerNode.getConsumerId(); + if (!currentWeights.containsKey(consumerId)) { + metrics.registerCurrentWeightGauge( + consumerId, currentWeights, weights -> weights.getOrDefault(consumerId, 0d)); + } + currentWeights.put(consumerId, consumerNode.getWeight().getOperationsPerSecond()); } + } - void reportProposedWeights(Map newWeights) { - for (Map.Entry entry : newWeights.entrySet()) { - String consumerId = entry.getKey(); - if (!proposedWeights.containsKey(consumerId)) { - metrics.registerProposedWeightGauge(consumerId, proposedWeights, weights -> weights.getOrDefault(consumerId, 0d)); - } - proposedWeights.put(consumerId, entry.getValue().getOperationsPerSecond()); - } + void reportProposedWeights(Map newWeights) { + for (Map.Entry entry : newWeights.entrySet()) { + String consumerId = entry.getKey(); + if (!proposedWeights.containsKey(consumerId)) { + metrics.registerProposedWeightGauge( + consumerId, proposedWeights, weights -> weights.getOrDefault(consumerId, 0d)); + } + proposedWeights.put(consumerId, entry.getValue().getOperationsPerSecond()); } + } - void unregisterLeaderMetrics() { - unregisterMetricsForConsumersOtherThan(emptySet()); - } + void unregisterLeaderMetrics() { + unregisterMetricsForConsumersOtherThan(emptySet()); + } - void unregisterMetricsForConsumersOtherThan(Set consumerIds) { - metrics.unregisterAllWorkloadWeightedGaugesForConsumerIds(findConsumerIdsToRemove(consumerIds)); - } + void unregisterMetricsForConsumersOtherThan(Set consumerIds) { + metrics.unregisterAllWorkloadWeightedGaugesForConsumerIds(findConsumerIdsToRemove(consumerIds)); + } - private Set findConsumerIdsToRemove(Set activeIds) { - return Sets.newHashSet( - Iterables.concat( - currentScores.keySet(), - proposedScores.keySet(), - scoringErrors.keySet(), - currentWeights.keySet(), - proposedWeights.keySet() - ) - ) - .stream() - .filter(consumerId -> !activeIds.contains(consumerId)) - .collect(toSet()); - } + private Set findConsumerIdsToRemove(Set activeIds) { + return Sets.newHashSet( + Iterables.concat( + currentScores.keySet(), + proposedScores.keySet(), + scoringErrors.keySet(), + currentWeights.keySet(), + proposedWeights.keySet())) + .stream() + .filter(consumerId -> !activeIds.contains(consumerId)) + .collect(toSet()); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ZookeeperConsumerNodeLoadRegistry.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ZookeeperConsumerNodeLoadRegistry.java index e2740e2f44..3606784abc 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ZookeeperConsumerNodeLoadRegistry.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ZookeeperConsumerNodeLoadRegistry.java @@ -1,6 +1,21 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload.weighted; +import static java.util.Collections.newSetFromMap; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.stream.Collectors.toMap; +import static pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths.CONSUMERS_WORKLOAD_PATH; +import static pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths.CONSUMER_LOAD_PATH; + import com.sun.management.OperatingSystemMXBean; +import java.lang.management.ManagementFactory; +import java.time.Clock; +import java.time.Duration; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.atomic.LongAdder; import org.apache.curator.framework.CuratorFramework; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; @@ -13,191 +28,191 @@ import pl.allegro.tech.hermes.consumers.subscription.id.SubscriptionIds; import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; -import java.lang.management.ManagementFactory; -import java.time.Clock; -import java.time.Duration; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.atomic.LongAdder; +public class ZookeeperConsumerNodeLoadRegistry implements ConsumerNodeLoadRegistry { -import static java.util.Collections.newSetFromMap; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.stream.Collectors.toMap; -import static pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths.CONSUMERS_WORKLOAD_PATH; -import static pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths.CONSUMER_LOAD_PATH; + private static final Logger logger = + LoggerFactory.getLogger(ZookeeperConsumerNodeLoadRegistry.class); + + private final Duration interval; + private final CuratorFramework curator; + private final ZookeeperPaths zookeeperPaths; + private final Clock clock; + private final String basePath; + private final String currentConsumerPath; + private final ConsumerNodeLoadEncoder encoder; + private final ConsumerNodeLoadDecoder decoder; + private final ScheduledExecutorService executor; + private final OperatingSystemMXBean platformMXBean = + ManagementFactory.getPlatformMXBean(OperatingSystemMXBean.class); + + private final Set subscriptionLoadRecorders = + newSetFromMap(new ConcurrentHashMap<>()); + private volatile long lastReset; + private volatile double currentOperationsPerSecond = 0d; + private volatile double cpuUtilization = -1; + + public ZookeeperConsumerNodeLoadRegistry( + CuratorFramework curator, + SubscriptionIds subscriptionIds, + ZookeeperPaths zookeeperPaths, + String currentConsumerId, + String clusterName, + Duration interval, + ExecutorServiceFactory executorServiceFactory, + Clock clock, + MetricsFacade metrics, + int consumerLoadEncoderBufferSizeBytes) { + this.curator = curator; + this.zookeeperPaths = zookeeperPaths; + this.clock = clock; + this.basePath = + zookeeperPaths.join( + zookeeperPaths.basePath(), CONSUMERS_WORKLOAD_PATH, clusterName, CONSUMER_LOAD_PATH); + this.currentConsumerPath = resolveConsumerLoadPath(currentConsumerId); + this.interval = interval; + this.encoder = new ConsumerNodeLoadEncoder(subscriptionIds, consumerLoadEncoderBufferSizeBytes); + this.decoder = new ConsumerNodeLoadDecoder(subscriptionIds); + this.executor = + executorServiceFactory.createSingleThreadScheduledExecutor( + "consumer-node-load-reporter-%d"); + this.lastReset = clock.millis(); + metrics + .workload() + .registerOperationsPerSecondGauge(this, registry -> registry.currentOperationsPerSecond); + metrics.workload().registerCpuUtilizationGauge(this, registry -> registry.cpuUtilization); + if (platformMXBean.getProcessCpuLoad() < 0d) { + logger.warn("Process CPU load is not available."); + } + } + + @Override + public void start() { + executor.scheduleWithFixedDelay(this::report, 0, interval.toMillis(), MILLISECONDS); + } + + @Override + public void stop() { + executor.shutdown(); + } + + private void report() { + try { + ConsumerNodeLoad consumerNodeLoad = calculateConsumerNodeLoad(); + currentOperationsPerSecond = + consumerNodeLoad.getLoadPerSubscription().values().stream() + .mapToDouble(SubscriptionLoad::getOperationsPerSecond) + .sum(); + persist(consumerNodeLoad); + } catch (Exception e) { + logger.error("Error while reporting consumer node load", e); + } + } + + private ConsumerNodeLoad calculateConsumerNodeLoad() { + long now = clock.millis(); + long elapsedMillis = now - lastReset; + long elapsedSeconds = Math.max(MILLISECONDS.toSeconds(elapsedMillis), 1); + lastReset = now; + cpuUtilization = platformMXBean.getProcessCpuLoad(); + Map loadPerSubscription = + subscriptionLoadRecorders.stream() + .collect( + toMap( + ZookeeperSubscriptionLoadRecorder::getSubscriptionName, + recorder -> recorder.calculate(elapsedSeconds))); + return new ConsumerNodeLoad(cpuUtilization, loadPerSubscription); + } + + private void persist(ConsumerNodeLoad metrics) throws Exception { + byte[] encoded = encoder.encode(metrics); + try { + curator.setData().forPath(currentConsumerPath, encoded); + } catch (KeeperException.NoNodeException e) { + try { + curator + .create() + .creatingParentContainersIfNeeded() + .withMode(CreateMode.EPHEMERAL) + .forPath(currentConsumerPath, encoded); + } catch (KeeperException.NodeExistsException ex) { + // ignore + } + } + } + + @Override + public ConsumerNodeLoad get(String consumerId) { + String consumerLoadPath = resolveConsumerLoadPath(consumerId); + try { + if (curator.checkExists().forPath(consumerLoadPath) != null) { + byte[] bytes = curator.getData().forPath(consumerLoadPath); + return decoder.decode(bytes); + } + } catch (Exception e) { + logger.warn("Could not read node data on path " + consumerLoadPath, e); + } + return ConsumerNodeLoad.UNDEFINED; + } -public class ZookeeperConsumerNodeLoadRegistry implements ConsumerNodeLoadRegistry { + private String resolveConsumerLoadPath(String consumerId) { + return zookeeperPaths.join(basePath, consumerId); + } + + @Override + public SubscriptionLoadRecorder register(SubscriptionName subscriptionName) { + return new ZookeeperSubscriptionLoadRecorder(subscriptionName); + } + + private class ZookeeperSubscriptionLoadRecorder implements SubscriptionLoadRecorder { + + private final SubscriptionName subscriptionName; + private final LongAdder operationsCounter = new LongAdder(); - private static final Logger logger = LoggerFactory.getLogger(ZookeeperConsumerNodeLoadRegistry.class); - - private final Duration interval; - private final CuratorFramework curator; - private final ZookeeperPaths zookeeperPaths; - private final Clock clock; - private final String basePath; - private final String currentConsumerPath; - private final ConsumerNodeLoadEncoder encoder; - private final ConsumerNodeLoadDecoder decoder; - private final ScheduledExecutorService executor; - private final OperatingSystemMXBean platformMXBean = ManagementFactory.getPlatformMXBean(OperatingSystemMXBean.class); - - private final Set subscriptionLoadRecorders = newSetFromMap(new ConcurrentHashMap<>()); - private volatile long lastReset; - private volatile double currentOperationsPerSecond = 0d; - private volatile double cpuUtilization = -1; - - public ZookeeperConsumerNodeLoadRegistry(CuratorFramework curator, - SubscriptionIds subscriptionIds, - ZookeeperPaths zookeeperPaths, - String currentConsumerId, - String clusterName, - Duration interval, - ExecutorServiceFactory executorServiceFactory, - Clock clock, - MetricsFacade metrics, - int consumerLoadEncoderBufferSizeBytes) { - this.curator = curator; - this.zookeeperPaths = zookeeperPaths; - this.clock = clock; - this.basePath = zookeeperPaths.join(zookeeperPaths.basePath(), CONSUMERS_WORKLOAD_PATH, clusterName, CONSUMER_LOAD_PATH); - this.currentConsumerPath = resolveConsumerLoadPath(currentConsumerId); - this.interval = interval; - this.encoder = new ConsumerNodeLoadEncoder(subscriptionIds, consumerLoadEncoderBufferSizeBytes); - this.decoder = new ConsumerNodeLoadDecoder(subscriptionIds); - this.executor = executorServiceFactory.createSingleThreadScheduledExecutor("consumer-node-load-reporter-%d"); - this.lastReset = clock.millis(); - metrics.workload().registerOperationsPerSecondGauge(this, registry -> registry.currentOperationsPerSecond); - metrics.workload().registerCpuUtilizationGauge(this, registry -> registry.cpuUtilization); - if (platformMXBean.getProcessCpuLoad() < 0d) { - logger.warn("Process CPU load is not available."); - } + ZookeeperSubscriptionLoadRecorder(SubscriptionName subscriptionName) { + this.subscriptionName = subscriptionName; } @Override - public void start() { - executor.scheduleWithFixedDelay(this::report, 0, interval.toMillis(), MILLISECONDS); + public void initialize() { + operationsCounter.reset(); + subscriptionLoadRecorders.add(this); } @Override - public void stop() { - executor.shutdown(); + public void recordSingleOperation() { + operationsCounter.increment(); } - private void report() { - try { - ConsumerNodeLoad consumerNodeLoad = calculateConsumerNodeLoad(); - currentOperationsPerSecond = consumerNodeLoad.getLoadPerSubscription().values().stream() - .mapToDouble(SubscriptionLoad::getOperationsPerSecond) - .sum(); - persist(consumerNodeLoad); - } catch (Exception e) { - logger.error("Error while reporting consumer node load", e); - } + @Override + public void shutdown() { + operationsCounter.reset(); + subscriptionLoadRecorders.remove(this); } - private ConsumerNodeLoad calculateConsumerNodeLoad() { - long now = clock.millis(); - long elapsedMillis = now - lastReset; - long elapsedSeconds = Math.max(MILLISECONDS.toSeconds(elapsedMillis), 1); - lastReset = now; - cpuUtilization = platformMXBean.getProcessCpuLoad(); - Map loadPerSubscription = subscriptionLoadRecorders.stream() - .collect(toMap(ZookeeperSubscriptionLoadRecorder::getSubscriptionName, recorder -> recorder.calculate(elapsedSeconds))); - return new ConsumerNodeLoad(cpuUtilization, loadPerSubscription); + SubscriptionName getSubscriptionName() { + return subscriptionName; } - private void persist(ConsumerNodeLoad metrics) throws Exception { - byte[] encoded = encoder.encode(metrics); - try { - curator.setData().forPath(currentConsumerPath, encoded); - } catch (KeeperException.NoNodeException e) { - try { - curator.create() - .creatingParentContainersIfNeeded() - .withMode(CreateMode.EPHEMERAL) - .forPath(currentConsumerPath, encoded); - } catch (KeeperException.NodeExistsException ex) { - // ignore - } - } + SubscriptionLoad calculate(long elapsedSeconds) { + double operationsPerSecond = (double) operationsCounter.sumThenReset() / elapsedSeconds; + return new SubscriptionLoad(operationsPerSecond); } @Override - public ConsumerNodeLoad get(String consumerId) { - String consumerLoadPath = resolveConsumerLoadPath(consumerId); - try { - if (curator.checkExists().forPath(consumerLoadPath) != null) { - byte[] bytes = curator.getData().forPath(consumerLoadPath); - return decoder.decode(bytes); - } - } catch (Exception e) { - logger.warn("Could not read node data on path " + consumerLoadPath, e); - } - return ConsumerNodeLoad.UNDEFINED; - } - - private String resolveConsumerLoadPath(String consumerId) { - return zookeeperPaths.join(basePath, consumerId); + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ZookeeperSubscriptionLoadRecorder that = (ZookeeperSubscriptionLoadRecorder) o; + return subscriptionName.equals(that.subscriptionName); } @Override - public SubscriptionLoadRecorder register(SubscriptionName subscriptionName) { - return new ZookeeperSubscriptionLoadRecorder(subscriptionName); - } - - private class ZookeeperSubscriptionLoadRecorder implements SubscriptionLoadRecorder { - - private final SubscriptionName subscriptionName; - private final LongAdder operationsCounter = new LongAdder(); - - ZookeeperSubscriptionLoadRecorder(SubscriptionName subscriptionName) { - this.subscriptionName = subscriptionName; - } - - @Override - public void initialize() { - operationsCounter.reset(); - subscriptionLoadRecorders.add(this); - } - - @Override - public void recordSingleOperation() { - operationsCounter.increment(); - } - - @Override - public void shutdown() { - operationsCounter.reset(); - subscriptionLoadRecorders.remove(this); - } - - SubscriptionName getSubscriptionName() { - return subscriptionName; - } - - SubscriptionLoad calculate(long elapsedSeconds) { - double operationsPerSecond = (double) operationsCounter.sumThenReset() / elapsedSeconds; - return new SubscriptionLoad(operationsPerSecond); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ZookeeperSubscriptionLoadRecorder that = (ZookeeperSubscriptionLoadRecorder) o; - return subscriptionName.equals(that.subscriptionName); - } - - @Override - public int hashCode() { - return Objects.hash(subscriptionName); - } + public int hashCode() { + return Objects.hash(subscriptionName); } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ZookeeperSubscriptionProfileRegistry.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ZookeeperSubscriptionProfileRegistry.java index f48d70bfe1..f795bc7f9a 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ZookeeperSubscriptionProfileRegistry.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ZookeeperSubscriptionProfileRegistry.java @@ -1,5 +1,8 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload.weighted; +import static pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths.CONSUMERS_WORKLOAD_PATH; +import static pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths.SUBSCRIPTION_PROFILES_PATH; + import org.apache.curator.framework.CuratorFramework; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; @@ -8,65 +11,71 @@ import pl.allegro.tech.hermes.consumers.subscription.id.SubscriptionIds; import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; -import static pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths.CONSUMERS_WORKLOAD_PATH; -import static pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths.SUBSCRIPTION_PROFILES_PATH; - public class ZookeeperSubscriptionProfileRegistry implements SubscriptionProfileRegistry { - private static final Logger logger = LoggerFactory.getLogger(ZookeeperSubscriptionProfileRegistry.class); + private static final Logger logger = + LoggerFactory.getLogger(ZookeeperSubscriptionProfileRegistry.class); - private final CuratorFramework curator; - private final SubscriptionProfilesEncoder encoder; - private final SubscriptionProfilesDecoder decoder; - private final String profilesPath; + private final CuratorFramework curator; + private final SubscriptionProfilesEncoder encoder; + private final SubscriptionProfilesDecoder decoder; + private final String profilesPath; - public ZookeeperSubscriptionProfileRegistry(CuratorFramework curator, - SubscriptionIds subscriptionIds, - ZookeeperPaths zookeeperPaths, - String clusterName, - int subscriptionProfilesEncoderBufferSizeBytes) { - this.curator = curator; - this.encoder = new SubscriptionProfilesEncoder(subscriptionIds, subscriptionProfilesEncoderBufferSizeBytes); - this.decoder = new SubscriptionProfilesDecoder(subscriptionIds); - this.profilesPath = - zookeeperPaths.join(zookeeperPaths.basePath(), CONSUMERS_WORKLOAD_PATH, clusterName, SUBSCRIPTION_PROFILES_PATH); - } + public ZookeeperSubscriptionProfileRegistry( + CuratorFramework curator, + SubscriptionIds subscriptionIds, + ZookeeperPaths zookeeperPaths, + String clusterName, + int subscriptionProfilesEncoderBufferSizeBytes) { + this.curator = curator; + this.encoder = + new SubscriptionProfilesEncoder( + subscriptionIds, subscriptionProfilesEncoderBufferSizeBytes); + this.decoder = new SubscriptionProfilesDecoder(subscriptionIds); + this.profilesPath = + zookeeperPaths.join( + zookeeperPaths.basePath(), + CONSUMERS_WORKLOAD_PATH, + clusterName, + SUBSCRIPTION_PROFILES_PATH); + } - @Override - public SubscriptionProfiles fetch() { - try { - if (curator.checkExists().forPath(profilesPath) != null) { - byte[] bytes = curator.getData().forPath(profilesPath); - return decoder.decode(bytes); - } - } catch (Exception e) { - logger.warn("Could not read node data on path " + profilesPath, e); - } - return SubscriptionProfiles.EMPTY; + @Override + public SubscriptionProfiles fetch() { + try { + if (curator.checkExists().forPath(profilesPath) != null) { + byte[] bytes = curator.getData().forPath(profilesPath); + return decoder.decode(bytes); + } + } catch (Exception e) { + logger.warn("Could not read node data on path " + profilesPath, e); } + return SubscriptionProfiles.EMPTY; + } - @Override - public void persist(SubscriptionProfiles profiles) { - byte[] encoded = encoder.encode(profiles); - try { - updateOrCreate(encoded); - } catch (Exception e) { - logger.error("An error while saving subscription profiles", e); - } + @Override + public void persist(SubscriptionProfiles profiles) { + byte[] encoded = encoder.encode(profiles); + try { + updateOrCreate(encoded); + } catch (Exception e) { + logger.error("An error while saving subscription profiles", e); } + } - private void updateOrCreate(byte[] encoded) throws Exception { - try { - curator.setData().forPath(profilesPath, encoded); - } catch (KeeperException.NoNodeException e) { - try { - curator.create() - .creatingParentContainersIfNeeded() - .withMode(CreateMode.PERSISTENT) - .forPath(profilesPath, encoded); - } catch (KeeperException.NodeExistsException ex) { - // ignore - } - } + private void updateOrCreate(byte[] encoded) throws Exception { + try { + curator.setData().forPath(profilesPath, encoded); + } catch (KeeperException.NoNodeException e) { + try { + curator + .create() + .creatingParentContainersIfNeeded() + .withMode(CreateMode.PERSISTENT) + .forPath(profilesPath, encoded); + } catch (KeeperException.NodeExistsException ex) { + // ignore + } } + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/uri/InvalidHostException.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/uri/InvalidHostException.java index 8abbea1d78..af19729441 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/uri/InvalidHostException.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/uri/InvalidHostException.java @@ -2,8 +2,7 @@ public class InvalidHostException extends RuntimeException { - public InvalidHostException(String message) { - super(message); - } - + public InvalidHostException(String message) { + super(message); + } } diff --git a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/uri/UriUtils.java b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/uri/UriUtils.java index 287068cd3f..8e98518e11 100644 --- a/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/uri/UriUtils.java +++ b/hermes-consumers/src/main/java/pl/allegro/tech/hermes/consumers/uri/UriUtils.java @@ -1,77 +1,74 @@ package pl.allegro.tech.hermes.consumers.uri; import com.google.common.base.Splitter; -import org.apache.commons.lang3.StringUtils; - import java.net.URI; import java.util.List; import java.util.Optional; +import org.apache.commons.lang3.StringUtils; public final class UriUtils { - private UriUtils() { - } + private UriUtils() {} - public static String extractHostFromUri(URI uri) { - // we are handling here bug from jdk http://bugs.java.com/view_bug.do?bug_id=6587184 - return Optional - .ofNullable(uri.getHost()) - .orElseThrow(() -> new InvalidHostException("Host name contains invalid chars. Underscore is one of them.")) - .replace("/", ""); - } + public static String extractHostFromUri(URI uri) { + // we are handling here bug from jdk http://bugs.java.com/view_bug.do?bug_id=6587184 + return Optional.ofNullable(uri.getHost()) + .orElseThrow( + () -> + new InvalidHostException( + "Host name contains invalid chars. Underscore is one of them.")) + .replace("/", ""); + } - public static Integer extractPortFromUri(URI uri) { - return uri.getPort() > 0 ? uri.getPort() : null; - } + public static Integer extractPortFromUri(URI uri) { + return uri.getPort() > 0 ? uri.getPort() : null; + } - public static String extractAddressFromUri(URI uri) { - String address = extractHostFromUri(uri); - if (uri.getPort() > 0) { - address += ":" + uri.getPort(); - } - return address; + public static String extractAddressFromUri(URI uri) { + String address = extractHostFromUri(uri); + if (uri.getPort() > 0) { + address += ":" + uri.getPort(); } + return address; + } - public static String extractUserNameFromUri(URI uri) { - if (uri.getRawUserInfo() == null) { - return null; - } - List userInfoParts = splitUserInfo(uri); - if (userInfoParts.isEmpty()) { - return null; - } - return userInfoParts.get(0); + public static String extractUserNameFromUri(URI uri) { + if (uri.getRawUserInfo() == null) { + return null; } - - public static String extractPasswordFromUri(URI uri) { - if (uri.getRawUserInfo() == null) { - return null; - } - List userInfoParts = splitUserInfo(uri); - if (userInfoParts.size() <= 1) { - return null; - } - - return userInfoParts.get(1); + List userInfoParts = splitUserInfo(uri); + if (userInfoParts.isEmpty()) { + return null; } + return userInfoParts.get(0); + } - public static String extractContextFromUri(URI uri) { - return StringUtils.substringAfter(uri.toString(), uri.getAuthority()); + public static String extractPasswordFromUri(URI uri) { + if (uri.getRawUserInfo() == null) { + return null; } - - public static String extractDestinationTopicFromUri(URI uri) { - return uri.getPath().replace("/", ""); + List userInfoParts = splitUserInfo(uri); + if (userInfoParts.size() <= 1) { + return null; } - public static URI appendContext(URI uri, String context) { - return URI.create(StringUtils.removeEnd(uri.toString(), "/") - + "/" - + StringUtils.removeStart(context, "/") - ); - } + return userInfoParts.get(1); + } - private static List splitUserInfo(URI uri) { - return Splitter.on(":").splitToList(uri.getRawUserInfo()); - } + public static String extractContextFromUri(URI uri) { + return StringUtils.substringAfter(uri.toString(), uri.getAuthority()); + } + + public static String extractDestinationTopicFromUri(URI uri) { + return uri.getPath().replace("/", ""); + } + + public static URI appendContext(URI uri, String context) { + return URI.create( + StringUtils.removeEnd(uri.toString(), "/") + "/" + StringUtils.removeStart(context, "/")); + } + private static List splitUserInfo(URI uri) { + return Splitter.on(":").splitToList(uri.getRawUserInfo()); + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/ConsumerMessageSenderTest.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/ConsumerMessageSenderTest.java index 67791d41d5..df2cfed1cb 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/ConsumerMessageSenderTest.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/ConsumerMessageSenderTest.java @@ -1,5 +1,24 @@ package pl.allegro.tech.hermes.consumers.consumer; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; +import static pl.allegro.tech.hermes.api.SubscriptionOAuthPolicy.passwordGrantOAuthPolicy; +import static pl.allegro.tech.hermes.api.SubscriptionPolicy.Builder.subscriptionPolicy; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; + +import java.nio.charset.StandardCharsets; +import java.time.Clock; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -26,571 +45,569 @@ import pl.allegro.tech.hermes.metrics.HermesTimerContext; import pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder; -import java.nio.charset.StandardCharsets; -import java.time.Clock; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executors; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.timeout; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; -import static org.mockito.Mockito.when; -import static pl.allegro.tech.hermes.api.SubscriptionOAuthPolicy.passwordGrantOAuthPolicy; -import static pl.allegro.tech.hermes.api.SubscriptionPolicy.Builder.subscriptionPolicy; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; - @RunWith(MockitoJUnitRunner.class) public class ConsumerMessageSenderTest { - public static final int ASYNC_TIMEOUT_MS = 2000; - private final Subscription subscription = subscriptionWithTtl(10); + public static final int ASYNC_TIMEOUT_MS = 2000; + private final Subscription subscription = subscriptionWithTtl(10); + + private final Subscription subscriptionWith4xxRetry = subscriptionWithTtlAndClientErrorRetry(10); + + @Mock private MessageSender messageSender; + + @Mock private MessageSenderFactory messageSenderFactory; + + @Mock private SuccessHandler successHandler; + + @Mock private ErrorHandler errorHandler; + + @Mock private SerialConsumerRateLimiter rateLimiter; + + @Mock private PendingOffsets pendingOffsets; - private final Subscription subscriptionWith4xxRetry = subscriptionWithTtlAndClientErrorRetry(10); + @Mock private HermesTimer consumerLatencyTimer; - @Mock - private MessageSender messageSender; + @Mock private HermesCounter retries; - @Mock - private MessageSenderFactory messageSenderFactory; + @Mock private HermesTimer rateLimiterAcquireTimer; - @Mock - private SuccessHandler successHandler; + @Mock private HermesTimerContext consumerLatencyTimerContext; - @Mock - private ErrorHandler errorHandler; + @Mock private HermesTimerContext rateLimiterAcquireTimerContext; + + @Mock private HermesCounter failedMeter; - @Mock - private SerialConsumerRateLimiter rateLimiter; + @Mock private HermesCounter errors; + + private final ConsumerProfiler profiler = new NoOpConsumerProfiler(); - @Mock - private PendingOffsets pendingOffsets; - - @Mock - private HermesTimer consumerLatencyTimer; - - @Mock - private HermesCounter retries; - - @Mock - private HermesTimer rateLimiterAcquireTimer; - - @Mock - private HermesTimerContext consumerLatencyTimerContext; - - @Mock - private HermesTimerContext rateLimiterAcquireTimerContext; - - @Mock - private HermesCounter failedMeter; - - @Mock - private HermesCounter errors; - - private final ConsumerProfiler profiler = new NoOpConsumerProfiler(); - - private ConsumerMessageSender sender; - - @Mock - private SubscriptionMetrics subscriptionMetrics; - - @Mock - private MetricsFacade metricsFacade; - - @Before - public void setUp() { - when(metricsFacade.subscriptions()).thenReturn(subscriptionMetrics); - setUpMetrics(subscription); - setUpMetrics(subscriptionWith4xxRetry); - sender = consumerMessageSender(subscription); - } - - private void setUpMetrics(Subscription subscription) { - when(metricsFacade.subscriptions().latency(subscription.getQualifiedName())).thenReturn(consumerLatencyTimer); - when(metricsFacade.subscriptions().rateLimiterAcquire(subscription.getQualifiedName())).thenReturn(rateLimiterAcquireTimer); - when(metricsFacade.subscriptions().otherErrorsCounter(subscription.getQualifiedName())).thenReturn(errors); - when(consumerLatencyTimer.time()).thenReturn(consumerLatencyTimerContext); - when(rateLimiterAcquireTimer.time()).thenReturn(rateLimiterAcquireTimerContext); - when(metricsFacade.subscriptions().failuresCounter(subscription.getQualifiedName())).thenReturn(failedMeter); - when(metricsFacade.subscriptions().retries(subscription.getQualifiedName())).thenReturn(retries); - } - - @Test - public void shouldHandleSuccessfulSending() { - // given - Message message = message(); - when(messageSender.send(message)).thenReturn(success()); - - // when - sender.sendAsync(message, profiler); - verify(successHandler, timeout(1000)).handleSuccess(eq(message), eq(subscription), any(MessageSendingResult.class)); - - // then - verify(pendingOffsets).markAsProcessed(any(SubscriptionPartitionOffset.class)); - verifyLatencyTimersCountedTimes(subscription, 1, 1); - verifyRateLimiterAcquireTimersCountedTimes(subscription, 1, 1); - verifyNoInteractions(errorHandler); - verifyNoInteractions(failedMeter); - verifyRateLimiterAcquired(); - verifyNoInteractions(retries); - } - - @Test - public void shouldKeepTryingToSendMessageFailedSending() { - // given - Message message = message(); - doReturn(failure()).doReturn(failure()).doReturn(success()).when(messageSender).send(message); - - // when - sender.sendAsync(message, profiler); - verify(successHandler, timeout(1000)).handleSuccess(eq(message), eq(subscription), any(MessageSendingResult.class)); - - // then - verify(pendingOffsets).markAsProcessed(any(SubscriptionPartitionOffset.class)); - verifyLatencyTimersCountedTimes(subscription, 3, 3); - verifyRateLimiterAcquireTimersCountedTimes(subscription, 3, 3); - verifyErrorHandlerHandleFailed(message, subscription, 2); - verifyRateLimiterAcquired(3); - verifyRetryCounted(2); - } - - @Test - public void shouldDiscardMessageWhenTTLIsExceeded() { - // given - Message message = messageWithTimestamp(System.currentTimeMillis() - 11000); - doReturn(failure()).when(messageSender).send(message); - - // when - sender.sendAsync(message, profiler); - - // then - verify(errorHandler, timeout(1000)).handleDiscarded(eq(message), eq(subscription), any(MessageSendingResult.class)); - verify(pendingOffsets).markAsProcessed(any(SubscriptionPartitionOffset.class)); - verifyNoInteractions(successHandler); - verifyLatencyTimersCountedTimes(subscription, 1, 1); - verifyRateLimiterAcquireTimersCountedTimes(subscription, 1, 1); - verifyRateLimiterAcquired(); - verifyNoInteractions(retries); - } - - @Test - public void shouldNotKeepTryingToSendMessageFailedWithStatusCode4xx() { - // given - Message message = message(); - doReturn(failure(403)).doReturn(success()).when(messageSender).send(message); - - // when - sender.sendAsync(message, profiler); - - // then - verify(errorHandler, timeout(1000)).handleDiscarded(eq(message), eq(subscription), any(MessageSendingResult.class)); - verify(pendingOffsets).markAsProcessed(any(SubscriptionPartitionOffset.class)); - verifyNoInteractions(successHandler); - verifyLatencyTimersCountedTimes(subscription, 1, 1); - verifyRateLimiterAcquireTimersCountedTimes(subscription, 1, 1); - verifyRateLimiterAcquired(); - verifyNoInteractions(retries); - } - - @Test - public void shouldKeepTryingToSendMessageFailedWithStatusCode4xxForSubscriptionWith4xxRetry() { - // given - final int expectedNumbersOfFailures = 3; - ConsumerMessageSender sender = consumerMessageSender(subscriptionWith4xxRetry); - Message message = message(); - doReturn(failure(403)).doReturn(failure(403)).doReturn(failure(403)).doReturn(success()).when(messageSender).send(message); - - // when - sender.sendAsync(message, profiler); - verify(successHandler, timeout(1000)).handleSuccess(eq(message), eq(subscriptionWith4xxRetry), any(MessageSendingResult.class)); - - // then - verify(pendingOffsets).markAsProcessed(any(SubscriptionPartitionOffset.class)); - verify(errorHandler, - timeout(1000).times(expectedNumbersOfFailures)).handleFailed(eq(message), - eq(subscriptionWith4xxRetry), - any(MessageSendingResult.class) - ); - verifyRateLimiterAcquired(expectedNumbersOfFailures + 1); - verifyRetryCounted(expectedNumbersOfFailures); - } - - @Test - public void shouldRetryOn401UnauthorizedForOAuthSecuredSubscription() { - // given - final int expectedNumbersOfFailures = 2; - Subscription subscription = subscriptionWithout4xxRetryAndWithOAuthPolicy(); - setUpMetrics(subscription); - ConsumerMessageSender sender = consumerMessageSender(subscription); - Message message = message(); - doReturn(failure(401)).doReturn(failure(401)).doReturn(success()).when(messageSender).send(message); - - // when - sender.sendAsync(message, profiler); - - // then - verifyErrorHandlerHandleFailed(message, subscription, expectedNumbersOfFailures); - verify(successHandler, timeout(1000)).handleSuccess(eq(message), eq(subscription), any(MessageSendingResult.class)); - verifyRetryCounted(expectedNumbersOfFailures); - verifyRateLimiterAcquired(expectedNumbersOfFailures + 1); - } - - @Test - public void shouldBackoffRetriesWhenEndpointFails() throws InterruptedException { - // given - final int executionTime = 100; - final int senderBackoffTime = 50; - final int expectedNumberOfFailures = 1 + executionTime / senderBackoffTime; - Subscription subscriptionWithBackoff = subscriptionWithBackoff(senderBackoffTime); - setUpMetrics(subscriptionWithBackoff); - - sender = consumerMessageSender(subscriptionWithBackoff); - Message message = message(); - doReturn(failure(500)).when(messageSender).send(message); - - //when - sender.sendAsync(message, profiler); - - //then - Thread.sleep(executionTime); - verifyErrorHandlerHandleFailed(message, subscriptionWithBackoff, expectedNumberOfFailures); - verifyRateLimiterAcquired(expectedNumberOfFailures); - verifyRetryCounted(expectedNumberOfFailures); - } - - @Test - public void shouldNotRetryOnRetryAfterAboveTtl() { - // given - int retrySeconds = subscription.getSerialSubscriptionPolicy().getMessageTtl(); - Message message = messageWithTimestamp(System.currentTimeMillis() - 1); - doReturn(backoff(retrySeconds + 1)).doReturn(success()).when(messageSender).send(message); - - // when - sender.sendAsync(message, profiler); - - // then - verify(errorHandler, timeout(1000)).handleDiscarded(eq(message), eq(subscription), any(MessageSendingResult.class)); - verify(pendingOffsets).markAsProcessed(any(SubscriptionPartitionOffset.class)); - verifyNoInteractions(successHandler); - verifyLatencyTimersCountedTimes(subscription, 1, 1); - verifyRateLimiterAcquireTimersCountedTimes(subscription, 1, 1); - verifyRateLimiterAcquired(); - verifyNoInteractions(retries); - } - - @Test - public void shouldDeliverToModifiedEndpoint() { - // given - Message message = message(); - Subscription subscriptionWithModfiedEndpoint = subscriptionWithEndpoint("http://somewhere:9876"); - MessageSender otherMessageSender = mock(MessageSender.class); - - when(messageSenderFactory.create(eq(subscriptionWithModfiedEndpoint), any(ResilientMessageSender.class))) - .thenReturn(otherMessageSender); - when(otherMessageSender.send(message)).thenReturn(success()); - - // when - sender.updateSubscription(subscriptionWithModfiedEndpoint); - sender.sendAsync(message, profiler); - - // then - verify(otherMessageSender, timeout(1000)).send(message); - verifyRateLimiterAcquired(); - verifyNoInteractions(retries); - } - - @Test - public void shouldDeliverToNewSenderAfterModifiedTimeout() { - // given - Message message = message(); - Subscription subscriptionWithModifiedTimeout = subscriptionWithRequestTimeout(2000); - MessageSender otherMessageSender = mock(MessageSender.class); - - when(messageSenderFactory.create(eq(subscriptionWithModifiedTimeout), any(ResilientMessageSender.class))) - .thenReturn(otherMessageSender); - when(otherMessageSender.send(message)).thenReturn(success()); - - // when - sender.updateSubscription(subscriptionWithModifiedTimeout); - sender.sendAsync(message, profiler); - - // then - verify(otherMessageSender, timeout(1000)).send(message); - verifyRateLimiterAcquired(); - verifyNoInteractions(retries); - } - - @Test - public void shouldDelaySendingMessageForHalfSecond() { - // given - Subscription subscription = subscriptionBuilderWithTestValues() - .withSubscriptionPolicy(subscriptionPolicy().applyDefaults() - .withSendingDelay(500) - .build()) - .build(); - setUpMetrics(subscription); - - Message message = message(); - when(messageSender.send(message)).thenReturn(success()); - ConsumerMessageSender sender = consumerMessageSender(subscription); - - // when - long sendingStartTime = System.currentTimeMillis(); - sender.sendAsync(message, profiler); - verify(successHandler, timeout(1000)).handleSuccess(eq(message), eq(subscription), any(MessageSendingResult.class)); - - // then - long sendingTime = System.currentTimeMillis() - sendingStartTime; - assertThat(sendingTime).isGreaterThanOrEqualTo(500); - verifyRateLimiterAcquired(); - verifyNoInteractions(retries); - } - - @Test - public void shouldCalculateSendingDelayBasingOnPublishingTimestamp() { - // given - Subscription subscription = subscriptionBuilderWithTestValues() - .withSubscriptionPolicy(subscriptionPolicy().applyDefaults() - .withSendingDelay(2000) - .build()) - .build(); - setUpMetrics(subscription); - - Message message = messageWithTimestamp(System.currentTimeMillis() - 1800); - when(messageSender.send(message)).thenReturn(success()); - ConsumerMessageSender sender = consumerMessageSender(subscription); - - // when - long sendingStartTime = System.currentTimeMillis(); - sender.sendAsync(message, profiler); - verify(successHandler, timeout(500)).handleSuccess(eq(message), eq(subscription), any(MessageSendingResult.class)); - - // then - long sendingTime = System.currentTimeMillis() - sendingStartTime; - assertThat(sendingTime).isLessThan(300); - verifyRateLimiterAcquired(); - verifyNoInteractions(retries); - } - - @Test - public void shouldIncreaseRetryBackoffExponentially() throws InterruptedException { - // given - final int expectedNumberOfFailures = 2; - final int backoff = 500; - final double multiplier = 2; - Subscription subscription = subscriptionWithExponentialRetryBackoff(backoff, multiplier); - setUpMetrics(subscription); - Message message = message(); - doReturn(failure()).doReturn(failure()).doReturn(success()).when(messageSender).send(message); - ConsumerMessageSender sender = consumerMessageSender(subscription); - - // when - sender.sendAsync(message, profiler); - Thread.sleep(backoff + (long) multiplier * backoff - 100); - - // then - verifyNoInteractions(successHandler); - verifyRateLimiterAcquired(expectedNumberOfFailures); - verifyRetryCounted(expectedNumberOfFailures); - } - - @Test - public void shouldIgnoreExponentialRetryBackoffWithRetryAfter() { - // given - final int expectedNumberOfRetries = 2; - final int retrySeconds = 1; - final int backoff = 5000; - final double multiplier = 3; - Subscription subscription = subscriptionWithExponentialRetryBackoff(backoff, multiplier); - setUpMetrics(subscription); - Message message = message(); - doReturn(backoff(retrySeconds)).doReturn(backoff(retrySeconds)).doReturn(success()).when(messageSender).send(message); - ConsumerMessageSender sender = consumerMessageSender(subscription); - - // when - sender.sendAsync(message, profiler); - - //then - verify(successHandler, timeout(retrySeconds * 1000 * 2 + 500)) - .handleSuccess(eq(message), eq(subscription), any(MessageSendingResult.class)); - verifyRateLimiterAcquired(expectedNumberOfRetries + 1); - verifyRetryCounted(expectedNumberOfRetries); - } - - @Test - public void shouldIgnoreExponentialRetryBackoffAfterExceededTtl() throws InterruptedException { - final int backoff = 1000; - final double multiplier = 2; - final int ttl = 2; - Subscription subscription = subscriptionWithExponentialRetryBackoff(backoff, multiplier, ttl); - setUpMetrics(subscription); - Message message = message(); - doReturn(failure()).doReturn(failure()).doReturn(success()).when(messageSender).send(message); - ConsumerMessageSender sender = consumerMessageSender(subscription); - - // when - sender.sendAsync(message, profiler); - Thread.sleep(backoff + (long) multiplier * backoff + 1000); - - //then - verifyNoInteractions(successHandler); - verifyRateLimiterAcquired(2); - verifyRetryCounted(); - } - - private ConsumerMessageSender consumerMessageSender(Subscription subscription) { - when(messageSenderFactory.create(eq(subscription), any(ResilientMessageSender.class))).thenReturn(messageSender); - ConsumerMessageSender sender = new ConsumerMessageSender( - subscription, - messageSenderFactory, - List.of(successHandler), - List.of(errorHandler), - rateLimiter, - Executors.newSingleThreadExecutor(), - pendingOffsets, - metricsFacade, - ASYNC_TIMEOUT_MS, - new FutureAsyncTimeout(Executors.newSingleThreadScheduledExecutor()), - Clock.systemUTC(), - new NoOpConsumerNodeLoadRegistry().register(subscription.getQualifiedName()) - ); - sender.initialize(); - - return sender; - } - - private void verifyErrorHandlerHandleFailed(Message message, Subscription subscription, int times) { - verifyErrorHandlerHandleFailed(message, subscription, times, 1000); - } - - private void verifyErrorHandlerHandleFailed(Message message, Subscription subscription, int times, int timeout) { - verify(errorHandler, timeout(timeout).times(times)).handleFailed(eq(message), eq(subscription), any(MessageSendingResult.class)); - } - - private void verifyLatencyTimersCountedTimes(Subscription subscription, int timeCount, int closeCount) { - verify(metricsFacade.subscriptions(), times(1)).latency(subscription.getQualifiedName()); - verify(consumerLatencyTimer, times(timeCount)).time(); - verify(consumerLatencyTimerContext, times(closeCount)).close(); - } - - private void verifyRateLimiterAcquireTimersCountedTimes(Subscription subscription, int timeCount, int closeCount) { - verify(metricsFacade.subscriptions(), times(1)).rateLimiterAcquire(subscription.getQualifiedName()); - verify(rateLimiterAcquireTimer, times(timeCount)).time(); - verify(rateLimiterAcquireTimerContext, times(closeCount)).close(); - } - - private Subscription subscriptionWithTtl(int ttl) { - return subscriptionBuilderWithTestValues() - .withSubscriptionPolicy(subscriptionPolicy().applyDefaults() - .withMessageTtl(ttl) - .build()) - .build(); - } - - private Subscription subscriptionWithTtlAndClientErrorRetry(int ttl) { - return subscriptionBuilderWithTestValues() - .withSubscriptionPolicy(subscriptionPolicy().applyDefaults() - .withMessageTtl(ttl) - .withClientErrorRetry() - .build()) - .build(); - } - - private Subscription subscriptionWithBackoff(int backoff) { - return subscriptionBuilderWithTestValues() - .withSubscriptionPolicy(subscriptionPolicy().applyDefaults() - .withMessageBackoff(backoff) - .build()) - .build(); - } - - private Subscription subscriptionWithout4xxRetryAndWithOAuthPolicy() { - return subscriptionBuilderWithTestValues() - .withOAuthPolicy(passwordGrantOAuthPolicy("myOAuthProvider") - .withUsername("user1") - .withPassword("abc123") - .build()) - .build(); - } - - private Subscription subscriptionWithEndpoint(String endpoint) { - return subscriptionBuilderWithTestValues().withEndpoint(endpoint).build(); - } - - private Subscription subscriptionWithRequestTimeout(int timeout) { - return subscriptionBuilderWithTestValues().withRequestTimeout(timeout).build(); - } - - private Subscription subscriptionWithExponentialRetryBackoff(int messageBackoff, double backoffMultiplier) { - return subscriptionWithExponentialRetryBackoff(messageBackoff, backoffMultiplier, 3600); - } - - private Subscription subscriptionWithExponentialRetryBackoff(int messageBackoff, double backoffMultiplier, int ttl) { - return subscriptionBuilderWithTestValues() - .withSubscriptionPolicy(subscriptionPolicy().applyDefaults() - .withMessageBackoff(messageBackoff) - .withBackoffMultiplier(backoffMultiplier) - .withMessageTtl(ttl) - .build()) - .build(); - } - - private SubscriptionBuilder subscriptionBuilderWithTestValues() { - return subscription("group.topic", "subscription"); - } - - private RuntimeException exception() { - return new RuntimeException("problem"); - } - - private CompletableFuture success() { - return CompletableFuture.completedFuture(MessageSendingResult.succeededResult()); - } - - private CompletableFuture failure() { - return CompletableFuture.completedFuture(MessageSendingResult.failedResult(exception())); - } - - private CompletableFuture failure(int statusCode) { - return CompletableFuture.completedFuture(MessageSendingResult.failedResult(statusCode)); - } - - private CompletableFuture backoff(int seconds) { - return CompletableFuture.completedFuture(MessageSendingResult.retryAfter(seconds)); - } - - private void verifyRateLimiterAcquired() { - verifyRateLimiterAcquired(1); - } - - private void verifyRateLimiterAcquired(int times) { - verify(rateLimiter, times(times)).acquire(); - } - - private void verifyRetryCounted() { - verifyRetryCounted(1); - } - - private void verifyRetryCounted(int times) { - verify(retries, times(times)).increment(); - } - - private Message message() { - return messageWithTimestamp(System.currentTimeMillis()); - } - - private Message messageWithTimestamp(long timestamp) { - return MessageBuilder - .withTestMessage() - .withContent("{\"username\":\"ala\"}", StandardCharsets.UTF_8) - .withReadingTimestamp(timestamp) - .withPublishingTimestamp(timestamp) - .build(); - } + private ConsumerMessageSender sender; + + @Mock private SubscriptionMetrics subscriptionMetrics; + + @Mock private MetricsFacade metricsFacade; + + @Before + public void setUp() { + when(metricsFacade.subscriptions()).thenReturn(subscriptionMetrics); + setUpMetrics(subscription); + setUpMetrics(subscriptionWith4xxRetry); + sender = consumerMessageSender(subscription); + } + + private void setUpMetrics(Subscription subscription) { + when(metricsFacade.subscriptions().latency(subscription.getQualifiedName())) + .thenReturn(consumerLatencyTimer); + when(metricsFacade.subscriptions().rateLimiterAcquire(subscription.getQualifiedName())) + .thenReturn(rateLimiterAcquireTimer); + when(metricsFacade.subscriptions().otherErrorsCounter(subscription.getQualifiedName())) + .thenReturn(errors); + when(consumerLatencyTimer.time()).thenReturn(consumerLatencyTimerContext); + when(rateLimiterAcquireTimer.time()).thenReturn(rateLimiterAcquireTimerContext); + when(metricsFacade.subscriptions().failuresCounter(subscription.getQualifiedName())) + .thenReturn(failedMeter); + when(metricsFacade.subscriptions().retries(subscription.getQualifiedName())) + .thenReturn(retries); + } + + @Test + public void shouldHandleSuccessfulSending() { + // given + Message message = message(); + when(messageSender.send(message)).thenReturn(success()); + + // when + sender.sendAsync(message, profiler); + verify(successHandler, timeout(1000)) + .handleSuccess(eq(message), eq(subscription), any(MessageSendingResult.class)); + + // then + verify(pendingOffsets).markAsProcessed(any(SubscriptionPartitionOffset.class)); + verifyLatencyTimersCountedTimes(subscription, 1, 1); + verifyRateLimiterAcquireTimersCountedTimes(subscription, 1, 1); + verifyNoInteractions(errorHandler); + verifyNoInteractions(failedMeter); + verifyRateLimiterAcquired(); + verifyNoInteractions(retries); + } + + @Test + public void shouldKeepTryingToSendMessageFailedSending() { + // given + Message message = message(); + doReturn(failure()).doReturn(failure()).doReturn(success()).when(messageSender).send(message); + + // when + sender.sendAsync(message, profiler); + verify(successHandler, timeout(1000)) + .handleSuccess(eq(message), eq(subscription), any(MessageSendingResult.class)); + + // then + verify(pendingOffsets).markAsProcessed(any(SubscriptionPartitionOffset.class)); + verifyLatencyTimersCountedTimes(subscription, 3, 3); + verifyRateLimiterAcquireTimersCountedTimes(subscription, 3, 3); + verifyErrorHandlerHandleFailed(message, subscription, 2); + verifyRateLimiterAcquired(3); + verifyRetryCounted(2); + } + + @Test + public void shouldDiscardMessageWhenTTLIsExceeded() { + // given + Message message = messageWithTimestamp(System.currentTimeMillis() - 11000); + doReturn(failure()).when(messageSender).send(message); + + // when + sender.sendAsync(message, profiler); + + // then + verify(errorHandler, timeout(1000)) + .handleDiscarded(eq(message), eq(subscription), any(MessageSendingResult.class)); + verify(pendingOffsets).markAsProcessed(any(SubscriptionPartitionOffset.class)); + verifyNoInteractions(successHandler); + verifyLatencyTimersCountedTimes(subscription, 1, 1); + verifyRateLimiterAcquireTimersCountedTimes(subscription, 1, 1); + verifyRateLimiterAcquired(); + verifyNoInteractions(retries); + } + + @Test + public void shouldNotKeepTryingToSendMessageFailedWithStatusCode4xx() { + // given + Message message = message(); + doReturn(failure(403)).doReturn(success()).when(messageSender).send(message); + + // when + sender.sendAsync(message, profiler); + + // then + verify(errorHandler, timeout(1000)) + .handleDiscarded(eq(message), eq(subscription), any(MessageSendingResult.class)); + verify(pendingOffsets).markAsProcessed(any(SubscriptionPartitionOffset.class)); + verifyNoInteractions(successHandler); + verifyLatencyTimersCountedTimes(subscription, 1, 1); + verifyRateLimiterAcquireTimersCountedTimes(subscription, 1, 1); + verifyRateLimiterAcquired(); + verifyNoInteractions(retries); + } + + @Test + public void shouldKeepTryingToSendMessageFailedWithStatusCode4xxForSubscriptionWith4xxRetry() { + // given + final int expectedNumbersOfFailures = 3; + ConsumerMessageSender sender = consumerMessageSender(subscriptionWith4xxRetry); + Message message = message(); + doReturn(failure(403)) + .doReturn(failure(403)) + .doReturn(failure(403)) + .doReturn(success()) + .when(messageSender) + .send(message); + + // when + sender.sendAsync(message, profiler); + verify(successHandler, timeout(1000)) + .handleSuccess(eq(message), eq(subscriptionWith4xxRetry), any(MessageSendingResult.class)); + + // then + verify(pendingOffsets).markAsProcessed(any(SubscriptionPartitionOffset.class)); + verify(errorHandler, timeout(1000).times(expectedNumbersOfFailures)) + .handleFailed(eq(message), eq(subscriptionWith4xxRetry), any(MessageSendingResult.class)); + verifyRateLimiterAcquired(expectedNumbersOfFailures + 1); + verifyRetryCounted(expectedNumbersOfFailures); + } + + @Test + public void shouldRetryOn401UnauthorizedForOAuthSecuredSubscription() { + // given + final int expectedNumbersOfFailures = 2; + Subscription subscription = subscriptionWithout4xxRetryAndWithOAuthPolicy(); + setUpMetrics(subscription); + ConsumerMessageSender sender = consumerMessageSender(subscription); + Message message = message(); + doReturn(failure(401)) + .doReturn(failure(401)) + .doReturn(success()) + .when(messageSender) + .send(message); + + // when + sender.sendAsync(message, profiler); + + // then + verifyErrorHandlerHandleFailed(message, subscription, expectedNumbersOfFailures); + verify(successHandler, timeout(1000)) + .handleSuccess(eq(message), eq(subscription), any(MessageSendingResult.class)); + verifyRetryCounted(expectedNumbersOfFailures); + verifyRateLimiterAcquired(expectedNumbersOfFailures + 1); + } + + @Test + public void shouldBackoffRetriesWhenEndpointFails() throws InterruptedException { + // given + final int executionTime = 100; + final int senderBackoffTime = 50; + final int expectedNumberOfFailures = 1 + executionTime / senderBackoffTime; + Subscription subscriptionWithBackoff = subscriptionWithBackoff(senderBackoffTime); + setUpMetrics(subscriptionWithBackoff); + + sender = consumerMessageSender(subscriptionWithBackoff); + Message message = message(); + doReturn(failure(500)).when(messageSender).send(message); + + // when + sender.sendAsync(message, profiler); + + // then + Thread.sleep(executionTime); + verifyErrorHandlerHandleFailed(message, subscriptionWithBackoff, expectedNumberOfFailures); + verifyRateLimiterAcquired(expectedNumberOfFailures); + verifyRetryCounted(expectedNumberOfFailures); + } + + @Test + public void shouldNotRetryOnRetryAfterAboveTtl() { + // given + int retrySeconds = subscription.getSerialSubscriptionPolicy().getMessageTtl(); + Message message = messageWithTimestamp(System.currentTimeMillis() - 1); + doReturn(backoff(retrySeconds + 1)).doReturn(success()).when(messageSender).send(message); + + // when + sender.sendAsync(message, profiler); + + // then + verify(errorHandler, timeout(1000)) + .handleDiscarded(eq(message), eq(subscription), any(MessageSendingResult.class)); + verify(pendingOffsets).markAsProcessed(any(SubscriptionPartitionOffset.class)); + verifyNoInteractions(successHandler); + verifyLatencyTimersCountedTimes(subscription, 1, 1); + verifyRateLimiterAcquireTimersCountedTimes(subscription, 1, 1); + verifyRateLimiterAcquired(); + verifyNoInteractions(retries); + } + + @Test + public void shouldDeliverToModifiedEndpoint() { + // given + Message message = message(); + Subscription subscriptionWithModfiedEndpoint = + subscriptionWithEndpoint("http://somewhere:9876"); + MessageSender otherMessageSender = mock(MessageSender.class); + + when(messageSenderFactory.create( + eq(subscriptionWithModfiedEndpoint), any(ResilientMessageSender.class))) + .thenReturn(otherMessageSender); + when(otherMessageSender.send(message)).thenReturn(success()); + + // when + sender.updateSubscription(subscriptionWithModfiedEndpoint); + sender.sendAsync(message, profiler); + + // then + verify(otherMessageSender, timeout(1000)).send(message); + verifyRateLimiterAcquired(); + verifyNoInteractions(retries); + } + + @Test + public void shouldDeliverToNewSenderAfterModifiedTimeout() { + // given + Message message = message(); + Subscription subscriptionWithModifiedTimeout = subscriptionWithRequestTimeout(2000); + MessageSender otherMessageSender = mock(MessageSender.class); + + when(messageSenderFactory.create( + eq(subscriptionWithModifiedTimeout), any(ResilientMessageSender.class))) + .thenReturn(otherMessageSender); + when(otherMessageSender.send(message)).thenReturn(success()); + + // when + sender.updateSubscription(subscriptionWithModifiedTimeout); + sender.sendAsync(message, profiler); + + // then + verify(otherMessageSender, timeout(1000)).send(message); + verifyRateLimiterAcquired(); + verifyNoInteractions(retries); + } + + @Test + public void shouldDelaySendingMessageForHalfSecond() { + // given + Subscription subscription = + subscriptionBuilderWithTestValues() + .withSubscriptionPolicy( + subscriptionPolicy().applyDefaults().withSendingDelay(500).build()) + .build(); + setUpMetrics(subscription); + + Message message = message(); + when(messageSender.send(message)).thenReturn(success()); + ConsumerMessageSender sender = consumerMessageSender(subscription); + + // when + long sendingStartTime = System.currentTimeMillis(); + sender.sendAsync(message, profiler); + verify(successHandler, timeout(1000)) + .handleSuccess(eq(message), eq(subscription), any(MessageSendingResult.class)); + + // then + long sendingTime = System.currentTimeMillis() - sendingStartTime; + assertThat(sendingTime).isGreaterThanOrEqualTo(500); + verifyRateLimiterAcquired(); + verifyNoInteractions(retries); + } + + @Test + public void shouldCalculateSendingDelayBasingOnPublishingTimestamp() { + // given + Subscription subscription = + subscriptionBuilderWithTestValues() + .withSubscriptionPolicy( + subscriptionPolicy().applyDefaults().withSendingDelay(2000).build()) + .build(); + setUpMetrics(subscription); + + Message message = messageWithTimestamp(System.currentTimeMillis() - 1800); + when(messageSender.send(message)).thenReturn(success()); + ConsumerMessageSender sender = consumerMessageSender(subscription); + + // when + long sendingStartTime = System.currentTimeMillis(); + sender.sendAsync(message, profiler); + verify(successHandler, timeout(500)) + .handleSuccess(eq(message), eq(subscription), any(MessageSendingResult.class)); + + // then + long sendingTime = System.currentTimeMillis() - sendingStartTime; + assertThat(sendingTime).isLessThan(300); + verifyRateLimiterAcquired(); + verifyNoInteractions(retries); + } + + @Test + public void shouldIncreaseRetryBackoffExponentially() throws InterruptedException { + // given + final int expectedNumberOfFailures = 2; + final int backoff = 500; + final double multiplier = 2; + Subscription subscription = subscriptionWithExponentialRetryBackoff(backoff, multiplier); + setUpMetrics(subscription); + Message message = message(); + doReturn(failure()).doReturn(failure()).doReturn(success()).when(messageSender).send(message); + ConsumerMessageSender sender = consumerMessageSender(subscription); + + // when + sender.sendAsync(message, profiler); + Thread.sleep(backoff + (long) multiplier * backoff - 100); + + // then + verifyNoInteractions(successHandler); + verifyRateLimiterAcquired(expectedNumberOfFailures); + verifyRetryCounted(expectedNumberOfFailures); + } + + @Test + public void shouldIgnoreExponentialRetryBackoffWithRetryAfter() { + // given + final int expectedNumberOfRetries = 2; + final int retrySeconds = 1; + final int backoff = 5000; + final double multiplier = 3; + Subscription subscription = subscriptionWithExponentialRetryBackoff(backoff, multiplier); + setUpMetrics(subscription); + Message message = message(); + doReturn(backoff(retrySeconds)) + .doReturn(backoff(retrySeconds)) + .doReturn(success()) + .when(messageSender) + .send(message); + ConsumerMessageSender sender = consumerMessageSender(subscription); + + // when + sender.sendAsync(message, profiler); + + // then + verify(successHandler, timeout(retrySeconds * 1000 * 2 + 500)) + .handleSuccess(eq(message), eq(subscription), any(MessageSendingResult.class)); + verifyRateLimiterAcquired(expectedNumberOfRetries + 1); + verifyRetryCounted(expectedNumberOfRetries); + } + + @Test + public void shouldIgnoreExponentialRetryBackoffAfterExceededTtl() throws InterruptedException { + final int backoff = 1000; + final double multiplier = 2; + final int ttl = 2; + Subscription subscription = subscriptionWithExponentialRetryBackoff(backoff, multiplier, ttl); + setUpMetrics(subscription); + Message message = message(); + doReturn(failure()).doReturn(failure()).doReturn(success()).when(messageSender).send(message); + ConsumerMessageSender sender = consumerMessageSender(subscription); + + // when + sender.sendAsync(message, profiler); + Thread.sleep(backoff + (long) multiplier * backoff + 1000); + + // then + verifyNoInteractions(successHandler); + verifyRateLimiterAcquired(2); + verifyRetryCounted(); + } + + private ConsumerMessageSender consumerMessageSender(Subscription subscription) { + when(messageSenderFactory.create(eq(subscription), any(ResilientMessageSender.class))) + .thenReturn(messageSender); + ConsumerMessageSender sender = + new ConsumerMessageSender( + subscription, + messageSenderFactory, + List.of(successHandler), + List.of(errorHandler), + rateLimiter, + Executors.newSingleThreadExecutor(), + pendingOffsets, + metricsFacade, + ASYNC_TIMEOUT_MS, + new FutureAsyncTimeout(Executors.newSingleThreadScheduledExecutor()), + Clock.systemUTC(), + new NoOpConsumerNodeLoadRegistry().register(subscription.getQualifiedName())); + sender.initialize(); + + return sender; + } + + private void verifyErrorHandlerHandleFailed( + Message message, Subscription subscription, int times) { + verifyErrorHandlerHandleFailed(message, subscription, times, 1000); + } + + private void verifyErrorHandlerHandleFailed( + Message message, Subscription subscription, int times, int timeout) { + verify(errorHandler, timeout(timeout).times(times)) + .handleFailed(eq(message), eq(subscription), any(MessageSendingResult.class)); + } + + private void verifyLatencyTimersCountedTimes( + Subscription subscription, int timeCount, int closeCount) { + verify(metricsFacade.subscriptions(), times(1)).latency(subscription.getQualifiedName()); + verify(consumerLatencyTimer, times(timeCount)).time(); + verify(consumerLatencyTimerContext, times(closeCount)).close(); + } + + private void verifyRateLimiterAcquireTimersCountedTimes( + Subscription subscription, int timeCount, int closeCount) { + verify(metricsFacade.subscriptions(), times(1)) + .rateLimiterAcquire(subscription.getQualifiedName()); + verify(rateLimiterAcquireTimer, times(timeCount)).time(); + verify(rateLimiterAcquireTimerContext, times(closeCount)).close(); + } + + private Subscription subscriptionWithTtl(int ttl) { + return subscriptionBuilderWithTestValues() + .withSubscriptionPolicy(subscriptionPolicy().applyDefaults().withMessageTtl(ttl).build()) + .build(); + } + + private Subscription subscriptionWithTtlAndClientErrorRetry(int ttl) { + return subscriptionBuilderWithTestValues() + .withSubscriptionPolicy( + subscriptionPolicy().applyDefaults().withMessageTtl(ttl).withClientErrorRetry().build()) + .build(); + } + + private Subscription subscriptionWithBackoff(int backoff) { + return subscriptionBuilderWithTestValues() + .withSubscriptionPolicy( + subscriptionPolicy().applyDefaults().withMessageBackoff(backoff).build()) + .build(); + } + + private Subscription subscriptionWithout4xxRetryAndWithOAuthPolicy() { + return subscriptionBuilderWithTestValues() + .withOAuthPolicy( + passwordGrantOAuthPolicy("myOAuthProvider") + .withUsername("user1") + .withPassword("abc123") + .build()) + .build(); + } + + private Subscription subscriptionWithEndpoint(String endpoint) { + return subscriptionBuilderWithTestValues().withEndpoint(endpoint).build(); + } + + private Subscription subscriptionWithRequestTimeout(int timeout) { + return subscriptionBuilderWithTestValues().withRequestTimeout(timeout).build(); + } + + private Subscription subscriptionWithExponentialRetryBackoff( + int messageBackoff, double backoffMultiplier) { + return subscriptionWithExponentialRetryBackoff(messageBackoff, backoffMultiplier, 3600); + } + + private Subscription subscriptionWithExponentialRetryBackoff( + int messageBackoff, double backoffMultiplier, int ttl) { + return subscriptionBuilderWithTestValues() + .withSubscriptionPolicy( + subscriptionPolicy() + .applyDefaults() + .withMessageBackoff(messageBackoff) + .withBackoffMultiplier(backoffMultiplier) + .withMessageTtl(ttl) + .build()) + .build(); + } + + private SubscriptionBuilder subscriptionBuilderWithTestValues() { + return subscription("group.topic", "subscription"); + } + + private RuntimeException exception() { + return new RuntimeException("problem"); + } + + private CompletableFuture success() { + return CompletableFuture.completedFuture(MessageSendingResult.succeededResult()); + } + + private CompletableFuture failure() { + return CompletableFuture.completedFuture(MessageSendingResult.failedResult(exception())); + } + + private CompletableFuture failure(int statusCode) { + return CompletableFuture.completedFuture(MessageSendingResult.failedResult(statusCode)); + } + + private CompletableFuture backoff(int seconds) { + return CompletableFuture.completedFuture(MessageSendingResult.retryAfter(seconds)); + } + + private void verifyRateLimiterAcquired() { + verifyRateLimiterAcquired(1); + } + + private void verifyRateLimiterAcquired(int times) { + verify(rateLimiter, times(times)).acquire(); + } + + private void verifyRetryCounted() { + verifyRetryCounted(1); + } + + private void verifyRetryCounted(int times) { + verify(retries, times(times)).increment(); + } + + private Message message() { + return messageWithTimestamp(System.currentTimeMillis()); + } + + private Message messageWithTimestamp(long timestamp) { + return MessageBuilder.withTestMessage() + .withContent("{\"username\":\"ala\"}", StandardCharsets.UTF_8) + .withReadingTimestamp(timestamp) + .withPublishingTimestamp(timestamp) + .build(); + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/batch/DirectBufferPoolTest.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/batch/DirectBufferPoolTest.java index fc5b54a42a..37fc9901c3 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/batch/DirectBufferPoolTest.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/batch/DirectBufferPoolTest.java @@ -1,8 +1,7 @@ package pl.allegro.tech.hermes.consumers.consumer.batch; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import java.nio.BufferOverflowException; import java.nio.ByteBuffer; @@ -12,207 +11,218 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import java.util.stream.IntStream; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; public class DirectBufferPoolTest { - long totalMemory = 10 * 1024; - int poolableSize = 1024; - - DirectBufferPool pool; - ByteBuffer buffer; - - @Before - public void setup() { - this.pool = new DirectBufferPool(totalMemory, poolableSize, false); - } - - - @After - public void cleanup() { - if (buffer != null) { - pool.deallocate(buffer); - } - } - - @Test - public void shouldAllocateGivenAmountOfBytes() throws InterruptedException { - // given - int size = 512; - - // when - buffer = pool.allocate(size); - - // then - assertEquals("Buffer size should equal requested size.", size, buffer.limit()); - assertEquals("Unallocated memory should have shrunk", totalMemory - size, pool.unallocatedMemory()); - assertEquals("Available memory should have shrunk", totalMemory - size, pool.availableMemory()); - } - - @Test - public void shouldDeallocateAllMemoryFromBuffer() throws InterruptedException { - // given - buffer = pool.allocate(512).putInt(1); - buffer.flip(); - - // when - pool.deallocate(buffer); - - // then - assertEquals("All memory should be available", totalMemory, pool.availableMemory()); - assertEquals("Nothing is on a free list", totalMemory, pool.unallocatedMemory()); - } - - @Test - public void shouldRecycleBuffersOfPoolableSize() throws InterruptedException { - // given - pool.deallocate(pool.allocate(poolableSize)); - - // when - buffer = pool.allocate(poolableSize); - - // then - assertEquals("Recycled buffer should be cleared.", 0, buffer.position()); - assertEquals("Recycled buffer should be cleared.", buffer.capacity(), buffer.limit()); - assertEquals("Still a single buffer on the free list", totalMemory - poolableSize, pool.unallocatedMemory()); - } - - @Test - public void shouldDeallocateNonStandardBufferSize() throws InterruptedException { - // given - buffer = pool.allocate(2 * poolableSize); - - // when - pool.deallocate(buffer); - - // then - assertEquals("All memory should be available", totalMemory, pool.availableMemory()); - assertEquals("Non-standard size didn't go to the free list.", totalMemory, pool.unallocatedMemory()); - } - - @Test(expected = IllegalArgumentException.class) - public void shouldNotAllocateMoreMemoryThanTotalMemoryWeHave() throws InterruptedException { - // given - DirectBufferPool pool = new DirectBufferPool(1024, 512, true); + long totalMemory = 10 * 1024; + int poolableSize = 1024; - // when - pool.allocate(1025); - } + DirectBufferPool pool; + ByteBuffer buffer; - @Test(expected = BufferOverflowException.class) - public void shouldThrowExceptionOnBufferExhaustion() throws InterruptedException { - // given - DirectBufferPool pool = new DirectBufferPool(1024, 512, false); + @Before + public void setup() { + this.pool = new DirectBufferPool(totalMemory, poolableSize, false); + } - // when - pool.allocate(512); - pool.allocate(768); + @After + public void cleanup() { + if (buffer != null) { + pool.deallocate(buffer); } - - @Test - public void shouldBlockOnAllocationUntilMemoryIsAvailable() throws Exception { - // given - int total = 5 * 1024; - DirectBufferPool pool = new DirectBufferPool(total, 1024, true); - ByteBuffer buffer = pool.allocate(1024); - - // when - CountDownLatch doDeallocation = asyncDeallocate(pool, buffer); - CountDownLatch allocation = asyncAllocate(pool, total); - - // then - assertEquals("Allocation shouldn't have happened yet, waiting on memory.", 1L, allocation.getCount()); - doDeallocation.countDown(); - allocation.await(); - } - - private CountDownLatch asyncDeallocate(final DirectBufferPool pool, final ByteBuffer buffer) { - final CountDownLatch latch = new CountDownLatch(1); - Thread thread = new Thread() { - public void run() { - try { - latch.await(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - pool.deallocate(buffer); + } + + @Test + public void shouldAllocateGivenAmountOfBytes() throws InterruptedException { + // given + int size = 512; + + // when + buffer = pool.allocate(size); + + // then + assertEquals("Buffer size should equal requested size.", size, buffer.limit()); + assertEquals( + "Unallocated memory should have shrunk", totalMemory - size, pool.unallocatedMemory()); + assertEquals("Available memory should have shrunk", totalMemory - size, pool.availableMemory()); + } + + @Test + public void shouldDeallocateAllMemoryFromBuffer() throws InterruptedException { + // given + buffer = pool.allocate(512).putInt(1); + buffer.flip(); + + // when + pool.deallocate(buffer); + + // then + assertEquals("All memory should be available", totalMemory, pool.availableMemory()); + assertEquals("Nothing is on a free list", totalMemory, pool.unallocatedMemory()); + } + + @Test + public void shouldRecycleBuffersOfPoolableSize() throws InterruptedException { + // given + pool.deallocate(pool.allocate(poolableSize)); + + // when + buffer = pool.allocate(poolableSize); + + // then + assertEquals("Recycled buffer should be cleared.", 0, buffer.position()); + assertEquals("Recycled buffer should be cleared.", buffer.capacity(), buffer.limit()); + assertEquals( + "Still a single buffer on the free list", + totalMemory - poolableSize, + pool.unallocatedMemory()); + } + + @Test + public void shouldDeallocateNonStandardBufferSize() throws InterruptedException { + // given + buffer = pool.allocate(2 * poolableSize); + + // when + pool.deallocate(buffer); + + // then + assertEquals("All memory should be available", totalMemory, pool.availableMemory()); + assertEquals( + "Non-standard size didn't go to the free list.", totalMemory, pool.unallocatedMemory()); + } + + @Test(expected = IllegalArgumentException.class) + public void shouldNotAllocateMoreMemoryThanTotalMemoryWeHave() throws InterruptedException { + // given + DirectBufferPool pool = new DirectBufferPool(1024, 512, true); + + // when + pool.allocate(1025); + } + + @Test(expected = BufferOverflowException.class) + public void shouldThrowExceptionOnBufferExhaustion() throws InterruptedException { + // given + DirectBufferPool pool = new DirectBufferPool(1024, 512, false); + + // when + pool.allocate(512); + pool.allocate(768); + } + + @Test + public void shouldBlockOnAllocationUntilMemoryIsAvailable() throws Exception { + // given + int total = 5 * 1024; + DirectBufferPool pool = new DirectBufferPool(total, 1024, true); + ByteBuffer buffer = pool.allocate(1024); + + // when + CountDownLatch doDeallocation = asyncDeallocate(pool, buffer); + CountDownLatch allocation = asyncAllocate(pool, total); + + // then + assertEquals( + "Allocation shouldn't have happened yet, waiting on memory.", 1L, allocation.getCount()); + doDeallocation.countDown(); + allocation.await(); + } + + private CountDownLatch asyncDeallocate(final DirectBufferPool pool, final ByteBuffer buffer) { + final CountDownLatch latch = new CountDownLatch(1); + Thread thread = + new Thread() { + public void run() { + try { + latch.await(); + } catch (InterruptedException e) { + e.printStackTrace(); } + pool.deallocate(buffer); + } }; - thread.start(); - return latch; - } - - private CountDownLatch asyncAllocate(final DirectBufferPool pool, final int size) { - final CountDownLatch completed = new CountDownLatch(1); - Thread thread = new Thread() { - public void run() { - try { - pool.allocate(size); - } catch (InterruptedException e) { - e.printStackTrace(); - } finally { - completed.countDown(); - } + thread.start(); + return latch; + } + + private CountDownLatch asyncAllocate(final DirectBufferPool pool, final int size) { + final CountDownLatch completed = new CountDownLatch(1); + Thread thread = + new Thread() { + public void run() { + try { + pool.allocate(size); + } catch (InterruptedException e) { + e.printStackTrace(); + } finally { + completed.countDown(); } + } }; - thread.start(); - return completed; - } - - @Test - public void shouldSurviveHammeringOfLotsOfThreadsOnBufferPool() throws Exception { - // given - int numThreads = 10; - final int iterations = 50000; - final int poolableSize = 1024; - final int totalMemory = numThreads / 2 * poolableSize; - final DirectBufferPool pool = new DirectBufferPool(totalMemory, poolableSize, true); - - List threads = IntStream.range(0, numThreads) - .mapToObj(i -> new StressTestThread(pool, iterations)) - .collect(Collectors.toList()); - - // when - threads.forEach(StressTestThread::start); - threads.forEach(StressTestThread::joinQuietly); - - // then - threads.forEach(thread -> - assertTrue("Thread should have completed all iterations successfully.", thread.success.get())); - assertEquals(totalMemory, pool.availableMemory()); + thread.start(); + return completed; + } + + @Test + public void shouldSurviveHammeringOfLotsOfThreadsOnBufferPool() throws Exception { + // given + int numThreads = 10; + final int iterations = 50000; + final int poolableSize = 1024; + final int totalMemory = numThreads / 2 * poolableSize; + final DirectBufferPool pool = new DirectBufferPool(totalMemory, poolableSize, true); + + List threads = + IntStream.range(0, numThreads) + .mapToObj(i -> new StressTestThread(pool, iterations)) + .collect(Collectors.toList()); + + // when + threads.forEach(StressTestThread::start); + threads.forEach(StressTestThread::joinQuietly); + + // then + threads.forEach( + thread -> + assertTrue( + "Thread should have completed all iterations successfully.", thread.success.get())); + assertEquals(totalMemory, pool.availableMemory()); + } + + public static class StressTestThread extends Thread { + private final int iterations; + private final DirectBufferPool pool; + public final AtomicBoolean success = new AtomicBoolean(false); + Random random = new Random(); + + public StressTestThread(DirectBufferPool pool, int iterations) { + this.iterations = iterations; + this.pool = pool; } - public static class StressTestThread extends Thread { - private final int iterations; - private final DirectBufferPool pool; - public final AtomicBoolean success = new AtomicBoolean(false); - Random random = new Random(); - - public StressTestThread(DirectBufferPool pool, int iterations) { - this.iterations = iterations; - this.pool = pool; - } - - public void run() { - try { - for (int i = 0; i < iterations; i++) { - int size = random.nextBoolean() ? pool.poolableSize() : random.nextInt((int) pool.totalMemory()); - ByteBuffer buffer = pool.allocate(size); - pool.deallocate(buffer); - } - success.set(true); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + public void run() { + try { + for (int i = 0; i < iterations; i++) { + int size = + random.nextBoolean() ? pool.poolableSize() : random.nextInt((int) pool.totalMemory()); + ByteBuffer buffer = pool.allocate(size); + pool.deallocate(buffer); } + success.set(true); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } - public void joinQuietly() { - try { - join(); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } + public void joinQuietly() { + try { + join(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } } + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/batch/DirectBufferUtilsTest.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/batch/DirectBufferUtilsTest.java index eb0e7ea56a..2723d76154 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/batch/DirectBufferUtilsTest.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/batch/DirectBufferUtilsTest.java @@ -1,32 +1,29 @@ package pl.allegro.tech.hermes.consumers.consumer.batch; -import org.assertj.core.api.Assertions; -import org.junit.Test; +import static org.junit.Assert.assertEquals; import java.nio.ByteBuffer; - -import static org.junit.Assert.assertEquals; +import org.assertj.core.api.Assertions; +import org.junit.Test; public class DirectBufferUtilsTest { - @Test - public void shouldReleaseDirectByteBuffer() { - // given - ByteBuffer buffer = ByteBuffer.allocateDirect(128); + @Test + public void shouldReleaseDirectByteBuffer() { + // given + ByteBuffer buffer = ByteBuffer.allocateDirect(128); - // when & then - assertEquals(DirectBufferUtils.supportsReleasing(), true); - Assertions.assertThatCode(() -> DirectBufferUtils.release(buffer)) - .doesNotThrowAnyException(); - } + // when & then + assertEquals(DirectBufferUtils.supportsReleasing(), true); + Assertions.assertThatCode(() -> DirectBufferUtils.release(buffer)).doesNotThrowAnyException(); + } - @Test - public void shouldNotReleaseByteBuffer() { - // given - ByteBuffer buffer = ByteBuffer.allocate(128); + @Test + public void shouldNotReleaseByteBuffer() { + // given + ByteBuffer buffer = ByteBuffer.allocate(128); - // when & then - assertEquals(DirectBufferUtils.supportsReleasing(), true); - Assertions.assertThatCode(() -> DirectBufferUtils.release(buffer)) - .doesNotThrowAnyException(); - } + // when & then + assertEquals(DirectBufferUtils.supportsReleasing(), true); + Assertions.assertThatCode(() -> DirectBufferUtils.release(buffer)).doesNotThrowAnyException(); + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/converter/AvroToJsonMessageConverterTest.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/converter/AvroToJsonMessageConverterTest.java index d53c82e68d..8a6e56a28a 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/converter/AvroToJsonMessageConverterTest.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/converter/AvroToJsonMessageConverterTest.java @@ -1,5 +1,11 @@ package pl.allegro.tech.hermes.consumers.consumer.converter; +import static com.google.common.collect.ImmutableMap.of; +import static net.javacrumbs.jsonunit.fluent.JsonFluentAssert.assertThatJson; +import static pl.allegro.tech.hermes.consumers.consumer.Message.message; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topic; + +import java.io.IOException; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.junit.MockitoJUnitRunner; @@ -8,33 +14,27 @@ import pl.allegro.tech.hermes.schema.CompiledSchema; import pl.allegro.tech.hermes.test.helper.avro.AvroUser; -import java.io.IOException; - -import static com.google.common.collect.ImmutableMap.of; -import static net.javacrumbs.jsonunit.fluent.JsonFluentAssert.assertThatJson; -import static pl.allegro.tech.hermes.consumers.consumer.Message.message; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topic; - @RunWith(MockitoJUnitRunner.class) public class AvroToJsonMessageConverterTest { - @Test - public void shouldConvertToJsonWithoutMetadata() throws IOException { - // given - Topic topic = topic("group.topic").build(); - AvroUser avroUser = new AvroUser("Bob", 18, "blue"); - Message source = message() - .withData(avroUser.asBytes()) - .withSchema(CompiledSchema.of(avroUser.getSchema(), 1, 0)) - .withExternalMetadata(of()) - .build(); - AvroToJsonMessageConverter converter = new AvroToJsonMessageConverter(); - - // when - Message target = converter.convert(source, topic); - - // then - assertThatJson(new String(target.getData())).isEqualTo("{\"name\": \"Bob\", \"age\": 18, \"favoriteColor\": \"blue\"}"); - } - + @Test + public void shouldConvertToJsonWithoutMetadata() throws IOException { + // given + Topic topic = topic("group.topic").build(); + AvroUser avroUser = new AvroUser("Bob", 18, "blue"); + Message source = + message() + .withData(avroUser.asBytes()) + .withSchema(CompiledSchema.of(avroUser.getSchema(), 1, 0)) + .withExternalMetadata(of()) + .build(); + AvroToJsonMessageConverter converter = new AvroToJsonMessageConverter(); + + // when + Message target = converter.convert(source, topic); + + // then + assertThatJson(new String(target.getData())) + .isEqualTo("{\"name\": \"Bob\", \"age\": 18, \"favoriteColor\": \"blue\"}"); + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/interpolation/MessageBodyInterpolatorTest.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/interpolation/MessageBodyInterpolatorTest.java index c34170a0c3..64cec1d44f 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/interpolation/MessageBodyInterpolatorTest.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/interpolation/MessageBodyInterpolatorTest.java @@ -1,131 +1,138 @@ package pl.allegro.tech.hermes.consumers.consumer.interpolation; +import static org.assertj.core.api.Assertions.assertThat; +import static pl.allegro.tech.hermes.consumers.test.MessageBuilder.withTestMessage; + +import java.net.URI; +import java.nio.charset.StandardCharsets; import org.junit.Test; import pl.allegro.tech.hermes.api.EndpointAddress; import pl.allegro.tech.hermes.common.kafka.KafkaTopicName; import pl.allegro.tech.hermes.common.kafka.offset.PartitionOffset; import pl.allegro.tech.hermes.consumers.consumer.Message; -import java.net.URI; -import java.nio.charset.StandardCharsets; - -import static org.assertj.core.api.Assertions.assertThat; -import static pl.allegro.tech.hermes.consumers.test.MessageBuilder.withTestMessage; - public class MessageBodyInterpolatorTest { - private static final Message SAMPLE_MSG = withTestMessage() + private static final Message SAMPLE_MSG = + withTestMessage() + .withTopic("some.topic") + .withContent("{\"a\": \"b\"}", StandardCharsets.UTF_8) + .withPublishingTimestamp(214312123L) + .withReadingTimestamp(2143121233L) + .withPartitionOffset(new PartitionOffset(KafkaTopicName.valueOf("kafka_topic"), 0, 0)) + .build(); + + private static final KafkaTopicName KAFKA_TOPIC = KafkaTopicName.valueOf("kafka_topic"); + + @Test + public void willReturnURIAsIsIfNoTemplate() throws InterpolationException { + // given + EndpointAddress endpoint = EndpointAddress.of("http://some.endpoint.com/"); + + // when + URI interpolated = new MessageBodyInterpolator().interpolate(endpoint, SAMPLE_MSG); + + // then + assertThat(interpolated).isEqualTo(endpoint.getUri()); + } + + @Test + public void willInterpolateJsonPathFromTemplate() throws InterpolationException { + // given + EndpointAddress endpoint = EndpointAddress.of("http://some.endpoint.com/{some.object}"); + URI expectedEndpoint = URI.create("http://some.endpoint.com/100"); + String jsonMessage = "{\"some\": {\"object\": 100}}"; + Message msg = + withTestMessage() .withTopic("some.topic") - .withContent("{\"a\": \"b\"}", StandardCharsets.UTF_8) - .withPublishingTimestamp(214312123L) - .withReadingTimestamp(2143121233L) + .withContent(jsonMessage, StandardCharsets.UTF_8) + .withPublishingTimestamp(121422L) + .withReadingTimestamp(121423L) .withPartitionOffset(new PartitionOffset(KafkaTopicName.valueOf("kafka_topic"), 0, 0)) .build(); - private static final KafkaTopicName KAFKA_TOPIC = KafkaTopicName.valueOf("kafka_topic"); - - @Test - public void willReturnURIAsIsIfNoTemplate() throws InterpolationException { - // given - EndpointAddress endpoint = EndpointAddress.of("http://some.endpoint.com/"); - - // when - URI interpolated = new MessageBodyInterpolator().interpolate(endpoint, SAMPLE_MSG); - - // then - assertThat(interpolated).isEqualTo(endpoint.getUri()); - } - - @Test - public void willInterpolateJsonPathFromTemplate() throws InterpolationException { - // given - EndpointAddress endpoint = EndpointAddress.of("http://some.endpoint.com/{some.object}"); - URI expectedEndpoint = URI.create("http://some.endpoint.com/100"); - String jsonMessage = "{\"some\": {\"object\": 100}}"; - Message msg = withTestMessage() - .withTopic("some.topic") - .withContent(jsonMessage, StandardCharsets.UTF_8) - .withPublishingTimestamp(121422L) - .withReadingTimestamp(121423L) - .withPartitionOffset(new PartitionOffset(KafkaTopicName.valueOf("kafka_topic"), 0, 0)) - .build(); - - // when - URI interpolated = new MessageBodyInterpolator().interpolate(endpoint, msg); - - // then - assertThat(interpolated).isEqualTo(expectedEndpoint); - } - - @Test - public void willReturnURIOnEmptyEndpoint() throws InterpolationException { - // given - EndpointAddress endpoint = EndpointAddress.of(""); - URI expectedEndpoint = URI.create(""); - - // when - URI interpolated = new MessageBodyInterpolator().interpolate(endpoint, SAMPLE_MSG); - - // then - assertThat(interpolated).isEqualTo(expectedEndpoint); - } - - @Test - public void willInterpolateMultipleJsonPathsFromTemplate() throws InterpolationException { - // given - EndpointAddress endpoint = EndpointAddress.of("http://some.endpoint.com/{some.object}?test={some.test}"); - URI expectedEndpoint = URI.create("http://some.endpoint.com/100?test=hello"); - String jsonMessage = "{\"some\": {\"object\": 100, \"test\": \"hello\"}}"; - Message msg = withTestMessage() - .withTopic("some.topic") - .withContent(jsonMessage, StandardCharsets.UTF_8) - .withPublishingTimestamp(12323L) - .withReadingTimestamp(123234L) - .build(); - - // when - URI interpolated = new MessageBodyInterpolator().interpolate(endpoint, msg); - - // then - assertThat(interpolated).isEqualTo(expectedEndpoint); - } - - @Test(expected = InterpolationException.class) - public void willThrowExceptionOnInvalidPayload() throws InterpolationException { - // given - EndpointAddress endpoint = EndpointAddress.of("http://some.endpoint.com/{some.object}"); - - // when - new MessageBodyInterpolator().interpolate(endpoint, SAMPLE_MSG); - } - - @Test(expected = InterpolationException.class) - public void willThrowExceptionOnInterpolationException() throws InterpolationException { - // given - EndpointAddress endpoint = EndpointAddress.of("http://some.endpoint.com/{some.object}?test={some.test}"); - - // when - new MessageBodyInterpolator().interpolate(endpoint, SAMPLE_MSG); - } - - @Test - public void willInterpolateMultipleJsonPathsFromTemplateInReverseOrder() throws InterpolationException { - // given - EndpointAddress endpoint = EndpointAddress.of("http://some.endpoint.com/{some.object}?test={some.test}"); - URI expectedEndpoint = URI.create("http://some.endpoint.com/100?test=hello"); - String jsonMessage = "{\"some\": {\"test\": \"hello\", \"object\": 100}}"; - Message msg = withTestMessage() - .withTopic("some.topic") - .withContent(jsonMessage, StandardCharsets.UTF_8) - .withPublishingTimestamp(1232443L) - .withReadingTimestamp(12324434L) - .withPartitionOffset(new PartitionOffset(KAFKA_TOPIC, 0, 0)) - .build(); - - // when - URI interpolated = new MessageBodyInterpolator().interpolate(endpoint, msg); - - // then - assertThat(interpolated).isEqualTo(expectedEndpoint); - } + // when + URI interpolated = new MessageBodyInterpolator().interpolate(endpoint, msg); + + // then + assertThat(interpolated).isEqualTo(expectedEndpoint); + } + + @Test + public void willReturnURIOnEmptyEndpoint() throws InterpolationException { + // given + EndpointAddress endpoint = EndpointAddress.of(""); + URI expectedEndpoint = URI.create(""); + + // when + URI interpolated = new MessageBodyInterpolator().interpolate(endpoint, SAMPLE_MSG); + + // then + assertThat(interpolated).isEqualTo(expectedEndpoint); + } + + @Test + public void willInterpolateMultipleJsonPathsFromTemplate() throws InterpolationException { + // given + EndpointAddress endpoint = + EndpointAddress.of("http://some.endpoint.com/{some.object}?test={some.test}"); + URI expectedEndpoint = URI.create("http://some.endpoint.com/100?test=hello"); + String jsonMessage = "{\"some\": {\"object\": 100, \"test\": \"hello\"}}"; + Message msg = + withTestMessage() + .withTopic("some.topic") + .withContent(jsonMessage, StandardCharsets.UTF_8) + .withPublishingTimestamp(12323L) + .withReadingTimestamp(123234L) + .build(); + + // when + URI interpolated = new MessageBodyInterpolator().interpolate(endpoint, msg); + + // then + assertThat(interpolated).isEqualTo(expectedEndpoint); + } + + @Test(expected = InterpolationException.class) + public void willThrowExceptionOnInvalidPayload() throws InterpolationException { + // given + EndpointAddress endpoint = EndpointAddress.of("http://some.endpoint.com/{some.object}"); + + // when + new MessageBodyInterpolator().interpolate(endpoint, SAMPLE_MSG); + } + + @Test(expected = InterpolationException.class) + public void willThrowExceptionOnInterpolationException() throws InterpolationException { + // given + EndpointAddress endpoint = + EndpointAddress.of("http://some.endpoint.com/{some.object}?test={some.test}"); + + // when + new MessageBodyInterpolator().interpolate(endpoint, SAMPLE_MSG); + } + + @Test + public void willInterpolateMultipleJsonPathsFromTemplateInReverseOrder() + throws InterpolationException { + // given + EndpointAddress endpoint = + EndpointAddress.of("http://some.endpoint.com/{some.object}?test={some.test}"); + URI expectedEndpoint = URI.create("http://some.endpoint.com/100?test=hello"); + String jsonMessage = "{\"some\": {\"test\": \"hello\", \"object\": 100}}"; + Message msg = + withTestMessage() + .withTopic("some.topic") + .withContent(jsonMessage, StandardCharsets.UTF_8) + .withPublishingTimestamp(1232443L) + .withReadingTimestamp(12324434L) + .withPartitionOffset(new PartitionOffset(KAFKA_TOPIC, 0, 0)) + .build(); + + // when + URI interpolated = new MessageBodyInterpolator().interpolate(endpoint, msg); + + // then + assertThat(interpolated).isEqualTo(expectedEndpoint); + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/HeartbeatModeOutputRateCalculatorTest.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/HeartbeatModeOutputRateCalculatorTest.java index c5880cf8a5..429db6e643 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/HeartbeatModeOutputRateCalculatorTest.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/HeartbeatModeOutputRateCalculatorTest.java @@ -1,36 +1,39 @@ package pl.allegro.tech.hermes.consumers.consumer.rate.calculator; -import org.junit.Test; -import pl.allegro.tech.hermes.consumers.consumer.rate.SendCounters; +import static pl.allegro.tech.hermes.consumers.test.HermesConsumersAssertions.assertThat; import java.time.Clock; - -import static pl.allegro.tech.hermes.consumers.test.HermesConsumersAssertions.assertThat; +import org.junit.Test; +import pl.allegro.tech.hermes.consumers.consumer.rate.SendCounters; public class HeartbeatModeOutputRateCalculatorTest { - private static final int SLOW_RATE = 10; + private static final int SLOW_RATE = 10; - private final HeartbeatModeOutputRateCalculator calculator = new HeartbeatModeOutputRateCalculator(SLOW_RATE); + private final HeartbeatModeOutputRateCalculator calculator = + new HeartbeatModeOutputRateCalculator(SLOW_RATE); - private final SendCounters counters = new SendCounters(Clock.systemDefaultZone()); + private final SendCounters counters = new SendCounters(Clock.systemDefaultZone()); - @Test - public void shouldNotChangeAnythingIfThereWereAnyFailures() { - // given - counters.incrementSuccesses() - .incrementFailures(); + @Test + public void shouldNotChangeAnythingIfThereWereAnyFailures() { + // given + counters.incrementSuccesses().incrementFailures(); - // when then - assertThat(calculator.calculateOutputRate(1, 20, counters)).hasRate(1).isInMode(OutputRateCalculator.Mode.HEARTBEAT); - } + // when then + assertThat(calculator.calculateOutputRate(1, 20, counters)) + .hasRate(1) + .isInMode(OutputRateCalculator.Mode.HEARTBEAT); + } - @Test - public void shouldSwitchToSlowModeIfOnlySuccesses() { - // given - counters.incrementSuccesses(); + @Test + public void shouldSwitchToSlowModeIfOnlySuccesses() { + // given + counters.incrementSuccesses(); - // when then - assertThat(calculator.calculateOutputRate(1, 20, counters)).hasRate(SLOW_RATE).isInMode(OutputRateCalculator.Mode.SLOW); - } + // when then + assertThat(calculator.calculateOutputRate(1, 20, counters)) + .hasRate(SLOW_RATE) + .isInMode(OutputRateCalculator.Mode.SLOW); + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/NormalModeOutputRateCalculatorTest.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/NormalModeOutputRateCalculatorTest.java index 042499ec5f..a1bd8effbf 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/NormalModeOutputRateCalculatorTest.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/NormalModeOutputRateCalculatorTest.java @@ -1,83 +1,89 @@ package pl.allegro.tech.hermes.consumers.consumer.rate.calculator; -import org.junit.Test; -import pl.allegro.tech.hermes.consumers.consumer.rate.SendCounters; +import static pl.allegro.tech.hermes.consumers.test.HermesConsumersAssertions.assertThat; import java.time.Clock; - -import static pl.allegro.tech.hermes.consumers.test.HermesConsumersAssertions.assertThat; +import org.junit.Test; +import pl.allegro.tech.hermes.consumers.consumer.rate.SendCounters; public class NormalModeOutputRateCalculatorTest { - private static final int SLOW_RATE = 5; - - private final NormalModeOutputRateCalculator calculator = new NormalModeOutputRateCalculator(0.5, SLOW_RATE, 0.1, 0.4); - - private final SendCounters counters = new SendCounters(Clock.systemDefaultZone()); - - @Test - public void shouldIncreaseRateWhenFailuresBelowToleranceOccuredAndNotAboveMaximumRate() { - // given - NormalModeOutputRateCalculator calculator = new NormalModeOutputRateCalculator(0.5, SLOW_RATE, 0.4, 0.5); - counters.incrementSuccesses().incrementSuccesses().incrementFailures(); - - // when then - assertThat(calculator.calculateOutputRate(10, 20, counters).rate()).isEqualTo(15); - } - - @Test - public void shouldNotIncreaseRateWhenNoFailuresOccuredAndAlreadyAtMaximum() { - // given - counters.incrementSuccesses(); - - // when then - assertThat(calculator.calculateOutputRate(20, 20, counters)).hasRate(20); - } - - @Test - public void shouldNotSlowDownWhenRatioOfFailuresIsBelowGivenThreshold() { - // given - NormalModeOutputRateCalculator calculator = new NormalModeOutputRateCalculator(0.5, SLOW_RATE, 0.0, 0.38); - counters.incrementSuccesses().incrementSuccesses() - .incrementFailures(); - - // when then - assertThat(calculator.calculateOutputRate(20, 20, counters)).hasRate(20); - } - - @Test - public void shouldSlowDownWhenRatioOfFailuresExceedsGivenThreshold() { - // given - NormalModeOutputRateCalculator calculator = new NormalModeOutputRateCalculator(0.5, SLOW_RATE, 0.1, 0.38); - counters.incrementSuccesses().incrementSuccesses().incrementSuccesses() - .incrementFailures().incrementFailures(); - - // when then - assertThat(calculator.calculateOutputRate(20, 20, counters)).hasRate(10); - } - - @Test - public void shouldNotLimitRateWhenThereWereNoEvents() { - assertThat(calculator.calculateOutputRate(20, 20, counters)).hasRate(20); - } - - @Test - public void shouldSwitchToSlowModeWhenCalculatedRateIsBelowSlowModeRate() { - // given - counters.incrementSuccesses() - .incrementFailures(); - - // when then - assertThat(calculator.calculateOutputRate(8, 20, counters)).hasRate(SLOW_RATE); - } - - @Test - public void shouldSwitchToSlowModeWhenThereWereMoreFailuresThanSuccesses() { - // given - counters.incrementSuccesses() - .incrementFailures().incrementFailures(); - - // when then - assertThat(calculator.calculateOutputRate(10, 20, counters)).hasRate(5).isInMode(OutputRateCalculator.Mode.SLOW); - } + private static final int SLOW_RATE = 5; + + private final NormalModeOutputRateCalculator calculator = + new NormalModeOutputRateCalculator(0.5, SLOW_RATE, 0.1, 0.4); + + private final SendCounters counters = new SendCounters(Clock.systemDefaultZone()); + + @Test + public void shouldIncreaseRateWhenFailuresBelowToleranceOccuredAndNotAboveMaximumRate() { + // given + NormalModeOutputRateCalculator calculator = + new NormalModeOutputRateCalculator(0.5, SLOW_RATE, 0.4, 0.5); + counters.incrementSuccesses().incrementSuccesses().incrementFailures(); + + // when then + assertThat(calculator.calculateOutputRate(10, 20, counters).rate()).isEqualTo(15); + } + + @Test + public void shouldNotIncreaseRateWhenNoFailuresOccuredAndAlreadyAtMaximum() { + // given + counters.incrementSuccesses(); + + // when then + assertThat(calculator.calculateOutputRate(20, 20, counters)).hasRate(20); + } + + @Test + public void shouldNotSlowDownWhenRatioOfFailuresIsBelowGivenThreshold() { + // given + NormalModeOutputRateCalculator calculator = + new NormalModeOutputRateCalculator(0.5, SLOW_RATE, 0.0, 0.38); + counters.incrementSuccesses().incrementSuccesses().incrementFailures(); + + // when then + assertThat(calculator.calculateOutputRate(20, 20, counters)).hasRate(20); + } + + @Test + public void shouldSlowDownWhenRatioOfFailuresExceedsGivenThreshold() { + // given + NormalModeOutputRateCalculator calculator = + new NormalModeOutputRateCalculator(0.5, SLOW_RATE, 0.1, 0.38); + counters + .incrementSuccesses() + .incrementSuccesses() + .incrementSuccesses() + .incrementFailures() + .incrementFailures(); + + // when then + assertThat(calculator.calculateOutputRate(20, 20, counters)).hasRate(10); + } + + @Test + public void shouldNotLimitRateWhenThereWereNoEvents() { + assertThat(calculator.calculateOutputRate(20, 20, counters)).hasRate(20); + } + + @Test + public void shouldSwitchToSlowModeWhenCalculatedRateIsBelowSlowModeRate() { + // given + counters.incrementSuccesses().incrementFailures(); + + // when then + assertThat(calculator.calculateOutputRate(8, 20, counters)).hasRate(SLOW_RATE); + } + + @Test + public void shouldSwitchToSlowModeWhenThereWereMoreFailuresThanSuccesses() { + // given + counters.incrementSuccesses().incrementFailures().incrementFailures(); + + // when then + assertThat(calculator.calculateOutputRate(10, 20, counters)) + .hasRate(5) + .isInMode(OutputRateCalculator.Mode.SLOW); + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/OutputRateCalculationResultAssert.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/OutputRateCalculationResultAssert.java index 523e08c0a0..ddd2ecc194 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/OutputRateCalculationResultAssert.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/OutputRateCalculationResultAssert.java @@ -4,23 +4,23 @@ import org.assertj.core.api.Assertions; public final class OutputRateCalculationResultAssert - extends AbstractAssert { + extends AbstractAssert { - private OutputRateCalculationResultAssert(OutputRateCalculationResult actual) { - super(actual, OutputRateCalculationResultAssert.class); - } + private OutputRateCalculationResultAssert(OutputRateCalculationResult actual) { + super(actual, OutputRateCalculationResultAssert.class); + } - public static OutputRateCalculationResultAssert assertThat(OutputRateCalculationResult result) { - return new OutputRateCalculationResultAssert(result); - } + public static OutputRateCalculationResultAssert assertThat(OutputRateCalculationResult result) { + return new OutputRateCalculationResultAssert(result); + } - public OutputRateCalculationResultAssert isInMode(OutputRateCalculator.Mode mode) { - Assertions.assertThat(actual.mode()).isEqualTo(mode); - return this; - } + public OutputRateCalculationResultAssert isInMode(OutputRateCalculator.Mode mode) { + Assertions.assertThat(actual.mode()).isEqualTo(mode); + return this; + } - public OutputRateCalculationResultAssert hasRate(double rate) { - Assertions.assertThat(actual.rate()).isEqualTo(rate); - return this; - } + public OutputRateCalculationResultAssert hasRate(double rate) { + Assertions.assertThat(actual.rate()).isEqualTo(rate); + return this; + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/OutputRateCalculationScenario.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/OutputRateCalculationScenario.java index 177604667b..5d74756671 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/OutputRateCalculationScenario.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/OutputRateCalculationScenario.java @@ -1,105 +1,105 @@ package pl.allegro.tech.hermes.consumers.consumer.rate.calculator; +import java.time.Clock; +import java.util.ArrayList; +import java.util.List; import org.assertj.core.api.Assertions; import org.assertj.core.data.Offset; import pl.allegro.tech.hermes.consumers.consumer.rate.SendCounters; import pl.allegro.tech.hermes.consumers.consumer.rate.calculator.OutputRateCalculator.Mode; -import java.time.Clock; -import java.util.ArrayList; -import java.util.List; - public class OutputRateCalculationScenario { - private final List results = new ArrayList<>(); + private final List results = new ArrayList<>(); - private OutputRateCalculationResult previous; + private OutputRateCalculationResult previous; - private final OutputRateCalculator calculator; + private final OutputRateCalculator calculator; - private final SendCounters counters = new SendCounters(Clock.systemDefaultZone()); + private final SendCounters counters = new SendCounters(Clock.systemDefaultZone()); - public OutputRateCalculationScenario(OutputRateCalculator calculator) { - this.calculator = calculator; - } + public OutputRateCalculationScenario(OutputRateCalculator calculator) { + this.calculator = calculator; + } - public OutputRateCalculationScenario start(double initialRate) { - previous = calculator.recalculateRate(counters, OutputRateCalculator.Mode.NORMAL, initialRate); - results.add(previous); - return this; - } + public OutputRateCalculationScenario start(double initialRate) { + previous = calculator.recalculateRate(counters, OutputRateCalculator.Mode.NORMAL, initialRate); + results.add(previous); + return this; + } - public OutputRateCalculationScenario nextInteration(int successes, int failures) { - incrementSuccesses(successes); - incrementFailures(failures); - previous = calculator.recalculateRate(counters, previous.mode(), previous.rate()); - results.add(previous); - counters.reset(); - return this; - } + public OutputRateCalculationScenario nextInteration(int successes, int failures) { + incrementSuccesses(successes); + incrementFailures(failures); + previous = calculator.recalculateRate(counters, previous.mode(), previous.rate()); + results.add(previous); + counters.reset(); + return this; + } - private void incrementSuccesses(int times) { - for (int i = 0; i < times; ++i) { - counters.incrementSuccesses(); - } + private void incrementSuccesses(int times) { + for (int i = 0; i < times; ++i) { + counters.incrementSuccesses(); } + } - private void incrementFailures(int times) { - for (int i = 0; i < times; ++i) { - counters.incrementFailures(); - } + private void incrementFailures(int times) { + for (int i = 0; i < times; ++i) { + counters.incrementFailures(); } + } - public OutputRateCalculationScenario nextSuccessIteration() { - return nextInteration(1, 0); - } + public OutputRateCalculationScenario nextSuccessIteration() { + return nextInteration(1, 0); + } - public OutputRateCalculationScenario nextFailureIteration() { - return nextInteration(0, 1); - } + public OutputRateCalculationScenario nextFailureIteration() { + return nextInteration(0, 1); + } - public OutputRateCalculationScenario fastForwardWithSuccesses(int iterations) { - for (int i = 0; i < iterations; ++i) { - nextSuccessIteration(); - } - return this; + public OutputRateCalculationScenario fastForwardWithSuccesses(int iterations) { + for (int i = 0; i < iterations; ++i) { + nextSuccessIteration(); } + return this; + } - public OutputRateCalculationScenario fastForwardWithFailures(int iterations) { - for (int i = 0; i < iterations; ++i) { - nextFailureIteration(); - } - return this; + public OutputRateCalculationScenario fastForwardWithFailures(int iterations) { + for (int i = 0; i < iterations; ++i) { + nextFailureIteration(); } + return this; + } - public void verifyModes(int offset, Mode... modes) { - for (int i = offset; i < modes.length; ++i) { - Assertions.assertThat(results.get(i).mode()).isEqualTo(modes[i]); - } + public void verifyModes(int offset, Mode... modes) { + for (int i = offset; i < modes.length; ++i) { + Assertions.assertThat(results.get(i).mode()).isEqualTo(modes[i]); } + } - public void verifyRates(int offset, double... rates) { - for (int i = offset; i < rates.length; ++i) { - Assertions.assertThat(results.get(i).rate()).isEqualTo(rates[i]); - } + public void verifyRates(int offset, double... rates) { + for (int i = offset; i < rates.length; ++i) { + Assertions.assertThat(results.get(i).rate()).isEqualTo(rates[i]); } + } - public OutputRateCalculationScenario verifyIntermediateResult(int iteration, double rate, Mode mode) { - Assertions.assertThat(results.get(iteration).rate()).isEqualTo(rate, Offset.offset(0.1)); - Assertions.assertThat(results.get(iteration).mode()).isEqualTo(mode); - return this; - } + public OutputRateCalculationScenario verifyIntermediateResult( + int iteration, double rate, Mode mode) { + Assertions.assertThat(results.get(iteration).rate()).isEqualTo(rate, Offset.offset(0.1)); + Assertions.assertThat(results.get(iteration).mode()).isEqualTo(mode); + return this; + } - public OutputRateCalculationScenario verifyFinalResult(double rate, Mode mode) { + public OutputRateCalculationScenario verifyFinalResult(double rate, Mode mode) { - Assertions.assertThat(results.get(results.size() - 1).mode()).isEqualTo(mode); + Assertions.assertThat(results.get(results.size() - 1).mode()).isEqualTo(mode); - double actualRate = results.get(results.size() - 1).rate(); - if (mode == Mode.NORMAL) { - Assertions.assertThat(actualRate).isEqualTo(rate, Offset.offset(0.1)); - } else { - Assertions.assertThat(actualRate).isEqualTo(rate); - } - return this; + double actualRate = results.get(results.size() - 1).rate(); + if (mode == Mode.NORMAL) { + Assertions.assertThat(actualRate).isEqualTo(rate, Offset.offset(0.1)); + } else { + Assertions.assertThat(actualRate).isEqualTo(rate); } + return this; + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/OutputRateCalculatorTest.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/OutputRateCalculatorTest.java index e6e80c18d1..8b5e557c65 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/OutputRateCalculatorTest.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/OutputRateCalculatorTest.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.consumers.consumer.rate.calculator; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; + +import java.time.Duration; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -8,88 +13,88 @@ import pl.allegro.tech.hermes.consumers.config.RateProperties; import pl.allegro.tech.hermes.consumers.consumer.rate.maxrate.NegotiatedMaxRateProvider; -import java.time.Duration; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; - @RunWith(MockitoJUnitRunner.class) public class OutputRateCalculatorTest { - private static final double HEARTBEAT_RATE = 1.0 / 60.0; - - private static final double MAX_RATE = 100; - - - private OutputRateCalculator calculator; - - @Before - public void setup() { - RateProperties rateProperties = new RateProperties(); - rateProperties.setLimiterSlowModeDelay(Duration.ofSeconds(1)); - rateProperties.setConvergenceFactor(0.5); - - subscription("group.topic", "subscription").withSubscriptionPolicy( - SubscriptionPolicy.Builder.subscriptionPolicy().withRate(200).build() - ).build(); - - NegotiatedMaxRateProvider maxRateProvider = mock(NegotiatedMaxRateProvider.class); - when(maxRateProvider.get()).thenReturn(100D); - - calculator = new OutputRateCalculator(rateProperties, maxRateProvider); - } - - @Test - public void shouldStartInNormalModeAndGraduallyIncreaseToMaximumWhenOnlySuccessfulDeliveries() { - //given - OutputRateCalculationScenario scenario = new OutputRateCalculationScenario(calculator); - - // when - scenario.start(10).fastForwardWithSuccesses(10); - - // then - scenario.verifyFinalResult(MAX_RATE, OutputRateCalculator.Mode.NORMAL); - } - - @Test - public void shouldStartInNormalModeReachMaximumAndGoToHeartbeatViaSlowWhenDeliveryStartsFailing() { - // given - OutputRateCalculationScenario scenario = new OutputRateCalculationScenario(calculator); - - // when - scenario.start(10).fastForwardWithSuccesses(10).nextInteration(2, 10).fastForwardWithFailures(20); - - // then - scenario.verifyIntermediateResult(11, 1, OutputRateCalculator.Mode.SLOW) - .verifyFinalResult(HEARTBEAT_RATE, OutputRateCalculator.Mode.HEARTBEAT); - } - - @Test - public void shouldRecoverFromHeartbeatModeAndGetToMaximumViaSlowWhenDeliveringWithSuccess() { - // given - OutputRateCalculationScenario scenario = new OutputRateCalculationScenario(calculator); - - // when - scenario.start(10).fastForwardWithFailures(10) - .fastForwardWithSuccesses(15); - - // then - scenario.verifyIntermediateResult(11, 1, OutputRateCalculator.Mode.SLOW) - .verifyFinalResult(MAX_RATE, OutputRateCalculator.Mode.NORMAL); - } - - @Test - public void shouldImmediatelySwitchToHeartbeatModeWhenAllMessagesFail() { - // given - OutputRateCalculationScenario scenario = new OutputRateCalculationScenario(calculator); - - // when - scenario.start(10).fastForwardWithSuccesses(10).fastForwardWithFailures(2); - - // then - scenario.verifyIntermediateResult(10, MAX_RATE, OutputRateCalculator.Mode.NORMAL) - .verifyFinalResult(HEARTBEAT_RATE, OutputRateCalculator.Mode.HEARTBEAT); - } - + private static final double HEARTBEAT_RATE = 1.0 / 60.0; + + private static final double MAX_RATE = 100; + + private OutputRateCalculator calculator; + + @Before + public void setup() { + RateProperties rateProperties = new RateProperties(); + rateProperties.setLimiterSlowModeDelay(Duration.ofSeconds(1)); + rateProperties.setConvergenceFactor(0.5); + + subscription("group.topic", "subscription") + .withSubscriptionPolicy( + SubscriptionPolicy.Builder.subscriptionPolicy().withRate(200).build()) + .build(); + + NegotiatedMaxRateProvider maxRateProvider = mock(NegotiatedMaxRateProvider.class); + when(maxRateProvider.get()).thenReturn(100D); + + calculator = new OutputRateCalculator(rateProperties, maxRateProvider); + } + + @Test + public void shouldStartInNormalModeAndGraduallyIncreaseToMaximumWhenOnlySuccessfulDeliveries() { + // given + OutputRateCalculationScenario scenario = new OutputRateCalculationScenario(calculator); + + // when + scenario.start(10).fastForwardWithSuccesses(10); + + // then + scenario.verifyFinalResult(MAX_RATE, OutputRateCalculator.Mode.NORMAL); + } + + @Test + public void + shouldStartInNormalModeReachMaximumAndGoToHeartbeatViaSlowWhenDeliveryStartsFailing() { + // given + OutputRateCalculationScenario scenario = new OutputRateCalculationScenario(calculator); + + // when + scenario + .start(10) + .fastForwardWithSuccesses(10) + .nextInteration(2, 10) + .fastForwardWithFailures(20); + + // then + scenario + .verifyIntermediateResult(11, 1, OutputRateCalculator.Mode.SLOW) + .verifyFinalResult(HEARTBEAT_RATE, OutputRateCalculator.Mode.HEARTBEAT); + } + + @Test + public void shouldRecoverFromHeartbeatModeAndGetToMaximumViaSlowWhenDeliveringWithSuccess() { + // given + OutputRateCalculationScenario scenario = new OutputRateCalculationScenario(calculator); + + // when + scenario.start(10).fastForwardWithFailures(10).fastForwardWithSuccesses(15); + + // then + scenario + .verifyIntermediateResult(11, 1, OutputRateCalculator.Mode.SLOW) + .verifyFinalResult(MAX_RATE, OutputRateCalculator.Mode.NORMAL); + } + + @Test + public void shouldImmediatelySwitchToHeartbeatModeWhenAllMessagesFail() { + // given + OutputRateCalculationScenario scenario = new OutputRateCalculationScenario(calculator); + + // when + scenario.start(10).fastForwardWithSuccesses(10).fastForwardWithFailures(2); + + // then + scenario + .verifyIntermediateResult(10, MAX_RATE, OutputRateCalculator.Mode.NORMAL) + .verifyFinalResult(HEARTBEAT_RATE, OutputRateCalculator.Mode.HEARTBEAT); + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/SlowModeOutputRateCalculatorTest.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/SlowModeOutputRateCalculatorTest.java index 79cc3dbae6..0063025ffa 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/SlowModeOutputRateCalculatorTest.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/calculator/SlowModeOutputRateCalculatorTest.java @@ -1,46 +1,50 @@ package pl.allegro.tech.hermes.consumers.consumer.rate.calculator; -import org.junit.Test; -import pl.allegro.tech.hermes.consumers.consumer.rate.SendCounters; +import static pl.allegro.tech.hermes.consumers.test.HermesConsumersAssertions.assertThat; import java.time.Clock; - -import static pl.allegro.tech.hermes.consumers.test.HermesConsumersAssertions.assertThat; +import org.junit.Test; +import pl.allegro.tech.hermes.consumers.consumer.rate.SendCounters; public class SlowModeOutputRateCalculatorTest { - private static final double HEARTBEAT_RATE = 1; - - private final SlowModeOutputRateCalculator calculator = new SlowModeOutputRateCalculator(HEARTBEAT_RATE); - - private final SendCounters counters = new SendCounters(Clock.systemDefaultZone()); - - @Test - public void shouldStayInSlowModeIfThereWereSomeFailures() { - // given - counters.incrementSuccesses() - .incrementFailures(); - - // when then - assertThat(calculator.calculateOutputRate(10, 20, counters)).hasRate(10).isInMode(OutputRateCalculator.Mode.SLOW); - } - - @Test - public void shouldSwitchToNormalModeWithoutModifyingRateWhenOnlySuccesses() { - // given - counters.incrementSuccesses(); - - // when then - assertThat(calculator.calculateOutputRate(10, 20, counters)).hasRate(10).isInMode(OutputRateCalculator.Mode.NORMAL); - } - - @Test - public void shouldSwitchToHeartbeatModeWhenThereWereOnlyFailures() { - // given - counters.incrementFailures(); - - // when then - assertThat(calculator.calculateOutputRate(10, 20, counters)).hasRate(HEARTBEAT_RATE).isInMode(OutputRateCalculator.Mode.HEARTBEAT); - } - + private static final double HEARTBEAT_RATE = 1; + + private final SlowModeOutputRateCalculator calculator = + new SlowModeOutputRateCalculator(HEARTBEAT_RATE); + + private final SendCounters counters = new SendCounters(Clock.systemDefaultZone()); + + @Test + public void shouldStayInSlowModeIfThereWereSomeFailures() { + // given + counters.incrementSuccesses().incrementFailures(); + + // when then + assertThat(calculator.calculateOutputRate(10, 20, counters)) + .hasRate(10) + .isInMode(OutputRateCalculator.Mode.SLOW); + } + + @Test + public void shouldSwitchToNormalModeWithoutModifyingRateWhenOnlySuccesses() { + // given + counters.incrementSuccesses(); + + // when then + assertThat(calculator.calculateOutputRate(10, 20, counters)) + .hasRate(10) + .isInMode(OutputRateCalculator.Mode.NORMAL); + } + + @Test + public void shouldSwitchToHeartbeatModeWhenThereWereOnlyFailures() { + // given + counters.incrementFailures(); + + // when then + assertThat(calculator.calculateOutputRate(10, 20, counters)) + .hasRate(HEARTBEAT_RATE) + .isInMode(OutputRateCalculator.Mode.HEARTBEAT); + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateRegistryTest.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateRegistryTest.java index d7a2e132a5..5d44bad381 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateRegistryTest.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/rate/maxrate/MaxRateRegistryTest.java @@ -1,8 +1,15 @@ package pl.allegro.tech.hermes.consumers.consumer.rate.maxrate; +import static org.awaitility.Awaitility.await; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import java.util.Collections; +import java.util.concurrent.TimeUnit; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -19,183 +26,197 @@ import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; import pl.allegro.tech.hermes.test.helper.zookeeper.ZookeeperBaseTest; -import java.util.Collections; -import java.util.concurrent.TimeUnit; - -import static org.awaitility.Awaitility.await; -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - public class MaxRateRegistryTest extends ZookeeperBaseTest { - private final SubscriptionName subscription1 = SubscriptionName.fromString("pl.allegro.tech.hermes$testSubscription"); - private final SubscriptionId subscriptionId1 = SubscriptionId.from(subscription1, 1L); - private final SubscriptionName subscription2 = SubscriptionName.fromString("pl.allegro.tech.hermes$testSubscription2"); - private final SubscriptionId subscriptionId2 = SubscriptionId.from(subscription2, 2L); - - private final SubscriptionsCache subscriptionsCache = mock(SubscriptionsCache.class); - private final SubscriptionIds subscriptionIds = new TestSubscriptionIds(ImmutableList.of(subscriptionId1, subscriptionId2)); - - private final ZookeeperPaths zookeeperPaths = new ZookeeperPaths("/hermes"); - private final MaxRateProperties maxRateProperties = new MaxRateProperties(); - private final String consumerId = new WorkloadProperties().getNodeId(); - private final String cluster = new KafkaProperties().getClusterName(); - - private final ConsumerAssignmentCache consumerAssignmentCache = mock(ConsumerAssignmentCache.class); - private final ClusterAssignmentCache clusterAssignmentCache = mock(ClusterAssignmentCache.class); - - private final MaxRateRegistry maxRateRegistry = new MaxRateRegistry( - maxRateProperties.getRegistryBinaryEncoder().getHistoryBufferSizeBytes(), - maxRateProperties.getRegistryBinaryEncoder().getMaxRateBufferSizeBytes(), - consumerId, - cluster, - clusterAssignmentCache, - consumerAssignmentCache, - zookeeperClient, - zookeeperPaths, - subscriptionIds - ); - - private final MaxRateRegistryPaths paths = new MaxRateRegistryPaths(zookeeperPaths, consumerId, cluster); - - @Before - public void setUp() { - when(subscriptionsCache.listActiveSubscriptionNames()).thenReturn(ImmutableList.of(subscription1, subscription2)); - - when(clusterAssignmentCache.getAssignedConsumers()).thenReturn(ImmutableSet.of(consumerId)); - when(consumerAssignmentCache.getConsumerSubscriptions()).thenReturn(ImmutableSet.of(subscription1, subscription2)); - maxRateRegistry.start(); - } - - @After - public void cleanup() { - try { - deleteAllNodes(); - } catch (Exception e) { - e.printStackTrace(); - } - maxRateRegistry.stop(); - } - - @Test - public void shouldHaveEmptyRateHistoryOnInit() { - // given - maxRateRegistry.onBeforeMaxRateCalculation(); // read - - // and - maxRateRegistry.ensureCorrectAssignments(subscription1, ImmutableSet.of(consumerId)); - - // then - assertEquals(RateHistory.empty(), maxRateRegistry.getRateHistory(new ConsumerInstance(consumerId, subscription1))); - } - - @Test - public void shouldWriteAndReadRateHistoryForCurrentConsumer() { - // given - ConsumerInstance consumer = new ConsumerInstance(consumerId, subscription1); - - RateHistory rateHistory = RateHistory.create(0.5); - - // when - maxRateRegistry.writeRateHistory(consumer, rateHistory); - - // and - maxRateRegistry.onAfterWriteRateHistories(); // store - - // then - wait.untilZookeeperPathIsCreated(paths.currentConsumerRateHistoryPath()); - - // when - maxRateRegistry.onBeforeMaxRateCalculation(); // read - - // and - RateHistory readHistory = maxRateRegistry.getRateHistory(consumer); - - // then - assertEquals(rateHistory, readHistory); - } - - @Test - public void shouldWriteAndReadMaxRateForCurrentConsumer() { - // when - maxRateRegistry.update(subscription1, ImmutableMap.of(consumerId, new MaxRate(0.5))); - maxRateRegistry.update(subscription2, ImmutableMap.of(consumerId, new MaxRate(350.0))); - - // and - maxRateRegistry.onAfterMaxRateCalculation(); // store - - // then - wait.untilZookeeperPathIsCreated(paths.consumerMaxRatePath(consumerId)); - - // when - maxRateRegistry.onBeforeMaxRateCalculation(); // read - - // then - assertEquals(new MaxRate(0.5), maxRateRegistry.getMaxRate(new ConsumerInstance(consumerId, subscription1)).get()); - assertEquals(new MaxRate(350.0), maxRateRegistry.getMaxRate(new ConsumerInstance(consumerId, subscription2)).get()); - } - - @Test - public void shouldCleanRegistryFromInactiveConsumerNodes() { - // when - maxRateRegistry.update(subscription1, ImmutableMap.of(consumerId, new MaxRate(350.0))); - - // and - maxRateRegistry.onAfterMaxRateCalculation(); // store - - // then - wait.untilZookeeperPathIsCreated(paths.consumerMaxRatePath(consumerId)); - - // when - when(clusterAssignmentCache.getAssignedConsumers()).thenReturn(Collections.emptySet()); - maxRateRegistry.onBeforeMaxRateCalculation(); // cleanup - - // then - wait.untilZookeeperPathNotExists(paths.consumerMaxRatePath(consumerId)); + private final SubscriptionName subscription1 = + SubscriptionName.fromString("pl.allegro.tech.hermes$testSubscription"); + private final SubscriptionId subscriptionId1 = SubscriptionId.from(subscription1, 1L); + private final SubscriptionName subscription2 = + SubscriptionName.fromString("pl.allegro.tech.hermes$testSubscription2"); + private final SubscriptionId subscriptionId2 = SubscriptionId.from(subscription2, 2L); + + private final SubscriptionsCache subscriptionsCache = mock(SubscriptionsCache.class); + private final SubscriptionIds subscriptionIds = + new TestSubscriptionIds(ImmutableList.of(subscriptionId1, subscriptionId2)); + + private final ZookeeperPaths zookeeperPaths = new ZookeeperPaths("/hermes"); + private final MaxRateProperties maxRateProperties = new MaxRateProperties(); + private final String consumerId = new WorkloadProperties().getNodeId(); + private final String cluster = new KafkaProperties().getClusterName(); + + private final ConsumerAssignmentCache consumerAssignmentCache = + mock(ConsumerAssignmentCache.class); + private final ClusterAssignmentCache clusterAssignmentCache = mock(ClusterAssignmentCache.class); + + private final MaxRateRegistry maxRateRegistry = + new MaxRateRegistry( + maxRateProperties.getRegistryBinaryEncoder().getHistoryBufferSizeBytes(), + maxRateProperties.getRegistryBinaryEncoder().getMaxRateBufferSizeBytes(), + consumerId, + cluster, + clusterAssignmentCache, + consumerAssignmentCache, + zookeeperClient, + zookeeperPaths, + subscriptionIds); + + private final MaxRateRegistryPaths paths = + new MaxRateRegistryPaths(zookeeperPaths, consumerId, cluster); + + @Before + public void setUp() { + when(subscriptionsCache.listActiveSubscriptionNames()) + .thenReturn(ImmutableList.of(subscription1, subscription2)); + + when(clusterAssignmentCache.getAssignedConsumers()).thenReturn(ImmutableSet.of(consumerId)); + when(consumerAssignmentCache.getConsumerSubscriptions()) + .thenReturn(ImmutableSet.of(subscription1, subscription2)); + maxRateRegistry.start(); + } + + @After + public void cleanup() { + try { + deleteAllNodes(); + } catch (Exception e) { + e.printStackTrace(); } + maxRateRegistry.stop(); + } - @Test - public void shouldCleanupConsumerMaxRateFromPreviouslyAssignedSubscriptions() { - // when - maxRateRegistry.update(subscription1, ImmutableMap.of(consumerId, new MaxRate(350.0))); - maxRateRegistry.update(subscription2, ImmutableMap.of(consumerId, new MaxRate(5.0))); - - // and - maxRateRegistry.onAfterMaxRateCalculation(); // store + @Test + public void shouldHaveEmptyRateHistoryOnInit() { + // given + maxRateRegistry.onBeforeMaxRateCalculation(); // read - // then - wait.untilZookeeperPathIsCreated(paths.consumerMaxRatePath(consumerId)); + // and + maxRateRegistry.ensureCorrectAssignments(subscription1, ImmutableSet.of(consumerId)); - // when - when(clusterAssignmentCache.getConsumerSubscriptions(consumerId)) - .thenReturn(Collections.singleton(subscription1)); - - // and - maxRateRegistry.onBeforeMaxRateCalculation(); // read and cleanup - maxRateRegistry.onAfterMaxRateCalculation(); // store - - // then - await().atMost(2, TimeUnit.SECONDS) - .until((() -> maxRateRegistry.getMaxRate(new ConsumerInstance(consumerId, subscription1)).isPresent() - && maxRateRegistry.getMaxRate(new ConsumerInstance(consumerId, subscription2)).isEmpty())); - } - - @Test(expected = IllegalStateException.class) - public void shouldNotAllowForSavingRateHistoryOfOtherThanCurrentConsumerNode() { - // when - maxRateRegistry.writeRateHistory(new ConsumerInstance("otherConsumer", subscription1), RateHistory.create(0.5)); - } - - @Test(expected = IllegalStateException.class) - public void shouldNotAllowForReadingRateHistoryOfOtherThanCurrentConsumerNode() { - // when - maxRateRegistry.getRateHistory(new ConsumerInstance("otherConsumer", subscription1)); - } - - @Test(expected = IllegalStateException.class) - public void shouldNotAllowForReadingMaxRateOfOtherThanCurrentConsumerNodeConsumerRateHistoriesDecoder() { - // when - maxRateRegistry.getMaxRate(new ConsumerInstance("otherConsumer", subscription1)); - } + // then + assertEquals( + RateHistory.empty(), + maxRateRegistry.getRateHistory(new ConsumerInstance(consumerId, subscription1))); + } + + @Test + public void shouldWriteAndReadRateHistoryForCurrentConsumer() { + // given + ConsumerInstance consumer = new ConsumerInstance(consumerId, subscription1); + + RateHistory rateHistory = RateHistory.create(0.5); + + // when + maxRateRegistry.writeRateHistory(consumer, rateHistory); + + // and + maxRateRegistry.onAfterWriteRateHistories(); // store + + // then + wait.untilZookeeperPathIsCreated(paths.currentConsumerRateHistoryPath()); + + // when + maxRateRegistry.onBeforeMaxRateCalculation(); // read + + // and + RateHistory readHistory = maxRateRegistry.getRateHistory(consumer); + + // then + assertEquals(rateHistory, readHistory); + } + + @Test + public void shouldWriteAndReadMaxRateForCurrentConsumer() { + // when + maxRateRegistry.update(subscription1, ImmutableMap.of(consumerId, new MaxRate(0.5))); + maxRateRegistry.update(subscription2, ImmutableMap.of(consumerId, new MaxRate(350.0))); + + // and + maxRateRegistry.onAfterMaxRateCalculation(); // store + + // then + wait.untilZookeeperPathIsCreated(paths.consumerMaxRatePath(consumerId)); + + // when + maxRateRegistry.onBeforeMaxRateCalculation(); // read + + // then + assertEquals( + new MaxRate(0.5), + maxRateRegistry.getMaxRate(new ConsumerInstance(consumerId, subscription1)).get()); + assertEquals( + new MaxRate(350.0), + maxRateRegistry.getMaxRate(new ConsumerInstance(consumerId, subscription2)).get()); + } + + @Test + public void shouldCleanRegistryFromInactiveConsumerNodes() { + // when + maxRateRegistry.update(subscription1, ImmutableMap.of(consumerId, new MaxRate(350.0))); + + // and + maxRateRegistry.onAfterMaxRateCalculation(); // store + + // then + wait.untilZookeeperPathIsCreated(paths.consumerMaxRatePath(consumerId)); + + // when + when(clusterAssignmentCache.getAssignedConsumers()).thenReturn(Collections.emptySet()); + maxRateRegistry.onBeforeMaxRateCalculation(); // cleanup + + // then + wait.untilZookeeperPathNotExists(paths.consumerMaxRatePath(consumerId)); + } + + @Test + public void shouldCleanupConsumerMaxRateFromPreviouslyAssignedSubscriptions() { + // when + maxRateRegistry.update(subscription1, ImmutableMap.of(consumerId, new MaxRate(350.0))); + maxRateRegistry.update(subscription2, ImmutableMap.of(consumerId, new MaxRate(5.0))); + + // and + maxRateRegistry.onAfterMaxRateCalculation(); // store + + // then + wait.untilZookeeperPathIsCreated(paths.consumerMaxRatePath(consumerId)); + + // when + when(clusterAssignmentCache.getConsumerSubscriptions(consumerId)) + .thenReturn(Collections.singleton(subscription1)); + + // and + maxRateRegistry.onBeforeMaxRateCalculation(); // read and cleanup + maxRateRegistry.onAfterMaxRateCalculation(); // store + + // then + await() + .atMost(2, TimeUnit.SECONDS) + .until( + (() -> + maxRateRegistry + .getMaxRate(new ConsumerInstance(consumerId, subscription1)) + .isPresent() + && maxRateRegistry + .getMaxRate(new ConsumerInstance(consumerId, subscription2)) + .isEmpty())); + } + + @Test(expected = IllegalStateException.class) + public void shouldNotAllowForSavingRateHistoryOfOtherThanCurrentConsumerNode() { + // when + maxRateRegistry.writeRateHistory( + new ConsumerInstance("otherConsumer", subscription1), RateHistory.create(0.5)); + } + + @Test(expected = IllegalStateException.class) + public void shouldNotAllowForReadingRateHistoryOfOtherThanCurrentConsumerNode() { + // when + maxRateRegistry.getRateHistory(new ConsumerInstance("otherConsumer", subscription1)); + } + + @Test(expected = IllegalStateException.class) + public void + shouldNotAllowForReadingMaxRateOfOtherThanCurrentConsumerNodeConsumerRateHistoriesDecoder() { + // when + maxRateRegistry.getMaxRate(new ConsumerInstance("otherConsumer", subscription1)); + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageSenderFactoryTest.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageSenderFactoryTest.java index f96e21422a..89caa055a4 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageSenderFactoryTest.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/sender/MessageSenderFactoryTest.java @@ -1,7 +1,12 @@ package pl.allegro.tech.hermes.consumers.consumer.sender; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertThrows; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; + import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import java.util.Set; import org.junit.Test; import org.mockito.Mockito; import pl.allegro.tech.hermes.api.EndpointAddress; @@ -9,62 +14,61 @@ import pl.allegro.tech.hermes.common.exception.EndpointProtocolNotSupportedException; import pl.allegro.tech.hermes.consumers.consumer.ResilientMessageSender; -import java.util.Set; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.assertThrows; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; - public class MessageSenderFactoryTest { - private final MessageSender referenceMessageSender = Mockito.mock(MessageSender.class); - private final ResilientMessageSender resilientMessageSender = Mockito.mock(ResilientMessageSender.class); + private final MessageSender referenceMessageSender = Mockito.mock(MessageSender.class); + private final ResilientMessageSender resilientMessageSender = + Mockito.mock(ResilientMessageSender.class); - @Test - public void shouldCreateCustomProtocolMessageSender() { - // given - EndpointAddress endpoint = EndpointAddress.of("myProtocol://service"); - Subscription subscription = subscription("group.topic", "subscription", endpoint).build(); - MessageSenderFactory factory = new MessageSenderFactory(ImmutableList.of( - protocolMessageSenderProviderReturning(referenceMessageSender, "myProtocol")) - ); + @Test + public void shouldCreateCustomProtocolMessageSender() { + // given + EndpointAddress endpoint = EndpointAddress.of("myProtocol://service"); + Subscription subscription = subscription("group.topic", "subscription", endpoint).build(); + MessageSenderFactory factory = + new MessageSenderFactory( + ImmutableList.of( + protocolMessageSenderProviderReturning(referenceMessageSender, "myProtocol"))); - // when - MessageSender sender = factory.create(subscription, resilientMessageSender); + // when + MessageSender sender = factory.create(subscription, resilientMessageSender); - // then - assertThat(sender).isEqualTo(referenceMessageSender); - } + // then + assertThat(sender).isEqualTo(referenceMessageSender); + } - @Test - public void shouldGetProtocolNotSupportedExceptionWhenPassingUnknownUri() { - // given - MessageSenderFactory factory = new MessageSenderFactory(ImmutableList.of()); - Subscription subscription = subscription("group.topic", "subscription", "unknown://localhost:8080/test").build(); + @Test + public void shouldGetProtocolNotSupportedExceptionWhenPassingUnknownUri() { + // given + MessageSenderFactory factory = new MessageSenderFactory(ImmutableList.of()); + Subscription subscription = + subscription("group.topic", "subscription", "unknown://localhost:8080/test").build(); - // then - assertThrows(EndpointProtocolNotSupportedException.class, () -> factory.create(subscription, resilientMessageSender)); - } + // then + assertThrows( + EndpointProtocolNotSupportedException.class, + () -> factory.create(subscription, resilientMessageSender)); + } - private ProtocolMessageSenderProvider protocolMessageSenderProviderReturning(Object createdMessageSender, String protocol) { - return new ProtocolMessageSenderProvider() { - @Override - public MessageSender create(Subscription endpoint, ResilientMessageSender resilientMessageSender) { - return (MessageSender) createdMessageSender; - } + private ProtocolMessageSenderProvider protocolMessageSenderProviderReturning( + Object createdMessageSender, String protocol) { + return new ProtocolMessageSenderProvider() { + @Override + public MessageSender create( + Subscription endpoint, ResilientMessageSender resilientMessageSender) { + return (MessageSender) createdMessageSender; + } - @Override - public Set getSupportedProtocols() { - return ImmutableSet.of(protocol); - } + @Override + public Set getSupportedProtocols() { + return ImmutableSet.of(protocol); + } - @Override - public void start() throws Exception { - } + @Override + public void start() throws Exception {} - @Override - public void stop() throws Exception { - } - }; - } + @Override + public void stop() throws Exception {} + }; + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyMessageSenderTest.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyMessageSenderTest.java index 7123eca3a5..80ad652b60 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyMessageSenderTest.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/sender/http/JettyMessageSenderTest.java @@ -1,6 +1,19 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.http; +import static jakarta.ws.rs.core.Response.Status.ACCEPTED; +import static jakarta.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR; +import static java.lang.String.format; +import static org.assertj.core.api.Assertions.assertThat; +import static pl.allegro.tech.hermes.consumers.test.MessageBuilder.TEST_MESSAGE_CONTENT; +import static pl.allegro.tech.hermes.consumers.test.MessageBuilder.testMessage; + import com.github.tomakehurst.wiremock.WireMockServer; +import java.util.Collections; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.eclipse.jetty.client.HttpClient; import org.junit.AfterClass; import org.junit.Before; @@ -25,234 +38,237 @@ import pl.allegro.tech.hermes.test.helper.metrics.TestMetricsFacadeFactory; import pl.allegro.tech.hermes.test.helper.util.Ports; -import java.util.Collections; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import static jakarta.ws.rs.core.Response.Status.ACCEPTED; -import static jakarta.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR; -import static java.lang.String.format; -import static org.assertj.core.api.Assertions.assertThat; -import static pl.allegro.tech.hermes.consumers.test.MessageBuilder.TEST_MESSAGE_CONTENT; -import static pl.allegro.tech.hermes.consumers.test.MessageBuilder.testMessage; - public class JettyMessageSenderTest { - private static final int ENDPOINT_PORT = Ports.nextAvailable(); - private static final EndpointAddress ENDPOINT = EndpointAddress.of(format("http://localhost:%d/", ENDPOINT_PORT)); - private static final EndpointAddressResolverMetadata METADATA = EndpointAddressResolverMetadata.empty(); - - private static HttpClient client; - private static WireMockServer wireMockServer; - - private ResolvableEndpointAddress address; - private RemoteServiceEndpoint remoteServiceEndpoint; - private JettyMessageSender messageSender; - - private final HttpHeadersProvider headersProvider = new HermesHeadersProvider(Collections.singleton(new Http1HeadersProvider())); - - @BeforeClass - public static void setupEnvironment() throws Exception { - wireMockServer = new WireMockServer(ENDPOINT_PORT); - wireMockServer.start(); - - SslContextFactoryProvider sslContextFactoryProvider = new SslContextFactoryProvider(null, new SslContextProperties()); - ConsumerSenderConfiguration consumerConfiguration = new ConsumerSenderConfiguration(); - client = consumerConfiguration.http1SerialClient( - new HttpClientsFactory( - new InstrumentedExecutorServiceFactory( - TestMetricsFacadeFactory.create() - ), - sslContextFactoryProvider), - new Http1ClientProperties() - ); - client.start(); - } - - @AfterClass - public static void cleanEnvironment() throws Exception { - wireMockServer.shutdown(); - client.stop(); - } - - @Before - public void setUp() throws Exception { - remoteServiceEndpoint = new RemoteServiceEndpoint(wireMockServer); - address = new ResolvableEndpointAddress(ENDPOINT, new SimpleEndpointAddressResolver(), METADATA); - HttpRequestFactory httpRequestFactory = new DefaultHttpRequestFactory(client, 1000, 1000, new DefaultHttpMetadataAppender()); - messageSender = new JettyMessageSender(httpRequestFactory, address, headersProvider, new DefaultSendingResultHandlers()); - } - - @Test - public void shouldSendMessageSuccessfully() throws Exception { - // given - remoteServiceEndpoint.expectMessages(TEST_MESSAGE_CONTENT); - - // when - CompletableFuture future = new CompletableFuture<>(); - messageSender.send(testMessage(), future); - - // then - remoteServiceEndpoint.waitUntilReceived(); - assertThat(future.get(1, TimeUnit.SECONDS).succeeded()).isTrue(); - } - - @Test - public void shouldReturnTrueWhenOtherSuccessfulCodeReturned() throws Exception { - // given - remoteServiceEndpoint.setReturnedStatusCode(ACCEPTED.getStatusCode()); - remoteServiceEndpoint.expectMessages(TEST_MESSAGE_CONTENT); - - // when - CompletableFuture future = new CompletableFuture<>(); - messageSender.send(testMessage(), future); - - // then - assertThat(future.get(1, TimeUnit.SECONDS).succeeded()).isTrue(); - } - - @Test - public void shouldNotRedirectMessage() throws InterruptedException, ExecutionException, TimeoutException { - // given - final int numberOfExpectedMessages = 1; - final int maximumWaitTimeInSeconds = 1; - remoteServiceEndpoint.redirectMessage(TEST_MESSAGE_CONTENT); - - // when - CompletableFuture future = new CompletableFuture<>(); - messageSender.send(testMessage(), future); - - // then - assertThat(future.get(maximumWaitTimeInSeconds, TimeUnit.SECONDS).succeeded()).isFalse(); - remoteServiceEndpoint.waitUntilReceived(maximumWaitTimeInSeconds, numberOfExpectedMessages); - } - - @Test - public void shouldReturnFalseWhenSendingFails() throws Exception { - // given - remoteServiceEndpoint.setReturnedStatusCode(INTERNAL_SERVER_ERROR.getStatusCode()); - remoteServiceEndpoint.expectMessages(TEST_MESSAGE_CONTENT); - - // when - CompletableFuture future = new CompletableFuture<>(); - messageSender.send(testMessage(), future); - - // then - remoteServiceEndpoint.waitUntilReceived(); - assertThat(future.get(1, TimeUnit.SECONDS).succeeded()).isFalse(); - } - - @Test - public void shouldSendMessageIdHeader() { - // given - remoteServiceEndpoint.expectMessages(TEST_MESSAGE_CONTENT); - - // when - CompletableFuture future = new CompletableFuture<>(); - messageSender.send(testMessage(), future); - - // then - remoteServiceEndpoint.waitUntilReceived(); - assertThat(remoteServiceEndpoint.getLastReceivedRequest().getHeader("Hermes-Message-Id")).isEqualTo("id"); - } - - @Test - public void shouldSendTraceIdHeader() { - // given - remoteServiceEndpoint.expectMessages(TEST_MESSAGE_CONTENT); - - // when - CompletableFuture future = new CompletableFuture<>(); - messageSender.send(testMessage(), future); - - // then - remoteServiceEndpoint.waitUntilReceived(); - assertThat(remoteServiceEndpoint.getLastReceivedRequest().getHeader("Trace-Id")).isEqualTo("traceId"); - } - - @Test - public void shouldSendRetryCounterInHeader() throws InterruptedException, ExecutionException, TimeoutException { - // given - remoteServiceEndpoint.expectMessages(TEST_MESSAGE_CONTENT); - - // when - CompletableFuture future = new CompletableFuture<>(); - messageSender.send(testMessage(), future); - - // then - assertThat(future.get(1000, TimeUnit.MILLISECONDS).getStatusCode()).isEqualTo(200); - remoteServiceEndpoint.waitUntilReceived(); - assertThat(remoteServiceEndpoint.getLastReceivedRequest().getHeader("Hermes-Retry-Count")).isEqualTo("0"); - } - - @Test - public void shouldSendAuthorizationHeaderIfAuthorizationProviderAttached() { - // given - HttpRequestFactory httpRequestFactory = new DefaultHttpRequestFactory(client, 1000, 1000, new DefaultHttpMetadataAppender()); - - JettyMessageSender messageSender = - new JettyMessageSender(httpRequestFactory, address, new HermesHeadersProvider(Collections.singleton( - new AuthHeadersProvider( - new Http1HeadersProvider(), - () -> Optional.of("Basic Auth Hello!") - )) - ), new DefaultSendingResultHandlers()); - Message message = MessageBuilder.withTestMessage().build(); - remoteServiceEndpoint.expectMessages(TEST_MESSAGE_CONTENT); - - // when - CompletableFuture future = new CompletableFuture<>(); - messageSender.send(message, future); - - // then - remoteServiceEndpoint.waitUntilReceived(); - assertThat(remoteServiceEndpoint.getLastReceivedRequest().getHeader("Authorization")).isEqualTo("Basic Auth Hello!"); - } - - @Test - public void shouldUseSuppliedRequestTimeout() throws ExecutionException, InterruptedException, TimeoutException { - // given - HttpRequestFactory httpRequestFactory = new DefaultHttpRequestFactory(client, - 100, 1000, - new DefaultHttpMetadataAppender()); - remoteServiceEndpoint.setDelay(500); - - JettyMessageSender messageSender = - new JettyMessageSender(httpRequestFactory, address, headersProvider, new DefaultSendingResultHandlers()); - Message message = MessageBuilder.withTestMessage().build(); - remoteServiceEndpoint.expectMessages(TEST_MESSAGE_CONTENT); - - // when - CompletableFuture future = new CompletableFuture<>(); - messageSender.send(message, future); - MessageSendingResult messageSendingResult = future.get(1000, TimeUnit.MILLISECONDS); - - // then - assertThat(messageSendingResult.isTimeout()).isTrue(); - } - - @Test - public void shouldUseSuppliedSocketTimeout() throws ExecutionException, InterruptedException, TimeoutException { - // given - HttpRequestFactory httpRequestFactory = new DefaultHttpRequestFactory(client, - 1000, 100, - new DefaultHttpMetadataAppender()); - remoteServiceEndpoint.setDelay(200); - - JettyMessageSender messageSender = - new JettyMessageSender(httpRequestFactory, address, headersProvider, new DefaultSendingResultHandlers()); - Message message = MessageBuilder.withTestMessage().build(); - remoteServiceEndpoint.expectMessages(TEST_MESSAGE_CONTENT); - - // when - CompletableFuture future = new CompletableFuture<>(); - messageSender.send(message, future); - MessageSendingResult messageSendingResult = future.get(1000, TimeUnit.MILLISECONDS); - - // then - assertThat(messageSendingResult.isTimeout()).isTrue(); - } -} \ No newline at end of file + private static final int ENDPOINT_PORT = Ports.nextAvailable(); + private static final EndpointAddress ENDPOINT = + EndpointAddress.of(format("http://localhost:%d/", ENDPOINT_PORT)); + private static final EndpointAddressResolverMetadata METADATA = + EndpointAddressResolverMetadata.empty(); + + private static HttpClient client; + private static WireMockServer wireMockServer; + + private ResolvableEndpointAddress address; + private RemoteServiceEndpoint remoteServiceEndpoint; + private JettyMessageSender messageSender; + + private final HttpHeadersProvider headersProvider = + new HermesHeadersProvider(Collections.singleton(new Http1HeadersProvider())); + + @BeforeClass + public static void setupEnvironment() throws Exception { + wireMockServer = new WireMockServer(ENDPOINT_PORT); + wireMockServer.start(); + + SslContextFactoryProvider sslContextFactoryProvider = + new SslContextFactoryProvider(null, new SslContextProperties()); + ConsumerSenderConfiguration consumerConfiguration = new ConsumerSenderConfiguration(); + client = + consumerConfiguration.http1SerialClient( + new HttpClientsFactory( + new InstrumentedExecutorServiceFactory(TestMetricsFacadeFactory.create()), + sslContextFactoryProvider), + new Http1ClientProperties()); + client.start(); + } + + @AfterClass + public static void cleanEnvironment() throws Exception { + wireMockServer.shutdown(); + client.stop(); + } + + @Before + public void setUp() throws Exception { + remoteServiceEndpoint = new RemoteServiceEndpoint(wireMockServer); + address = + new ResolvableEndpointAddress(ENDPOINT, new SimpleEndpointAddressResolver(), METADATA); + HttpRequestFactory httpRequestFactory = + new DefaultHttpRequestFactory(client, 1000, 1000, new DefaultHttpMetadataAppender()); + messageSender = + new JettyMessageSender( + httpRequestFactory, address, headersProvider, new DefaultSendingResultHandlers()); + } + + @Test + public void shouldSendMessageSuccessfully() throws Exception { + // given + remoteServiceEndpoint.expectMessages(TEST_MESSAGE_CONTENT); + + // when + CompletableFuture future = new CompletableFuture<>(); + messageSender.send(testMessage(), future); + + // then + remoteServiceEndpoint.waitUntilReceived(); + assertThat(future.get(1, TimeUnit.SECONDS).succeeded()).isTrue(); + } + + @Test + public void shouldReturnTrueWhenOtherSuccessfulCodeReturned() throws Exception { + // given + remoteServiceEndpoint.setReturnedStatusCode(ACCEPTED.getStatusCode()); + remoteServiceEndpoint.expectMessages(TEST_MESSAGE_CONTENT); + + // when + CompletableFuture future = new CompletableFuture<>(); + messageSender.send(testMessage(), future); + + // then + assertThat(future.get(1, TimeUnit.SECONDS).succeeded()).isTrue(); + } + + @Test + public void shouldNotRedirectMessage() + throws InterruptedException, ExecutionException, TimeoutException { + // given + final int numberOfExpectedMessages = 1; + final int maximumWaitTimeInSeconds = 1; + remoteServiceEndpoint.redirectMessage(TEST_MESSAGE_CONTENT); + + // when + CompletableFuture future = new CompletableFuture<>(); + messageSender.send(testMessage(), future); + + // then + assertThat(future.get(maximumWaitTimeInSeconds, TimeUnit.SECONDS).succeeded()).isFalse(); + remoteServiceEndpoint.waitUntilReceived(maximumWaitTimeInSeconds, numberOfExpectedMessages); + } + + @Test + public void shouldReturnFalseWhenSendingFails() throws Exception { + // given + remoteServiceEndpoint.setReturnedStatusCode(INTERNAL_SERVER_ERROR.getStatusCode()); + remoteServiceEndpoint.expectMessages(TEST_MESSAGE_CONTENT); + + // when + CompletableFuture future = new CompletableFuture<>(); + messageSender.send(testMessage(), future); + + // then + remoteServiceEndpoint.waitUntilReceived(); + assertThat(future.get(1, TimeUnit.SECONDS).succeeded()).isFalse(); + } + + @Test + public void shouldSendMessageIdHeader() { + // given + remoteServiceEndpoint.expectMessages(TEST_MESSAGE_CONTENT); + + // when + CompletableFuture future = new CompletableFuture<>(); + messageSender.send(testMessage(), future); + + // then + remoteServiceEndpoint.waitUntilReceived(); + assertThat(remoteServiceEndpoint.getLastReceivedRequest().getHeader("Hermes-Message-Id")) + .isEqualTo("id"); + } + + @Test + public void shouldSendTraceIdHeader() { + // given + remoteServiceEndpoint.expectMessages(TEST_MESSAGE_CONTENT); + + // when + CompletableFuture future = new CompletableFuture<>(); + messageSender.send(testMessage(), future); + + // then + remoteServiceEndpoint.waitUntilReceived(); + assertThat(remoteServiceEndpoint.getLastReceivedRequest().getHeader("Trace-Id")) + .isEqualTo("traceId"); + } + + @Test + public void shouldSendRetryCounterInHeader() + throws InterruptedException, ExecutionException, TimeoutException { + // given + remoteServiceEndpoint.expectMessages(TEST_MESSAGE_CONTENT); + + // when + CompletableFuture future = new CompletableFuture<>(); + messageSender.send(testMessage(), future); + + // then + assertThat(future.get(1000, TimeUnit.MILLISECONDS).getStatusCode()).isEqualTo(200); + remoteServiceEndpoint.waitUntilReceived(); + assertThat(remoteServiceEndpoint.getLastReceivedRequest().getHeader("Hermes-Retry-Count")) + .isEqualTo("0"); + } + + @Test + public void shouldSendAuthorizationHeaderIfAuthorizationProviderAttached() { + // given + HttpRequestFactory httpRequestFactory = + new DefaultHttpRequestFactory(client, 1000, 1000, new DefaultHttpMetadataAppender()); + + JettyMessageSender messageSender = + new JettyMessageSender( + httpRequestFactory, + address, + new HermesHeadersProvider( + Collections.singleton( + new AuthHeadersProvider( + new Http1HeadersProvider(), () -> Optional.of("Basic Auth Hello!")))), + new DefaultSendingResultHandlers()); + Message message = MessageBuilder.withTestMessage().build(); + remoteServiceEndpoint.expectMessages(TEST_MESSAGE_CONTENT); + + // when + CompletableFuture future = new CompletableFuture<>(); + messageSender.send(message, future); + + // then + remoteServiceEndpoint.waitUntilReceived(); + assertThat(remoteServiceEndpoint.getLastReceivedRequest().getHeader("Authorization")) + .isEqualTo("Basic Auth Hello!"); + } + + @Test + public void shouldUseSuppliedRequestTimeout() + throws ExecutionException, InterruptedException, TimeoutException { + // given + HttpRequestFactory httpRequestFactory = + new DefaultHttpRequestFactory(client, 100, 1000, new DefaultHttpMetadataAppender()); + remoteServiceEndpoint.setDelay(500); + + JettyMessageSender messageSender = + new JettyMessageSender( + httpRequestFactory, address, headersProvider, new DefaultSendingResultHandlers()); + Message message = MessageBuilder.withTestMessage().build(); + remoteServiceEndpoint.expectMessages(TEST_MESSAGE_CONTENT); + + // when + CompletableFuture future = new CompletableFuture<>(); + messageSender.send(message, future); + MessageSendingResult messageSendingResult = future.get(1000, TimeUnit.MILLISECONDS); + + // then + assertThat(messageSendingResult.isTimeout()).isTrue(); + } + + @Test + public void shouldUseSuppliedSocketTimeout() + throws ExecutionException, InterruptedException, TimeoutException { + // given + HttpRequestFactory httpRequestFactory = + new DefaultHttpRequestFactory(client, 1000, 100, new DefaultHttpMetadataAppender()); + remoteServiceEndpoint.setDelay(200); + + JettyMessageSender messageSender = + new JettyMessageSender( + httpRequestFactory, address, headersProvider, new DefaultSendingResultHandlers()); + Message message = MessageBuilder.withTestMessage().build(); + remoteServiceEndpoint.expectMessages(TEST_MESSAGE_CONTENT); + + // when + CompletableFuture future = new CompletableFuture<>(); + messageSender.send(message, future); + MessageSendingResult messageSendingResult = future.get(1000, TimeUnit.MILLISECONDS); + + // then + assertThat(messageSendingResult.isTimeout()).isTrue(); + } +} diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/sender/jms/JmsMessageSenderTest.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/sender/jms/JmsMessageSenderTest.java index 7255e9a62e..aa44d32b77 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/sender/jms/JmsMessageSenderTest.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/sender/jms/JmsMessageSenderTest.java @@ -1,5 +1,21 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.jms; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static pl.allegro.tech.hermes.common.http.MessageMetadataHeaders.MESSAGE_ID; +import static pl.allegro.tech.hermes.consumers.test.MessageBuilder.withTestMessage; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import javax.jms.BytesMessage; +import javax.jms.CompletionListener; +import javax.jms.JMSContext; +import javax.jms.JMSException; +import javax.jms.JMSProducer; +import javax.jms.JMSRuntimeException; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -12,118 +28,99 @@ import pl.allegro.tech.hermes.consumers.consumer.Message; import pl.allegro.tech.hermes.consumers.consumer.sender.MessageSendingResult; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; -import javax.jms.BytesMessage; -import javax.jms.CompletionListener; -import javax.jms.JMSContext; -import javax.jms.JMSException; -import javax.jms.JMSProducer; -import javax.jms.JMSRuntimeException; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import static pl.allegro.tech.hermes.common.http.MessageMetadataHeaders.MESSAGE_ID; -import static pl.allegro.tech.hermes.consumers.test.MessageBuilder.withTestMessage; - @RunWith(MockitoJUnitRunner.class) public class JmsMessageSenderTest { - private static final Message SOME_MESSAGE = withTestMessage().build(); - - @Mock(answer = Answers.RETURNS_DEEP_STUBS) - private JMSContext jmsContextMock; - - @Mock(answer = Answers.RETURNS_DEEP_STUBS) - private JMSProducer jmsProducerMock; - - @Mock - private BytesMessage messageMock; - - @Spy - private JmsMetadataAppender metadataAppender; - - @InjectMocks - private JmsMessageSender messageSender; - - @Before - public void setUp() throws Exception { - when(jmsContextMock.createBytesMessage()).thenReturn(messageMock); - when(jmsContextMock.createProducer()).thenReturn(jmsProducerMock); - } - - - @Test - public void shouldReturnTrueWhenMessageSuccessfullyPublished() throws Exception { - // when - CompletableFuture future = new CompletableFuture<>(); - messageSender.send(SOME_MESSAGE, future); - - // then - ArgumentCaptor listenerCaptor = ArgumentCaptor.forClass(CompletionListener.class); - verify(jmsProducerMock).setAsync(listenerCaptor.capture()); - listenerCaptor.getValue().onCompletion(messageMock); - assertTrue(future.get(1, TimeUnit.SECONDS).succeeded()); - } - - @Test - public void shouldReturnFalseWhenOnExceptionCalledOnListener() throws Exception { - // when - CompletableFuture future = new CompletableFuture<>(); - messageSender.send(SOME_MESSAGE, future); - - // then - ArgumentCaptor listenerCaptor = ArgumentCaptor.forClass(CompletionListener.class); - verify(jmsProducerMock).setAsync(listenerCaptor.capture()); - listenerCaptor.getValue().onException(messageMock, new RuntimeException()); - assertFalse(future.get(1, TimeUnit.SECONDS).succeeded()); - } - - @Test - public void shouldReturnFalseWhenJMSThrowsCheckedException() throws Exception { - // given - doThrow(new JMSException("test")).when(messageMock).writeBytes(SOME_MESSAGE.getData()); - - // when - CompletableFuture future = new CompletableFuture<>(); - messageSender.send(SOME_MESSAGE, future); - - // then - assertFalse(future.get(1, TimeUnit.SECONDS).succeeded()); - } - - @Test - public void shouldReturnFalseWhenJMSThrowsRuntimeException() throws Exception { - doThrow(new JMSRuntimeException("test")).when(jmsContextMock).createProducer(); - - // when - CompletableFuture future = new CompletableFuture<>(); - messageSender.send(SOME_MESSAGE, future); - - // then - assertFalse(future.get(1, TimeUnit.SECONDS).succeeded()); - } - - @Test - public void shouldSetMessageIdInProperty() throws JMSException { - // when - CompletableFuture future = new CompletableFuture<>(); - messageSender.send(SOME_MESSAGE, future); - - // then - verify(messageMock).setStringProperty(MESSAGE_ID.getCamelCaseName(), "id"); - } - - @Test - public void shouldSetMessageTraceIdInProperty() throws JMSException { - // when - CompletableFuture future = new CompletableFuture<>(); - messageSender.send(SOME_MESSAGE, future); - - // then - verify(messageMock).setStringProperty("TraceId", "traceId"); - } -} \ No newline at end of file + private static final Message SOME_MESSAGE = withTestMessage().build(); + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private JMSContext jmsContextMock; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private JMSProducer jmsProducerMock; + + @Mock private BytesMessage messageMock; + + @Spy private JmsMetadataAppender metadataAppender; + + @InjectMocks private JmsMessageSender messageSender; + + @Before + public void setUp() throws Exception { + when(jmsContextMock.createBytesMessage()).thenReturn(messageMock); + when(jmsContextMock.createProducer()).thenReturn(jmsProducerMock); + } + + @Test + public void shouldReturnTrueWhenMessageSuccessfullyPublished() throws Exception { + // when + CompletableFuture future = new CompletableFuture<>(); + messageSender.send(SOME_MESSAGE, future); + + // then + ArgumentCaptor listenerCaptor = + ArgumentCaptor.forClass(CompletionListener.class); + verify(jmsProducerMock).setAsync(listenerCaptor.capture()); + listenerCaptor.getValue().onCompletion(messageMock); + assertTrue(future.get(1, TimeUnit.SECONDS).succeeded()); + } + + @Test + public void shouldReturnFalseWhenOnExceptionCalledOnListener() throws Exception { + // when + CompletableFuture future = new CompletableFuture<>(); + messageSender.send(SOME_MESSAGE, future); + + // then + ArgumentCaptor listenerCaptor = + ArgumentCaptor.forClass(CompletionListener.class); + verify(jmsProducerMock).setAsync(listenerCaptor.capture()); + listenerCaptor.getValue().onException(messageMock, new RuntimeException()); + assertFalse(future.get(1, TimeUnit.SECONDS).succeeded()); + } + + @Test + public void shouldReturnFalseWhenJMSThrowsCheckedException() throws Exception { + // given + doThrow(new JMSException("test")).when(messageMock).writeBytes(SOME_MESSAGE.getData()); + + // when + CompletableFuture future = new CompletableFuture<>(); + messageSender.send(SOME_MESSAGE, future); + + // then + assertFalse(future.get(1, TimeUnit.SECONDS).succeeded()); + } + + @Test + public void shouldReturnFalseWhenJMSThrowsRuntimeException() throws Exception { + doThrow(new JMSRuntimeException("test")).when(jmsContextMock).createProducer(); + + // when + CompletableFuture future = new CompletableFuture<>(); + messageSender.send(SOME_MESSAGE, future); + + // then + assertFalse(future.get(1, TimeUnit.SECONDS).succeeded()); + } + + @Test + public void shouldSetMessageIdInProperty() throws JMSException { + // when + CompletableFuture future = new CompletableFuture<>(); + messageSender.send(SOME_MESSAGE, future); + + // then + verify(messageMock).setStringProperty(MESSAGE_ID.getCamelCaseName(), "id"); + } + + @Test + public void shouldSetMessageTraceIdInProperty() throws JMSException { + // when + CompletableFuture future = new CompletableFuture<>(); + messageSender.send(SOME_MESSAGE, future); + + // then + verify(messageMock).setStringProperty("TraceId", "traceId"); + } +} diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/sender/resolver/InterpolatingEndpointAddressResolverTest.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/sender/resolver/InterpolatingEndpointAddressResolverTest.java index 5f4c432aa8..00b9aac336 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/sender/resolver/InterpolatingEndpointAddressResolverTest.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/consumer/sender/resolver/InterpolatingEndpointAddressResolverTest.java @@ -1,5 +1,13 @@ package pl.allegro.tech.hermes.consumers.consumer.sender.resolver; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static pl.allegro.tech.hermes.consumers.test.MessageBuilder.withTestMessage; + +import java.net.URI; +import java.nio.charset.StandardCharsets; import org.junit.Test; import pl.allegro.tech.hermes.api.EndpointAddress; import pl.allegro.tech.hermes.api.EndpointAddressResolverMetadata; @@ -7,51 +15,49 @@ import pl.allegro.tech.hermes.consumers.consumer.interpolation.InterpolationException; import pl.allegro.tech.hermes.consumers.consumer.interpolation.UriInterpolator; -import java.net.URI; -import java.nio.charset.StandardCharsets; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.assertThrows; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static pl.allegro.tech.hermes.consumers.test.MessageBuilder.withTestMessage; - public class InterpolatingEndpointAddressResolverTest { - private final UriInterpolator interpolator = mock(UriInterpolator.class); - - private final InterpolatingEndpointAddressResolver resolver = new InterpolatingEndpointAddressResolver(interpolator); - - private final EndpointAddressResolverMetadata metadata = EndpointAddressResolverMetadata.empty(); - - @Test - public void shouldUseInterpolatorToInterpolateURI() throws InterpolationException, EndpointAddressResolutionException { - // given - EndpointAddress address = EndpointAddress.of("http://localhost/{a}"); - Message message = withTestMessage().withContent("content", StandardCharsets.UTF_8).build(); - when(interpolator.interpolate(address, message)).thenReturn(URI.create("http://localhost/hello")); - - // when - URI uri = resolver.resolve(EndpointAddress.of("http://localhost/{a}"), - withTestMessage().withContent("content", StandardCharsets.UTF_8).build(), metadata); - - // then - assertThat(uri).isEqualTo(URI.create("http://localhost/hello")); - } - - @Test - public void shouldThrowResolvingExceptionWhenInterpolationFails() throws InterpolationException { - // given - EndpointAddress address = EndpointAddress.of("http://localhost/{a}"); - Message message = withTestMessage().withContent("content", StandardCharsets.UTF_8).build(); - when(interpolator.interpolate(address, message)).thenThrow(InterpolationException.class); - - // then - assertThrows(EndpointAddressResolutionException.class, - () -> resolver.resolve( - EndpointAddress.of("http://localhost/{a}"), - withTestMessage().withContent("content", StandardCharsets.UTF_8).build(), - metadata) - ); - } + private final UriInterpolator interpolator = mock(UriInterpolator.class); + + private final InterpolatingEndpointAddressResolver resolver = + new InterpolatingEndpointAddressResolver(interpolator); + + private final EndpointAddressResolverMetadata metadata = EndpointAddressResolverMetadata.empty(); + + @Test + public void shouldUseInterpolatorToInterpolateURI() + throws InterpolationException, EndpointAddressResolutionException { + // given + EndpointAddress address = EndpointAddress.of("http://localhost/{a}"); + Message message = withTestMessage().withContent("content", StandardCharsets.UTF_8).build(); + when(interpolator.interpolate(address, message)) + .thenReturn(URI.create("http://localhost/hello")); + + // when + URI uri = + resolver.resolve( + EndpointAddress.of("http://localhost/{a}"), + withTestMessage().withContent("content", StandardCharsets.UTF_8).build(), + metadata); + + // then + assertThat(uri).isEqualTo(URI.create("http://localhost/hello")); + } + + @Test + public void shouldThrowResolvingExceptionWhenInterpolationFails() throws InterpolationException { + // given + EndpointAddress address = EndpointAddress.of("http://localhost/{a}"); + Message message = withTestMessage().withContent("content", StandardCharsets.UTF_8).build(); + when(interpolator.interpolate(address, message)).thenThrow(InterpolationException.class); + + // then + assertThrows( + EndpointAddressResolutionException.class, + () -> + resolver.resolve( + EndpointAddress.of("http://localhost/{a}"), + withTestMessage().withContent("content", StandardCharsets.UTF_8).build(), + metadata)); + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/queue/FullDrainMpscQueueTest.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/queue/FullDrainMpscQueueTest.java index c4ea984bab..475c55cf81 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/queue/FullDrainMpscQueueTest.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/queue/FullDrainMpscQueueTest.java @@ -2,8 +2,8 @@ public class FullDrainMpscQueueTest extends MpscQueuesAbstractTest { - @Override - protected MpscQueue createMpscQueue(int size) { - return new FullDrainMpscQueue<>(size); - } + @Override + protected MpscQueue createMpscQueue(int size) { + return new FullDrainMpscQueue<>(size); + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/queue/MpscQueuesAbstractTest.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/queue/MpscQueuesAbstractTest.java index ee0a22a2d9..e9f262966f 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/queue/MpscQueuesAbstractTest.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/queue/MpscQueuesAbstractTest.java @@ -1,51 +1,50 @@ package pl.allegro.tech.hermes.consumers.queue; -import org.junit.Test; +import static org.assertj.core.api.Assertions.assertThat; import java.util.ArrayList; import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; +import org.junit.Test; public abstract class MpscQueuesAbstractTest { - protected abstract MpscQueue createMpscQueue(int size); + protected abstract MpscQueue createMpscQueue(int size); - @Test - public void shouldDrainItemsFromNonEmptyQueue() { - // given - MpscQueue queue = createMpscQueue(16); - assertThat(queue.capacity()).isEqualTo(16); + @Test + public void shouldDrainItemsFromNonEmptyQueue() { + // given + MpscQueue queue = createMpscQueue(16); + assertThat(queue.capacity()).isEqualTo(16); - // when - queue.offer(1); - queue.offer(2); - queue.offer(3); + // when + queue.offer(1); + queue.offer(2); + queue.offer(3); - // then - assertThat(queue.size()).isEqualTo(3); + // then + assertThat(queue.size()).isEqualTo(3); - // when - List drained = new ArrayList<>(); - queue.drain(drained::add); + // when + List drained = new ArrayList<>(); + queue.drain(drained::add); - // then - assertThat(drained).contains(1, 2, 3); + // then + assertThat(drained).contains(1, 2, 3); - // and - assertThat(queue.size()).isZero(); - } + // and + assertThat(queue.size()).isZero(); + } - @Test - public void shouldDrainEmptyQueue() { - // given - MpscQueue queue = createMpscQueue(16); + @Test + public void shouldDrainEmptyQueue() { + // given + MpscQueue queue = createMpscQueue(16); - // when - List drained = new ArrayList<>(); - queue.drain(drained::add); + // when + List drained = new ArrayList<>(); + queue.drain(drained::add); - // then - assertThat(drained).isEmpty(); - } -} \ No newline at end of file + // then + assertThat(drained).isEmpty(); + } +} diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/queue/WaitFreeDrainMpscQueueTest.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/queue/WaitFreeDrainMpscQueueTest.java index 6923f10a3f..c6f3320806 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/queue/WaitFreeDrainMpscQueueTest.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/queue/WaitFreeDrainMpscQueueTest.java @@ -2,8 +2,8 @@ public class WaitFreeDrainMpscQueueTest extends MpscQueuesAbstractTest { - @Override - protected MpscQueue createMpscQueue(int size) { - return new WaitFreeDrainMpscQueue<>(size); - } + @Override + protected MpscQueue createMpscQueue(int size) { + return new WaitFreeDrainMpscQueue<>(size); + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ConsumerTestRuntimeEnvironment.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ConsumerTestRuntimeEnvironment.java index 878522aa0b..2deff0501a 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ConsumerTestRuntimeEnvironment.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/ConsumerTestRuntimeEnvironment.java @@ -1,7 +1,24 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload; +import static java.util.stream.Collectors.toList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.mockito.Mockito.mock; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topic; +import static pl.allegro.tech.hermes.test.helper.endpoint.TimeoutAdjuster.adjust; + import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Maps; +import java.time.Clock; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.Supplier; +import java.util.stream.IntStream; import org.apache.curator.framework.CuratorFramework; import pl.allegro.tech.hermes.api.Group; import pl.allegro.tech.hermes.api.Subscription; @@ -45,293 +62,299 @@ import pl.allegro.tech.hermes.infrastructure.zookeeper.notifications.ZookeeperInternalNotificationBus; import pl.allegro.tech.hermes.test.helper.metrics.TestMetricsFacadeFactory; -import java.time.Clock; -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.function.Supplier; -import java.util.stream.IntStream; - -import static java.util.stream.Collectors.toList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; -import static org.mockito.Mockito.mock; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topic; -import static pl.allegro.tech.hermes.test.helper.endpoint.TimeoutAdjuster.adjust; - class ConsumerTestRuntimeEnvironment { - private static final int DEATH_OF_CONSUMER_AFTER_SECONDS = 300; - private final KafkaProperties kafkaProperties = new KafkaProperties(); - - private final ConsumerNodesRegistryPaths nodesRegistryPaths; - - private int consumerIdSequence = 0; - - private int subscriptionIdSequence = 0; - - private final ZookeeperPaths zookeeperPaths = new ZookeeperPaths("/hermes"); - private final Supplier curatorSupplier; - - private final GroupRepository groupRepository; - private final TopicRepository topicRepository; - private final SubscriptionRepository subscriptionRepository; - private final ObjectMapper objectMapper = new ObjectMapper(); - private final ExecutorService executorService = Executors.newSingleThreadExecutor(); - private final Supplier metricsSupplier; - private final WorkloadConstraintsRepository workloadConstraintsRepository; - private final CuratorFramework curator; - private final ConsumerPartitionAssignmentState partitionAssignmentState; - private final ZookeeperProperties zookeeperProperties; - - private final Map consumerZookeeperConnections = Maps.newHashMap(); - private final List subscriptionsCaches = new ArrayList<>(); - - ConsumerTestRuntimeEnvironment(Supplier curatorSupplier) { - this.curatorSupplier = curatorSupplier; - this.curator = curatorSupplier.get(); - this.partitionAssignmentState = new ConsumerPartitionAssignmentState(); - this.groupRepository = new ZookeeperGroupRepository(curator, objectMapper, zookeeperPaths); - this.topicRepository = new ZookeeperTopicRepository(curator, objectMapper, zookeeperPaths, groupRepository); - this.subscriptionRepository = new ZookeeperSubscriptionRepository( - curator, objectMapper, zookeeperPaths, topicRepository - ); - - this.workloadConstraintsRepository = new ZookeeperWorkloadConstraintsRepository(curator, objectMapper, zookeeperPaths); - - this.metricsSupplier = TestMetricsFacadeFactory::create; - this.nodesRegistryPaths = new ConsumerNodesRegistryPaths(zookeeperPaths, kafkaProperties.getClusterName()); - this.zookeeperProperties = new ZookeeperProperties(); - } - - WorkloadSupervisor findLeader(List supervisors) { - return supervisors.stream().filter(WorkloadSupervisor::isLeader).findAny().get(); + private static final int DEATH_OF_CONSUMER_AFTER_SECONDS = 300; + private final KafkaProperties kafkaProperties = new KafkaProperties(); + + private final ConsumerNodesRegistryPaths nodesRegistryPaths; + + private int consumerIdSequence = 0; + + private int subscriptionIdSequence = 0; + + private final ZookeeperPaths zookeeperPaths = new ZookeeperPaths("/hermes"); + private final Supplier curatorSupplier; + + private final GroupRepository groupRepository; + private final TopicRepository topicRepository; + private final SubscriptionRepository subscriptionRepository; + private final ObjectMapper objectMapper = new ObjectMapper(); + private final ExecutorService executorService = Executors.newSingleThreadExecutor(); + private final Supplier metricsSupplier; + private final WorkloadConstraintsRepository workloadConstraintsRepository; + private final CuratorFramework curator; + private final ConsumerPartitionAssignmentState partitionAssignmentState; + private final ZookeeperProperties zookeeperProperties; + + private final Map consumerZookeeperConnections = Maps.newHashMap(); + private final List subscriptionsCaches = new ArrayList<>(); + + ConsumerTestRuntimeEnvironment(Supplier curatorSupplier) { + this.curatorSupplier = curatorSupplier; + this.curator = curatorSupplier.get(); + this.partitionAssignmentState = new ConsumerPartitionAssignmentState(); + this.groupRepository = new ZookeeperGroupRepository(curator, objectMapper, zookeeperPaths); + this.topicRepository = + new ZookeeperTopicRepository(curator, objectMapper, zookeeperPaths, groupRepository); + this.subscriptionRepository = + new ZookeeperSubscriptionRepository(curator, objectMapper, zookeeperPaths, topicRepository); + + this.workloadConstraintsRepository = + new ZookeeperWorkloadConstraintsRepository(curator, objectMapper, zookeeperPaths); + + this.metricsSupplier = TestMetricsFacadeFactory::create; + this.nodesRegistryPaths = + new ConsumerNodesRegistryPaths(zookeeperPaths, kafkaProperties.getClusterName()); + this.zookeeperProperties = new ZookeeperProperties(); + } + + WorkloadSupervisor findLeader(List supervisors) { + return supervisors.stream().filter(WorkloadSupervisor::isLeader).findAny().get(); + } + + private WorkloadSupervisor createConsumer( + String consumerId, ConsumersSupervisor consumersSupervisor) { + CuratorFramework curator = curatorSupplier.get(); + consumerZookeeperConnections.put(consumerId, curator); + ConsumerNodesRegistry nodesRegistry = + new ConsumerNodesRegistry( + curator, + executorService, + nodesRegistryPaths, + consumerId, + DEATH_OF_CONSUMER_AFTER_SECONDS, + Clock.systemDefaultZone()); + + try { + nodesRegistry.start(); + } catch (Exception e) { + throw new IllegalStateException(e); } - private WorkloadSupervisor createConsumer(String consumerId, - ConsumersSupervisor consumersSupervisor) { - CuratorFramework curator = curatorSupplier.get(); - consumerZookeeperConnections.put(consumerId, curator); - ConsumerNodesRegistry nodesRegistry = new ConsumerNodesRegistry( - curator, - executorService, - nodesRegistryPaths, - consumerId, - DEATH_OF_CONSUMER_AFTER_SECONDS, - Clock.systemDefaultZone()); - - try { - nodesRegistry.start(); - } catch (Exception e) { - throw new IllegalStateException(e); - } - - WorkloadProperties workloadProperties = new WorkloadProperties(); - workloadProperties.setNodeId(consumerId); - workloadProperties.setRebalanceInterval(Duration.ofSeconds(1)); - workloadProperties.setConsumersPerSubscription(2); - workloadProperties.setMonitorScanInterval(Duration.ofSeconds(1)); - - ModelAwareZookeeperNotifyingCache modelAwareCache = new ModelAwareZookeeperNotifyingCacheFactory( - curator, metricsSupplier.get(), zookeeperProperties - ).provide(); - - InternalNotificationsBus notificationsBus = - new ZookeeperInternalNotificationBus(objectMapper, modelAwareCache); - - SubscriptionsCache subscriptionsCache = new NotificationsBasedSubscriptionCache( - notificationsBus, groupRepository, topicRepository, subscriptionRepository - ); - subscriptionsCaches.add(subscriptionsCache); - - SubscriptionConfiguration subscriptionConfiguration = new SubscriptionConfiguration(); - SubscriptionIds subscriptionIds = subscriptionConfiguration.subscriptionIds(notificationsBus, subscriptionsCache, - new ZookeeperSubscriptionIdProvider(curator, zookeeperPaths), new CommonConsumerProperties()); - - ConsumerAssignmentCache consumerAssignmentCache = - new ConsumerAssignmentCache( - curator, - workloadProperties.getNodeId(), - kafkaProperties.getClusterName(), - zookeeperPaths, - subscriptionIds - ); - try { - consumerAssignmentCache.start(); - } catch (Exception e) { - throw new IllegalStateException(e); - } - - ClusterAssignmentCache clusterAssignmentCache = - new ClusterAssignmentCache( - curator, - kafkaProperties.getClusterName(), - zookeeperPaths, - subscriptionIds, - nodesRegistry - ); - - ConsumerAssignmentRegistry consumerAssignmentRegistry = - new ConsumerAssignmentRegistry( - curator, - workloadProperties.getRegistryBinaryEncoderAssignmentsBufferSizeBytes(), - kafkaProperties.getClusterName(), - zookeeperPaths, - subscriptionIds - ); - - return new WorkloadSupervisor( - consumersSupervisor, - notificationsBus, - subscriptionsCache, - consumerAssignmentCache, - consumerAssignmentRegistry, - clusterAssignmentCache, - nodesRegistry, - mock(ZookeeperAdminCache.class), - executorService, - workloadProperties, - kafkaProperties.getClusterName(), - metricsSupplier.get(), - workloadConstraintsRepository, - new SelectiveWorkBalancer(), - new NoOpBalancingListener() - ); + WorkloadProperties workloadProperties = new WorkloadProperties(); + workloadProperties.setNodeId(consumerId); + workloadProperties.setRebalanceInterval(Duration.ofSeconds(1)); + workloadProperties.setConsumersPerSubscription(2); + workloadProperties.setMonitorScanInterval(Duration.ofSeconds(1)); + + ModelAwareZookeeperNotifyingCache modelAwareCache = + new ModelAwareZookeeperNotifyingCacheFactory( + curator, metricsSupplier.get(), zookeeperProperties) + .provide(); + + InternalNotificationsBus notificationsBus = + new ZookeeperInternalNotificationBus(objectMapper, modelAwareCache); + + SubscriptionsCache subscriptionsCache = + new NotificationsBasedSubscriptionCache( + notificationsBus, groupRepository, topicRepository, subscriptionRepository); + subscriptionsCaches.add(subscriptionsCache); + + SubscriptionConfiguration subscriptionConfiguration = new SubscriptionConfiguration(); + SubscriptionIds subscriptionIds = + subscriptionConfiguration.subscriptionIds( + notificationsBus, + subscriptionsCache, + new ZookeeperSubscriptionIdProvider(curator, zookeeperPaths), + new CommonConsumerProperties()); + + ConsumerAssignmentCache consumerAssignmentCache = + new ConsumerAssignmentCache( + curator, + workloadProperties.getNodeId(), + kafkaProperties.getClusterName(), + zookeeperPaths, + subscriptionIds); + try { + consumerAssignmentCache.start(); + } catch (Exception e) { + throw new IllegalStateException(e); } - WorkloadSupervisor spawnConsumer() { - return spawnConsumers(1).get(0); + ClusterAssignmentCache clusterAssignmentCache = + new ClusterAssignmentCache( + curator, + kafkaProperties.getClusterName(), + zookeeperPaths, + subscriptionIds, + nodesRegistry); + + ConsumerAssignmentRegistry consumerAssignmentRegistry = + new ConsumerAssignmentRegistry( + curator, + workloadProperties.getRegistryBinaryEncoderAssignmentsBufferSizeBytes(), + kafkaProperties.getClusterName(), + zookeeperPaths, + subscriptionIds); + + return new WorkloadSupervisor( + consumersSupervisor, + notificationsBus, + subscriptionsCache, + consumerAssignmentCache, + consumerAssignmentRegistry, + clusterAssignmentCache, + nodesRegistry, + mock(ZookeeperAdminCache.class), + executorService, + workloadProperties, + kafkaProperties.getClusterName(), + metricsSupplier.get(), + workloadConstraintsRepository, + new SelectiveWorkBalancer(), + new NoOpBalancingListener()); + } + + WorkloadSupervisor spawnConsumer() { + return spawnConsumers(1).get(0); + } + + WorkloadSupervisor spawnConsumer(String consumerId, ConsumersSupervisor consumersSupervisor) { + WorkloadSupervisor workloadSupervisor = createConsumer(consumerId, consumersSupervisor); + return startNode(workloadSupervisor); + } + + ConsumersSupervisor consumersSupervisor(ConsumerFactory consumerFactory) { + MetricsFacade metrics = metricsSupplier.get(); + CommonConsumerProperties commonConsumerProperties = new CommonConsumerProperties(); + CommonConsumerProperties.BackgroundSupervisor supervisorParameters = + new CommonConsumerProperties.BackgroundSupervisor(); + supervisorParameters.setInterval(Duration.ofSeconds(1)); + commonConsumerProperties.setBackgroundSupervisor(supervisorParameters); + return new NonblockingConsumersSupervisor( + commonConsumerProperties, + new ConsumersExecutorService(new CommonConsumerProperties().getThreadPoolSize(), metrics), + consumerFactory, + partitionAssignmentState, + mock(Retransmitter.class), + mock(UndeliveredMessageLogPersister.class), + subscriptionRepository, + metrics, + mock(ConsumerMonitor.class), + Clock.systemDefaultZone()); + } + + ConsumersRuntimeMonitor monitor( + String consumerId, + ConsumersSupervisor consumersSupervisor, + WorkloadSupervisor workloadSupervisor, + Duration monitorScanInterval) { + CuratorFramework curator = consumerZookeeperConnections.get(consumerId); + ModelAwareZookeeperNotifyingCache modelAwareCache = + new ModelAwareZookeeperNotifyingCacheFactory( + curator, metricsSupplier.get(), zookeeperProperties) + .provide(); + InternalNotificationsBus notificationsBus = + new ZookeeperInternalNotificationBus(objectMapper, modelAwareCache); + SubscriptionsCache subscriptionsCache = + new NotificationsBasedSubscriptionCache( + notificationsBus, groupRepository, topicRepository, subscriptionRepository); + subscriptionsCaches.add(subscriptionsCache); + subscriptionsCache.start(); + return new ConsumersRuntimeMonitor( + consumersSupervisor, + workloadSupervisor, + metricsSupplier.get(), + subscriptionsCache, + monitorScanInterval); + } + + List spawnConsumers(int howMany) { + return IntStream.range(0, howMany) + .mapToObj( + i -> { + String consumerId = nextConsumerId(); + ConsumersSupervisor consumersSupervisor = + consumersSupervisor(mock(ConsumerFactory.class)); + WorkloadSupervisor consumer = createConsumer(consumerId, consumersSupervisor); + startNode(consumer); + return consumer; + }) + .collect(toList()); + } + + void kill(WorkloadSupervisor node) { + consumerZookeeperConnections.get(node.consumerId()).close(); + } + + void cleanState() { + consumerZookeeperConnections.values().forEach(CuratorFramework::close); + subscriptionsCaches.clear(); + } + + private WorkloadSupervisor startNode(WorkloadSupervisor workloadSupervisor) { + try { + workloadSupervisor.start(); + waitForRegistration(workloadSupervisor.consumerId()); + return workloadSupervisor; + } catch (Exception e) { + throw new InternalProcessingException(e); } + } - WorkloadSupervisor spawnConsumer(String consumerId, ConsumersSupervisor consumersSupervisor) { - WorkloadSupervisor workloadSupervisor = createConsumer(consumerId, consumersSupervisor); - return startNode(workloadSupervisor); - } + void waitForRegistration(String consumerId) { + await().atMost(adjust(Duration.ofSeconds(1))).until(() -> isRegistered(consumerId)); + } - ConsumersSupervisor consumersSupervisor(ConsumerFactory consumerFactory) { - MetricsFacade metrics = metricsSupplier.get(); - CommonConsumerProperties commonConsumerProperties = new CommonConsumerProperties(); - CommonConsumerProperties.BackgroundSupervisor supervisorParameters = new CommonConsumerProperties.BackgroundSupervisor(); - supervisorParameters.setInterval(Duration.ofSeconds(1)); - commonConsumerProperties.setBackgroundSupervisor(supervisorParameters); - return new NonblockingConsumersSupervisor(commonConsumerProperties, - new ConsumersExecutorService(new CommonConsumerProperties().getThreadPoolSize(), metrics), - consumerFactory, - partitionAssignmentState, - mock(Retransmitter.class), - mock(UndeliveredMessageLogPersister.class), - subscriptionRepository, - metrics, - mock(ConsumerMonitor.class), - Clock.systemDefaultZone()); + private boolean isRegistered(String nodeId) { + try { + return curator.checkExists().forPath(nodesRegistryPaths.nodePath(nodeId)) != null; + } catch (Exception e) { + return false; } - - ConsumersRuntimeMonitor monitor(String consumerId, - ConsumersSupervisor consumersSupervisor, - WorkloadSupervisor workloadSupervisor, - Duration monitorScanInterval) { - CuratorFramework curator = consumerZookeeperConnections.get(consumerId); - ModelAwareZookeeperNotifyingCache modelAwareCache = - new ModelAwareZookeeperNotifyingCacheFactory(curator, metricsSupplier.get(), zookeeperProperties).provide(); - InternalNotificationsBus notificationsBus = - new ZookeeperInternalNotificationBus(objectMapper, modelAwareCache); - SubscriptionsCache subscriptionsCache = new NotificationsBasedSubscriptionCache( - notificationsBus, groupRepository, topicRepository, subscriptionRepository); - subscriptionsCaches.add(subscriptionsCache); - subscriptionsCache.start(); - return new ConsumersRuntimeMonitor( - consumersSupervisor, - workloadSupervisor, - metricsSupplier.get(), - subscriptionsCache, - monitorScanInterval); - } - - List spawnConsumers(int howMany) { - return IntStream.range(0, howMany) - .mapToObj(i -> { - String consumerId = nextConsumerId(); - ConsumersSupervisor consumersSupervisor = consumersSupervisor(mock(ConsumerFactory.class)); - WorkloadSupervisor consumer = createConsumer(consumerId, consumersSupervisor); - startNode(consumer); - return consumer; - }) - .collect(toList()); - } - - void kill(WorkloadSupervisor node) { - consumerZookeeperConnections.get(node.consumerId()).close(); - } - - void cleanState() { - consumerZookeeperConnections.values().forEach(CuratorFramework::close); - subscriptionsCaches.clear(); + } + + void awaitUntilAssignmentExists(SubscriptionName subscription, WorkloadSupervisor node) { + await() + .atMost(adjust(Duration.ofSeconds(2))) + .until(() -> node.assignedSubscriptions().contains(subscription)); + } + + List createSubscription(int howMany) { + return IntStream.range(0, howMany) + .mapToObj(i -> createSubscription(nextSubscriptionName())) + .collect(toList()); + } + + SubscriptionName createSubscription() { + return createSubscription(1).get(0); + } + + private SubscriptionName createSubscription(SubscriptionName subscriptionName) { + Subscription subscription = subscription(subscriptionName).build(); + Group group = Group.from(subscription.getTopicName().getGroupName()); + if (!groupRepository.groupExists(group.getGroupName())) { + groupRepository.createGroup(group); } - - private WorkloadSupervisor startNode(WorkloadSupervisor workloadSupervisor) { - try { - workloadSupervisor.start(); - waitForRegistration(workloadSupervisor.consumerId()); - return workloadSupervisor; - } catch (Exception e) { - throw new InternalProcessingException(e); - } - } - - void waitForRegistration(String consumerId) { - await().atMost(adjust(Duration.ofSeconds(1))).until(() -> isRegistered(consumerId)); - } - - private boolean isRegistered(String nodeId) { - try { - return curator.checkExists().forPath(nodesRegistryPaths.nodePath(nodeId)) != null; - } catch (Exception e) { - return false; - } - } - - void awaitUntilAssignmentExists(SubscriptionName subscription, WorkloadSupervisor node) { - await().atMost(adjust(Duration.ofSeconds(2))).until(() -> node.assignedSubscriptions().contains(subscription)); - } - - List createSubscription(int howMany) { - return IntStream.range(0, howMany).mapToObj(i -> - createSubscription(nextSubscriptionName())).collect(toList()); - } - - SubscriptionName createSubscription() { - return createSubscription(1).get(0); - } - - private SubscriptionName createSubscription(SubscriptionName subscriptionName) { - Subscription subscription = subscription(subscriptionName).build(); - Group group = Group.from(subscription.getTopicName().getGroupName()); - if (!groupRepository.groupExists(group.getGroupName())) { - groupRepository.createGroup(group); - } - if (!topicRepository.topicExists(subscription.getTopicName())) { - topicRepository.createTopic(topic(subscription.getTopicName()).build()); - } - subscriptionRepository.createSubscription(subscription); - await().atMost(adjust(Duration.ofSeconds(2))).untilAsserted( - () -> { - assertThat(subscriptionRepository.subscriptionExists(subscription.getTopicName(), subscription.getName())).isTrue(); - subscriptionsCaches.forEach(subscriptionsCache -> - assertThat(subscriptionsCache.listActiveSubscriptionNames().contains(subscriptionName)).isTrue()); - } - ); - return subscription.getQualifiedName(); - } - - private String nextConsumerId() { - return String.valueOf(consumerIdSequence++); - } - - private SubscriptionName nextSubscriptionName() { - return SubscriptionName.fromString("com.example.topic$test" + subscriptionIdSequence++); + if (!topicRepository.topicExists(subscription.getTopicName())) { + topicRepository.createTopic(topic(subscription.getTopicName()).build()); } + subscriptionRepository.createSubscription(subscription); + await() + .atMost(adjust(Duration.ofSeconds(2))) + .untilAsserted( + () -> { + assertThat( + subscriptionRepository.subscriptionExists( + subscription.getTopicName(), subscription.getName())) + .isTrue(); + subscriptionsCaches.forEach( + subscriptionsCache -> + assertThat( + subscriptionsCache + .listActiveSubscriptionNames() + .contains(subscriptionName)) + .isTrue()); + }); + return subscription.getQualifiedName(); + } + + private String nextConsumerId() { + return String.valueOf(consumerIdSequence++); + } + + private SubscriptionName nextSubscriptionName() { + return SubscriptionName.fromString("com.example.topic$test" + subscriptionIdSequence++); + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/SelectiveWorkBalancerTest.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/SelectiveWorkBalancerTest.java index 06119b5865..8a68c95ab4 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/SelectiveWorkBalancerTest.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/SelectiveWorkBalancerTest.java @@ -1,322 +1,358 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload; +import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static java.util.stream.Collectors.toList; +import static org.assertj.core.api.Assertions.assertThat; + import com.google.common.collect.ImmutableList; import com.tngtech.java.junit.dataprovider.DataProvider; import com.tngtech.java.junit.dataprovider.DataProviderRunner; import com.tngtech.java.junit.dataprovider.UseDataProvider; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.IntStream; import org.junit.Test; import org.junit.runner.RunWith; import pl.allegro.tech.hermes.api.Constraints; import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.consumers.supervisor.workload.selective.SelectiveWorkBalancer; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.stream.IntStream; - -import static java.util.Arrays.asList; -import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; -import static java.util.stream.Collectors.toList; -import static org.assertj.core.api.Assertions.assertThat; - @RunWith(DataProviderRunner.class) public class SelectiveWorkBalancerTest { - private SelectiveWorkBalancer workBalancer = new SelectiveWorkBalancer(); - - @Test - public void shouldPerformSubscriptionsCleanup() { - // given - List subscriptions = someSubscriptions(1); - List supervisors = someSupervisors(1); - WorkloadConstraints constraints = WorkloadConstraints.builder() - .withActiveConsumers(supervisors.size()) - .withConsumersPerSubscription(2) - .withMaxSubscriptionsPerConsumer(2) - .build(); - SubscriptionAssignmentView currentState = initialState(subscriptions, supervisors, constraints); - - // when - WorkBalancingResult target = workBalancer.balance(emptyList(), supervisors, currentState, constraints); - - // then - assertThat(target.getAssignmentsView().getSubscriptions()).isEmpty(); - assertThat(currentState.deletions(target.getAssignmentsView()).getAllAssignments()).hasSize(subscriptions.size()); - } - - @Test - public void shouldPerformSupervisorsCleanup() { - // given - List supervisors = someSupervisors(2); - List subscriptions = someSubscriptions(1); - WorkloadConstraints constraints = WorkloadConstraints.builder() - .withActiveConsumers(supervisors.size()) - .withConsumersPerSubscription(2) - .withMaxSubscriptionsPerConsumer(2) - .build(); - SubscriptionAssignmentView currentState = initialState(subscriptions, supervisors, constraints); - - // when - supervisors.remove(1); - WorkBalancingResult work = workBalancer.balance(subscriptions, supervisors, currentState, constraints); - - // then - assertThat(currentState.deletions(work.getAssignmentsView()).getAllAssignments()).hasSize(1); - assertThatSubscriptionIsAssignedTo(work.getAssignmentsView(), subscriptions.get(0), supervisors.get(0)); - } - - @Test - public void shouldBalanceWorkForSingleSubscription() { - // given - List supervisors = someSupervisors(1); - List subscriptions = someSubscriptions(1); - WorkloadConstraints constraints = WorkloadConstraints.builder() - .withActiveConsumers(supervisors.size()) - .withConsumersPerSubscription(2) - .withMaxSubscriptionsPerConsumer(2) - .build(); - - // when - SubscriptionAssignmentView view = initialState(subscriptions, supervisors, constraints); - - // then - assertThatSubscriptionIsAssignedTo(view, subscriptions.get(0), supervisors.get(0)); - } - - @Test - public void shouldBalanceWorkForMultipleConsumersAndSingleSubscription() { - // given - List supervisors = someSupervisors(2); - List subscriptions = someSubscriptions(1); - WorkloadConstraints constraints = WorkloadConstraints.builder() - .withActiveConsumers(supervisors.size()) - .withConsumersPerSubscription(2) - .withMaxSubscriptionsPerConsumer(2) - .build(); - - // when - SubscriptionAssignmentView view = initialState(subscriptions, supervisors, constraints); - - // then - assertThatSubscriptionIsAssignedTo(view, subscriptions.get(0), supervisors); - } - - @Test - public void shouldBalanceWorkForMultipleConsumersAndMultipleSubscriptions() { - // given - List supervisors = someSupervisors(2); - List subscriptions = someSubscriptions(2); - WorkloadConstraints constraints = WorkloadConstraints.builder() - .withActiveConsumers(supervisors.size()) - .withConsumersPerSubscription(2) - .withMaxSubscriptionsPerConsumer(2) - .build(); - - // when - SubscriptionAssignmentView view = initialState(subscriptions, supervisors, constraints); - - // then - assertThatSubscriptionIsAssignedTo(view, subscriptions.get(0), supervisors); - assertThatSubscriptionIsAssignedTo(view, subscriptions.get(1), supervisors); - } - - @Test - public void shouldNotOverloadConsumers() { - // given - List supervisors = someSupervisors(1); - List subscriptions = someSubscriptions(3); - WorkloadConstraints constraints = WorkloadConstraints.builder() - .withActiveConsumers(supervisors.size()) - .withConsumersPerSubscription(2) - .withMaxSubscriptionsPerConsumer(2) - .build(); - - // when - SubscriptionAssignmentView view = initialState(subscriptions, supervisors, constraints); - - // then - assertThat(view.getAssignmentsForConsumerNode(supervisors.get(0))).hasSize(2); - } - - @Test - public void shouldRebalanceAfterConsumerDisappearing() { - // given - List supervisors = ImmutableList.of("c1", "c2"); - List subscriptions = someSubscriptions(2); - WorkloadConstraints constraints = WorkloadConstraints.builder() - .withActiveConsumers(supervisors.size()) - .withConsumersPerSubscription(2) - .withMaxSubscriptionsPerConsumer(2) - .build(); - SubscriptionAssignmentView currentState = initialState(subscriptions, supervisors, constraints); - - // when - List extendedSupervisorsList = ImmutableList.of("c1", "c3"); - SubscriptionAssignmentView stateAfterRebalance = workBalancer - .balance(subscriptions, extendedSupervisorsList, currentState, constraints) - .getAssignmentsView(); - - // then - assertThat(stateAfterRebalance.getSubscriptionsForConsumerNode("c3")).containsOnly(subscriptions.get(0), subscriptions.get(1)); - } - - @Test - public void shouldAssignWorkToNewConsumersByWorkStealing() { - // given - List supervisors = someSupervisors(2); - List subscriptions = someSubscriptions(2); - WorkloadConstraints constraints = WorkloadConstraints.builder() - .withActiveConsumers(supervisors.size()) - .withConsumersPerSubscription(2) - .withMaxSubscriptionsPerConsumer(2) - .build(); - SubscriptionAssignmentView currentState = initialState(subscriptions, supervisors, constraints); - - // when - supervisors.add("new-supervisor"); - SubscriptionAssignmentView stateAfterRebalance = workBalancer - .balance(subscriptions, supervisors, currentState, constraints) - .getAssignmentsView(); - - // then - assertThat(stateAfterRebalance.getAssignmentsForConsumerNode("new-supervisor").size()).isGreaterThan(0); - assertThat(stateAfterRebalance.getAssignmentsForSubscription(subscriptions.get(0))).hasSize(2); - assertThat(stateAfterRebalance.getAssignmentsForSubscription(subscriptions.get(1))).hasSize(2); - } - - @Test - public void shouldEquallyAssignWorkToConsumers() { - // given - List supervisors = ImmutableList.of("c1", "c2"); - List subscriptions = someSubscriptions(50); - WorkloadConstraints constraints = WorkloadConstraints.builder() - .withActiveConsumers(supervisors.size()) - .withConsumersPerSubscription(2) - .withMaxSubscriptionsPerConsumer(200) - .build(); - SubscriptionAssignmentView currentState = initialState(subscriptions, supervisors, constraints); - - // when - List extendedSupervisorsList = ImmutableList.of("c1", "c2", "c3"); - SubscriptionAssignmentView stateAfterRebalance = workBalancer - .balance(subscriptions, extendedSupervisorsList, currentState, constraints) - .getAssignmentsView(); - - // then - assertThat(stateAfterRebalance.getAssignmentsForConsumerNode("c3")).hasSize(50 * 2 / 3); - } - - @Test - public void shouldReassignWorkToFreeConsumers() { - // given - List supervisors = ImmutableList.of("c1"); - List subscriptions = someSubscriptions(10); - WorkloadConstraints constraints = WorkloadConstraints.builder() - .withActiveConsumers(supervisors.size()) - .withConsumersPerSubscription(1) - .withMaxSubscriptionsPerConsumer(100) - .build(); - SubscriptionAssignmentView currentState = initialState(subscriptions, supervisors, constraints); - - // when - ImmutableList extendedSupervisorsList = ImmutableList.of("c1", "c2", "c3", "c4", "c5"); - SubscriptionAssignmentView stateAfterRebalance = workBalancer - .balance(subscriptions, extendedSupervisorsList, currentState, constraints) - .getAssignmentsView(); - - // then - assertThat(stateAfterRebalance.getAssignmentsForConsumerNode("c5")).hasSize(2); - } - - @Test - public void shouldRemoveRedundantWorkAssignmentsToKeepWorkloadMinimal() { - // given - List supervisors = ImmutableList.of("c1", "c2", "c3"); - List subscriptions = someSubscriptions(10); - WorkloadConstraints initialConstraints = WorkloadConstraints.builder() - .withActiveConsumers(supervisors.size()) - .withConsumersPerSubscription(3) - .withMaxSubscriptionsPerConsumer(100) - .build(); - SubscriptionAssignmentView currentState = initialState(subscriptions, supervisors, initialConstraints); - - // when - WorkloadConstraints newConstraints = WorkloadConstraints.builder() - .withActiveConsumers(supervisors.size()) - .withConsumersPerSubscription(3) - .withMaxSubscriptionsPerConsumer(100) - .withSubscriptionConstraints(Map.of(subscriptions.get(0), new Constraints(1))) - .build(); - SubscriptionAssignmentView stateAfterRebalance = workBalancer - .balance(subscriptions, supervisors, currentState, newConstraints) - .getAssignmentsView(); - - // then - assertThat(stateAfterRebalance.getAssignmentsCountForSubscription(subscriptions.get(0))).isEqualTo(1); - } - - @DataProvider - public static Object[][] subscriptionConstraints() { - return new Object[][] { - { 1 }, { 3 } - }; - } - - @Test - @UseDataProvider("subscriptionConstraints") - public void shouldAssignConsumersForSubscriptionsAccordingToConstraints(int requiredConsumersNumber) { - // given - SubscriptionAssignmentView initialState = new SubscriptionAssignmentView(emptyMap()); - - List supervisors = ImmutableList.of("c1", "c2", "c3"); - List subscriptions = someSubscriptions(4); - SelectiveWorkBalancer workBalancer = new SelectiveWorkBalancer(); - WorkloadConstraints subscriptionConstraints = WorkloadConstraints.builder() - .withActiveConsumers(supervisors.size()) - .withConsumersPerSubscription(2) - .withMaxSubscriptionsPerConsumer(4) - .withSubscriptionConstraints(Map.of(subscriptions.get(0), new Constraints(requiredConsumersNumber))) - .build(); - - // when - SubscriptionAssignmentView state = workBalancer - .balance(subscriptions, supervisors, initialState, subscriptionConstraints) - .getAssignmentsView(); - - // then - assertThat(state.getAssignmentsForSubscription(subscriptions.get(0)).size()).isEqualTo(requiredConsumersNumber); - assertThat(state.getAssignmentsForSubscription(subscriptions.get(1)).size()).isEqualTo(2); - assertThat(state.getAssignmentsForSubscription(subscriptions.get(2)).size()).isEqualTo(2); - assertThat(state.getAssignmentsForSubscription(subscriptions.get(3)).size()).isEqualTo(2); - } - - private SubscriptionAssignmentView initialState(List subscriptions, List supervisors, - WorkloadConstraints workloadConstraints) { - return workBalancer.balance(subscriptions, supervisors, new SubscriptionAssignmentView(emptyMap()), workloadConstraints) - .getAssignmentsView(); - } - - private List someSubscriptions(int count) { - return IntStream.range(0, count).mapToObj(i -> anySubscription()).collect(toList()); - } - - private List someSupervisors(int count) { - return IntStream.range(0, count).mapToObj(i -> "c" + i).collect(toList()); - } - - private SubscriptionName anySubscription() { - return SubscriptionName.fromString("tech.topic$s" + UUID.randomUUID().getMostSignificantBits()); - } - - private void assertThatSubscriptionIsAssignedTo(SubscriptionAssignmentView work, SubscriptionName sub, String... nodeIds) { - assertThatSubscriptionIsAssignedTo(work, sub, asList(nodeIds)); - } - - private void assertThatSubscriptionIsAssignedTo(SubscriptionAssignmentView work, SubscriptionName sub, List nodeIds) { - assertThat(work.getAssignmentsForSubscription(sub)) - .extracting(SubscriptionAssignment::getConsumerNodeId) - .containsOnly(nodeIds.toArray(String[]::new)); - } + private SelectiveWorkBalancer workBalancer = new SelectiveWorkBalancer(); + + @Test + public void shouldPerformSubscriptionsCleanup() { + // given + List subscriptions = someSubscriptions(1); + List supervisors = someSupervisors(1); + WorkloadConstraints constraints = + WorkloadConstraints.builder() + .withActiveConsumers(supervisors.size()) + .withConsumersPerSubscription(2) + .withMaxSubscriptionsPerConsumer(2) + .build(); + SubscriptionAssignmentView currentState = initialState(subscriptions, supervisors, constraints); + + // when + WorkBalancingResult target = + workBalancer.balance(emptyList(), supervisors, currentState, constraints); + + // then + assertThat(target.getAssignmentsView().getSubscriptions()).isEmpty(); + assertThat(currentState.deletions(target.getAssignmentsView()).getAllAssignments()) + .hasSize(subscriptions.size()); + } + + @Test + public void shouldPerformSupervisorsCleanup() { + // given + List supervisors = someSupervisors(2); + List subscriptions = someSubscriptions(1); + WorkloadConstraints constraints = + WorkloadConstraints.builder() + .withActiveConsumers(supervisors.size()) + .withConsumersPerSubscription(2) + .withMaxSubscriptionsPerConsumer(2) + .build(); + SubscriptionAssignmentView currentState = initialState(subscriptions, supervisors, constraints); + + // when + supervisors.remove(1); + WorkBalancingResult work = + workBalancer.balance(subscriptions, supervisors, currentState, constraints); + + // then + assertThat(currentState.deletions(work.getAssignmentsView()).getAllAssignments()).hasSize(1); + assertThatSubscriptionIsAssignedTo( + work.getAssignmentsView(), subscriptions.get(0), supervisors.get(0)); + } + + @Test + public void shouldBalanceWorkForSingleSubscription() { + // given + List supervisors = someSupervisors(1); + List subscriptions = someSubscriptions(1); + WorkloadConstraints constraints = + WorkloadConstraints.builder() + .withActiveConsumers(supervisors.size()) + .withConsumersPerSubscription(2) + .withMaxSubscriptionsPerConsumer(2) + .build(); + + // when + SubscriptionAssignmentView view = initialState(subscriptions, supervisors, constraints); + + // then + assertThatSubscriptionIsAssignedTo(view, subscriptions.get(0), supervisors.get(0)); + } + + @Test + public void shouldBalanceWorkForMultipleConsumersAndSingleSubscription() { + // given + List supervisors = someSupervisors(2); + List subscriptions = someSubscriptions(1); + WorkloadConstraints constraints = + WorkloadConstraints.builder() + .withActiveConsumers(supervisors.size()) + .withConsumersPerSubscription(2) + .withMaxSubscriptionsPerConsumer(2) + .build(); + + // when + SubscriptionAssignmentView view = initialState(subscriptions, supervisors, constraints); + + // then + assertThatSubscriptionIsAssignedTo(view, subscriptions.get(0), supervisors); + } + + @Test + public void shouldBalanceWorkForMultipleConsumersAndMultipleSubscriptions() { + // given + List supervisors = someSupervisors(2); + List subscriptions = someSubscriptions(2); + WorkloadConstraints constraints = + WorkloadConstraints.builder() + .withActiveConsumers(supervisors.size()) + .withConsumersPerSubscription(2) + .withMaxSubscriptionsPerConsumer(2) + .build(); + + // when + SubscriptionAssignmentView view = initialState(subscriptions, supervisors, constraints); + + // then + assertThatSubscriptionIsAssignedTo(view, subscriptions.get(0), supervisors); + assertThatSubscriptionIsAssignedTo(view, subscriptions.get(1), supervisors); + } + + @Test + public void shouldNotOverloadConsumers() { + // given + List supervisors = someSupervisors(1); + List subscriptions = someSubscriptions(3); + WorkloadConstraints constraints = + WorkloadConstraints.builder() + .withActiveConsumers(supervisors.size()) + .withConsumersPerSubscription(2) + .withMaxSubscriptionsPerConsumer(2) + .build(); + + // when + SubscriptionAssignmentView view = initialState(subscriptions, supervisors, constraints); + + // then + assertThat(view.getAssignmentsForConsumerNode(supervisors.get(0))).hasSize(2); + } + + @Test + public void shouldRebalanceAfterConsumerDisappearing() { + // given + List supervisors = ImmutableList.of("c1", "c2"); + List subscriptions = someSubscriptions(2); + WorkloadConstraints constraints = + WorkloadConstraints.builder() + .withActiveConsumers(supervisors.size()) + .withConsumersPerSubscription(2) + .withMaxSubscriptionsPerConsumer(2) + .build(); + SubscriptionAssignmentView currentState = initialState(subscriptions, supervisors, constraints); + + // when + List extendedSupervisorsList = ImmutableList.of("c1", "c3"); + SubscriptionAssignmentView stateAfterRebalance = + workBalancer + .balance(subscriptions, extendedSupervisorsList, currentState, constraints) + .getAssignmentsView(); + + // then + assertThat(stateAfterRebalance.getSubscriptionsForConsumerNode("c3")) + .containsOnly(subscriptions.get(0), subscriptions.get(1)); + } + + @Test + public void shouldAssignWorkToNewConsumersByWorkStealing() { + // given + List supervisors = someSupervisors(2); + List subscriptions = someSubscriptions(2); + WorkloadConstraints constraints = + WorkloadConstraints.builder() + .withActiveConsumers(supervisors.size()) + .withConsumersPerSubscription(2) + .withMaxSubscriptionsPerConsumer(2) + .build(); + SubscriptionAssignmentView currentState = initialState(subscriptions, supervisors, constraints); + + // when + supervisors.add("new-supervisor"); + SubscriptionAssignmentView stateAfterRebalance = + workBalancer + .balance(subscriptions, supervisors, currentState, constraints) + .getAssignmentsView(); + + // then + assertThat(stateAfterRebalance.getAssignmentsForConsumerNode("new-supervisor").size()) + .isGreaterThan(0); + assertThat(stateAfterRebalance.getAssignmentsForSubscription(subscriptions.get(0))).hasSize(2); + assertThat(stateAfterRebalance.getAssignmentsForSubscription(subscriptions.get(1))).hasSize(2); + } + + @Test + public void shouldEquallyAssignWorkToConsumers() { + // given + List supervisors = ImmutableList.of("c1", "c2"); + List subscriptions = someSubscriptions(50); + WorkloadConstraints constraints = + WorkloadConstraints.builder() + .withActiveConsumers(supervisors.size()) + .withConsumersPerSubscription(2) + .withMaxSubscriptionsPerConsumer(200) + .build(); + SubscriptionAssignmentView currentState = initialState(subscriptions, supervisors, constraints); + + // when + List extendedSupervisorsList = ImmutableList.of("c1", "c2", "c3"); + SubscriptionAssignmentView stateAfterRebalance = + workBalancer + .balance(subscriptions, extendedSupervisorsList, currentState, constraints) + .getAssignmentsView(); + + // then + assertThat(stateAfterRebalance.getAssignmentsForConsumerNode("c3")).hasSize(50 * 2 / 3); + } + + @Test + public void shouldReassignWorkToFreeConsumers() { + // given + List supervisors = ImmutableList.of("c1"); + List subscriptions = someSubscriptions(10); + WorkloadConstraints constraints = + WorkloadConstraints.builder() + .withActiveConsumers(supervisors.size()) + .withConsumersPerSubscription(1) + .withMaxSubscriptionsPerConsumer(100) + .build(); + SubscriptionAssignmentView currentState = initialState(subscriptions, supervisors, constraints); + + // when + ImmutableList extendedSupervisorsList = ImmutableList.of("c1", "c2", "c3", "c4", "c5"); + SubscriptionAssignmentView stateAfterRebalance = + workBalancer + .balance(subscriptions, extendedSupervisorsList, currentState, constraints) + .getAssignmentsView(); + + // then + assertThat(stateAfterRebalance.getAssignmentsForConsumerNode("c5")).hasSize(2); + } + + @Test + public void shouldRemoveRedundantWorkAssignmentsToKeepWorkloadMinimal() { + // given + List supervisors = ImmutableList.of("c1", "c2", "c3"); + List subscriptions = someSubscriptions(10); + WorkloadConstraints initialConstraints = + WorkloadConstraints.builder() + .withActiveConsumers(supervisors.size()) + .withConsumersPerSubscription(3) + .withMaxSubscriptionsPerConsumer(100) + .build(); + SubscriptionAssignmentView currentState = + initialState(subscriptions, supervisors, initialConstraints); + + // when + WorkloadConstraints newConstraints = + WorkloadConstraints.builder() + .withActiveConsumers(supervisors.size()) + .withConsumersPerSubscription(3) + .withMaxSubscriptionsPerConsumer(100) + .withSubscriptionConstraints(Map.of(subscriptions.get(0), new Constraints(1))) + .build(); + SubscriptionAssignmentView stateAfterRebalance = + workBalancer + .balance(subscriptions, supervisors, currentState, newConstraints) + .getAssignmentsView(); + + // then + assertThat(stateAfterRebalance.getAssignmentsCountForSubscription(subscriptions.get(0))) + .isEqualTo(1); + } + + @DataProvider + public static Object[][] subscriptionConstraints() { + return new Object[][] {{1}, {3}}; + } + + @Test + @UseDataProvider("subscriptionConstraints") + public void shouldAssignConsumersForSubscriptionsAccordingToConstraints( + int requiredConsumersNumber) { + // given + SubscriptionAssignmentView initialState = new SubscriptionAssignmentView(emptyMap()); + + List supervisors = ImmutableList.of("c1", "c2", "c3"); + List subscriptions = someSubscriptions(4); + SelectiveWorkBalancer workBalancer = new SelectiveWorkBalancer(); + WorkloadConstraints subscriptionConstraints = + WorkloadConstraints.builder() + .withActiveConsumers(supervisors.size()) + .withConsumersPerSubscription(2) + .withMaxSubscriptionsPerConsumer(4) + .withSubscriptionConstraints( + Map.of(subscriptions.get(0), new Constraints(requiredConsumersNumber))) + .build(); + + // when + SubscriptionAssignmentView state = + workBalancer + .balance(subscriptions, supervisors, initialState, subscriptionConstraints) + .getAssignmentsView(); + + // then + assertThat(state.getAssignmentsForSubscription(subscriptions.get(0)).size()) + .isEqualTo(requiredConsumersNumber); + assertThat(state.getAssignmentsForSubscription(subscriptions.get(1)).size()).isEqualTo(2); + assertThat(state.getAssignmentsForSubscription(subscriptions.get(2)).size()).isEqualTo(2); + assertThat(state.getAssignmentsForSubscription(subscriptions.get(3)).size()).isEqualTo(2); + } + + private SubscriptionAssignmentView initialState( + List subscriptions, + List supervisors, + WorkloadConstraints workloadConstraints) { + return workBalancer + .balance( + subscriptions, + supervisors, + new SubscriptionAssignmentView(emptyMap()), + workloadConstraints) + .getAssignmentsView(); + } + + private List someSubscriptions(int count) { + return IntStream.range(0, count).mapToObj(i -> anySubscription()).collect(toList()); + } + + private List someSupervisors(int count) { + return IntStream.range(0, count).mapToObj(i -> "c" + i).collect(toList()); + } + + private SubscriptionName anySubscription() { + return SubscriptionName.fromString("tech.topic$s" + UUID.randomUUID().getMostSignificantBits()); + } + + private void assertThatSubscriptionIsAssignedTo( + SubscriptionAssignmentView work, SubscriptionName sub, String... nodeIds) { + assertThatSubscriptionIsAssignedTo(work, sub, asList(nodeIds)); + } + + private void assertThatSubscriptionIsAssignedTo( + SubscriptionAssignmentView work, SubscriptionName sub, List nodeIds) { + assertThat(work.getAssignmentsForSubscription(sub)) + .extracting(SubscriptionAssignment::getConsumerNodeId) + .containsOnly(nodeIds.toArray(String[]::new)); + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/SubscriptionAssignmentViewBuilder.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/SubscriptionAssignmentViewBuilder.java index c0a6886681..0577910a94 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/SubscriptionAssignmentViewBuilder.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/SubscriptionAssignmentViewBuilder.java @@ -1,29 +1,32 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload; -import pl.allegro.tech.hermes.api.SubscriptionName; - import java.util.Collections; +import pl.allegro.tech.hermes.api.SubscriptionName; public class SubscriptionAssignmentViewBuilder { - private SubscriptionAssignmentView assignmentView; + private SubscriptionAssignmentView assignmentView; - public SubscriptionAssignmentViewBuilder() { - assignmentView = new SubscriptionAssignmentView(Collections.emptyMap()); - } + public SubscriptionAssignmentViewBuilder() { + assignmentView = new SubscriptionAssignmentView(Collections.emptyMap()); + } - public SubscriptionAssignmentView build() { - return SubscriptionAssignmentView.copyOf(assignmentView); - } + public SubscriptionAssignmentView build() { + return SubscriptionAssignmentView.copyOf(assignmentView); + } - public SubscriptionAssignmentViewBuilder withAssignment(SubscriptionName subscriptionName, String... consumerNodeIds) { - for (String consumerNodeId : consumerNodeIds) { - assignmentView = assignmentView.transform((view, transformer) -> { + public SubscriptionAssignmentViewBuilder withAssignment( + SubscriptionName subscriptionName, String... consumerNodeIds) { + for (String consumerNodeId : consumerNodeIds) { + assignmentView = + assignmentView.transform( + (view, transformer) -> { transformer.addSubscription(subscriptionName); transformer.addConsumerNode(consumerNodeId); - transformer.addAssignment(new SubscriptionAssignment(consumerNodeId, subscriptionName)); - }); - } - return this; + transformer.addAssignment( + new SubscriptionAssignment(consumerNodeId, subscriptionName)); + }); } + return this; + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/SubscriptionAssignmentViewTest.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/SubscriptionAssignmentViewTest.java index f9ad4e50cd..58909f41f2 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/SubscriptionAssignmentViewTest.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/SubscriptionAssignmentViewTest.java @@ -1,142 +1,146 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload; -import org.junit.Test; -import pl.allegro.tech.hermes.api.SubscriptionName; +import static org.assertj.core.api.Assertions.assertThat; import java.util.UUID; - -import static org.assertj.core.api.Assertions.assertThat; +import org.junit.Test; +import pl.allegro.tech.hermes.api.SubscriptionName; public class SubscriptionAssignmentViewTest { - @Test - public void shouldNotDeleteFromEmptyView() { - // given - SubscriptionName s1 = anySubscriptionName(); - SubscriptionAssignmentView current = emptySubscriptionAssignmentView(); - SubscriptionAssignmentView target = assignmentView().withAssignment(s1, "c1").build(); - - // when - SubscriptionAssignmentView deletions = current.deletions(target); - - // then - assertThat(deletions.getSubscriptions()).isEmpty(); - } - - @Test - public void shouldAddToEmptyView() { - // given - SubscriptionName s1 = anySubscriptionName(); - SubscriptionAssignmentView current = emptySubscriptionAssignmentView(); - SubscriptionAssignmentView target = assignmentView().withAssignment(s1, "c1").build(); - - // when - SubscriptionAssignmentView additions = current.additions(target); - - // then - assertThat(additions).isEqualTo(target); - } - - @Test - public void shouldNotAddToUnchangedView() { - // given - SubscriptionName s1 = anySubscriptionName(); - SubscriptionAssignmentView current = assignmentView().withAssignment(s1, "c1").build(); - SubscriptionAssignmentView target = assignmentView().withAssignment(s1, "c1").build(); - - // when - SubscriptionAssignmentView additions = current.additions(target); - - // then - assertThat(additions.getSubscriptions()).isEmpty(); - } - - @Test - public void shouldNotDeleteFromUnchangedView() { - // given - SubscriptionName s1 = anySubscriptionName(); - SubscriptionAssignmentView current = assignmentView().withAssignment(s1, "c1").build(); - SubscriptionAssignmentView target = assignmentView().withAssignment(s1, "c1").build(); - - // when - SubscriptionAssignmentView deletions = current.deletions(target); - - // then - assertThat(deletions.getSubscriptions()).isEmpty(); - } - - @Test - public void shouldAddAssignmentToExistingSubscription() { - // given - SubscriptionName s1 = anySubscriptionName(); - SubscriptionAssignmentView current = assignmentView().withAssignment(s1, "c1").build(); - SubscriptionAssignmentView target = assignmentView().withAssignment(s1, "c1").withAssignment(s1, "c2").build(); - - // when - SubscriptionAssignmentView additions = current.additions(target); - - // then - assertThat(additions.getAssignmentsForSubscription(s1)).containsOnly(assignment(s1, "c2")); - } - - @Test - public void shouldDeleteAssignmentFromExistingSubscription() { - // given - SubscriptionName s1 = anySubscriptionName(); - SubscriptionAssignmentView current = assignmentView().withAssignment(s1, "c1").withAssignment(s1, "c2").build(); - SubscriptionAssignmentView target = assignmentView().withAssignment(s1, "c1").build(); - - // when - SubscriptionAssignmentView deletions = current.deletions(target); - - // then - assertThat(deletions.getAssignmentsForSubscription(s1)).containsOnly(assignment(s1, "c2")); - } - - @Test - public void shouldAddNewSubscription() { - // given - SubscriptionName s1 = anySubscriptionName(); - SubscriptionName s2 = anySubscriptionName(); - SubscriptionAssignmentView current = assignmentView().withAssignment(s1, "c1").build(); - SubscriptionAssignmentView target = assignmentView().withAssignment(s1, "c1").withAssignment(s2, "c1").build(); - - // when - SubscriptionAssignmentView additions = current.additions(target); - - // then - assertThat(additions.getSubscriptions()).containsOnly(s2); - assertThat(additions.getAssignmentsForSubscription(s2)).containsOnly(assignment(s2, "c1")); - } - - @Test - public void shouldDeleteOldSubscription() { - // given - SubscriptionName s1 = anySubscriptionName(); - SubscriptionAssignmentView current = assignmentView().withAssignment(s1, "c1").build(); - SubscriptionAssignmentView target = emptySubscriptionAssignmentView(); - - // when - SubscriptionAssignmentView deletions = current.deletions(target); - - // then - assertThat(deletions.getSubscriptions()).isEqualTo(current.getSubscriptions()); - assertThat(deletions.getAssignmentsForSubscription(s1)).isEqualTo(current.getAssignmentsForSubscription(s1)); - } - - private SubscriptionAssignment assignment(SubscriptionName s1, String supervisorId) { - return new SubscriptionAssignment(supervisorId, s1); - } - - private SubscriptionName anySubscriptionName() { - return SubscriptionName.fromString("com.example.topic$" + Math.abs(UUID.randomUUID().getMostSignificantBits())); - } - - private static SubscriptionAssignmentView emptySubscriptionAssignmentView() { - return assignmentView().build(); - } - - private static SubscriptionAssignmentViewBuilder assignmentView() { - return new SubscriptionAssignmentViewBuilder(); - } + @Test + public void shouldNotDeleteFromEmptyView() { + // given + SubscriptionName s1 = anySubscriptionName(); + SubscriptionAssignmentView current = emptySubscriptionAssignmentView(); + SubscriptionAssignmentView target = assignmentView().withAssignment(s1, "c1").build(); + + // when + SubscriptionAssignmentView deletions = current.deletions(target); + + // then + assertThat(deletions.getSubscriptions()).isEmpty(); + } + + @Test + public void shouldAddToEmptyView() { + // given + SubscriptionName s1 = anySubscriptionName(); + SubscriptionAssignmentView current = emptySubscriptionAssignmentView(); + SubscriptionAssignmentView target = assignmentView().withAssignment(s1, "c1").build(); + + // when + SubscriptionAssignmentView additions = current.additions(target); + + // then + assertThat(additions).isEqualTo(target); + } + + @Test + public void shouldNotAddToUnchangedView() { + // given + SubscriptionName s1 = anySubscriptionName(); + SubscriptionAssignmentView current = assignmentView().withAssignment(s1, "c1").build(); + SubscriptionAssignmentView target = assignmentView().withAssignment(s1, "c1").build(); + + // when + SubscriptionAssignmentView additions = current.additions(target); + + // then + assertThat(additions.getSubscriptions()).isEmpty(); + } + + @Test + public void shouldNotDeleteFromUnchangedView() { + // given + SubscriptionName s1 = anySubscriptionName(); + SubscriptionAssignmentView current = assignmentView().withAssignment(s1, "c1").build(); + SubscriptionAssignmentView target = assignmentView().withAssignment(s1, "c1").build(); + + // when + SubscriptionAssignmentView deletions = current.deletions(target); + + // then + assertThat(deletions.getSubscriptions()).isEmpty(); + } + + @Test + public void shouldAddAssignmentToExistingSubscription() { + // given + SubscriptionName s1 = anySubscriptionName(); + SubscriptionAssignmentView current = assignmentView().withAssignment(s1, "c1").build(); + SubscriptionAssignmentView target = + assignmentView().withAssignment(s1, "c1").withAssignment(s1, "c2").build(); + + // when + SubscriptionAssignmentView additions = current.additions(target); + + // then + assertThat(additions.getAssignmentsForSubscription(s1)).containsOnly(assignment(s1, "c2")); + } + + @Test + public void shouldDeleteAssignmentFromExistingSubscription() { + // given + SubscriptionName s1 = anySubscriptionName(); + SubscriptionAssignmentView current = + assignmentView().withAssignment(s1, "c1").withAssignment(s1, "c2").build(); + SubscriptionAssignmentView target = assignmentView().withAssignment(s1, "c1").build(); + + // when + SubscriptionAssignmentView deletions = current.deletions(target); + + // then + assertThat(deletions.getAssignmentsForSubscription(s1)).containsOnly(assignment(s1, "c2")); + } + + @Test + public void shouldAddNewSubscription() { + // given + SubscriptionName s1 = anySubscriptionName(); + SubscriptionName s2 = anySubscriptionName(); + SubscriptionAssignmentView current = assignmentView().withAssignment(s1, "c1").build(); + SubscriptionAssignmentView target = + assignmentView().withAssignment(s1, "c1").withAssignment(s2, "c1").build(); + + // when + SubscriptionAssignmentView additions = current.additions(target); + + // then + assertThat(additions.getSubscriptions()).containsOnly(s2); + assertThat(additions.getAssignmentsForSubscription(s2)).containsOnly(assignment(s2, "c1")); + } + + @Test + public void shouldDeleteOldSubscription() { + // given + SubscriptionName s1 = anySubscriptionName(); + SubscriptionAssignmentView current = assignmentView().withAssignment(s1, "c1").build(); + SubscriptionAssignmentView target = emptySubscriptionAssignmentView(); + + // when + SubscriptionAssignmentView deletions = current.deletions(target); + + // then + assertThat(deletions.getSubscriptions()).isEqualTo(current.getSubscriptions()); + assertThat(deletions.getAssignmentsForSubscription(s1)) + .isEqualTo(current.getAssignmentsForSubscription(s1)); + } + + private SubscriptionAssignment assignment(SubscriptionName s1, String supervisorId) { + return new SubscriptionAssignment(supervisorId, s1); + } + + private SubscriptionName anySubscriptionName() { + return SubscriptionName.fromString( + "com.example.topic$" + Math.abs(UUID.randomUUID().getMostSignificantBits())); + } + + private static SubscriptionAssignmentView emptySubscriptionAssignmentView() { + return assignmentView().build(); + } + + private static SubscriptionAssignmentViewBuilder assignmentView() { + return new SubscriptionAssignmentViewBuilder(); + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/TestSubscriptionIds.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/TestSubscriptionIds.java index 88e31bdf09..be1b779548 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/TestSubscriptionIds.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/TestSubscriptionIds.java @@ -1,38 +1,36 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload; -import pl.allegro.tech.hermes.api.SubscriptionName; -import pl.allegro.tech.hermes.consumers.subscription.id.SubscriptionId; -import pl.allegro.tech.hermes.consumers.subscription.id.SubscriptionIds; - import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; +import pl.allegro.tech.hermes.api.SubscriptionName; +import pl.allegro.tech.hermes.consumers.subscription.id.SubscriptionId; +import pl.allegro.tech.hermes.consumers.subscription.id.SubscriptionIds; public class TestSubscriptionIds implements SubscriptionIds { - private final Map nameToIdMap = new ConcurrentHashMap<>(); - private final Map valueToIdMap = new ConcurrentHashMap<>(); + private final Map nameToIdMap = new ConcurrentHashMap<>(); + private final Map valueToIdMap = new ConcurrentHashMap<>(); - public TestSubscriptionIds(List ids) { - ids.forEach(id -> { - nameToIdMap.put(id.getSubscriptionName(), id); - valueToIdMap.put(id.getValue(), id); + public TestSubscriptionIds(List ids) { + ids.forEach( + id -> { + nameToIdMap.put(id.getSubscriptionName(), id); + valueToIdMap.put(id.getValue(), id); }); - } - - @Override - public Optional getSubscriptionId(SubscriptionName subscriptionName) { - return Optional.ofNullable(nameToIdMap.get(subscriptionName)); - } + } - @Override - public Optional getSubscriptionId(long id) { - return Optional.ofNullable(valueToIdMap.get(id)); - } + @Override + public Optional getSubscriptionId(SubscriptionName subscriptionName) { + return Optional.ofNullable(nameToIdMap.get(subscriptionName)); + } - @Override - public void start() { + @Override + public Optional getSubscriptionId(long id) { + return Optional.ofNullable(valueToIdMap.get(id)); + } - } + @Override + public void start() {} } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkloadRegistryTest.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkloadRegistryTest.java index 6362dabe33..c30556861d 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkloadRegistryTest.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkloadRegistryTest.java @@ -1,6 +1,12 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import com.google.common.collect.ImmutableList; +import java.util.Collections; +import java.util.Set; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -14,188 +20,202 @@ import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; import pl.allegro.tech.hermes.test.helper.zookeeper.ZookeeperBaseTest; -import java.util.Collections; -import java.util.Set; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - public class WorkloadRegistryTest extends ZookeeperBaseTest { - private static final SubscriptionName subscription1 = SubscriptionName.fromString("pl.allegro.tech.hermes$testSubscription"); - private static final SubscriptionName subscription2 = SubscriptionName.fromString("pl.allegro.tech.hermes$testSubscription2"); - private static final SubscriptionName subscription3 = SubscriptionName.fromString("pl.allegro.tech.hermes$testSubscription3"); - - private static final String consumer1 = "consumer1"; - private static final String consumer2 = "consumer2"; - private static final String clusterName = new KafkaProperties().getClusterName(); - - private static final ZookeeperPaths zookeeperPaths = new ZookeeperPaths("/hermes"); - - private static final WorkloadRegistryPaths registryPaths = - new WorkloadRegistryPaths(zookeeperPaths, clusterName); - - private static final SubscriptionIds subscriptionIds = - new TestSubscriptionIds(ImmutableList.of( - SubscriptionId.from(subscription1, 1L), - SubscriptionId.from(subscription2, 2L), - SubscriptionId.from(subscription2, 3L) - )); - - private static final ConsumerAssignmentRegistry registry = - new ConsumerAssignmentRegistry( - zookeeperClient, - new WorkloadProperties().getRegistryBinaryEncoderAssignmentsBufferSizeBytes(), - clusterName, - zookeeperPaths, - subscriptionIds - ); - - private static final KafkaProperties kafkaProperties = new KafkaProperties(); - - private static final String cluster = kafkaProperties.getClusterName(); - - private static final ConsumerAssignmentCache assignmentCacheOfConsumer1 = - new ConsumerAssignmentCache(zookeeperClient, consumer1, cluster, zookeeperPaths, subscriptionIds); - - private static final ConsumerAssignmentCache assignmentCacheOfConsumer2 = - new ConsumerAssignmentCache(zookeeperClient, consumer2, cluster, zookeeperPaths, subscriptionIds); - - private static final ConsumerNodesRegistry consumerNodesRegistry = mock(ConsumerNodesRegistry.class); - - private static final ClusterAssignmentCache clusterAssignmentCache = - new ClusterAssignmentCache(zookeeperClient, cluster, zookeeperPaths, subscriptionIds, consumerNodesRegistry); - - @BeforeClass - public static void setUp() throws Exception { - assignmentCacheOfConsumer1.start(); - assignmentCacheOfConsumer2.start(); - } - - @Before - public void beforeEach() { - when(consumerNodesRegistry.listConsumerNodes()).thenReturn(ImmutableList.of(consumer1, consumer2)); - } - - @AfterClass - public static void cleanup() throws Exception { - assignmentCacheOfConsumer1.stop(); - assignmentCacheOfConsumer2.stop(); - } - - @Test - public void shouldSaveAssignmentsAndNotifyConsumers() { - // when - registry.updateAssignments(consumer1, Set.of(subscription1, subscription2)); - registry.updateAssignments(consumer2, Set.of(subscription1)); - - wait.untilZookeeperPathIsCreated(registryPaths.consumerWorkloadPath(consumer1)); - wait.untilZookeeperPathIsCreated(registryPaths.consumerWorkloadPath(consumer2)); - - // then - assertThat(assignmentCacheOfConsumer1.isAssignedTo(subscription1)).isTrue(); - assertThat(assignmentCacheOfConsumer1.isAssignedTo(subscription2)).isTrue(); - assertThat(assignmentCacheOfConsumer1.isAssignedTo(subscription3)).isFalse(); - - assertThat(assignmentCacheOfConsumer2.isAssignedTo(subscription1)).isTrue(); - assertThat(assignmentCacheOfConsumer2.isAssignedTo(subscription2)).isFalse(); - } - - @Test - public void shouldSaveAssignmentsAndReadThroughClusterAssignmentCache() { - // when - registry.updateAssignments(consumer1, Set.of(subscription1, subscription2)); - registry.updateAssignments(consumer2, Set.of(subscription1)); - - // and - clusterAssignmentCache.refresh(); - - // then - assertThat(clusterAssignmentCache.getAssignedConsumers()).containsOnly(consumer1, consumer2); - assertThat(clusterAssignmentCache.getConsumerSubscriptions(consumer1)).containsOnly(subscription1, subscription2); - assertThat(clusterAssignmentCache.getConsumerSubscriptions(consumer2)).containsOnly(subscription1); - assertThat(clusterAssignmentCache.getSubscriptionConsumers().get(subscription1)).containsOnly(consumer1, consumer2); - assertThat(clusterAssignmentCache.getSubscriptionConsumers().get(subscription2)).containsOnly(consumer1); - assertThat(clusterAssignmentCache.getSubscriptionConsumers().get(subscription3)).isNull(); - assertThat(clusterAssignmentCache.isAssignedTo(consumer1, subscription1)).isTrue(); - assertThat(clusterAssignmentCache.isAssignedTo(consumer1, subscription3)).isFalse(); - } - - @Test - public void shouldApplyChangesToAssignments() { - // when - registry.updateAssignments(consumer1, Set.of(subscription1, subscription2)); - registry.updateAssignments(consumer2, Set.of(subscription1)); - - // and - clusterAssignmentCache.refresh(); - - // then - assertThat(clusterAssignmentCache.getAssignedConsumers()).containsOnly(consumer1, consumer2); - assertThat(clusterAssignmentCache.getConsumerSubscriptions(consumer1)).containsOnly(subscription1, subscription2); - assertThat(clusterAssignmentCache.getConsumerSubscriptions(consumer2)).containsOnly(subscription1); - assertThat(clusterAssignmentCache.getSubscriptionConsumers().get(subscription1)).containsOnly(consumer1, consumer2); - - // when - registry.updateAssignments(consumer1, Set.of(subscription2)); - registry.updateAssignments(consumer2, Set.of(subscription1)); - - // and - clusterAssignmentCache.refresh(); - - // then - assertThat(clusterAssignmentCache.getAssignedConsumers()).containsOnly(consumer1, consumer2); - assertThat(clusterAssignmentCache.getConsumerSubscriptions(consumer1)).containsOnly(subscription2); - assertThat(clusterAssignmentCache.getConsumerSubscriptions(consumer2)).containsOnly(subscription1); - } - - @Test - public void shouldCleanStaleNodesFromRegistryOnRefresh() { - // when - registry.updateAssignments(consumer1, Set.of(subscription1, subscription2)); - registry.updateAssignments(consumer2, Set.of(subscription1)); - - // and - clusterAssignmentCache.refresh(); - - // then - assertThat(clusterAssignmentCache.getAssignedConsumers()).containsOnly(consumer1, consumer2); - - // when - when(consumerNodesRegistry.listConsumerNodes()).thenReturn(Collections.singletonList(consumer2)); - - // and - clusterAssignmentCache.refresh(); - - // then - assertThat(clusterAssignmentCache.getAssignedConsumers()).containsOnly(consumer2); - } - - @Test - public void shouldCleanRegistryFromStaleAssignments() { - // when - registry.updateAssignments(consumer1, Set.of(subscription1, subscription2)); - registry.updateAssignments(consumer2, Set.of(subscription1)); - - // then - wait.untilZookeeperPathIsCreated(registryPaths.consumerWorkloadPath(consumer1)); - wait.untilZookeeperPathIsCreated(registryPaths.consumerWorkloadPath(consumer2)); - - // when - clusterAssignmentCache.refresh(); - - // then - assertThat(clusterAssignmentCache.createSnapshot().getAssignmentsCountForConsumerNode(consumer1)).isEqualTo(2); - - // when subscription2 is removed - registry.updateAssignments(consumer1, Set.of(subscription1)); - registry.updateAssignments(consumer2, Set.of(subscription1)); - - // and - clusterAssignmentCache.refresh(); - - // then - assertThat(clusterAssignmentCache.createSnapshot().getAssignmentsCountForConsumerNode(consumer1)).isEqualTo(1); - } + private static final SubscriptionName subscription1 = + SubscriptionName.fromString("pl.allegro.tech.hermes$testSubscription"); + private static final SubscriptionName subscription2 = + SubscriptionName.fromString("pl.allegro.tech.hermes$testSubscription2"); + private static final SubscriptionName subscription3 = + SubscriptionName.fromString("pl.allegro.tech.hermes$testSubscription3"); + + private static final String consumer1 = "consumer1"; + private static final String consumer2 = "consumer2"; + private static final String clusterName = new KafkaProperties().getClusterName(); + + private static final ZookeeperPaths zookeeperPaths = new ZookeeperPaths("/hermes"); + + private static final WorkloadRegistryPaths registryPaths = + new WorkloadRegistryPaths(zookeeperPaths, clusterName); + + private static final SubscriptionIds subscriptionIds = + new TestSubscriptionIds( + ImmutableList.of( + SubscriptionId.from(subscription1, 1L), + SubscriptionId.from(subscription2, 2L), + SubscriptionId.from(subscription2, 3L))); + + private static final ConsumerAssignmentRegistry registry = + new ConsumerAssignmentRegistry( + zookeeperClient, + new WorkloadProperties().getRegistryBinaryEncoderAssignmentsBufferSizeBytes(), + clusterName, + zookeeperPaths, + subscriptionIds); + + private static final KafkaProperties kafkaProperties = new KafkaProperties(); + + private static final String cluster = kafkaProperties.getClusterName(); + + private static final ConsumerAssignmentCache assignmentCacheOfConsumer1 = + new ConsumerAssignmentCache( + zookeeperClient, consumer1, cluster, zookeeperPaths, subscriptionIds); + + private static final ConsumerAssignmentCache assignmentCacheOfConsumer2 = + new ConsumerAssignmentCache( + zookeeperClient, consumer2, cluster, zookeeperPaths, subscriptionIds); + + private static final ConsumerNodesRegistry consumerNodesRegistry = + mock(ConsumerNodesRegistry.class); + + private static final ClusterAssignmentCache clusterAssignmentCache = + new ClusterAssignmentCache( + zookeeperClient, cluster, zookeeperPaths, subscriptionIds, consumerNodesRegistry); + + @BeforeClass + public static void setUp() throws Exception { + assignmentCacheOfConsumer1.start(); + assignmentCacheOfConsumer2.start(); + } + + @Before + public void beforeEach() { + when(consumerNodesRegistry.listConsumerNodes()) + .thenReturn(ImmutableList.of(consumer1, consumer2)); + } + + @AfterClass + public static void cleanup() throws Exception { + assignmentCacheOfConsumer1.stop(); + assignmentCacheOfConsumer2.stop(); + } + + @Test + public void shouldSaveAssignmentsAndNotifyConsumers() { + // when + registry.updateAssignments(consumer1, Set.of(subscription1, subscription2)); + registry.updateAssignments(consumer2, Set.of(subscription1)); + + wait.untilZookeeperPathIsCreated(registryPaths.consumerWorkloadPath(consumer1)); + wait.untilZookeeperPathIsCreated(registryPaths.consumerWorkloadPath(consumer2)); + + // then + assertThat(assignmentCacheOfConsumer1.isAssignedTo(subscription1)).isTrue(); + assertThat(assignmentCacheOfConsumer1.isAssignedTo(subscription2)).isTrue(); + assertThat(assignmentCacheOfConsumer1.isAssignedTo(subscription3)).isFalse(); + + assertThat(assignmentCacheOfConsumer2.isAssignedTo(subscription1)).isTrue(); + assertThat(assignmentCacheOfConsumer2.isAssignedTo(subscription2)).isFalse(); + } + + @Test + public void shouldSaveAssignmentsAndReadThroughClusterAssignmentCache() { + // when + registry.updateAssignments(consumer1, Set.of(subscription1, subscription2)); + registry.updateAssignments(consumer2, Set.of(subscription1)); + + // and + clusterAssignmentCache.refresh(); + + // then + assertThat(clusterAssignmentCache.getAssignedConsumers()).containsOnly(consumer1, consumer2); + assertThat(clusterAssignmentCache.getConsumerSubscriptions(consumer1)) + .containsOnly(subscription1, subscription2); + assertThat(clusterAssignmentCache.getConsumerSubscriptions(consumer2)) + .containsOnly(subscription1); + assertThat(clusterAssignmentCache.getSubscriptionConsumers().get(subscription1)) + .containsOnly(consumer1, consumer2); + assertThat(clusterAssignmentCache.getSubscriptionConsumers().get(subscription2)) + .containsOnly(consumer1); + assertThat(clusterAssignmentCache.getSubscriptionConsumers().get(subscription3)).isNull(); + assertThat(clusterAssignmentCache.isAssignedTo(consumer1, subscription1)).isTrue(); + assertThat(clusterAssignmentCache.isAssignedTo(consumer1, subscription3)).isFalse(); + } + + @Test + public void shouldApplyChangesToAssignments() { + // when + registry.updateAssignments(consumer1, Set.of(subscription1, subscription2)); + registry.updateAssignments(consumer2, Set.of(subscription1)); + + // and + clusterAssignmentCache.refresh(); + + // then + assertThat(clusterAssignmentCache.getAssignedConsumers()).containsOnly(consumer1, consumer2); + assertThat(clusterAssignmentCache.getConsumerSubscriptions(consumer1)) + .containsOnly(subscription1, subscription2); + assertThat(clusterAssignmentCache.getConsumerSubscriptions(consumer2)) + .containsOnly(subscription1); + assertThat(clusterAssignmentCache.getSubscriptionConsumers().get(subscription1)) + .containsOnly(consumer1, consumer2); + + // when + registry.updateAssignments(consumer1, Set.of(subscription2)); + registry.updateAssignments(consumer2, Set.of(subscription1)); + + // and + clusterAssignmentCache.refresh(); + + // then + assertThat(clusterAssignmentCache.getAssignedConsumers()).containsOnly(consumer1, consumer2); + assertThat(clusterAssignmentCache.getConsumerSubscriptions(consumer1)) + .containsOnly(subscription2); + assertThat(clusterAssignmentCache.getConsumerSubscriptions(consumer2)) + .containsOnly(subscription1); + } + + @Test + public void shouldCleanStaleNodesFromRegistryOnRefresh() { + // when + registry.updateAssignments(consumer1, Set.of(subscription1, subscription2)); + registry.updateAssignments(consumer2, Set.of(subscription1)); + + // and + clusterAssignmentCache.refresh(); + + // then + assertThat(clusterAssignmentCache.getAssignedConsumers()).containsOnly(consumer1, consumer2); + + // when + when(consumerNodesRegistry.listConsumerNodes()) + .thenReturn(Collections.singletonList(consumer2)); + + // and + clusterAssignmentCache.refresh(); + + // then + assertThat(clusterAssignmentCache.getAssignedConsumers()).containsOnly(consumer2); + } + + @Test + public void shouldCleanRegistryFromStaleAssignments() { + // when + registry.updateAssignments(consumer1, Set.of(subscription1, subscription2)); + registry.updateAssignments(consumer2, Set.of(subscription1)); + + // then + wait.untilZookeeperPathIsCreated(registryPaths.consumerWorkloadPath(consumer1)); + wait.untilZookeeperPathIsCreated(registryPaths.consumerWorkloadPath(consumer2)); + + // when + clusterAssignmentCache.refresh(); + + // then + assertThat( + clusterAssignmentCache.createSnapshot().getAssignmentsCountForConsumerNode(consumer1)) + .isEqualTo(2); + + // when subscription2 is removed + registry.updateAssignments(consumer1, Set.of(subscription1)); + registry.updateAssignments(consumer2, Set.of(subscription1)); + + // and + clusterAssignmentCache.refresh(); + + // then + assertThat( + clusterAssignmentCache.createSnapshot().getAssignmentsCountForConsumerNode(consumer1)) + .isEqualTo(1); + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkloadSupervisorIntegrationTest.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkloadSupervisorIntegrationTest.java index 8e609aad90..02323f0cc4 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkloadSupervisorIntegrationTest.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/WorkloadSupervisorIntegrationTest.java @@ -1,5 +1,16 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static pl.allegro.tech.hermes.test.helper.endpoint.TimeoutAdjuster.adjust; + +import java.time.Duration; +import java.util.List; import org.junit.Before; import org.junit.Test; import pl.allegro.tech.hermes.api.Subscription; @@ -11,154 +22,144 @@ import pl.allegro.tech.hermes.consumers.supervisor.monitor.ConsumersRuntimeMonitor; import pl.allegro.tech.hermes.test.helper.zookeeper.ZookeeperBaseTest; -import java.time.Duration; -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import static pl.allegro.tech.hermes.test.helper.endpoint.TimeoutAdjuster.adjust; - public class WorkloadSupervisorIntegrationTest extends ZookeeperBaseTest { - private static final ConsumerTestRuntimeEnvironment runtime = new ConsumerTestRuntimeEnvironment(ZookeeperBaseTest::newClient); + private static final ConsumerTestRuntimeEnvironment runtime = + new ConsumerTestRuntimeEnvironment(ZookeeperBaseTest::newClient); - @Before - public void setup() throws Exception { - runtime.cleanState(); - deleteData("/hermes"); - createPath("/hermes/groups"); - } + @Before + public void setup() throws Exception { + runtime.cleanState(); + deleteData("/hermes"); + createPath("/hermes/groups"); + } - @Test - public void shouldRegisterConsumerInActiveNodesRegistryOnStartup() throws Exception { - // when - WorkloadSupervisor controller = runtime.spawnConsumer(); + @Test + public void shouldRegisterConsumerInActiveNodesRegistryOnStartup() throws Exception { + // when + WorkloadSupervisor controller = runtime.spawnConsumer(); - // then - runtime.waitForRegistration(controller.consumerId()); + // then + runtime.waitForRegistration(controller.consumerId()); - shutdown(controller); - } + shutdown(controller); + } - @Test - public void shouldElectOnlyOneLeaderFromRegisteredConsumers() throws Exception { - // when - List supervisors = runtime.spawnConsumers(3); + @Test + public void shouldElectOnlyOneLeaderFromRegisteredConsumers() throws Exception { + // when + List supervisors = runtime.spawnConsumers(3); - // then - assertThat(supervisors.stream().filter(WorkloadSupervisor::isLeader).count()).isEqualTo(1); + // then + assertThat(supervisors.stream().filter(WorkloadSupervisor::isLeader).count()).isEqualTo(1); - shutdown(supervisors); - } + shutdown(supervisors); + } - @Test - public void shouldElectNewLeaderAfterShutdown() { - // given - List supervisors = runtime.spawnConsumers(3); - WorkloadSupervisor leader = runtime.findLeader(supervisors); + @Test + public void shouldElectNewLeaderAfterShutdown() { + // given + List supervisors = runtime.spawnConsumers(3); + WorkloadSupervisor leader = runtime.findLeader(supervisors); - // when - runtime.kill(leader); + // when + runtime.kill(leader); - // then - await().atMost(adjust(Duration.ofSeconds(5))).until(() -> runtime.findLeader(supervisors) != leader); - await().atMost(adjust(Duration.ofSeconds(5))).until(() -> !leader.isLeader()); - } + // then + await() + .atMost(adjust(Duration.ofSeconds(5))) + .until(() -> runtime.findLeader(supervisors) != leader); + await().atMost(adjust(Duration.ofSeconds(5))).until(() -> !leader.isLeader()); + } - @Test - public void shouldAssignConsumerToSubscription() throws Exception { - // given - WorkloadSupervisor node = runtime.spawnConsumer(); + @Test + public void shouldAssignConsumerToSubscription() throws Exception { + // given + WorkloadSupervisor node = runtime.spawnConsumer(); - // when - SubscriptionName subscription = runtime.createSubscription(); + // when + SubscriptionName subscription = runtime.createSubscription(); - // then - runtime.awaitUntilAssignmentExists(subscription, node); + // then + runtime.awaitUntilAssignmentExists(subscription, node); - shutdown(node); - } - - @Test - public void shouldAssignSubscriptionToMultipleConsumers() throws Exception { - // given - List nodes = runtime.spawnConsumers(2); + shutdown(node); + } - // when - SubscriptionName subscription = runtime.createSubscription(); + @Test + public void shouldAssignSubscriptionToMultipleConsumers() throws Exception { + // given + List nodes = runtime.spawnConsumers(2); - // then - nodes.forEach(node -> runtime.awaitUntilAssignmentExists(subscription, node)); + // when + SubscriptionName subscription = runtime.createSubscription(); - shutdown(nodes); - } + // then + nodes.forEach(node -> runtime.awaitUntilAssignmentExists(subscription, node)); - @Test - public void shouldAssignConsumerToMultipleSubscriptions() throws Exception { - // given - WorkloadSupervisor node = runtime.spawnConsumer(); + shutdown(nodes); + } - // when - List subscriptions = runtime.createSubscription(2); + @Test + public void shouldAssignConsumerToMultipleSubscriptions() throws Exception { + // given + WorkloadSupervisor node = runtime.spawnConsumer(); - // then - subscriptions.forEach(subscription -> runtime.awaitUntilAssignmentExists(subscription, node)); + // when + List subscriptions = runtime.createSubscription(2); - shutdown(node); - } + // then + subscriptions.forEach(subscription -> runtime.awaitUntilAssignmentExists(subscription, node)); - @Test - public void shouldRecreateMissingConsumer() throws Exception { - // given - ConsumerFactory consumerFactory = mock(ConsumerFactory.class); + shutdown(node); + } - when(consumerFactory.createConsumer(any(Subscription.class))) - .thenThrow( - new InternalProcessingException("failed to create consumer")) - .thenReturn( - mock(SerialConsumer.class)); + @Test + public void shouldRecreateMissingConsumer() throws Exception { + // given + ConsumerFactory consumerFactory = mock(ConsumerFactory.class); - String consumerId = "consumer"; + when(consumerFactory.createConsumer(any(Subscription.class))) + .thenThrow(new InternalProcessingException("failed to create consumer")) + .thenReturn(mock(SerialConsumer.class)); + String consumerId = "consumer"; - ConsumersSupervisor supervisor = runtime.consumersSupervisor(consumerFactory); - WorkloadSupervisor node = runtime.spawnConsumer(consumerId, supervisor); + ConsumersSupervisor supervisor = runtime.consumersSupervisor(consumerFactory); + WorkloadSupervisor node = runtime.spawnConsumer(consumerId, supervisor); - runtime.awaitUntilAssignmentExists(runtime.createSubscription(), node); + runtime.awaitUntilAssignmentExists(runtime.createSubscription(), node); - // when - ConsumersRuntimeMonitor monitor = runtime.monitor(consumerId, supervisor, node, Duration.ofSeconds(1)); - monitor.start(); + // when + ConsumersRuntimeMonitor monitor = + runtime.monitor(consumerId, supervisor, node, Duration.ofSeconds(1)); + monitor.start(); - // then - await().atMost(Duration.ofSeconds(5)).untilAsserted( - () -> verify(consumerFactory, times(2)).createConsumer(any())); + // then + await() + .atMost(Duration.ofSeconds(5)) + .untilAsserted(() -> verify(consumerFactory, times(2)).createConsumer(any())); - shutdown(supervisor); - shutdown(node); - shutdown(monitor); - } + shutdown(supervisor); + shutdown(node); + shutdown(monitor); + } - private void shutdown(WorkloadSupervisor workloadSupervisor) throws Exception { - workloadSupervisor.shutdown(); - } + private void shutdown(WorkloadSupervisor workloadSupervisor) throws Exception { + workloadSupervisor.shutdown(); + } - private void shutdown(List workloadSupervisors) throws Exception { - for (WorkloadSupervisor s : workloadSupervisors) { - s.shutdown(); - } + private void shutdown(List workloadSupervisors) throws Exception { + for (WorkloadSupervisor s : workloadSupervisors) { + s.shutdown(); } + } - private void shutdown(ConsumersRuntimeMonitor monitor) throws InterruptedException { - monitor.shutdown(); - } + private void shutdown(ConsumersRuntimeMonitor monitor) throws InterruptedException { + monitor.shutdown(); + } - private void shutdown(ConsumersSupervisor supervisor) throws InterruptedException { - supervisor.shutdown(); - } + private void shutdown(ConsumersSupervisor supervisor) throws InterruptedException { + supervisor.shutdown(); + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ZookeeperConsumerNodeLoadRegistryTest.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ZookeeperConsumerNodeLoadRegistryTest.java index 5803f557d7..19dc4cc2bc 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ZookeeperConsumerNodeLoadRegistryTest.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ZookeeperConsumerNodeLoadRegistryTest.java @@ -1,5 +1,11 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload.weighted; +import static org.assertj.core.api.Assertions.assertThat; +import static pl.allegro.tech.hermes.api.SubscriptionName.fromString; + +import java.time.Duration; +import java.util.List; +import java.util.Map; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -15,97 +21,96 @@ import pl.allegro.tech.hermes.test.helper.time.ModifiableClock; import pl.allegro.tech.hermes.test.helper.zookeeper.ZookeeperBaseTest; -import java.time.Duration; -import java.util.List; -import java.util.Map; - -import static org.assertj.core.api.Assertions.assertThat; -import static pl.allegro.tech.hermes.api.SubscriptionName.fromString; - public class ZookeeperConsumerNodeLoadRegistryTest extends ZookeeperBaseTest { - private final SubscriptionName firstSubscription = fromString("pl.allegro.tech.hermes$testSubscription"); - private final SubscriptionName secondSubscription = fromString("pl.allegro.tech.hermes$testSubscription2"); - private final SubscriptionIds subscriptionIds = new TestSubscriptionIds(List.of( - SubscriptionId.from(firstSubscription, 1L), - SubscriptionId.from(secondSubscription, 2L) - )); - private final String currentConsumerId = "consumer-id"; - private final ManuallyTriggeredScheduledExecutorService scheduledExecutorService = new ManuallyTriggeredScheduledExecutorService(); - private final ModifiableClock clock = new ModifiableClock(); - private final ZookeeperConsumerNodeLoadRegistry registry = new ZookeeperConsumerNodeLoadRegistry( - zookeeperClient, - subscriptionIds, - new ZookeeperPaths("/hermes"), - currentConsumerId, - "kafka-cluster", - Duration.ofMillis(50), - new TestExecutorServiceFactory(scheduledExecutorService), - clock, - TestMetricsFacadeFactory.create(), - 100_000 - ); - - @Before - public void setUp() { - registry.start(); - } - - @After - public void cleanup() throws Exception { - registry.stop(); - deleteAllNodes(); - } - - @Test - public void shouldPeriodicallyReportConsumerLoad() { - // given - scheduledExecutorService.triggerScheduledTasks(); - - // and - SubscriptionLoadRecorder firstSubscriptionReporter = registry.register(firstSubscription); - firstSubscriptionReporter.initialize(); - firstSubscriptionReporter.recordSingleOperation(); - firstSubscriptionReporter.recordSingleOperation(); - - // and - SubscriptionLoadRecorder secondSubscriptionReporter = registry.register(secondSubscription); - secondSubscriptionReporter.initialize(); - - // and - clock.advanceMinutes(1); - - // when - scheduledExecutorService.triggerScheduledTasks(); - - // then - ConsumerNodeLoad consumerNodeLoad = registry.get(currentConsumerId); - Map loadPerSubscription = consumerNodeLoad.getLoadPerSubscription(); - assertThat(loadPerSubscription).hasSize(2); - assertThat(loadPerSubscription.get(firstSubscription).getOperationsPerSecond()).isEqualTo(2d / 60); - assertThat(loadPerSubscription.get(secondSubscription).getOperationsPerSecond()).isEqualTo(0d); - } - - @Test - public void shouldNotReportLoadOfRemovedSubscription() { - // given - scheduledExecutorService.triggerScheduledTasks(); - - // and - SubscriptionLoadRecorder firstSubscriptionReporter = registry.register(firstSubscription); - firstSubscriptionReporter.initialize(); - - // when - scheduledExecutorService.triggerScheduledTasks(); - - // then - assertThat(registry.get(currentConsumerId).getLoadPerSubscription()).hasSize(1); - - // when - firstSubscriptionReporter.shutdown(); - scheduledExecutorService.triggerScheduledTasks(); - - // then - assertThat(registry.get(currentConsumerId).getLoadPerSubscription()).hasSize(0); - } + private final SubscriptionName firstSubscription = + fromString("pl.allegro.tech.hermes$testSubscription"); + private final SubscriptionName secondSubscription = + fromString("pl.allegro.tech.hermes$testSubscription2"); + private final SubscriptionIds subscriptionIds = + new TestSubscriptionIds( + List.of( + SubscriptionId.from(firstSubscription, 1L), + SubscriptionId.from(secondSubscription, 2L))); + private final String currentConsumerId = "consumer-id"; + private final ManuallyTriggeredScheduledExecutorService scheduledExecutorService = + new ManuallyTriggeredScheduledExecutorService(); + private final ModifiableClock clock = new ModifiableClock(); + private final ZookeeperConsumerNodeLoadRegistry registry = + new ZookeeperConsumerNodeLoadRegistry( + zookeeperClient, + subscriptionIds, + new ZookeeperPaths("/hermes"), + currentConsumerId, + "kafka-cluster", + Duration.ofMillis(50), + new TestExecutorServiceFactory(scheduledExecutorService), + clock, + TestMetricsFacadeFactory.create(), + 100_000); + + @Before + public void setUp() { + registry.start(); + } + + @After + public void cleanup() throws Exception { + registry.stop(); + deleteAllNodes(); + } + + @Test + public void shouldPeriodicallyReportConsumerLoad() { + // given + scheduledExecutorService.triggerScheduledTasks(); + + // and + SubscriptionLoadRecorder firstSubscriptionReporter = registry.register(firstSubscription); + firstSubscriptionReporter.initialize(); + firstSubscriptionReporter.recordSingleOperation(); + firstSubscriptionReporter.recordSingleOperation(); + + // and + SubscriptionLoadRecorder secondSubscriptionReporter = registry.register(secondSubscription); + secondSubscriptionReporter.initialize(); + + // and + clock.advanceMinutes(1); + + // when + scheduledExecutorService.triggerScheduledTasks(); + + // then + ConsumerNodeLoad consumerNodeLoad = registry.get(currentConsumerId); + Map loadPerSubscription = + consumerNodeLoad.getLoadPerSubscription(); + assertThat(loadPerSubscription).hasSize(2); + assertThat(loadPerSubscription.get(firstSubscription).getOperationsPerSecond()) + .isEqualTo(2d / 60); + assertThat(loadPerSubscription.get(secondSubscription).getOperationsPerSecond()).isEqualTo(0d); + } + + @Test + public void shouldNotReportLoadOfRemovedSubscription() { + // given + scheduledExecutorService.triggerScheduledTasks(); + + // and + SubscriptionLoadRecorder firstSubscriptionReporter = registry.register(firstSubscription); + firstSubscriptionReporter.initialize(); + + // when + scheduledExecutorService.triggerScheduledTasks(); + + // then + assertThat(registry.get(currentConsumerId).getLoadPerSubscription()).hasSize(1); + + // when + firstSubscriptionReporter.shutdown(); + scheduledExecutorService.triggerScheduledTasks(); + + // then + assertThat(registry.get(currentConsumerId).getLoadPerSubscription()).hasSize(0); + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ZookeeperSubscriptionProfileRegistryTest.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ZookeeperSubscriptionProfileRegistryTest.java index 107d5cc0fe..ad3976e32d 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ZookeeperSubscriptionProfileRegistryTest.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/supervisor/workload/weighted/ZookeeperSubscriptionProfileRegistryTest.java @@ -1,5 +1,12 @@ package pl.allegro.tech.hermes.consumers.supervisor.workload.weighted; +import static java.time.temporal.ChronoUnit.MILLIS; +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Instant; +import java.util.Comparator; +import java.util.List; +import java.util.Map; import org.junit.Test; import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.consumers.subscription.id.SubscriptionId; @@ -8,80 +15,77 @@ import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; import pl.allegro.tech.hermes.test.helper.zookeeper.ZookeeperBaseTest; -import java.time.Instant; -import java.util.Comparator; -import java.util.List; -import java.util.Map; - -import static java.time.temporal.ChronoUnit.MILLIS; -import static org.assertj.core.api.Assertions.assertThat; - public class ZookeeperSubscriptionProfileRegistryTest extends ZookeeperBaseTest { - @Test - public void shouldPersistAndReadSubscriptionProfiles() { - // given - SubscriptionName firstSubscription = SubscriptionName.fromString("pl.allegro.tech.hermes$testSubscription"); - SubscriptionName secondSubscription = SubscriptionName.fromString("pl.allegro.tech.hermes$testSubscription2"); - SubscriptionIds subscriptionIds = new TestSubscriptionIds(List.of( + @Test + public void shouldPersistAndReadSubscriptionProfiles() { + // given + SubscriptionName firstSubscription = + SubscriptionName.fromString("pl.allegro.tech.hermes$testSubscription"); + SubscriptionName secondSubscription = + SubscriptionName.fromString("pl.allegro.tech.hermes$testSubscription2"); + SubscriptionIds subscriptionIds = + new TestSubscriptionIds( + List.of( SubscriptionId.from(firstSubscription, -1422951212L), - SubscriptionId.from(secondSubscription, 2L) - )); - ZookeeperSubscriptionProfileRegistry registry = new ZookeeperSubscriptionProfileRegistry( - zookeeperClient, - subscriptionIds, - new ZookeeperPaths("/hermes"), - "kafka-cluster", - 100_000 - ); - SubscriptionProfiles profiles = new SubscriptionProfiles( - Map.of( - firstSubscription, new SubscriptionProfile(Instant.now(), new Weight(100d)), - secondSubscription, SubscriptionProfile.UNDEFINED - ), - Instant.now() - ); + SubscriptionId.from(secondSubscription, 2L))); + ZookeeperSubscriptionProfileRegistry registry = + new ZookeeperSubscriptionProfileRegistry( + zookeeperClient, + subscriptionIds, + new ZookeeperPaths("/hermes"), + "kafka-cluster", + 100_000); + SubscriptionProfiles profiles = + new SubscriptionProfiles( + Map.of( + firstSubscription, + new SubscriptionProfile(Instant.now(), new Weight(100d)), + secondSubscription, + SubscriptionProfile.UNDEFINED), + Instant.now()); - // when - registry.persist(profiles); + // when + registry.persist(profiles); - // then - SubscriptionProfiles readProfiles = registry.fetch(); - assertThatProfilesAreEqual(readProfiles, profiles); - } + // then + SubscriptionProfiles readProfiles = registry.fetch(); + assertThatProfilesAreEqual(readProfiles, profiles); + } - @Test - public void shouldPersistAndReadEmptySubscriptionProfiles() { - // given - ZookeeperSubscriptionProfileRegistry registry = new ZookeeperSubscriptionProfileRegistry( - zookeeperClient, - new TestSubscriptionIds(List.of()), - new ZookeeperPaths("/hermes"), - "kafka-cluster", - 100_000 - ); - SubscriptionProfiles profiles = SubscriptionProfiles.EMPTY; + @Test + public void shouldPersistAndReadEmptySubscriptionProfiles() { + // given + ZookeeperSubscriptionProfileRegistry registry = + new ZookeeperSubscriptionProfileRegistry( + zookeeperClient, + new TestSubscriptionIds(List.of()), + new ZookeeperPaths("/hermes"), + "kafka-cluster", + 100_000); + SubscriptionProfiles profiles = SubscriptionProfiles.EMPTY; - // when - registry.persist(profiles); + // when + registry.persist(profiles); - // then - SubscriptionProfiles readProfiles = registry.fetch(); - assertThatProfilesAreEqual(readProfiles, profiles); - } + // then + SubscriptionProfiles readProfiles = registry.fetch(); + assertThatProfilesAreEqual(readProfiles, profiles); + } - private static void assertThatProfilesAreEqual(SubscriptionProfiles actual, SubscriptionProfiles expected) { - assertThat(actual) - .usingRecursiveComparison() - .withComparatorForType(new InstantComparatorIgnoringNanos(), Instant.class) - .isEqualTo(expected); - } + private static void assertThatProfilesAreEqual( + SubscriptionProfiles actual, SubscriptionProfiles expected) { + assertThat(actual) + .usingRecursiveComparison() + .withComparatorForType(new InstantComparatorIgnoringNanos(), Instant.class) + .isEqualTo(expected); + } - private static class InstantComparatorIgnoringNanos implements Comparator { + private static class InstantComparatorIgnoringNanos implements Comparator { - @Override - public int compare(Instant o1, Instant o2) { - return o1.truncatedTo(MILLIS).compareTo(o2.truncatedTo(MILLIS)); - } + @Override + public int compare(Instant o1, Instant o2) { + return o1.truncatedTo(MILLIS).compareTo(o2.truncatedTo(MILLIS)); } + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/test/HermesConsumersAssertions.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/test/HermesConsumersAssertions.java index 8cd0470248..8c54e26d64 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/test/HermesConsumersAssertions.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/test/HermesConsumersAssertions.java @@ -6,8 +6,7 @@ public class HermesConsumersAssertions extends Assertions { - public static OutputRateCalculationResultAssert assertThat(OutputRateCalculationResult actual) { - return OutputRateCalculationResultAssert.assertThat(actual); - } - + public static OutputRateCalculationResultAssert assertThat(OutputRateCalculationResult actual) { + return OutputRateCalculationResultAssert.assertThat(actual); + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/test/MessageBuilder.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/test/MessageBuilder.java index 1f794e19be..a9838c4b63 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/test/MessageBuilder.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/test/MessageBuilder.java @@ -1,12 +1,6 @@ package pl.allegro.tech.hermes.consumers.test; -import org.apache.avro.Schema; -import pl.allegro.tech.hermes.api.ContentType; -import pl.allegro.tech.hermes.api.Header; -import pl.allegro.tech.hermes.common.kafka.KafkaTopicName; -import pl.allegro.tech.hermes.common.kafka.offset.PartitionOffset; -import pl.allegro.tech.hermes.consumers.consumer.Message; -import pl.allegro.tech.hermes.schema.CompiledSchema; +import static com.google.common.collect.ImmutableMap.of; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; @@ -14,131 +8,147 @@ import java.util.List; import java.util.Map; import java.util.Optional; - -import static com.google.common.collect.ImmutableMap.of; +import org.apache.avro.Schema; +import pl.allegro.tech.hermes.api.ContentType; +import pl.allegro.tech.hermes.api.Header; +import pl.allegro.tech.hermes.common.kafka.KafkaTopicName; +import pl.allegro.tech.hermes.common.kafka.offset.PartitionOffset; +import pl.allegro.tech.hermes.consumers.consumer.Message; +import pl.allegro.tech.hermes.schema.CompiledSchema; public final class MessageBuilder { - public static final String TEST_MESSAGE_CONTENT = "Some test message"; - - private String id; - private String topic; - private ContentType contentType; - private long publishingTimestamp; - private long readingTimestamp; - private PartitionOffset partitionOffset; - private long partitionAssignmentTerm = -1L; - private byte[] content; - private Map externalMetadata; - private List
additionalHeaders; - private Optional> schema = Optional.empty(); - private String subscription; - private boolean hasSubscriptionIdentityHeaders; - - private MessageBuilder() { - } - - public static MessageBuilder newBuilder() { - return new MessageBuilder(); - } - - public static Message testMessage() { - return MessageBuilder.withTestMessage().build(); - } - - public static MessageBuilder withTestMessage() { - return new MessageBuilder() - .withId("id") - .withTopic("topicId") - .withContent(TEST_MESSAGE_CONTENT, StandardCharsets.UTF_8) - .withContentType(ContentType.JSON) - .withPublishingTimestamp(123L) - .withReadingTimestamp(123L) - .withPartitionOffset(new PartitionOffset(KafkaTopicName.valueOf("kafka_topic"), 123, 1)) - .withExternalMetadata(of("Trace-Id", "traceId")) - .withAdditionalHeaders(Collections.emptyList()) - .withSubscription("subscriptionId") - .withHasSubscriptionIdentityHeaders(true); - } - - public Message build() { - return new Message(id, topic, content, contentType, schema, publishingTimestamp, - readingTimestamp, partitionOffset, partitionAssignmentTerm, externalMetadata, additionalHeaders, - subscription, hasSubscriptionIdentityHeaders); - } - - public MessageBuilder withId(String id) { - this.id = id; - return this; - } - - public MessageBuilder withSchema(Schema schema, int id, int version) { - this.schema = Optional.of(CompiledSchema.of(schema, id, version)); - return this; - } - - public MessageBuilder withTopic(String topic) { - this.topic = topic; - return this; - } - - public MessageBuilder withContent(String content, Charset charset) { - this.content = content.getBytes(charset); - return this; - } - - public MessageBuilder withContent(byte[] content) { - this.content = content; - return this; - } - - public MessageBuilder withContentType(ContentType contentType) { - this.contentType = contentType; - return this; - } - - public MessageBuilder withPublishingTimestamp(long publishingTimestamp) { - this.publishingTimestamp = publishingTimestamp; - return this; - } - - public MessageBuilder withReadingTimestamp(long readingTimestamp) { - this.readingTimestamp = readingTimestamp; - return this; - } - - public MessageBuilder withPartitionOffset(PartitionOffset partitionOffset) { - this.partitionOffset = partitionOffset; - return this; - } - - public MessageBuilder withPartitionOffset(String kafkaTopic, int partition, long offset) { - this.partitionOffset = new PartitionOffset(KafkaTopicName.valueOf(kafkaTopic), offset, partition); - return this; - } - - public MessageBuilder withPartitionAssignmentTerm(long term) { - this.partitionAssignmentTerm = term; - return this; - } - - public MessageBuilder withExternalMetadata(Map externalMetadata) { - this.externalMetadata = externalMetadata; - return this; - } - - public MessageBuilder withAdditionalHeaders(List
additionalHeaders) { - this.additionalHeaders = additionalHeaders; - return this; - } - - public MessageBuilder withSubscription(String subscription) { - this.subscription = subscription; - return this; - } - - public MessageBuilder withHasSubscriptionIdentityHeaders(boolean hasSubscriptionIdentityHeaders) { - this.hasSubscriptionIdentityHeaders = hasSubscriptionIdentityHeaders; - return this; - } + public static final String TEST_MESSAGE_CONTENT = "Some test message"; + + private String id; + private String topic; + private ContentType contentType; + private long publishingTimestamp; + private long readingTimestamp; + private PartitionOffset partitionOffset; + private long partitionAssignmentTerm = -1L; + private byte[] content; + private Map externalMetadata; + private List
additionalHeaders; + private Optional> schema = Optional.empty(); + private String subscription; + private boolean hasSubscriptionIdentityHeaders; + + private MessageBuilder() {} + + public static MessageBuilder newBuilder() { + return new MessageBuilder(); + } + + public static Message testMessage() { + return MessageBuilder.withTestMessage().build(); + } + + public static MessageBuilder withTestMessage() { + return new MessageBuilder() + .withId("id") + .withTopic("topicId") + .withContent(TEST_MESSAGE_CONTENT, StandardCharsets.UTF_8) + .withContentType(ContentType.JSON) + .withPublishingTimestamp(123L) + .withReadingTimestamp(123L) + .withPartitionOffset(new PartitionOffset(KafkaTopicName.valueOf("kafka_topic"), 123, 1)) + .withExternalMetadata(of("Trace-Id", "traceId")) + .withAdditionalHeaders(Collections.emptyList()) + .withSubscription("subscriptionId") + .withHasSubscriptionIdentityHeaders(true); + } + + public Message build() { + return new Message( + id, + topic, + content, + contentType, + schema, + publishingTimestamp, + readingTimestamp, + partitionOffset, + partitionAssignmentTerm, + externalMetadata, + additionalHeaders, + subscription, + hasSubscriptionIdentityHeaders); + } + + public MessageBuilder withId(String id) { + this.id = id; + return this; + } + + public MessageBuilder withSchema(Schema schema, int id, int version) { + this.schema = Optional.of(CompiledSchema.of(schema, id, version)); + return this; + } + + public MessageBuilder withTopic(String topic) { + this.topic = topic; + return this; + } + + public MessageBuilder withContent(String content, Charset charset) { + this.content = content.getBytes(charset); + return this; + } + + public MessageBuilder withContent(byte[] content) { + this.content = content; + return this; + } + + public MessageBuilder withContentType(ContentType contentType) { + this.contentType = contentType; + return this; + } + + public MessageBuilder withPublishingTimestamp(long publishingTimestamp) { + this.publishingTimestamp = publishingTimestamp; + return this; + } + + public MessageBuilder withReadingTimestamp(long readingTimestamp) { + this.readingTimestamp = readingTimestamp; + return this; + } + + public MessageBuilder withPartitionOffset(PartitionOffset partitionOffset) { + this.partitionOffset = partitionOffset; + return this; + } + + public MessageBuilder withPartitionOffset(String kafkaTopic, int partition, long offset) { + this.partitionOffset = + new PartitionOffset(KafkaTopicName.valueOf(kafkaTopic), offset, partition); + return this; + } + + public MessageBuilder withPartitionAssignmentTerm(long term) { + this.partitionAssignmentTerm = term; + return this; + } + + public MessageBuilder withExternalMetadata(Map externalMetadata) { + this.externalMetadata = externalMetadata; + return this; + } + + public MessageBuilder withAdditionalHeaders(List
additionalHeaders) { + this.additionalHeaders = additionalHeaders; + return this; + } + + public MessageBuilder withSubscription(String subscription) { + this.subscription = subscription; + return this; + } + + public MessageBuilder withHasSubscriptionIdentityHeaders(boolean hasSubscriptionIdentityHeaders) { + this.hasSubscriptionIdentityHeaders = hasSubscriptionIdentityHeaders; + return this; + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/test/TestTrackers.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/test/TestTrackers.java index fb2350dd24..3febad1cf6 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/test/TestTrackers.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/test/TestTrackers.java @@ -5,7 +5,7 @@ public class TestTrackers extends Trackers { - public TestTrackers() { - super(ImmutableList.of()); - } + public TestTrackers() { + super(ImmutableList.of()); + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/test/Wait.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/test/Wait.java index 67026279f8..b30ba80d89 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/test/Wait.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/test/Wait.java @@ -2,18 +2,17 @@ public final class Wait { - private Wait() { - } + private Wait() {} - public static void forCacheInvalidation() { - sleep(700); - } + public static void forCacheInvalidation() { + sleep(700); + } - private static void sleep(int millis) { - try { - Thread.sleep(millis); - } catch (InterruptedException exception) { - throw new RuntimeException("Who dares to interrupt me?", exception); - } + private static void sleep(int millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException exception) { + throw new RuntimeException("Who dares to interrupt me?", exception); } + } } diff --git a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/uri/UriUtilsTest.java b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/uri/UriUtilsTest.java index e1fe81076b..8a438b904d 100644 --- a/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/uri/UriUtilsTest.java +++ b/hermes-consumers/src/test/java/pl/allegro/tech/hermes/consumers/uri/UriUtilsTest.java @@ -1,9 +1,5 @@ package pl.allegro.tech.hermes.consumers.uri; -import org.junit.Test; - -import java.net.URI; - import static java.net.URI.create; import static org.assertj.core.api.Assertions.assertThat; import static pl.allegro.tech.hermes.consumers.uri.UriUtils.appendContext; @@ -14,72 +10,78 @@ import static pl.allegro.tech.hermes.consumers.uri.UriUtils.extractPortFromUri; import static pl.allegro.tech.hermes.consumers.uri.UriUtils.extractUserNameFromUri; -public class UriUtilsTest { - - private static final URI FULL_URI = create("jms://user:pass@localhost:123123/12312312/312?param=test"); - private static final URI NO_PORT_URI = create("jms://user:pass@localhost/12312312/312?param=test"); - private static final URI NO_USER_URI = create("jms://localhost:123123/12312312/312?param=test"); - - @Test(expected = InvalidHostException.class) - public void shouldThrowExceptionForInvalidHost() { - extractHostFromUri(URI.create("jms://host_with_underscores/test")); - } - - @Test - public void shouldExtractHostFromUri() { - assertThat(extractHostFromUri(FULL_URI)).isEqualTo("localhost"); - } - - @Test - public void shouldExtractPortFromUri() { - assertThat(extractPortFromUri(FULL_URI).longValue()).isEqualTo(123123L); - assertThat(extractPortFromUri(NO_PORT_URI)).isNull(); - } - - @Test - public void shouldExtractAddressFromUri() { - assertThat(extractAddressFromUri(FULL_URI)).isEqualTo("localhost:123123"); - assertThat(extractAddressFromUri(NO_PORT_URI)).isEqualTo("localhost"); - } - - @Test - public void shouldExtractUsernameFromUri() { - assertThat(extractUserNameFromUri(FULL_URI)).isEqualTo("user"); - assertThat(extractUserNameFromUri(NO_USER_URI)).isNull(); - } - - @Test - public void shouldExtractPasswordFromUri() { - assertThat(extractPasswordFromUri(FULL_URI)).isEqualTo("pass"); - assertThat(extractPasswordFromUri(NO_USER_URI)).isNull(); - } - - @Test - public void shouldExtractContextFromUri() { - assertContext("http://localhost:8080", ""); - assertContext("http://localhost:8080/", "/"); - assertContext("http://localhost:8080/path/1", "/path/1"); - assertContext("http://localhost:8080/path/1?arg1=test1&arg2=test2", "/path/1?arg1=test1&arg2=test2"); - assertContext("http://localhost:8080/path/1?arg=test#fragment", "/path/1?arg=test#fragment"); - assertContext("http://localhost:8080?arg=test", "?arg=test"); - assertContext("http://localhost:8080?arg=test#fragment", "?arg=test#fragment"); - assertContext("http://localhost:8080#fragment", "#fragment"); - } - - private void assertContext(String uri, String context) { - assertThat(extractContextFromUri(create(uri))).isEqualTo(context); - } - - @Test - public void shouldAppendContextToURI() { - // given when then - assertThat(appendContext(URI.create("http://localhost:8080/"), "context")) - .isEqualTo(URI.create("http://localhost:8080/context")); +import java.net.URI; +import org.junit.Test; - assertThat(appendContext(URI.create("http://localhost:8080/"), "/context")) - .isEqualTo(URI.create("http://localhost:8080/context")); +public class UriUtilsTest { - assertThat(appendContext(URI.create("http://localhost:8080/sth"), "context")) - .isEqualTo(URI.create("http://localhost:8080/sth/context")); - } + private static final URI FULL_URI = + create("jms://user:pass@localhost:123123/12312312/312?param=test"); + private static final URI NO_PORT_URI = + create("jms://user:pass@localhost/12312312/312?param=test"); + private static final URI NO_USER_URI = create("jms://localhost:123123/12312312/312?param=test"); + + @Test(expected = InvalidHostException.class) + public void shouldThrowExceptionForInvalidHost() { + extractHostFromUri(URI.create("jms://host_with_underscores/test")); + } + + @Test + public void shouldExtractHostFromUri() { + assertThat(extractHostFromUri(FULL_URI)).isEqualTo("localhost"); + } + + @Test + public void shouldExtractPortFromUri() { + assertThat(extractPortFromUri(FULL_URI).longValue()).isEqualTo(123123L); + assertThat(extractPortFromUri(NO_PORT_URI)).isNull(); + } + + @Test + public void shouldExtractAddressFromUri() { + assertThat(extractAddressFromUri(FULL_URI)).isEqualTo("localhost:123123"); + assertThat(extractAddressFromUri(NO_PORT_URI)).isEqualTo("localhost"); + } + + @Test + public void shouldExtractUsernameFromUri() { + assertThat(extractUserNameFromUri(FULL_URI)).isEqualTo("user"); + assertThat(extractUserNameFromUri(NO_USER_URI)).isNull(); + } + + @Test + public void shouldExtractPasswordFromUri() { + assertThat(extractPasswordFromUri(FULL_URI)).isEqualTo("pass"); + assertThat(extractPasswordFromUri(NO_USER_URI)).isNull(); + } + + @Test + public void shouldExtractContextFromUri() { + assertContext("http://localhost:8080", ""); + assertContext("http://localhost:8080/", "/"); + assertContext("http://localhost:8080/path/1", "/path/1"); + assertContext( + "http://localhost:8080/path/1?arg1=test1&arg2=test2", "/path/1?arg1=test1&arg2=test2"); + assertContext("http://localhost:8080/path/1?arg=test#fragment", "/path/1?arg=test#fragment"); + assertContext("http://localhost:8080?arg=test", "?arg=test"); + assertContext("http://localhost:8080?arg=test#fragment", "?arg=test#fragment"); + assertContext("http://localhost:8080#fragment", "#fragment"); + } + + private void assertContext(String uri, String context) { + assertThat(extractContextFromUri(create(uri))).isEqualTo(context); + } + + @Test + public void shouldAppendContextToURI() { + // given when then + assertThat(appendContext(URI.create("http://localhost:8080/"), "context")) + .isEqualTo(URI.create("http://localhost:8080/context")); + + assertThat(appendContext(URI.create("http://localhost:8080/"), "/context")) + .isEqualTo(URI.create("http://localhost:8080/context")); + + assertThat(appendContext(URI.create("http://localhost:8080/sth"), "context")) + .isEqualTo(URI.create("http://localhost:8080/sth/context")); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/HermesFrontend.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/HermesFrontend.java index 0da5c5552d..adba6ccde3 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/HermesFrontend.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/HermesFrontend.java @@ -7,9 +7,9 @@ @SpringBootApplication public class HermesFrontend { - public static void main(String[] args) { - SpringApplication application = new SpringApplication(HermesFrontend.class); - application.setWebApplicationType(WebApplicationType.NONE); - application.run(args); - } -} \ No newline at end of file + public static void main(String[] args) { + SpringApplication application = new SpringApplication(HermesFrontend.class); + application.setWebApplicationType(WebApplicationType.NONE); + application.run(args); + } +} diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/blacklist/BlacklistZookeeperNotifyingCache.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/blacklist/BlacklistZookeeperNotifyingCache.java index cfb11574a2..ab9d720a4e 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/blacklist/BlacklistZookeeperNotifyingCache.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/blacklist/BlacklistZookeeperNotifyingCache.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.frontend.blacklist; +import java.util.ArrayList; +import java.util.List; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCache; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; @@ -8,56 +10,55 @@ import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; -import java.util.ArrayList; -import java.util.List; +public class BlacklistZookeeperNotifyingCache extends PathChildrenCache + implements PathChildrenCacheListener { -public class BlacklistZookeeperNotifyingCache extends PathChildrenCache implements PathChildrenCacheListener { + private static final Logger logger = + LoggerFactory.getLogger(BlacklistZookeeperNotifyingCache.class); - private static final Logger logger = LoggerFactory.getLogger(BlacklistZookeeperNotifyingCache.class); + private final List topicCallbacks = new ArrayList<>(); - private final List topicCallbacks = new ArrayList<>(); + public BlacklistZookeeperNotifyingCache(CuratorFramework curator, ZookeeperPaths paths) { + super(curator, paths.topicsBlacklistPath(), true); + getListenable().addListener(this); + } - public BlacklistZookeeperNotifyingCache(CuratorFramework curator, ZookeeperPaths paths) { - super(curator, paths.topicsBlacklistPath(), true); - getListenable().addListener(this); + @Override + public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception { + if (event == null || event.getData() == null) { + return; } - @Override - public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception { - if (event == null || event.getData() == null) { - return; - } - - logger.info("Got {} event for path {}", event.getType(), event.getData().getPath()); - - String qualifiedTopicName = getTopicName(event); - - switch (event.getType()) { - case CHILD_ADDED: - topicCallbacks.forEach(callback -> callback.onTopicBlacklisted(qualifiedTopicName)); - break; - case CHILD_REMOVED: - topicCallbacks.forEach(callback -> callback.onTopicUnblacklisted(qualifiedTopicName)); - break; - default: - break; - } - } + logger.info("Got {} event for path {}", event.getType(), event.getData().getPath()); - public void startup() { - try { - this.start(); - } catch (Exception e) { - throw new IllegalStateException("Failed to start Zookeeper Topic Blacklist cache", e); - } - } + String qualifiedTopicName = getTopicName(event); - private String getTopicName(PathChildrenCacheEvent event) { - String[] paths = event.getData().getPath().split("/"); - return paths[paths.length - 1]; + switch (event.getType()) { + case CHILD_ADDED: + topicCallbacks.forEach(callback -> callback.onTopicBlacklisted(qualifiedTopicName)); + break; + case CHILD_REMOVED: + topicCallbacks.forEach(callback -> callback.onTopicUnblacklisted(qualifiedTopicName)); + break; + default: + break; } + } - public void addCallback(TopicBlacklistCallback callback) { - topicCallbacks.add(callback); + public void startup() { + try { + this.start(); + } catch (Exception e) { + throw new IllegalStateException("Failed to start Zookeeper Topic Blacklist cache", e); } + } + + private String getTopicName(PathChildrenCacheEvent event) { + String[] paths = event.getData().getPath().split("/"); + return paths[paths.length - 1]; + } + + public void addCallback(TopicBlacklistCallback callback) { + topicCallbacks.add(callback); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/blacklist/TopicBlacklistCallback.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/blacklist/TopicBlacklistCallback.java index f63e770198..8d7b3f7c82 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/blacklist/TopicBlacklistCallback.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/blacklist/TopicBlacklistCallback.java @@ -2,7 +2,7 @@ public interface TopicBlacklistCallback { - default void onTopicBlacklisted(String qualifiedTopicName) {} + default void onTopicBlacklisted(String qualifiedTopicName) {} - default void onTopicUnblacklisted(String qualifiedTopicName) {} + default void onTopicUnblacklisted(String qualifiedTopicName) {} } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/BackupFilesManager.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/BackupFilesManager.java index 2ead3bd3d1..eb2141469b 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/BackupFilesManager.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/BackupFilesManager.java @@ -1,10 +1,10 @@ package pl.allegro.tech.hermes.frontend.buffer; -import com.google.common.io.PatternFilenameFilter; -import org.apache.commons.io.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static com.google.common.collect.Lists.newArrayList; +import static java.lang.String.format; +import static java.util.stream.Collectors.toList; +import com.google.common.io.PatternFilenameFilter; import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -14,82 +14,85 @@ import java.util.Collections; import java.util.List; import java.util.Optional; - -import static com.google.common.collect.Lists.newArrayList; -import static java.lang.String.format; -import static java.util.stream.Collectors.toList; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class BackupFilesManager { - private static final Logger logger = LoggerFactory.getLogger(BackupFilesManager.class); + private static final Logger logger = LoggerFactory.getLogger(BackupFilesManager.class); - private static final String FILE_NAME = "hermes-buffer-v3"; - private static final String TIMESTAMPED_BACKUP_FILE_PATTERN = FILE_NAME + "-\\d+\\.dat"; - private static final String OLD_V2_BACKUP_PREFIX = "hermes-buffer"; - private static final String OLD_V2_BACKUP_SUFFIX = "-v2-old.tmp"; + private static final String FILE_NAME = "hermes-buffer-v3"; + private static final String TIMESTAMPED_BACKUP_FILE_PATTERN = FILE_NAME + "-\\d+\\.dat"; + private static final String OLD_V2_BACKUP_PREFIX = "hermes-buffer"; + private static final String OLD_V2_BACKUP_SUFFIX = "-v2-old.tmp"; - private final String baseDir; - private final Clock clock; - private final File backupFile; + private final String baseDir; + private final Clock clock; + private final File backupFile; - public BackupFilesManager(String baseDir, Clock clock) { - this.baseDir = baseDir; - this.clock = clock; + public BackupFilesManager(String baseDir, Clock clock) { + this.baseDir = baseDir; + this.clock = clock; - backupFile = getBackupFile(); - } + backupFile = getBackupFile(); + } - public File getCurrentBackupFile() { - return backupFile; - } + public File getCurrentBackupFile() { + return backupFile; + } - public Optional rolloverBackupFileIfExists() { - if (!backupFile.exists()) { - logger.info("Backup file doesn't exist."); - return Optional.empty(); - } - - File timestampedBackupFile = new File(format("%s/%s-%s.dat", baseDir, FILE_NAME, clock.millis())); - - try { - FileUtils.moveFile(backupFile, timestampedBackupFile); - } catch (IOException e) { - logger.error("Error while moving backup file from path {} to path {}.", - backupFile.getAbsolutePath(), - timestampedBackupFile.getAbsolutePath(), - e); - return Optional.empty(); - } - - return Optional.of(timestampedBackupFile); + public Optional rolloverBackupFileIfExists() { + if (!backupFile.exists()) { + logger.info("Backup file doesn't exist."); + return Optional.empty(); } - public List getTemporaryBackupV2Files(String temporaryDir) { - try { - Path dir = Paths.get(temporaryDir); - return Files.list(dir) - .filter(file -> file.getFileName().toString().startsWith(OLD_V2_BACKUP_PREFIX)) - .filter(file -> file.getFileName().toString().endsWith(OLD_V2_BACKUP_SUFFIX)) - .map(Path::toFile) - .collect(toList()); - } catch (IOException e) { - logger.error("Error while scanning temporary backup v2 files from absolute path: {}", - backupFile.getAbsolutePath(), - e); - return Collections.emptyList(); - } + File timestampedBackupFile = + new File(format("%s/%s-%s.dat", baseDir, FILE_NAME, clock.millis())); + + try { + FileUtils.moveFile(backupFile, timestampedBackupFile); + } catch (IOException e) { + logger.error( + "Error while moving backup file from path {} to path {}.", + backupFile.getAbsolutePath(), + timestampedBackupFile.getAbsolutePath(), + e); + return Optional.empty(); } - public void delete(File file) { - boolean status = FileUtils.deleteQuietly(file); - logger.info("Deleted file from path {} with status {}", file.getAbsolutePath(), status); + return Optional.of(timestampedBackupFile); + } + + public List getTemporaryBackupV2Files(String temporaryDir) { + try { + Path dir = Paths.get(temporaryDir); + return Files.list(dir) + .filter(file -> file.getFileName().toString().startsWith(OLD_V2_BACKUP_PREFIX)) + .filter(file -> file.getFileName().toString().endsWith(OLD_V2_BACKUP_SUFFIX)) + .map(Path::toFile) + .collect(toList()); + } catch (IOException e) { + logger.error( + "Error while scanning temporary backup v2 files from absolute path: {}", + backupFile.getAbsolutePath(), + e); + return Collections.emptyList(); } + } - private File getBackupFile() { - return new File(format("%s/%s.dat", baseDir, FILE_NAME)); - } + public void delete(File file) { + boolean status = FileUtils.deleteQuietly(file); + logger.info("Deleted file from path {} with status {}", file.getAbsolutePath(), status); + } - public List getRolledBackupFiles() { - return newArrayList(new File(baseDir).listFiles(new PatternFilenameFilter(TIMESTAMPED_BACKUP_FILE_PATTERN))); - } + private File getBackupFile() { + return new File(format("%s/%s.dat", baseDir, FILE_NAME)); + } + + public List getRolledBackupFiles() { + return newArrayList( + new File(baseDir).listFiles(new PatternFilenameFilter(TIMESTAMPED_BACKUP_FILE_PATTERN))); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/BackupMessage.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/BackupMessage.java index 81971e7d8d..072d77c2d0 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/BackupMessage.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/BackupMessage.java @@ -8,79 +8,88 @@ public class BackupMessage implements Serializable { - private final String messageId; - private final byte[] data; - private final long timestamp; - private final String qualifiedTopicName; - private final String partitionKey; - private final Integer schemaVersion; - private final Integer schemaId; - private final Map propagatedHTTPHeaders; + private final String messageId; + private final byte[] data; + private final long timestamp; + private final String qualifiedTopicName; + private final String partitionKey; + private final Integer schemaVersion; + private final Integer schemaId; + private final Map propagatedHTTPHeaders; - public BackupMessage(String messageId, byte[] data, long timestamp, String qualifiedTopicName, String partitionKey, - Integer schemaVersion, Integer schemaId, Map propagatedHTTPHeaders) { - this.messageId = messageId; - this.data = data; - this.timestamp = timestamp; - this.qualifiedTopicName = qualifiedTopicName; - this.partitionKey = partitionKey; - this.schemaVersion = schemaVersion; - this.schemaId = schemaId; - this.propagatedHTTPHeaders = propagatedHTTPHeaders == null ? Collections.emptyMap() : propagatedHTTPHeaders; - } + public BackupMessage( + String messageId, + byte[] data, + long timestamp, + String qualifiedTopicName, + String partitionKey, + Integer schemaVersion, + Integer schemaId, + Map propagatedHTTPHeaders) { + this.messageId = messageId; + this.data = data; + this.timestamp = timestamp; + this.qualifiedTopicName = qualifiedTopicName; + this.partitionKey = partitionKey; + this.schemaVersion = schemaVersion; + this.schemaId = schemaId; + this.propagatedHTTPHeaders = + propagatedHTTPHeaders == null ? Collections.emptyMap() : propagatedHTTPHeaders; + } - public String getMessageId() { - return messageId; - } + public String getMessageId() { + return messageId; + } - public byte[] getData() { - return data; - } + public byte[] getData() { + return data; + } - public long getTimestamp() { - return timestamp; - } + public long getTimestamp() { + return timestamp; + } - public String getQualifiedTopicName() { - return qualifiedTopicName; - } + public String getQualifiedTopicName() { + return qualifiedTopicName; + } - public String getPartitionKey() { - return partitionKey; - } + public String getPartitionKey() { + return partitionKey; + } - public Integer getSchemaVersion() { - return schemaVersion; - } + public Integer getSchemaVersion() { + return schemaVersion; + } - public Integer getSchemaId() { - return schemaId; - } + public Integer getSchemaId() { + return schemaId; + } - public Map getPropagatedHTTPHeaders() { - return propagatedHTTPHeaders; - } + public Map getPropagatedHTTPHeaders() { + return propagatedHTTPHeaders; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof BackupMessage)) { - return false; - } - BackupMessage that = (BackupMessage) o; - return Objects.equals(timestamp, that.timestamp) - && Objects.equals(messageId, that.messageId) - && Arrays.equals(data, that.data) - && Objects.equals(qualifiedTopicName, that.qualifiedTopicName) - && Objects.equals(partitionKey, that.partitionKey) - && Objects.equals(schemaVersion, that.schemaVersion) - && Objects.equals(schemaId, that.schemaId); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(messageId, data, timestamp, qualifiedTopicName, partitionKey, schemaVersion, schemaId); + if (!(o instanceof BackupMessage)) { + return false; } + BackupMessage that = (BackupMessage) o; + return Objects.equals(timestamp, that.timestamp) + && Objects.equals(messageId, that.messageId) + && Arrays.equals(data, that.data) + && Objects.equals(qualifiedTopicName, that.qualifiedTopicName) + && Objects.equals(partitionKey, that.partitionKey) + && Objects.equals(schemaVersion, that.schemaVersion) + && Objects.equals(schemaId, that.schemaId); + } + + @Override + public int hashCode() { + return Objects.hash( + messageId, data, timestamp, qualifiedTopicName, partitionKey, schemaVersion, schemaId); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/BackupMessagesLoader.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/BackupMessagesLoader.java index 1ddff796f8..34882cb3ad 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/BackupMessagesLoader.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/BackupMessagesLoader.java @@ -1,6 +1,22 @@ package pl.allegro.tech.hermes.frontend.buffer; import com.google.common.collect.Lists; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.nio.charset.Charset; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicReference; import org.apache.avro.Schema; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; @@ -23,250 +39,296 @@ import pl.allegro.tech.hermes.schema.SchemaVersion; import pl.allegro.tech.hermes.tracker.frontend.Trackers; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.nio.charset.Charset; -import java.time.Duration; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.atomic.AtomicReference; - public class BackupMessagesLoader { - private static final Logger logger = LoggerFactory.getLogger(BackupMessagesLoader.class); - - private final BrokerTopicAvailabilityChecker brokerTopicAvailabilityChecker; - private final BrokerMessageProducer brokerMessageProducer; - private final BrokerListeners brokerListeners; - private final TopicsCache topicsCache; - private final SchemaRepository schemaRepository; - private final SchemaExistenceEnsurer schemaExistenceEnsurer; - private final Trackers trackers; - private final Duration messageMaxAgeHours; - private final int maxResendRetries; - private final Duration resendSleep; - private final Duration readTopicInfoSleep; - - private final Set topicsAvailabilityCache = new HashSet<>(); - private final AtomicReference>> toResend = new AtomicReference<>(); - - public BackupMessagesLoader(BrokerTopicAvailabilityChecker brokerTopicAvailabilityChecker, - BrokerMessageProducer brokerMessageProducer, - BrokerListeners brokerListeners, - TopicsCache topicsCache, - SchemaRepository schemaRepository, - SchemaExistenceEnsurer schemaExistenceEnsurer, - Trackers trackers, - BackupMessagesLoaderParameters backupMessagesLoaderParameters) { - this.brokerTopicAvailabilityChecker = brokerTopicAvailabilityChecker; - this.brokerMessageProducer = brokerMessageProducer; - this.brokerListeners = brokerListeners; - this.topicsCache = topicsCache; - this.schemaRepository = schemaRepository; - this.schemaExistenceEnsurer = schemaExistenceEnsurer; - this.trackers = trackers; - this.messageMaxAgeHours = backupMessagesLoaderParameters.getMaxAge(); - this.resendSleep = backupMessagesLoaderParameters.getLoadingPauseBetweenResend(); - this.readTopicInfoSleep = backupMessagesLoaderParameters.getLoadingWaitForBrokerTopicInfo(); - this.maxResendRetries = backupMessagesLoaderParameters.getMaxResendRetries(); - } - - public void loadMessages(List messages) { - logger.info("Loading {} messages from backup storage.", messages.size()); - toResend.set(new ConcurrentLinkedQueue<>()); - - sendMessages(messages); - - if (toResend.get().size() == 0) { - logger.info("No messages to resend."); - return; - } - - int retry = 0; - do { - if (retry > 0) { - List> retryMessages = Lists.newArrayList(toResend.getAndSet(new ConcurrentLinkedQueue<>())); - resendMessages(retryMessages, retry); - } - try { - Thread.sleep(resendSleep.toMillis()); - } catch (InterruptedException e) { - logger.warn("Sleep interrupted", e); - } - retry++; - } while (toResend.get().size() > 0 && retry <= maxResendRetries); - - logger.info("Finished resending messages from backup storage after retry #{} with {} unsent messages.", retry - 1, - toResend.get().size()); - } - - public void loadFromTemporaryBackupV2File(File file) { - try (FileInputStream fileInputStream = new FileInputStream(file); - ObjectInputStream objectInputStream = new ObjectInputStream(fileInputStream)) { - List messages = (List) objectInputStream.readObject(); - logger.info("Loaded {} messages from temporary v2 backup file: {}", messages.size(), file); - loadMessages(messages); - - } catch (IOException | ClassNotFoundException e) { - logger.error("Error reading temporary backup v2 files from path {}.", file.getAbsolutePath(), e); - } - } - - public void clearTopicsAvailabilityCache() { - topicsAvailabilityCache.clear(); + private static final Logger logger = LoggerFactory.getLogger(BackupMessagesLoader.class); + + private final BrokerTopicAvailabilityChecker brokerTopicAvailabilityChecker; + private final BrokerMessageProducer brokerMessageProducer; + private final BrokerListeners brokerListeners; + private final TopicsCache topicsCache; + private final SchemaRepository schemaRepository; + private final SchemaExistenceEnsurer schemaExistenceEnsurer; + private final Trackers trackers; + private final Duration messageMaxAgeHours; + private final int maxResendRetries; + private final Duration resendSleep; + private final Duration readTopicInfoSleep; + + private final Set topicsAvailabilityCache = new HashSet<>(); + private final AtomicReference>> toResend = + new AtomicReference<>(); + + public BackupMessagesLoader( + BrokerTopicAvailabilityChecker brokerTopicAvailabilityChecker, + BrokerMessageProducer brokerMessageProducer, + BrokerListeners brokerListeners, + TopicsCache topicsCache, + SchemaRepository schemaRepository, + SchemaExistenceEnsurer schemaExistenceEnsurer, + Trackers trackers, + BackupMessagesLoaderParameters backupMessagesLoaderParameters) { + this.brokerTopicAvailabilityChecker = brokerTopicAvailabilityChecker; + this.brokerMessageProducer = brokerMessageProducer; + this.brokerListeners = brokerListeners; + this.topicsCache = topicsCache; + this.schemaRepository = schemaRepository; + this.schemaExistenceEnsurer = schemaExistenceEnsurer; + this.trackers = trackers; + this.messageMaxAgeHours = backupMessagesLoaderParameters.getMaxAge(); + this.resendSleep = backupMessagesLoaderParameters.getLoadingPauseBetweenResend(); + this.readTopicInfoSleep = backupMessagesLoaderParameters.getLoadingWaitForBrokerTopicInfo(); + this.maxResendRetries = backupMessagesLoaderParameters.getMaxResendRetries(); + } + + public void loadMessages(List messages) { + logger.info("Loading {} messages from backup storage.", messages.size()); + toResend.set(new ConcurrentLinkedQueue<>()); + + sendMessages(messages); + + if (toResend.get().size() == 0) { + logger.info("No messages to resend."); + return; } - private void sendMessages(List messages) { - logger.info("Sending {} messages from backup storage.", messages.size()); - int sentCounter = 0; - int discardedCounter = 0; - for (BackupMessage backupMessage : messages) { - String topicQualifiedName = backupMessage.getQualifiedTopicName(); - Optional optionalCachedTopic = topicsCache.getTopic(topicQualifiedName); - if (sendBackupMessageIfNeeded(backupMessage, topicQualifiedName, optionalCachedTopic, "sending")) { - sentCounter++; - } else { - discardedCounter++; - } - } - logger.info("Loaded and sent {} messages and discarded {} messages from the backup storage.", sentCounter, discardedCounter); + int retry = 0; + do { + if (retry > 0) { + List> retryMessages = + Lists.newArrayList(toResend.getAndSet(new ConcurrentLinkedQueue<>())); + resendMessages(retryMessages, retry); + } + try { + Thread.sleep(resendSleep.toMillis()); + } catch (InterruptedException e) { + logger.warn("Sleep interrupted", e); + } + retry++; + } while (toResend.get().size() > 0 && retry <= maxResendRetries); + + logger.info( + "Finished resending messages from backup storage after retry #{} with {} unsent messages.", + retry - 1, + toResend.get().size()); + } + + public void loadFromTemporaryBackupV2File(File file) { + try (FileInputStream fileInputStream = new FileInputStream(file); + ObjectInputStream objectInputStream = new ObjectInputStream(fileInputStream)) { + List messages = (List) objectInputStream.readObject(); + logger.info("Loaded {} messages from temporary v2 backup file: {}", messages.size(), file); + loadMessages(messages); + + } catch (IOException | ClassNotFoundException e) { + logger.error( + "Error reading temporary backup v2 files from path {}.", file.getAbsolutePath(), e); } - - private void resendMessages(List> messageAndTopicList, int retry) { - logger.info("Resending {} messages from backup storage retry {}.", messageAndTopicList.size(), retry); - - int sentCounter = 0; - int discardedCounter = 0; - for (Pair messageAndTopic : messageAndTopicList) { - Message message = messageAndTopic.getKey(); - Optional cachedTopic = Optional.of(messageAndTopic.getValue()); - if (sendMessageIfNeeded(message, cachedTopic.get().getQualifiedName(), cachedTopic, "resending")) { - sentCounter++; - } else { - discardedCounter++; - } - } - - logger.info("Resent {}/{} messages and discarded {} messages from the backup storage retry {}.", sentCounter, - messageAndTopicList.size(), discardedCounter, retry); + } + + public void clearTopicsAvailabilityCache() { + topicsAvailabilityCache.clear(); + } + + private void sendMessages(List messages) { + logger.info("Sending {} messages from backup storage.", messages.size()); + int sentCounter = 0; + int discardedCounter = 0; + for (BackupMessage backupMessage : messages) { + String topicQualifiedName = backupMessage.getQualifiedTopicName(); + Optional optionalCachedTopic = topicsCache.getTopic(topicQualifiedName); + if (sendBackupMessageIfNeeded( + backupMessage, topicQualifiedName, optionalCachedTopic, "sending")) { + sentCounter++; + } else { + discardedCounter++; + } } - - private boolean sendBackupMessageIfNeeded(BackupMessage backupMessage, String topicQualifiedName, Optional cachedTopic, - String contextName) { - if (cachedTopic.isPresent()) { - Message message; - if (backupMessage.getSchemaVersion() != null) { - message = createAvroMessageFromVersion(backupMessage, cachedTopic); - } else if (backupMessage.getSchemaId() != null) { - message = createAvroMessageFromSchemaId(backupMessage, cachedTopic); - } else { - message = createJsonMessage(backupMessage); - } - return sendMessageIfNeeded(message, topicQualifiedName, cachedTopic, contextName); - } - return false; + logger.info( + "Loaded and sent {} messages and discarded {} messages from the backup storage.", + sentCounter, + discardedCounter); + } + + private void resendMessages(List> messageAndTopicList, int retry) { + logger.info( + "Resending {} messages from backup storage retry {}.", messageAndTopicList.size(), retry); + + int sentCounter = 0; + int discardedCounter = 0; + for (Pair messageAndTopic : messageAndTopicList) { + Message message = messageAndTopic.getKey(); + Optional cachedTopic = Optional.of(messageAndTopic.getValue()); + if (sendMessageIfNeeded( + message, cachedTopic.get().getQualifiedName(), cachedTopic, "resending")) { + sentCounter++; + } else { + discardedCounter++; + } } - private Message createAvroMessageFromSchemaId(BackupMessage backupMessage, Optional cachedTopic) { - SchemaId schemaId = SchemaId.valueOf(backupMessage.getSchemaId()); - schemaExistenceEnsurer.ensureSchemaExists(cachedTopic.get().getTopic(), schemaId); - CompiledSchema schema = schemaRepository.getAvroSchema(cachedTopic.get().getTopic(), schemaId); - return createAvroMessage(backupMessage, schema); + logger.info( + "Resent {}/{} messages and discarded {} messages from the backup storage retry {}.", + sentCounter, + messageAndTopicList.size(), + discardedCounter, + retry); + } + + private boolean sendBackupMessageIfNeeded( + BackupMessage backupMessage, + String topicQualifiedName, + Optional cachedTopic, + String contextName) { + if (cachedTopic.isPresent()) { + Message message; + if (backupMessage.getSchemaVersion() != null) { + message = createAvroMessageFromVersion(backupMessage, cachedTopic); + } else if (backupMessage.getSchemaId() != null) { + message = createAvroMessageFromSchemaId(backupMessage, cachedTopic); + } else { + message = createJsonMessage(backupMessage); + } + return sendMessageIfNeeded(message, topicQualifiedName, cachedTopic, contextName); } - - private Message createAvroMessageFromVersion(BackupMessage backupMessage, Optional cachedTopic) { - SchemaVersion version = SchemaVersion.valueOf(backupMessage.getSchemaVersion()); - schemaExistenceEnsurer.ensureSchemaExists(cachedTopic.get().getTopic(), version); - CompiledSchema schema = schemaRepository.getAvroSchema(cachedTopic.get().getTopic(), version); - return createAvroMessage(backupMessage, schema); + return false; + } + + private Message createAvroMessageFromSchemaId( + BackupMessage backupMessage, Optional cachedTopic) { + SchemaId schemaId = SchemaId.valueOf(backupMessage.getSchemaId()); + schemaExistenceEnsurer.ensureSchemaExists(cachedTopic.get().getTopic(), schemaId); + CompiledSchema schema = + schemaRepository.getAvroSchema(cachedTopic.get().getTopic(), schemaId); + return createAvroMessage(backupMessage, schema); + } + + private Message createAvroMessageFromVersion( + BackupMessage backupMessage, Optional cachedTopic) { + SchemaVersion version = SchemaVersion.valueOf(backupMessage.getSchemaVersion()); + schemaExistenceEnsurer.ensureSchemaExists(cachedTopic.get().getTopic(), version); + CompiledSchema schema = + schemaRepository.getAvroSchema(cachedTopic.get().getTopic(), version); + return createAvroMessage(backupMessage, schema); + } + + private Message createAvroMessage(BackupMessage backupMessage, CompiledSchema schema) { + return new AvroMessage( + backupMessage.getMessageId(), + backupMessage.getData(), + backupMessage.getTimestamp(), + schema, + backupMessage.getPartitionKey(), + backupMessage.getPropagatedHTTPHeaders()); + } + + private Message createJsonMessage(BackupMessage backupMessage) { + return new JsonMessage( + backupMessage.getMessageId(), + backupMessage.getData(), + backupMessage.getTimestamp(), + backupMessage.getPartitionKey(), + backupMessage.getPropagatedHTTPHeaders()); + } + + private boolean sendMessageIfNeeded( + Message message, + String topicQualifiedName, + Optional cachedTopic, + String contextName) { + if (cachedTopic.isPresent()) { + if (isNotStale(message)) { + waitOnBrokerTopicAvailability(cachedTopic.get()); + sendMessage(message, cachedTopic.get()); + return true; + } + logger.warn( + "Not {} stale message {} {} {}", + contextName, + message.getId(), + topicQualifiedName, + new String(message.getData(), Charset.defaultCharset())); + return false; } - - private Message createAvroMessage(BackupMessage backupMessage, CompiledSchema schema) { - return new AvroMessage(backupMessage.getMessageId(), backupMessage.getData(), backupMessage.getTimestamp(), schema, - backupMessage.getPartitionKey(), backupMessage.getPropagatedHTTPHeaders()); + logger.error( + "Topic {} not present. Not {} message {} {}", + topicQualifiedName, + contextName, + message.getId(), + new String(message.getData(), Charset.defaultCharset())); + return false; + } + + private void waitOnBrokerTopicAvailability(CachedTopic cachedTopic) { + int tries = 0; + while (!isBrokerTopicAvailable(cachedTopic)) { + try { + tries++; + logger.info( + "Broker topic {} is not available, checked {} times.", + cachedTopic.getTopic().getQualifiedName(), + tries); + Thread.sleep(readTopicInfoSleep.toMillis()); + } catch (InterruptedException e) { + logger.warn( + "Waiting for broker topic availability interrupted. Topic: {}", + cachedTopic.getTopic().getQualifiedName()); + } } + } - private Message createJsonMessage(BackupMessage backupMessage) { - return new JsonMessage(backupMessage.getMessageId(), backupMessage.getData(), backupMessage.getTimestamp(), - backupMessage.getPartitionKey(), backupMessage.getPropagatedHTTPHeaders()); + private boolean isBrokerTopicAvailable(CachedTopic cachedTopic) { + if (topicsAvailabilityCache.contains(cachedTopic.getTopic())) { + return true; } - private boolean sendMessageIfNeeded(Message message, String topicQualifiedName, Optional cachedTopic, String contextName) { - if (cachedTopic.isPresent()) { - if (isNotStale(message)) { - waitOnBrokerTopicAvailability(cachedTopic.get()); - sendMessage(message, cachedTopic.get()); - return true; - } - logger.warn("Not {} stale message {} {} {}", contextName, message.getId(), topicQualifiedName, - new String(message.getData(), Charset.defaultCharset())); - return false; - } - logger.error("Topic {} not present. Not {} message {} {}", topicQualifiedName, contextName, message.getId(), - new String(message.getData(), Charset.defaultCharset())); - return false; + if (brokerTopicAvailabilityChecker.isTopicAvailable(cachedTopic)) { + topicsAvailabilityCache.add(cachedTopic.getTopic()); + logger.info("Broker topic {} is available.", cachedTopic.getTopic().getQualifiedName()); + return true; } - private void waitOnBrokerTopicAvailability(CachedTopic cachedTopic) { - int tries = 0; - while (!isBrokerTopicAvailable(cachedTopic)) { - try { - tries++; - logger.info("Broker topic {} is not available, checked {} times.", cachedTopic.getTopic().getQualifiedName(), tries); - Thread.sleep(readTopicInfoSleep.toMillis()); - } catch (InterruptedException e) { - logger.warn("Waiting for broker topic availability interrupted. Topic: {}", cachedTopic.getTopic().getQualifiedName()); - } - } - } - - private boolean isBrokerTopicAvailable(CachedTopic cachedTopic) { - if (topicsAvailabilityCache.contains(cachedTopic.getTopic())) { - return true; - } - - if (brokerTopicAvailabilityChecker.isTopicAvailable(cachedTopic)) { - topicsAvailabilityCache.add(cachedTopic.getTopic()); - logger.info("Broker topic {} is available.", cachedTopic.getTopic().getQualifiedName()); - return true; - } - - return false; - } - - private boolean isNotStale(Message message) { - return LocalDateTime.ofInstant(Instant.ofEpochMilli(message.getTimestamp()), ZoneId.systemDefault()) - .isAfter(LocalDateTime.now().minusHours(messageMaxAgeHours.toHours())); - } - - private void sendMessage(Message message, CachedTopic cachedTopic) { - brokerMessageProducer.send(message, cachedTopic, new PublishingCallback() { - @Override - public void onUnpublished(Message message, Topic topic, Exception exception) { - brokerListeners.onError(message, topic, exception); - trackers.get(topic).logError(message.getId(), topic.getName(), exception.getMessage(), "", Collections.emptyMap()); - toResend.get().add(ImmutablePair.of(message, cachedTopic)); - } - - @Override - public void onPublished(Message message, Topic topic) { - brokerListeners.onAcknowledge(message, topic); - } - - @Override - public void onEachPublished(Message message, Topic topic, String datacenter) { - cachedTopic.incrementPublished(datacenter); - trackers.get(topic).logPublished(message.getId(), topic.getName(), "", datacenter, Collections.emptyMap()); - } + return false; + } + + private boolean isNotStale(Message message) { + return LocalDateTime.ofInstant( + Instant.ofEpochMilli(message.getTimestamp()), ZoneId.systemDefault()) + .isAfter(LocalDateTime.now().minusHours(messageMaxAgeHours.toHours())); + } + + private void sendMessage(Message message, CachedTopic cachedTopic) { + brokerMessageProducer.send( + message, + cachedTopic, + new PublishingCallback() { + @Override + public void onUnpublished(Message message, Topic topic, Exception exception) { + brokerListeners.onError(message, topic, exception); + trackers + .get(topic) + .logError( + message.getId(), + topic.getName(), + exception.getMessage(), + "", + Collections.emptyMap()); + toResend.get().add(ImmutablePair.of(message, cachedTopic)); + } + + @Override + public void onPublished(Message message, Topic topic) { + brokerListeners.onAcknowledge(message, topic); + } + + @Override + public void onEachPublished(Message message, Topic topic, String datacenter) { + cachedTopic.incrementPublished(datacenter); + trackers + .get(topic) + .logPublished( + message.getId(), topic.getName(), "", datacenter, Collections.emptyMap()); + } }); - } + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/BackupMessagesLoaderParameters.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/BackupMessagesLoaderParameters.java index 0229227d45..671e0e7380 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/BackupMessagesLoaderParameters.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/BackupMessagesLoaderParameters.java @@ -4,11 +4,11 @@ public interface BackupMessagesLoaderParameters { - Duration getMaxAge(); + Duration getMaxAge(); - int getMaxResendRetries(); + int getMaxResendRetries(); - Duration getLoadingPauseBetweenResend(); + Duration getLoadingPauseBetweenResend(); - Duration getLoadingWaitForBrokerTopicInfo(); + Duration getLoadingWaitForBrokerTopicInfo(); } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/BrokerListener.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/BrokerListener.java index c44917e62f..91e124e339 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/BrokerListener.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/BrokerListener.java @@ -6,26 +6,27 @@ import pl.allegro.tech.hermes.frontend.listeners.BrokerTimeoutListener; import pl.allegro.tech.hermes.frontend.publishing.message.Message; -public class BrokerListener implements BrokerAcknowledgeListener, BrokerTimeoutListener, BrokerErrorListener { +public class BrokerListener + implements BrokerAcknowledgeListener, BrokerTimeoutListener, BrokerErrorListener { - private final MessageRepository messageRepository; + private final MessageRepository messageRepository; - public BrokerListener(MessageRepository messageRepository) { - this.messageRepository = messageRepository; - } + public BrokerListener(MessageRepository messageRepository) { + this.messageRepository = messageRepository; + } - @Override - public void onAcknowledge(Message message, Topic topic) { - messageRepository.delete(message.getId()); - } + @Override + public void onAcknowledge(Message message, Topic topic) { + messageRepository.delete(message.getId()); + } - @Override - public void onTimeout(Message message, Topic topic) { - messageRepository.save(message, topic); - } + @Override + public void onTimeout(Message message, Topic topic) { + messageRepository.save(message, topic); + } - @Override - public void onError(Message message, Topic topic, Exception ex) { - messageRepository.save(message, topic); - } + @Override + public void onError(Message message, Topic topic, Exception ex) { + messageRepository.save(message, topic); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/MessageRepository.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/MessageRepository.java index 9bbc307cc0..0ee1973e3e 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/MessageRepository.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/MessageRepository.java @@ -1,17 +1,16 @@ package pl.allegro.tech.hermes.frontend.buffer; +import java.util.List; import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.frontend.publishing.message.Message; -import java.util.List; - public interface MessageRepository { - void save(Message message, Topic topic); + void save(Message message, Topic topic); - void delete(String messageId); + void delete(String messageId); - List findAll(); + List findAll(); - void close(); + void close(); } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/PersistentBufferExtension.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/PersistentBufferExtension.java index 80fd1cda92..0fa1dded8e 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/PersistentBufferExtension.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/PersistentBufferExtension.java @@ -1,108 +1,114 @@ package pl.allegro.tech.hermes.frontend.buffer; +import static java.util.stream.Collectors.joining; + +import java.io.File; +import java.time.Clock; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.common.metric.MetricsFacade; import pl.allegro.tech.hermes.frontend.buffer.chronicle.ChronicleMapMessageRepository; import pl.allegro.tech.hermes.frontend.listeners.BrokerListeners; -import java.io.File; -import java.time.Clock; -import java.util.List; - -import static java.util.stream.Collectors.joining; - public class PersistentBufferExtension { - private static final Logger logger = LoggerFactory.getLogger(PersistentBufferExtension.class); + private static final Logger logger = LoggerFactory.getLogger(PersistentBufferExtension.class); - private final PersistentBufferExtensionParameters persistentBufferExtensionParameters; + private final PersistentBufferExtensionParameters persistentBufferExtensionParameters; - private final Clock clock; + private final Clock clock; - private final BrokerListeners listeners; + private final BrokerListeners listeners; - private final BackupMessagesLoader backupMessagesLoader; - private final MetricsFacade metricsFacade; + private final BackupMessagesLoader backupMessagesLoader; + private final MetricsFacade metricsFacade; - private int entries; - private int avgMessageSize; + private int entries; + private int avgMessageSize; - public PersistentBufferExtension(PersistentBufferExtensionParameters persistentBufferExtensionParameters, - Clock clock, - BrokerListeners listeners, - BackupMessagesLoader backupMessagesLoader, - MetricsFacade metricsFacade) { - this.persistentBufferExtensionParameters = persistentBufferExtensionParameters; - this.clock = clock; - this.listeners = listeners; - this.backupMessagesLoader = backupMessagesLoader; - this.metricsFacade = metricsFacade; - } - - public void extend() { - BackupFilesManager backupFilesManager = new BackupFilesManager( - persistentBufferExtensionParameters.getDirectory(), - clock); - - long backupStorageSizeInBytes = persistentBufferExtensionParameters.getBufferedSizeBytes(); - avgMessageSize = persistentBufferExtensionParameters.getAverageMessageSize(); - - entries = (int) (backupStorageSizeInBytes / avgMessageSize); - - if (persistentBufferExtensionParameters.isV2MigrationEnabled()) { - loadTemporaryBackupV2Files(backupFilesManager); - } - - backupFilesManager.rolloverBackupFileIfExists(); - List rolledBackupFiles = backupFilesManager.getRolledBackupFiles(); - if (!rolledBackupFiles.isEmpty()) { - rollBackupFiles(backupFilesManager, rolledBackupFiles); - } + public PersistentBufferExtension( + PersistentBufferExtensionParameters persistentBufferExtensionParameters, + Clock clock, + BrokerListeners listeners, + BackupMessagesLoader backupMessagesLoader, + MetricsFacade metricsFacade) { + this.persistentBufferExtensionParameters = persistentBufferExtensionParameters; + this.clock = clock; + this.listeners = listeners; + this.backupMessagesLoader = backupMessagesLoader; + this.metricsFacade = metricsFacade; + } - if (persistentBufferExtensionParameters.isEnabled()) { - enableLocalStorage(backupFilesManager); - } - } - - private void loadTemporaryBackupV2Files(BackupFilesManager backupFilesManager) { - String temporaryDir = persistentBufferExtensionParameters.getTemporaryDirectory(); - List temporaryBackupV2Files = backupFilesManager.getTemporaryBackupV2Files(temporaryDir); - temporaryBackupV2Files.forEach(f -> loadTemporaryBackupV2Messages(backupFilesManager, f)); - backupMessagesLoader.clearTopicsAvailabilityCache(); - } - - private void rollBackupFiles(BackupFilesManager backupFilesManager, List rolledBackupFiles) { - logger.info("Backup files were found. Number of files: {}. Files: {}", - rolledBackupFiles.size(), - rolledBackupFiles.stream().map(File::getName).collect(joining(", "))); - rolledBackupFiles.forEach(f -> loadOldMessages(backupFilesManager, f)); - backupMessagesLoader.clearTopicsAvailabilityCache(); - } + public void extend() { + BackupFilesManager backupFilesManager = + new BackupFilesManager(persistentBufferExtensionParameters.getDirectory(), clock); - private void enableLocalStorage(BackupFilesManager backupFilesManager) { - MessageRepository repository = persistentBufferExtensionParameters.isSizeReportingEnabled() - ? new ChronicleMapMessageRepository(backupFilesManager.getCurrentBackupFile(), entries, avgMessageSize, metricsFacade) - : new ChronicleMapMessageRepository(backupFilesManager.getCurrentBackupFile(), entries, avgMessageSize); + long backupStorageSizeInBytes = persistentBufferExtensionParameters.getBufferedSizeBytes(); + avgMessageSize = persistentBufferExtensionParameters.getAverageMessageSize(); - BrokerListener brokerListener = new BrokerListener(repository); + entries = (int) (backupStorageSizeInBytes / avgMessageSize); - listeners.addAcknowledgeListener(brokerListener); - listeners.addErrorListener(brokerListener); - listeners.addTimeoutListener(brokerListener); + if (persistentBufferExtensionParameters.isV2MigrationEnabled()) { + loadTemporaryBackupV2Files(backupFilesManager); } - private void loadTemporaryBackupV2Messages(BackupFilesManager backupFilesManager, File temporaryBackup) { - logger.info("Loading messages from temporary backup v2 file: {}", temporaryBackup.getName()); - backupMessagesLoader.loadFromTemporaryBackupV2File(temporaryBackup); - backupFilesManager.delete(temporaryBackup); + backupFilesManager.rolloverBackupFileIfExists(); + List rolledBackupFiles = backupFilesManager.getRolledBackupFiles(); + if (!rolledBackupFiles.isEmpty()) { + rollBackupFiles(backupFilesManager, rolledBackupFiles); } - private void loadOldMessages(BackupFilesManager backupFilesManager, File oldBackup) { - logger.info("Loading messages from backup file: {}", oldBackup.getName()); - MessageRepository oldMessageRepository = new ChronicleMapMessageRepository(oldBackup, entries, avgMessageSize); - backupMessagesLoader.loadMessages(oldMessageRepository.findAll()); - oldMessageRepository.close(); - backupFilesManager.delete(oldBackup); + if (persistentBufferExtensionParameters.isEnabled()) { + enableLocalStorage(backupFilesManager); } + } + + private void loadTemporaryBackupV2Files(BackupFilesManager backupFilesManager) { + String temporaryDir = persistentBufferExtensionParameters.getTemporaryDirectory(); + List temporaryBackupV2Files = backupFilesManager.getTemporaryBackupV2Files(temporaryDir); + temporaryBackupV2Files.forEach(f -> loadTemporaryBackupV2Messages(backupFilesManager, f)); + backupMessagesLoader.clearTopicsAvailabilityCache(); + } + + private void rollBackupFiles( + BackupFilesManager backupFilesManager, List rolledBackupFiles) { + logger.info( + "Backup files were found. Number of files: {}. Files: {}", + rolledBackupFiles.size(), + rolledBackupFiles.stream().map(File::getName).collect(joining(", "))); + rolledBackupFiles.forEach(f -> loadOldMessages(backupFilesManager, f)); + backupMessagesLoader.clearTopicsAvailabilityCache(); + } + + private void enableLocalStorage(BackupFilesManager backupFilesManager) { + MessageRepository repository = + persistentBufferExtensionParameters.isSizeReportingEnabled() + ? new ChronicleMapMessageRepository( + backupFilesManager.getCurrentBackupFile(), entries, avgMessageSize, metricsFacade) + : new ChronicleMapMessageRepository( + backupFilesManager.getCurrentBackupFile(), entries, avgMessageSize); + + BrokerListener brokerListener = new BrokerListener(repository); + + listeners.addAcknowledgeListener(brokerListener); + listeners.addErrorListener(brokerListener); + listeners.addTimeoutListener(brokerListener); + } + + private void loadTemporaryBackupV2Messages( + BackupFilesManager backupFilesManager, File temporaryBackup) { + logger.info("Loading messages from temporary backup v2 file: {}", temporaryBackup.getName()); + backupMessagesLoader.loadFromTemporaryBackupV2File(temporaryBackup); + backupFilesManager.delete(temporaryBackup); + } + + private void loadOldMessages(BackupFilesManager backupFilesManager, File oldBackup) { + logger.info("Loading messages from backup file: {}", oldBackup.getName()); + MessageRepository oldMessageRepository = + new ChronicleMapMessageRepository(oldBackup, entries, avgMessageSize); + backupMessagesLoader.loadMessages(oldMessageRepository.findAll()); + oldMessageRepository.close(); + backupFilesManager.delete(oldBackup); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/PersistentBufferExtensionParameters.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/PersistentBufferExtensionParameters.java index eff019cdb7..e7c5fae57f 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/PersistentBufferExtensionParameters.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/PersistentBufferExtensionParameters.java @@ -2,17 +2,17 @@ public interface PersistentBufferExtensionParameters { - long getBufferedSizeBytes(); + long getBufferedSizeBytes(); - boolean isV2MigrationEnabled(); + boolean isV2MigrationEnabled(); - boolean isEnabled(); + boolean isEnabled(); - String getDirectory(); + String getDirectory(); - String getTemporaryDirectory(); + String getTemporaryDirectory(); - int getAverageMessageSize(); + int getAverageMessageSize(); - boolean isSizeReportingEnabled(); + boolean isSizeReportingEnabled(); } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/chronicle/ChronicleMapClosedException.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/chronicle/ChronicleMapClosedException.java index 98fad069bd..026ac42049 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/chronicle/ChronicleMapClosedException.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/chronicle/ChronicleMapClosedException.java @@ -2,7 +2,7 @@ public class ChronicleMapClosedException extends RuntimeException { - public ChronicleMapClosedException(String message) { - super(message); - } + public ChronicleMapClosedException(String message) { + super(message); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/chronicle/ChronicleMapCreationException.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/chronicle/ChronicleMapCreationException.java index 8e32700812..80e177100d 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/chronicle/ChronicleMapCreationException.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/chronicle/ChronicleMapCreationException.java @@ -2,7 +2,7 @@ public class ChronicleMapCreationException extends RuntimeException { - public ChronicleMapCreationException(Exception e) { - super("Exception while creating ChronicleMap", e); - } + public ChronicleMapCreationException(Exception e) { + super("Exception while creating ChronicleMap", e); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/chronicle/ChronicleMapEntryValue.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/chronicle/ChronicleMapEntryValue.java index 80a21001fc..ad1e716ce0 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/chronicle/ChronicleMapEntryValue.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/chronicle/ChronicleMapEntryValue.java @@ -6,75 +6,86 @@ import java.util.Objects; public class ChronicleMapEntryValue implements Serializable { - static final long serialVersionUID = -2149667159974528954L; + static final long serialVersionUID = -2149667159974528954L; - private final byte[] data; - private final long timestamp; - private final String qualifiedTopicName; - private final String partitionKey; - private final Integer schemaVersion; - private final Integer schemaId; - private final Map propagatedHttpHeaders; + private final byte[] data; + private final long timestamp; + private final String qualifiedTopicName; + private final String partitionKey; + private final Integer schemaVersion; + private final Integer schemaId; + private final Map propagatedHttpHeaders; - public ChronicleMapEntryValue(byte[] data, long timestamp, String qualifiedTopicName, String partitionKey, - Integer schemaVersion, Integer schemaId, Map propagatedHttpHeaders) { - this.data = data; - this.timestamp = timestamp; - this.qualifiedTopicName = qualifiedTopicName; - this.partitionKey = partitionKey; - this.schemaVersion = schemaVersion; - this.schemaId = schemaId; - this.propagatedHttpHeaders = propagatedHttpHeaders; - } + public ChronicleMapEntryValue( + byte[] data, + long timestamp, + String qualifiedTopicName, + String partitionKey, + Integer schemaVersion, + Integer schemaId, + Map propagatedHttpHeaders) { + this.data = data; + this.timestamp = timestamp; + this.qualifiedTopicName = qualifiedTopicName; + this.partitionKey = partitionKey; + this.schemaVersion = schemaVersion; + this.schemaId = schemaId; + this.propagatedHttpHeaders = propagatedHttpHeaders; + } - public byte[] getData() { - return data; - } + public byte[] getData() { + return data; + } - public long getTimestamp() { - return timestamp; - } + public long getTimestamp() { + return timestamp; + } - public String getQualifiedTopicName() { - return qualifiedTopicName; - } + public String getQualifiedTopicName() { + return qualifiedTopicName; + } - public String getPartitionKey() { - return partitionKey; - } + public String getPartitionKey() { + return partitionKey; + } - public Integer getSchemaVersion() { - return schemaVersion; - } + public Integer getSchemaVersion() { + return schemaVersion; + } - public Integer getSchemaId() { - return schemaId; - } + public Integer getSchemaId() { + return schemaId; + } - public Map getPropagatedHttpHeaders() { - return propagatedHttpHeaders; - } + public Map getPropagatedHttpHeaders() { + return propagatedHttpHeaders; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof ChronicleMapEntryValue)) { - return false; - } - ChronicleMapEntryValue that = (ChronicleMapEntryValue) o; - return Objects.equals(timestamp, that.timestamp) - && Arrays.equals(data, that.data) - && Objects.equals(qualifiedTopicName, that.qualifiedTopicName) - && Objects.equals(partitionKey, that.partitionKey) - && Objects.equals(propagatedHttpHeaders, that.propagatedHttpHeaders); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(data, timestamp, qualifiedTopicName, partitionKey, schemaVersion, schemaId, - propagatedHttpHeaders); + if (!(o instanceof ChronicleMapEntryValue)) { + return false; } + ChronicleMapEntryValue that = (ChronicleMapEntryValue) o; + return Objects.equals(timestamp, that.timestamp) + && Arrays.equals(data, that.data) + && Objects.equals(qualifiedTopicName, that.qualifiedTopicName) + && Objects.equals(partitionKey, that.partitionKey) + && Objects.equals(propagatedHttpHeaders, that.propagatedHttpHeaders); + } + @Override + public int hashCode() { + return Objects.hash( + data, + timestamp, + qualifiedTopicName, + partitionKey, + schemaVersion, + schemaId, + propagatedHttpHeaders); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/chronicle/ChronicleMapMessageRepository.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/chronicle/ChronicleMapMessageRepository.java index c4c70dd519..dab4eb2b65 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/chronicle/ChronicleMapMessageRepository.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/buffer/chronicle/ChronicleMapMessageRepository.java @@ -1,5 +1,13 @@ package pl.allegro.tech.hermes.frontend.buffer.chronicle; +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; import net.openhft.chronicle.map.ChronicleMap; import net.openhft.chronicle.map.ChronicleMapBuilder; import org.slf4j.Logger; @@ -11,104 +19,111 @@ import pl.allegro.tech.hermes.frontend.publishing.message.Message; import pl.allegro.tech.hermes.frontend.publishing.message.MessageIdGenerator; -import java.io.File; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.stream.Collectors; - public class ChronicleMapMessageRepository implements MessageRepository { - private static final Logger logger = LoggerFactory.getLogger(ChronicleMapMessageRepository.class); - - private static final boolean SAME_BUILDER_CONFIG = false; - - private final ChronicleMap map; - - private boolean closed = false; - private final ReadWriteLock closeLock = new ReentrantReadWriteLock(); - - static { - System.setProperty("chronicle.map.disable.locking", Boolean.TRUE.toString()); - } - - public ChronicleMapMessageRepository(File file, int entries, int averageMessageSize) { - logger.info("Creating backup storage in path: {}", file.getAbsolutePath()); - try { - map = ChronicleMapBuilder.of(String.class, ChronicleMapEntryValue.class) - .constantKeySizeBySample(MessageIdGenerator.generate()) - .averageValueSize(averageMessageSize) - .entries(entries) - .setPreShutdownAction(new LoggingMapSizePreShutdownHook()) - .sparseFile(true) - .createOrRecoverPersistedTo(file, SAME_BUILDER_CONFIG); - } catch (IOException e) { - logger.error("Failed to load backup storage file from path {}", file.getAbsoluteFile(), e); - throw new ChronicleMapCreationException(e); - } - } - - public ChronicleMapMessageRepository(File file, int entries, int averageMessageSize, MetricsFacade metricsFacade) { - this(file, entries, averageMessageSize); - metricsFacade.persistentBuffer().registerBackupStorageSizeGauge(map, Map::size); + private static final Logger logger = LoggerFactory.getLogger(ChronicleMapMessageRepository.class); + + private static final boolean SAME_BUILDER_CONFIG = false; + + private final ChronicleMap map; + + private boolean closed = false; + private final ReadWriteLock closeLock = new ReentrantReadWriteLock(); + + static { + System.setProperty("chronicle.map.disable.locking", Boolean.TRUE.toString()); + } + + public ChronicleMapMessageRepository(File file, int entries, int averageMessageSize) { + logger.info("Creating backup storage in path: {}", file.getAbsolutePath()); + try { + map = + ChronicleMapBuilder.of(String.class, ChronicleMapEntryValue.class) + .constantKeySizeBySample(MessageIdGenerator.generate()) + .averageValueSize(averageMessageSize) + .entries(entries) + .setPreShutdownAction(new LoggingMapSizePreShutdownHook()) + .sparseFile(true) + .createOrRecoverPersistedTo(file, SAME_BUILDER_CONFIG); + } catch (IOException e) { + logger.error("Failed to load backup storage file from path {}", file.getAbsoluteFile(), e); + throw new ChronicleMapCreationException(e); } - - @Override - public void save(Message message, Topic topic) { - Lock lock = closeLock.readLock(); - lock.lock(); - try { - if (closed) { - throw new ChronicleMapClosedException("Backup storage is closed. Unable to add new messages."); - } - map.put(message.getId(), - new ChronicleMapEntryValue( - message.getData(), message.getTimestamp(), topic.getQualifiedName(), - message.getPartitionKey(), message.getCompiledSchema().map(v -> v.getVersion().value()).orElse(null), - message.getCompiledSchema().map(v -> v.getId().value()).orElse(null), message.getHTTPHeaders())); - } finally { - lock.unlock(); - } + } + + public ChronicleMapMessageRepository( + File file, int entries, int averageMessageSize, MetricsFacade metricsFacade) { + this(file, entries, averageMessageSize); + metricsFacade.persistentBuffer().registerBackupStorageSizeGauge(map, Map::size); + } + + @Override + public void save(Message message, Topic topic) { + Lock lock = closeLock.readLock(); + lock.lock(); + try { + if (closed) { + throw new ChronicleMapClosedException( + "Backup storage is closed. Unable to add new messages."); + } + map.put( + message.getId(), + new ChronicleMapEntryValue( + message.getData(), + message.getTimestamp(), + topic.getQualifiedName(), + message.getPartitionKey(), + message.getCompiledSchema().map(v -> v.getVersion().value()).orElse(null), + message.getCompiledSchema().map(v -> v.getId().value()).orElse(null), + message.getHTTPHeaders())); + } finally { + lock.unlock(); } + } + + @Override + public void delete(String messageId) { + map.remove(messageId); + } + + @Override + public List findAll() { + return map.entrySet().stream() + .map((e) -> toBackupMessage(e.getKey(), e.getValue())) + .collect(Collectors.toList()); + } + + @Override + public void close() { + map.close(); + } + + private BackupMessage toBackupMessage(String id, ChronicleMapEntryValue entryValue) { + return new BackupMessage( + id, + entryValue.getData(), + entryValue.getTimestamp(), + entryValue.getQualifiedTopicName(), + entryValue.getPartitionKey(), + entryValue.getSchemaVersion(), + entryValue.getSchemaId(), + entryValue.getPropagatedHttpHeaders()); + } + + private class LoggingMapSizePreShutdownHook implements Runnable { @Override - public void delete(String messageId) { - map.remove(messageId); - } - - @Override - public List findAll() { - return map.entrySet().stream().map((e) -> toBackupMessage(e.getKey(), e.getValue())).collect(Collectors.toList()); - } - - @Override - public void close() { - map.close(); - } - - private BackupMessage toBackupMessage(String id, ChronicleMapEntryValue entryValue) { - return new BackupMessage(id, entryValue.getData(), entryValue.getTimestamp(), - entryValue.getQualifiedTopicName(), entryValue.getPartitionKey(), entryValue.getSchemaVersion(), - entryValue.getSchemaId(), entryValue.getPropagatedHttpHeaders()); - } - - private class LoggingMapSizePreShutdownHook implements Runnable { - - @Override - public void run() { - Lock lock = closeLock.writeLock(); - lock.lock(); - try { - closed = true; - if (map != null) { - logger.info("Closing backup storage with {} messages.", map.size()); - } - } finally { - lock.unlock(); - } + public void run() { + Lock lock = closeLock.writeLock(); + lock.lock(); + try { + closed = true; + if (map != null) { + logger.info("Closing backup storage with {} messages.", map.size()); } + } finally { + lock.unlock(); + } } + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/cache/topic/NotificationBasedTopicsCache.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/cache/topic/NotificationBasedTopicsCache.java index 5ac0b5d232..fb44476623 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/cache/topic/NotificationBasedTopicsCache.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/cache/topic/NotificationBasedTopicsCache.java @@ -1,6 +1,10 @@ package pl.allegro.tech.hermes.frontend.cache.topic; import com.google.common.collect.ImmutableList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.Topic; @@ -16,106 +20,116 @@ import pl.allegro.tech.hermes.frontend.metric.CachedTopic; import pl.allegro.tech.hermes.frontend.metric.ThroughputRegistry; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -public class NotificationBasedTopicsCache implements TopicCallback, TopicsCache, TopicBlacklistCallback { - - private static final Logger logger = LoggerFactory.getLogger(NotificationBasedTopicsCache.class); - - private final ConcurrentMap topicCache = new ConcurrentHashMap<>(); - - private final GroupRepository groupRepository; - private final TopicRepository topicRepository; - private final MetricsFacade metricsFacade; - private final KafkaNamesMapper kafkaNamesMapper; - private final ThroughputRegistry throughputRegistry; - - public NotificationBasedTopicsCache(InternalNotificationsBus notificationsBus, - BlacklistZookeeperNotifyingCache blacklistZookeeperNotifyingCache, - GroupRepository groupRepository, - TopicRepository topicRepository, - MetricsFacade metricsFacade, - ThroughputRegistry throughputRegistry, - KafkaNamesMapper kafkaNamesMapper) { - this.groupRepository = groupRepository; - this.topicRepository = topicRepository; - this.metricsFacade = metricsFacade; - this.kafkaNamesMapper = kafkaNamesMapper; - this.throughputRegistry = throughputRegistry; - notificationsBus.registerTopicCallback(this); - blacklistZookeeperNotifyingCache.addCallback(this); - } - - @Override - public void onTopicCreated(Topic topic) { - topicCache.put(topic.getName().qualifiedName(), cachedTopic(topic)); - } - - @Override - public void onTopicRemoved(Topic topic) { - if (topicCache.containsKey(topic.getName().qualifiedName())) { - Topic cachedTopic = topicCache.get(topic.getName().qualifiedName()).getTopic(); - if (cachedTopic.equals(topic)) { - topicCache.remove(topic.getName().qualifiedName()); - } else { - logger.warn("Received event about removed topic but cache contains different topic under the same name." - + "Cached topic {}, removed topic {}", cachedTopic, topic); - } - } - } - - @Override - public void onTopicChanged(Topic topic) { - topicCache.put(topic.getName().qualifiedName(), cachedTopic(topic)); - } - - @Override - public void onTopicBlacklisted(String qualifiedTopicName) { - Optional topic = Optional.ofNullable( - Optional.ofNullable( - topicCache.get(qualifiedTopicName)).map(CachedTopic::getTopic).orElseGet(() -> - topicRepository.getTopicDetails(TopicName.fromQualifiedName(qualifiedTopicName)))); - - topic.ifPresent(t -> topicCache.put(qualifiedTopicName, bannedTopic(t))); - } - - @Override - public void onTopicUnblacklisted(String qualifiedTopicName) { - Optional topic = Optional.ofNullable( - Optional.ofNullable( - topicCache.get(qualifiedTopicName)).map(CachedTopic::getTopic).orElseGet(() -> - topicRepository.getTopicDetails(TopicName.fromQualifiedName(qualifiedTopicName)))); - - topic.ifPresent(t -> topicCache.put(qualifiedTopicName, cachedTopic(t))); +public class NotificationBasedTopicsCache + implements TopicCallback, TopicsCache, TopicBlacklistCallback { + + private static final Logger logger = LoggerFactory.getLogger(NotificationBasedTopicsCache.class); + + private final ConcurrentMap topicCache = new ConcurrentHashMap<>(); + + private final GroupRepository groupRepository; + private final TopicRepository topicRepository; + private final MetricsFacade metricsFacade; + private final KafkaNamesMapper kafkaNamesMapper; + private final ThroughputRegistry throughputRegistry; + + public NotificationBasedTopicsCache( + InternalNotificationsBus notificationsBus, + BlacklistZookeeperNotifyingCache blacklistZookeeperNotifyingCache, + GroupRepository groupRepository, + TopicRepository topicRepository, + MetricsFacade metricsFacade, + ThroughputRegistry throughputRegistry, + KafkaNamesMapper kafkaNamesMapper) { + this.groupRepository = groupRepository; + this.topicRepository = topicRepository; + this.metricsFacade = metricsFacade; + this.kafkaNamesMapper = kafkaNamesMapper; + this.throughputRegistry = throughputRegistry; + notificationsBus.registerTopicCallback(this); + blacklistZookeeperNotifyingCache.addCallback(this); + } + + @Override + public void onTopicCreated(Topic topic) { + topicCache.put(topic.getName().qualifiedName(), cachedTopic(topic)); + } + + @Override + public void onTopicRemoved(Topic topic) { + if (topicCache.containsKey(topic.getName().qualifiedName())) { + Topic cachedTopic = topicCache.get(topic.getName().qualifiedName()).getTopic(); + if (cachedTopic.equals(topic)) { + topicCache.remove(topic.getName().qualifiedName()); + } else { + logger.warn( + "Received event about removed topic but cache contains different topic under the same name." + + "Cached topic {}, removed topic {}", + cachedTopic, + topic); + } } - - @Override - public Optional getTopic(String qualifiedTopicName) { - return Optional.ofNullable(topicCache.get(qualifiedTopicName)); - } - - @Override - public List getTopics() { - return ImmutableList.copyOf(topicCache.values()); + } + + @Override + public void onTopicChanged(Topic topic) { + topicCache.put(topic.getName().qualifiedName(), cachedTopic(topic)); + } + + @Override + public void onTopicBlacklisted(String qualifiedTopicName) { + Optional topic = + Optional.ofNullable( + Optional.ofNullable(topicCache.get(qualifiedTopicName)) + .map(CachedTopic::getTopic) + .orElseGet( + () -> + topicRepository.getTopicDetails( + TopicName.fromQualifiedName(qualifiedTopicName)))); + + topic.ifPresent(t -> topicCache.put(qualifiedTopicName, bannedTopic(t))); + } + + @Override + public void onTopicUnblacklisted(String qualifiedTopicName) { + Optional topic = + Optional.ofNullable( + Optional.ofNullable(topicCache.get(qualifiedTopicName)) + .map(CachedTopic::getTopic) + .orElseGet( + () -> + topicRepository.getTopicDetails( + TopicName.fromQualifiedName(qualifiedTopicName)))); + + topic.ifPresent(t -> topicCache.put(qualifiedTopicName, cachedTopic(t))); + } + + @Override + public Optional getTopic(String qualifiedTopicName) { + return Optional.ofNullable(topicCache.get(qualifiedTopicName)); + } + + @Override + public List getTopics() { + return ImmutableList.copyOf(topicCache.values()); + } + + @Override + public void start() { + for (String groupName : groupRepository.listGroupNames()) { + for (Topic topic : topicRepository.listTopics(groupName)) { + topicCache.put(topic.getQualifiedName(), cachedTopic(topic)); + } } + } - @Override - public void start() { - for (String groupName : groupRepository.listGroupNames()) { - for (Topic topic : topicRepository.listTopics(groupName)) { - topicCache.put(topic.getQualifiedName(), cachedTopic(topic)); - } - } - } + private CachedTopic cachedTopic(Topic topic) { + return new CachedTopic( + topic, metricsFacade, throughputRegistry, kafkaNamesMapper.toKafkaTopics(topic)); + } - private CachedTopic cachedTopic(Topic topic) { - return new CachedTopic(topic, metricsFacade, throughputRegistry, kafkaNamesMapper.toKafkaTopics(topic)); - } - - private CachedTopic bannedTopic(Topic topic) { - return new CachedTopic(topic, metricsFacade, throughputRegistry, kafkaNamesMapper.toKafkaTopics(topic), true); - } + private CachedTopic bannedTopic(Topic topic) { + return new CachedTopic( + topic, metricsFacade, throughputRegistry, kafkaNamesMapper.toKafkaTopics(topic), true); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/cache/topic/TopicsCache.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/cache/topic/TopicsCache.java index 520e33d536..39b8b56d42 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/cache/topic/TopicsCache.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/cache/topic/TopicsCache.java @@ -1,16 +1,14 @@ package pl.allegro.tech.hermes.frontend.cache.topic; -import pl.allegro.tech.hermes.frontend.metric.CachedTopic; - import java.util.List; import java.util.Optional; +import pl.allegro.tech.hermes.frontend.metric.CachedTopic; public interface TopicsCache { - Optional getTopic(String qualifiedTopicName); - - List getTopics(); + Optional getTopic(String qualifiedTopicName); - void start(); + List getTopics(); + void start(); } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/BrokerLatencyReporterConfiguration.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/BrokerLatencyReporterConfiguration.java index 6a257099f3..d92b07ae52 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/BrokerLatencyReporterConfiguration.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/BrokerLatencyReporterConfiguration.java @@ -1,5 +1,6 @@ package pl.allegro.tech.hermes.frontend.config; +import java.util.concurrent.ExecutorService; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -7,29 +8,26 @@ import pl.allegro.tech.hermes.common.metric.executor.InstrumentedExecutorServiceFactory; import pl.allegro.tech.hermes.frontend.producer.BrokerLatencyReporter; -import java.util.concurrent.ExecutorService; - - @Configuration @EnableConfigurationProperties(BrokerLatencyReporterProperties.class) public class BrokerLatencyReporterConfiguration { - @Bean - BrokerLatencyReporter brokerLatencyReporter(BrokerLatencyReporterProperties properties, - MetricsFacade metricsFacade, - InstrumentedExecutorServiceFactory executorServiceFactory) { - ExecutorService executorService = executorServiceFactory.getExecutorService( - "broker-latency-reporter", - properties.getThreadPoolSize(), - true, - properties.getThreadPoolQueueCapacity() - ); + @Bean + BrokerLatencyReporter brokerLatencyReporter( + BrokerLatencyReporterProperties properties, + MetricsFacade metricsFacade, + InstrumentedExecutorServiceFactory executorServiceFactory) { + ExecutorService executorService = + executorServiceFactory.getExecutorService( + "broker-latency-reporter", + properties.getThreadPoolSize(), + true, + properties.getThreadPoolQueueCapacity()); - return new BrokerLatencyReporter( - properties.isEnabled(), - metricsFacade, - properties.getSlowResponseLoggingThreshold(), - executorService - ); - } + return new BrokerLatencyReporter( + properties.isEnabled(), + metricsFacade, + properties.getSlowResponseLoggingThreshold(), + executorService); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/BrokerLatencyReporterProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/BrokerLatencyReporterProperties.java index 1688a8b271..dc54308c3c 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/BrokerLatencyReporterProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/BrokerLatencyReporterProperties.java @@ -1,45 +1,44 @@ package pl.allegro.tech.hermes.frontend.config; -import org.springframework.boot.context.properties.ConfigurationProperties; - import java.time.Duration; +import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties(prefix = "frontend.broker-latency-reporter") public class BrokerLatencyReporterProperties { - private boolean enabled; - private Duration slowResponseLoggingThreshold = Duration.ofMillis(100); - private int threadPoolSize = 8; - private int threadPoolQueueCapacity = 1_000_000; - - public int getThreadPoolSize() { - return threadPoolSize; - } - - public void setThreadPoolSize(int threadPoolSize) { - this.threadPoolSize = threadPoolSize; - } - - public int getThreadPoolQueueCapacity() { - return threadPoolQueueCapacity; - } - - public void setThreadPoolQueueCapacity(int threadPoolQueueCapacity) { - this.threadPoolQueueCapacity = threadPoolQueueCapacity; - } - - public boolean isEnabled() { - return enabled; - } - - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } - - public Duration getSlowResponseLoggingThreshold() { - return slowResponseLoggingThreshold; - } - - public void setSlowResponseLoggingThreshold(Duration slowResponseLoggingThreshold) { - this.slowResponseLoggingThreshold = slowResponseLoggingThreshold; - } + private boolean enabled; + private Duration slowResponseLoggingThreshold = Duration.ofMillis(100); + private int threadPoolSize = 8; + private int threadPoolQueueCapacity = 1_000_000; + + public int getThreadPoolSize() { + return threadPoolSize; + } + + public void setThreadPoolSize(int threadPoolSize) { + this.threadPoolSize = threadPoolSize; + } + + public int getThreadPoolQueueCapacity() { + return threadPoolQueueCapacity; + } + + public void setThreadPoolQueueCapacity(int threadPoolQueueCapacity) { + this.threadPoolQueueCapacity = threadPoolQueueCapacity; + } + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public Duration getSlowResponseLoggingThreshold() { + return slowResponseLoggingThreshold; + } + + public void setSlowResponseLoggingThreshold(Duration slowResponseLoggingThreshold) { + this.slowResponseLoggingThreshold = slowResponseLoggingThreshold; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/CommonConfiguration.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/CommonConfiguration.java index ce996a7052..dfc71aae5b 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/CommonConfiguration.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/CommonConfiguration.java @@ -1,10 +1,14 @@ package pl.allegro.tech.hermes.frontend.config; +import static io.micrometer.core.instrument.Clock.SYSTEM; + import com.fasterxml.jackson.databind.ObjectMapper; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.composite.CompositeMeterRegistry; import io.micrometer.prometheus.PrometheusConfig; import io.micrometer.prometheus.PrometheusMeterRegistry; +import java.time.Clock; +import java.util.List; import org.apache.curator.framework.CuratorFramework; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; @@ -69,305 +73,331 @@ import pl.allegro.tech.hermes.metrics.PathsCompiler; import pl.allegro.tech.hermes.schema.SchemaRepository; -import java.time.Clock; -import java.util.List; - -import static io.micrometer.core.instrument.Clock.SYSTEM; - @Configuration @EnableConfigurationProperties({ - MetricRegistryProperties.class, - MicrometerRegistryProperties.class, - PrometheusProperties.class, - SchemaProperties.class, - ZookeeperClustersProperties.class, - KafkaClustersProperties.class, - ContentRootProperties.class, - DatacenterNameProperties.class, - TopicDefaultsProperties.class + MetricRegistryProperties.class, + MicrometerRegistryProperties.class, + PrometheusProperties.class, + SchemaProperties.class, + ZookeeperClustersProperties.class, + KafkaClustersProperties.class, + ContentRootProperties.class, + DatacenterNameProperties.class, + TopicDefaultsProperties.class }) public class CommonConfiguration { - @Bean - public DatacenterNameProvider dcNameProvider(DatacenterNameProperties datacenterNameProperties) { - if (datacenterNameProperties.getSource() == DcNameSource.ENV) { - return new EnvironmentVariableDatacenterNameProvider(datacenterNameProperties.getEnv()); - } else { - return new DefaultDatacenterNameProvider(); - } - } - - @Bean - public SubscriptionRepository subscriptionRepository(CuratorFramework zookeeper, - ZookeeperPaths paths, - ObjectMapper mapper, - TopicRepository topicRepository) { - return new ZookeeperSubscriptionRepository(zookeeper, mapper, paths, topicRepository); - } - - @Bean - public OAuthProviderRepository oAuthProviderRepository(CuratorFramework zookeeper, - ZookeeperPaths paths, - ObjectMapper mapper) { - return new ZookeeperOAuthProviderRepository(zookeeper, mapper, paths); - } - - @Bean - public TopicRepository topicRepository(CuratorFramework zookeeper, - ZookeeperPaths paths, - ObjectMapper mapper, - GroupRepository groupRepository) { - return new ZookeeperTopicRepository(zookeeper, mapper, paths, groupRepository); - } - - @Bean - public GroupRepository groupRepository(CuratorFramework zookeeper, - ZookeeperPaths paths, - ObjectMapper mapper) { - return new ZookeeperGroupRepository(zookeeper, mapper, paths); - } - - @Bean(destroyMethod = "close") - public CuratorFramework hermesCurator(ZookeeperClustersProperties zookeeperClustersProperties, - CuratorClientFactory curatorClientFactory, - DatacenterNameProvider datacenterNameProvider) { - ZookeeperProperties zookeeperProperties = zookeeperClustersProperties.toZookeeperProperties(datacenterNameProvider); - return new HermesCuratorClientFactory(zookeeperProperties, curatorClientFactory).provide(); - } - - @Bean - public CuratorClientFactory curatorClientFactory(ZookeeperClustersProperties zookeeperClustersProperties, - DatacenterNameProvider datacenterNameProvider) { - ZookeeperProperties zookeeperProperties = zookeeperClustersProperties.toZookeeperProperties(datacenterNameProvider); - return new CuratorClientFactory(zookeeperProperties); - } - - @Bean - public FilterChainFactory filterChainFactory(MessageFilterSource filters) { - return new FilterChainFactory(filters); - } - - @Bean - public InternalNotificationsBus zookeeperInternalNotificationBus(ObjectMapper objectMapper, - ModelAwareZookeeperNotifyingCache modelNotifyingCache) { - return new ZookeeperInternalNotificationBus(objectMapper, modelNotifyingCache); - } - - @Bean(initMethod = "start", destroyMethod = "stop") - public ModelAwareZookeeperNotifyingCache modelAwareZookeeperNotifyingCache(CuratorFramework curator, - MetricsFacade metricsFacade, - ZookeeperClustersProperties zookeeperClustersProperties, - DatacenterNameProvider datacenterNameProvider) { - ZookeeperProperties zookeeperProperties = zookeeperClustersProperties.toZookeeperProperties(datacenterNameProvider); - return new ModelAwareZookeeperNotifyingCacheFactory(curator, metricsFacade, zookeeperProperties).provide(); - } - - @Bean - public UndeliveredMessageLog undeliveredMessageLog(CuratorFramework zookeeper, - ZookeeperPaths paths, - ObjectMapper mapper, - MetricsFacade metricsFacade) { - return new ZookeeperUndeliveredMessageLog(zookeeper, paths, mapper, metricsFacade); - } - - @Bean - public InstrumentedExecutorServiceFactory instrumentedExecutorServiceFactory(MetricsFacade metricsFacade) { - return new InstrumentedExecutorServiceFactory(metricsFacade); - } - - @Bean - public ZookeeperAdminCache zookeeperAdminCache(ZookeeperPaths zookeeperPaths, - CuratorFramework client, - ObjectMapper objectMapper, - Clock clock) { - return new ZookeeperAdminCache(zookeeperPaths, client, objectMapper, clock); - } - - @Bean - public ObjectMapper objectMapper(SchemaProperties schemaProperties, TopicDefaultsProperties topicDefaults) { - return new ObjectMapperFactory(schemaProperties.isIdSerializationEnabled(), topicDefaults.isFallbackToRemoteDatacenterEnabled()).provide(); - } - - @Bean - public CompositeMessageContentWrapper messageContentWrapper( - JsonMessageContentWrapper jsonMessageContentWrapper, - AvroMessageContentWrapper avroMessageContentWrapper, - AvroMessageSchemaIdAwareContentWrapper schemaIdAwareContentWrapper, - AvroMessageHeaderSchemaVersionContentWrapper headerSchemaVersionContentWrapper, - AvroMessageHeaderSchemaIdContentWrapper headerSchemaIdContentWrapper, - AvroMessageSchemaVersionTruncationContentWrapper schemaVersionTruncationContentWrapper) { - return new CompositeMessageContentWrapper( - jsonMessageContentWrapper, - avroMessageContentWrapper, - schemaIdAwareContentWrapper, - headerSchemaVersionContentWrapper, - headerSchemaIdContentWrapper, - schemaVersionTruncationContentWrapper); - } - - @Bean - public JsonMessageContentWrapper jsonMessageContentWrapper(ContentRootProperties contentRootProperties, - ObjectMapper mapper) { - return new JsonMessageContentWrapper(contentRootProperties.getMessage(), contentRootProperties.getMetadata(), mapper); - } - - @Bean - public AvroMessageContentWrapper avroMessageContentWrapper(Clock clock) { - return new AvroMessageContentWrapper(clock); - } - - @Bean - public AvroMessageSchemaVersionTruncationContentWrapper avroMessageSchemaVersionTruncationContentWrapper( - SchemaRepository schemaRepository, - AvroMessageContentWrapper avroMessageContentWrapper, - MetricsFacade metricsFacade, - SchemaProperties schemaProperties) { - return new AvroMessageSchemaVersionTruncationContentWrapper(schemaRepository, avroMessageContentWrapper, - metricsFacade, schemaProperties.isVersionTruncationEnabled()); - } - - @Bean - public AvroMessageHeaderSchemaIdContentWrapper avroMessageHeaderSchemaIdContentWrapper( - SchemaRepository schemaRepository, - AvroMessageContentWrapper avroMessageContentWrapper, - MetricsFacade metricsFacade, - SchemaProperties schemaProperties) { - return new AvroMessageHeaderSchemaIdContentWrapper(schemaRepository, avroMessageContentWrapper, - metricsFacade, schemaProperties.isIdHeaderEnabled()); - } - - @Bean - public AvroMessageHeaderSchemaVersionContentWrapper avroMessageHeaderSchemaVersionContentWrapper( - SchemaRepository schemaRepository, - AvroMessageContentWrapper avroMessageContentWrapper, - MetricsFacade metricsFacade) { - return new AvroMessageHeaderSchemaVersionContentWrapper(schemaRepository, avroMessageContentWrapper, - metricsFacade); - } - - @Bean - public AvroMessageSchemaIdAwareContentWrapper avroMessageSchemaIdAwareContentWrapper( - SchemaRepository schemaRepository, - AvroMessageContentWrapper avroMessageContentWrapper, - MetricsFacade metricsFacade) { - return new AvroMessageSchemaIdAwareContentWrapper(schemaRepository, avroMessageContentWrapper, - metricsFacade); - } - - @Bean - public KafkaNamesMapper prodKafkaNamesMapper(KafkaClustersProperties kafkaClustersProperties) { - return new NamespaceKafkaNamesMapper(kafkaClustersProperties.getNamespace(), kafkaClustersProperties.getNamespaceSeparator()); - } - - @Bean - public Clock clock() { - return new ClockFactory().provide(); - } - - @Bean - public ZookeeperPaths zookeeperPaths(ZookeeperClustersProperties zookeeperClustersProperties, - DatacenterNameProvider datacenterNameProvider) { - ZookeeperProperties zookeeperProperties = zookeeperClustersProperties.toZookeeperProperties(datacenterNameProvider); - return new ZookeeperPaths(zookeeperProperties.getRoot()); - } - - @Bean - public WorkloadConstraintsRepository workloadConstraintsRepository(CuratorFramework curator, - ObjectMapper mapper, - ZookeeperPaths paths) { - return new ZookeeperWorkloadConstraintsRepository(curator, mapper, paths); - } - - @Bean - public MetricsFacade micrometerHermesMetrics(MeterRegistry meterRegistry) { - return new MetricsFacade(meterRegistry); - } - - @Bean - PrometheusConfig prometheusConfig(PrometheusProperties properties) { - return new PrometheusConfigAdapter(properties); - } - - @Bean - public PrometheusMeterRegistry micrometerRegistry(MicrometerRegistryParameters micrometerRegistryParameters, - PrometheusConfig prometheusConfig, - CounterStorage counterStorage) { - return new PrometheusMeterRegistryFactory(micrometerRegistryParameters, - prometheusConfig, counterStorage, "hermes-frontend").provide(); - } - - @Bean - @Primary - public MeterRegistry compositeMeterRegistry(List registries) { - return new CompositeMeterRegistry(SYSTEM, registries); - } - - @Bean - public PathsCompiler metricRegistryPathsCompiler(InstanceIdResolver instanceIdResolver) { - return new PathsCompiler(instanceIdResolver.resolve()); - } - - @Bean - public CounterStorage zookeeperCounterStorage(SharedCounter sharedCounter, - SubscriptionRepository subscriptionRepository, - PathsCompiler pathsCompiler, - ZookeeperClustersProperties zookeeperClustersProperties, - DatacenterNameProvider datacenterNameProvider) { - ZookeeperProperties zookeeperProperties = zookeeperClustersProperties.toZookeeperProperties(datacenterNameProvider); - return new ZookeeperCounterStorage(sharedCounter, subscriptionRepository, pathsCompiler, zookeeperProperties.getRoot()); - } - - @Bean - public SharedCounter sharedCounter(CuratorFramework zookeeper, - ZookeeperClustersProperties zookeeperClustersProperties, - MetricRegistryProperties metricRegistryProperties, - DatacenterNameProvider datacenterNameProvider) { - ZookeeperProperties zookeeperProperties = zookeeperClustersProperties.toZookeeperProperties(datacenterNameProvider); - return new SharedCounter(zookeeper, - metricRegistryProperties.getCounterExpireAfterAccess(), - zookeeperProperties.getBaseSleepTime(), - zookeeperProperties.getMaxRetries() - ); - } - - @Bean - public InstanceIdResolver instanceIdResolver() { - return new InetAddressInstanceIdResolver(); - } - - @Bean - public SubscriptionOffsetChangeIndicator subscriptionOffsetChangeIndicatorFactory( - CuratorFramework zookeeper, - ZookeeperPaths paths, - SubscriptionRepository subscriptionRepository) { - return new ZookeeperSubscriptionOffsetChangeIndicator(zookeeper, paths, subscriptionRepository); - } - - @Bean - public MessageFilters messageFilters(List globalFilters, - List subscriptionMessageFilterCompilers) { - return new MessageFilters(globalFilters, subscriptionMessageFilterCompilers); - } - - @Bean - public SubscriptionMessageFilterCompiler jsonPathSubscriptionMessageFilterCompiler() { - return new JsonPathSubscriptionMessageFilterCompiler(); - } - - @Bean - public SubscriptionMessageFilterCompiler avroPathSubscriptionMessageFilterCompiler() { - return new AvroPathSubscriptionMessageFilterCompiler(); - } - - @Bean - public SubscriptionMessageFilterCompiler headerSubscriptionMessageFilterCompiler() { - return new HeaderSubscriptionMessageFilterCompiler(); - } - - @Bean - public MessagePreviewRepository zookeeperMessagePreviewRepository(CuratorFramework zookeeper, - ObjectMapper mapper, - ZookeeperPaths paths) { - return new ZookeeperMessagePreviewRepository(zookeeper, mapper, paths); - } + @Bean + public DatacenterNameProvider dcNameProvider(DatacenterNameProperties datacenterNameProperties) { + if (datacenterNameProperties.getSource() == DcNameSource.ENV) { + return new EnvironmentVariableDatacenterNameProvider(datacenterNameProperties.getEnv()); + } else { + return new DefaultDatacenterNameProvider(); + } + } + + @Bean + public SubscriptionRepository subscriptionRepository( + CuratorFramework zookeeper, + ZookeeperPaths paths, + ObjectMapper mapper, + TopicRepository topicRepository) { + return new ZookeeperSubscriptionRepository(zookeeper, mapper, paths, topicRepository); + } + + @Bean + public OAuthProviderRepository oAuthProviderRepository( + CuratorFramework zookeeper, ZookeeperPaths paths, ObjectMapper mapper) { + return new ZookeeperOAuthProviderRepository(zookeeper, mapper, paths); + } + + @Bean + public TopicRepository topicRepository( + CuratorFramework zookeeper, + ZookeeperPaths paths, + ObjectMapper mapper, + GroupRepository groupRepository) { + return new ZookeeperTopicRepository(zookeeper, mapper, paths, groupRepository); + } + + @Bean + public GroupRepository groupRepository( + CuratorFramework zookeeper, ZookeeperPaths paths, ObjectMapper mapper) { + return new ZookeeperGroupRepository(zookeeper, mapper, paths); + } + + @Bean(destroyMethod = "close") + public CuratorFramework hermesCurator( + ZookeeperClustersProperties zookeeperClustersProperties, + CuratorClientFactory curatorClientFactory, + DatacenterNameProvider datacenterNameProvider) { + ZookeeperProperties zookeeperProperties = + zookeeperClustersProperties.toZookeeperProperties(datacenterNameProvider); + return new HermesCuratorClientFactory(zookeeperProperties, curatorClientFactory).provide(); + } + + @Bean + public CuratorClientFactory curatorClientFactory( + ZookeeperClustersProperties zookeeperClustersProperties, + DatacenterNameProvider datacenterNameProvider) { + ZookeeperProperties zookeeperProperties = + zookeeperClustersProperties.toZookeeperProperties(datacenterNameProvider); + return new CuratorClientFactory(zookeeperProperties); + } + + @Bean + public FilterChainFactory filterChainFactory(MessageFilterSource filters) { + return new FilterChainFactory(filters); + } + + @Bean + public InternalNotificationsBus zookeeperInternalNotificationBus( + ObjectMapper objectMapper, ModelAwareZookeeperNotifyingCache modelNotifyingCache) { + return new ZookeeperInternalNotificationBus(objectMapper, modelNotifyingCache); + } + + @Bean(initMethod = "start", destroyMethod = "stop") + public ModelAwareZookeeperNotifyingCache modelAwareZookeeperNotifyingCache( + CuratorFramework curator, + MetricsFacade metricsFacade, + ZookeeperClustersProperties zookeeperClustersProperties, + DatacenterNameProvider datacenterNameProvider) { + ZookeeperProperties zookeeperProperties = + zookeeperClustersProperties.toZookeeperProperties(datacenterNameProvider); + return new ModelAwareZookeeperNotifyingCacheFactory(curator, metricsFacade, zookeeperProperties) + .provide(); + } + + @Bean + public UndeliveredMessageLog undeliveredMessageLog( + CuratorFramework zookeeper, + ZookeeperPaths paths, + ObjectMapper mapper, + MetricsFacade metricsFacade) { + return new ZookeeperUndeliveredMessageLog(zookeeper, paths, mapper, metricsFacade); + } + + @Bean + public InstrumentedExecutorServiceFactory instrumentedExecutorServiceFactory( + MetricsFacade metricsFacade) { + return new InstrumentedExecutorServiceFactory(metricsFacade); + } + + @Bean + public ZookeeperAdminCache zookeeperAdminCache( + ZookeeperPaths zookeeperPaths, + CuratorFramework client, + ObjectMapper objectMapper, + Clock clock) { + return new ZookeeperAdminCache(zookeeperPaths, client, objectMapper, clock); + } + + @Bean + public ObjectMapper objectMapper( + SchemaProperties schemaProperties, TopicDefaultsProperties topicDefaults) { + return new ObjectMapperFactory( + schemaProperties.isIdSerializationEnabled(), + topicDefaults.isFallbackToRemoteDatacenterEnabled()) + .provide(); + } + + @Bean + public CompositeMessageContentWrapper messageContentWrapper( + JsonMessageContentWrapper jsonMessageContentWrapper, + AvroMessageContentWrapper avroMessageContentWrapper, + AvroMessageSchemaIdAwareContentWrapper schemaIdAwareContentWrapper, + AvroMessageHeaderSchemaVersionContentWrapper headerSchemaVersionContentWrapper, + AvroMessageHeaderSchemaIdContentWrapper headerSchemaIdContentWrapper, + AvroMessageSchemaVersionTruncationContentWrapper schemaVersionTruncationContentWrapper) { + return new CompositeMessageContentWrapper( + jsonMessageContentWrapper, + avroMessageContentWrapper, + schemaIdAwareContentWrapper, + headerSchemaVersionContentWrapper, + headerSchemaIdContentWrapper, + schemaVersionTruncationContentWrapper); + } + + @Bean + public JsonMessageContentWrapper jsonMessageContentWrapper( + ContentRootProperties contentRootProperties, ObjectMapper mapper) { + return new JsonMessageContentWrapper( + contentRootProperties.getMessage(), contentRootProperties.getMetadata(), mapper); + } + + @Bean + public AvroMessageContentWrapper avroMessageContentWrapper(Clock clock) { + return new AvroMessageContentWrapper(clock); + } + + @Bean + public AvroMessageSchemaVersionTruncationContentWrapper + avroMessageSchemaVersionTruncationContentWrapper( + SchemaRepository schemaRepository, + AvroMessageContentWrapper avroMessageContentWrapper, + MetricsFacade metricsFacade, + SchemaProperties schemaProperties) { + return new AvroMessageSchemaVersionTruncationContentWrapper( + schemaRepository, + avroMessageContentWrapper, + metricsFacade, + schemaProperties.isVersionTruncationEnabled()); + } + + @Bean + public AvroMessageHeaderSchemaIdContentWrapper avroMessageHeaderSchemaIdContentWrapper( + SchemaRepository schemaRepository, + AvroMessageContentWrapper avroMessageContentWrapper, + MetricsFacade metricsFacade, + SchemaProperties schemaProperties) { + return new AvroMessageHeaderSchemaIdContentWrapper( + schemaRepository, + avroMessageContentWrapper, + metricsFacade, + schemaProperties.isIdHeaderEnabled()); + } + + @Bean + public AvroMessageHeaderSchemaVersionContentWrapper avroMessageHeaderSchemaVersionContentWrapper( + SchemaRepository schemaRepository, + AvroMessageContentWrapper avroMessageContentWrapper, + MetricsFacade metricsFacade) { + return new AvroMessageHeaderSchemaVersionContentWrapper( + schemaRepository, avroMessageContentWrapper, metricsFacade); + } + + @Bean + public AvroMessageSchemaIdAwareContentWrapper avroMessageSchemaIdAwareContentWrapper( + SchemaRepository schemaRepository, + AvroMessageContentWrapper avroMessageContentWrapper, + MetricsFacade metricsFacade) { + return new AvroMessageSchemaIdAwareContentWrapper( + schemaRepository, avroMessageContentWrapper, metricsFacade); + } + + @Bean + public KafkaNamesMapper prodKafkaNamesMapper(KafkaClustersProperties kafkaClustersProperties) { + return new NamespaceKafkaNamesMapper( + kafkaClustersProperties.getNamespace(), kafkaClustersProperties.getNamespaceSeparator()); + } + + @Bean + public Clock clock() { + return new ClockFactory().provide(); + } + + @Bean + public ZookeeperPaths zookeeperPaths( + ZookeeperClustersProperties zookeeperClustersProperties, + DatacenterNameProvider datacenterNameProvider) { + ZookeeperProperties zookeeperProperties = + zookeeperClustersProperties.toZookeeperProperties(datacenterNameProvider); + return new ZookeeperPaths(zookeeperProperties.getRoot()); + } + + @Bean + public WorkloadConstraintsRepository workloadConstraintsRepository( + CuratorFramework curator, ObjectMapper mapper, ZookeeperPaths paths) { + return new ZookeeperWorkloadConstraintsRepository(curator, mapper, paths); + } + + @Bean + public MetricsFacade micrometerHermesMetrics(MeterRegistry meterRegistry) { + return new MetricsFacade(meterRegistry); + } + + @Bean + PrometheusConfig prometheusConfig(PrometheusProperties properties) { + return new PrometheusConfigAdapter(properties); + } + + @Bean + public PrometheusMeterRegistry micrometerRegistry( + MicrometerRegistryParameters micrometerRegistryParameters, + PrometheusConfig prometheusConfig, + CounterStorage counterStorage) { + return new PrometheusMeterRegistryFactory( + micrometerRegistryParameters, prometheusConfig, counterStorage, "hermes-frontend") + .provide(); + } + + @Bean + @Primary + public MeterRegistry compositeMeterRegistry(List registries) { + return new CompositeMeterRegistry(SYSTEM, registries); + } + + @Bean + public PathsCompiler metricRegistryPathsCompiler(InstanceIdResolver instanceIdResolver) { + return new PathsCompiler(instanceIdResolver.resolve()); + } + + @Bean + public CounterStorage zookeeperCounterStorage( + SharedCounter sharedCounter, + SubscriptionRepository subscriptionRepository, + PathsCompiler pathsCompiler, + ZookeeperClustersProperties zookeeperClustersProperties, + DatacenterNameProvider datacenterNameProvider) { + ZookeeperProperties zookeeperProperties = + zookeeperClustersProperties.toZookeeperProperties(datacenterNameProvider); + return new ZookeeperCounterStorage( + sharedCounter, subscriptionRepository, pathsCompiler, zookeeperProperties.getRoot()); + } + + @Bean + public SharedCounter sharedCounter( + CuratorFramework zookeeper, + ZookeeperClustersProperties zookeeperClustersProperties, + MetricRegistryProperties metricRegistryProperties, + DatacenterNameProvider datacenterNameProvider) { + ZookeeperProperties zookeeperProperties = + zookeeperClustersProperties.toZookeeperProperties(datacenterNameProvider); + return new SharedCounter( + zookeeper, + metricRegistryProperties.getCounterExpireAfterAccess(), + zookeeperProperties.getBaseSleepTime(), + zookeeperProperties.getMaxRetries()); + } + + @Bean + public InstanceIdResolver instanceIdResolver() { + return new InetAddressInstanceIdResolver(); + } + + @Bean + public SubscriptionOffsetChangeIndicator subscriptionOffsetChangeIndicatorFactory( + CuratorFramework zookeeper, + ZookeeperPaths paths, + SubscriptionRepository subscriptionRepository) { + return new ZookeeperSubscriptionOffsetChangeIndicator(zookeeper, paths, subscriptionRepository); + } + + @Bean + public MessageFilters messageFilters( + List globalFilters, + List subscriptionMessageFilterCompilers) { + return new MessageFilters(globalFilters, subscriptionMessageFilterCompilers); + } + + @Bean + public SubscriptionMessageFilterCompiler jsonPathSubscriptionMessageFilterCompiler() { + return new JsonPathSubscriptionMessageFilterCompiler(); + } + + @Bean + public SubscriptionMessageFilterCompiler avroPathSubscriptionMessageFilterCompiler() { + return new AvroPathSubscriptionMessageFilterCompiler(); + } + + @Bean + public SubscriptionMessageFilterCompiler headerSubscriptionMessageFilterCompiler() { + return new HeaderSubscriptionMessageFilterCompiler(); + } + + @Bean + public MessagePreviewRepository zookeeperMessagePreviewRepository( + CuratorFramework zookeeper, ObjectMapper mapper, ZookeeperPaths paths) { + return new ZookeeperMessagePreviewRepository(zookeeper, mapper, paths); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/ContentRootProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/ContentRootProperties.java index 0cf7c61610..74433a00c8 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/ContentRootProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/ContentRootProperties.java @@ -5,23 +5,23 @@ @ConfigurationProperties(prefix = "frontend.content.root") public class ContentRootProperties { - private String message = "message"; + private String message = "message"; - private String metadata = "metadata"; + private String metadata = "metadata"; - public String getMessage() { - return message; - } + public String getMessage() { + return message; + } - public void setMessage(String message) { - this.message = message; - } + public void setMessage(String message) { + this.message = message; + } - public String getMetadata() { - return metadata; - } + public String getMetadata() { + return metadata; + } - public void setMetadata(String metadata) { - this.metadata = metadata; - } + public void setMetadata(String metadata) { + this.metadata = metadata; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/DatacenterNameProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/DatacenterNameProperties.java index 64075272a1..4630277cbd 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/DatacenterNameProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/DatacenterNameProperties.java @@ -6,23 +6,23 @@ @ConfigurationProperties(prefix = "frontend.datacenter.name") public class DatacenterNameProperties { - private DcNameSource source; + private DcNameSource source; - private String env = "DC"; + private String env = "DC"; - public DcNameSource getSource() { - return source; - } + public DcNameSource getSource() { + return source; + } - public void setSource(DcNameSource source) { - this.source = source; - } + public void setSource(DcNameSource source) { + this.source = source; + } - public String getEnv() { - return env; - } + public String getEnv() { + return env; + } - public void setEnv(String env) { - this.env = env; - } + public void setEnv(String env) { + this.env = env; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FailFastKafkaProducerProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FailFastKafkaProducerProperties.java index 01056b10d4..aa00a91037 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FailFastKafkaProducerProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FailFastKafkaProducerProperties.java @@ -1,73 +1,72 @@ package pl.allegro.tech.hermes.frontend.config; +import java.time.Duration; import org.springframework.boot.context.properties.ConfigurationProperties; import pl.allegro.tech.hermes.frontend.producer.kafka.KafkaProducerParameters; -import java.time.Duration; - @ConfigurationProperties(prefix = "frontend.kafka.fail-fast-producer") public class FailFastKafkaProducerProperties { - private KafkaProducerParameters local = new FailFastLocalKafkaProducerProperties(); + private KafkaProducerParameters local = new FailFastLocalKafkaProducerProperties(); - private KafkaProducerParameters remote = new FailFastRemoteKafkaProducerProperties(); + private KafkaProducerParameters remote = new FailFastRemoteKafkaProducerProperties(); - private Duration speculativeSendDelay = Duration.ofMillis(250); + private Duration speculativeSendDelay = Duration.ofMillis(250); - private FallbackSchedulerProperties fallbackScheduler = new FallbackSchedulerProperties(); + private FallbackSchedulerProperties fallbackScheduler = new FallbackSchedulerProperties(); - public Duration getSpeculativeSendDelay() { - return speculativeSendDelay; - } + public Duration getSpeculativeSendDelay() { + return speculativeSendDelay; + } - public void setSpeculativeSendDelay(Duration speculativeSendDelay) { - this.speculativeSendDelay = speculativeSendDelay; - } + public void setSpeculativeSendDelay(Duration speculativeSendDelay) { + this.speculativeSendDelay = speculativeSendDelay; + } - public FallbackSchedulerProperties getFallbackScheduler() { - return fallbackScheduler; - } + public FallbackSchedulerProperties getFallbackScheduler() { + return fallbackScheduler; + } - public void setFallbackScheduler(FallbackSchedulerProperties fallbackScheduler) { - this.fallbackScheduler = fallbackScheduler; - } + public void setFallbackScheduler(FallbackSchedulerProperties fallbackScheduler) { + this.fallbackScheduler = fallbackScheduler; + } - public KafkaProducerParameters getLocal() { - return local; - } + public KafkaProducerParameters getLocal() { + return local; + } - public void setLocal(KafkaProducerParameters local) { - this.local = local; - } + public void setLocal(KafkaProducerParameters local) { + this.local = local; + } - public KafkaProducerParameters getRemote() { - return remote; - } + public KafkaProducerParameters getRemote() { + return remote; + } - public void setRemote(KafkaProducerParameters remote) { - this.remote = remote; - } + public void setRemote(KafkaProducerParameters remote) { + this.remote = remote; + } - public static class FallbackSchedulerProperties { + public static class FallbackSchedulerProperties { - private int threadPoolSize = 16; + private int threadPoolSize = 16; - private boolean threadPoolMonitoringEnabled = false; + private boolean threadPoolMonitoringEnabled = false; - public int getThreadPoolSize() { - return threadPoolSize; - } + public int getThreadPoolSize() { + return threadPoolSize; + } - public void setThreadPoolSize(int threadPoolSize) { - this.threadPoolSize = threadPoolSize; - } + public void setThreadPoolSize(int threadPoolSize) { + this.threadPoolSize = threadPoolSize; + } - public boolean isThreadPoolMonitoringEnabled() { - return threadPoolMonitoringEnabled; - } + public boolean isThreadPoolMonitoringEnabled() { + return threadPoolMonitoringEnabled; + } - public void setThreadPoolMonitoringEnabled(boolean threadPoolMonitoringEnabled) { - this.threadPoolMonitoringEnabled = threadPoolMonitoringEnabled; - } + public void setThreadPoolMonitoringEnabled(boolean threadPoolMonitoringEnabled) { + this.threadPoolMonitoringEnabled = threadPoolMonitoringEnabled; } + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FailFastLocalKafkaProducerProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FailFastLocalKafkaProducerProperties.java index f5c53b047c..87811c5cb1 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FailFastLocalKafkaProducerProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FailFastLocalKafkaProducerProperties.java @@ -1,209 +1,206 @@ package pl.allegro.tech.hermes.frontend.config; -import pl.allegro.tech.hermes.frontend.producer.kafka.KafkaProducerParameters; - import java.time.Duration; - +import pl.allegro.tech.hermes.frontend.producer.kafka.KafkaProducerParameters; /** - Kafka producer maintains a single connection to each broker, over which produce request are sent. - When producer request duration exceeds requestTimeout, producer closes the connection to the broker - that the request was sent to. This causes all inflight requests that were sent to that broker to be cancelled. - The number of inflight requests is configured by maxInflightRequestsPerConnection property. - - Let's assume that we have requestTimeout set to 500ms, maxInflightRequestsPerConnection set to 5, - and there are following inflight batches in the producer being sent to broker1: - - batchId | time spent in send buffer (socket) - ------------------------------------ - batch1 | 10ms - batch2 | 200ms - batch3 | 300ms - batch4 | 400ms - batch5 | 501ms - - Batch5 exceeded the requestTimeout so producer will close the connection to broker1. This causes batch5 to be marked - as failed but also causes batches 1-4 to be retried. This has the following consequences: - 1. Batches 1-4 will probably get duplicated - even tough they were cancelled, they were probably sent to the broker, - just haven't been ACKd yet. Retry would cause them to be sent once again resulting in duplicates. - 2. On retry, batches 1-4 will have a smaller time budget to complete. Part of their budget was already wasted - in send buffer + retryBackoff will be applied to them. They will have little time to complete on retry which can cause - them to be timed out, potentially resulting in a vicious circle. - 3. Connection to the broker must be reestablished which takes time. - - To avoid problems described above we actually set requestTimeout and deliveryTimeout to be much higher than the - maximum frontend request duration (frontend.handlers.maxPublishRequestDuration). This means that when - maxPublishRequestDuration is exceeded for a message we received, a client will receive 5xx even tough the - corresponding message is still being processed in the producer. The message will eventually be ACKd by Kafka so upon client-side - retry the message will be duplicated. This however, would likely also happen if the message was promptly timed-out by producer - before maxPublishRequestDuration elapsed - the message was likely already sent to Kafka, there just haven't been a response yet. - - So by using large requestTimeout we cause the first slow message to be duplicated (by client-side retry) but: - - we protect other inflight messages from being duplicated, - - we prevent connections from being frequently dropped and reestablished. + * Kafka producer maintains a single connection to each broker, over which produce request are sent. + * When producer request duration exceeds requestTimeout, producer closes the connection to the + * broker that the request was sent to. This causes all inflight requests that were sent to that + * broker to be cancelled. The number of inflight requests is configured by + * maxInflightRequestsPerConnection property. + * + *

Let's assume that we have requestTimeout set to 500ms, maxInflightRequestsPerConnection set to + * 5, and there are following inflight batches in the producer being sent to broker1: + * + *

batchId | time spent in send buffer (socket) ------------------------------------ batch1 | + * 10ms batch2 | 200ms batch3 | 300ms batch4 | 400ms batch5 | 501ms + * + *

Batch5 exceeded the requestTimeout so producer will close the connection to broker1. This + * causes batch5 to be marked as failed but also causes batches 1-4 to be retried. This has the + * following consequences: 1. Batches 1-4 will probably get duplicated - even tough they were + * cancelled, they were probably sent to the broker, just haven't been ACKd yet. Retry would cause + * them to be sent once again resulting in duplicates. 2. On retry, batches 1-4 will have a smaller + * time budget to complete. Part of their budget was already wasted in send buffer + retryBackoff + * will be applied to them. They will have little time to complete on retry which can cause them to + * be timed out, potentially resulting in a vicious circle. 3. Connection to the broker must be + * reestablished which takes time. + * + *

To avoid problems described above we actually set requestTimeout and deliveryTimeout to be + * much higher than the maximum frontend request duration + * (frontend.handlers.maxPublishRequestDuration). This means that when maxPublishRequestDuration is + * exceeded for a message we received, a client will receive 5xx even tough the corresponding + * message is still being processed in the producer. The message will eventually be ACKd by Kafka so + * upon client-side retry the message will be duplicated. This however, would likely also happen if + * the message was promptly timed-out by producer before maxPublishRequestDuration elapsed - the + * message was likely already sent to Kafka, there just haven't been a response yet. + * + *

So by using large requestTimeout we cause the first slow message to be duplicated (by + * client-side retry) but: - we protect other inflight messages from being duplicated, - we prevent + * connections from being frequently dropped and reestablished. */ public class FailFastLocalKafkaProducerProperties implements KafkaProducerParameters { - private Duration maxBlock = Duration.ofMillis(500); + private Duration maxBlock = Duration.ofMillis(500); - private Duration metadataMaxAge = Duration.ofMinutes(5); + private Duration metadataMaxAge = Duration.ofMinutes(5); - private String compressionCodec = "none"; + private String compressionCodec = "none"; - private int retries = Integer.MAX_VALUE; + private int retries = Integer.MAX_VALUE; - private Duration retryBackoff = Duration.ofMillis(50); + private Duration retryBackoff = Duration.ofMillis(50); - private Duration requestTimeout = Duration.ofSeconds(30); + private Duration requestTimeout = Duration.ofSeconds(30); - private Duration deliveryTimeout = Duration.ofSeconds(30); + private Duration deliveryTimeout = Duration.ofSeconds(30); - private int batchSize = 16 * 1024; + private int batchSize = 16 * 1024; - private int tcpSendBuffer = 128 * 1024; + private int tcpSendBuffer = 128 * 1024; - private int maxRequestSize = 1024 * 1024; + private int maxRequestSize = 1024 * 1024; - private Duration linger = Duration.ofMillis(0); + private Duration linger = Duration.ofMillis(0); - private Duration metricsSampleWindow = Duration.ofSeconds(30); + private Duration metricsSampleWindow = Duration.ofSeconds(30); - private int maxInflightRequestsPerConnection = 5; + private int maxInflightRequestsPerConnection = 5; - private boolean reportNodeMetricsEnabled = false; + private boolean reportNodeMetricsEnabled = false; - private boolean idempotenceEnabled = false; + private boolean idempotenceEnabled = false; - @Override - public Duration getMaxBlock() { - return maxBlock; - } + @Override + public Duration getMaxBlock() { + return maxBlock; + } - public void setMaxBlock(Duration maxBlock) { - this.maxBlock = maxBlock; - } + public void setMaxBlock(Duration maxBlock) { + this.maxBlock = maxBlock; + } - @Override - public Duration getMetadataMaxAge() { - return metadataMaxAge; - } + @Override + public Duration getMetadataMaxAge() { + return metadataMaxAge; + } - public void setMetadataMaxAge(Duration metadataMaxAge) { - this.metadataMaxAge = metadataMaxAge; - } + public void setMetadataMaxAge(Duration metadataMaxAge) { + this.metadataMaxAge = metadataMaxAge; + } - @Override - public String getCompressionCodec() { - return compressionCodec; - } + @Override + public String getCompressionCodec() { + return compressionCodec; + } - public void setCompressionCodec(String compressionCodec) { - this.compressionCodec = compressionCodec; - } + public void setCompressionCodec(String compressionCodec) { + this.compressionCodec = compressionCodec; + } - @Override - public int getRetries() { - return retries; - } + @Override + public int getRetries() { + return retries; + } - public void setRetries(int retries) { - this.retries = retries; - } + public void setRetries(int retries) { + this.retries = retries; + } - @Override - public Duration getRetryBackoff() { - return retryBackoff; - } + @Override + public Duration getRetryBackoff() { + return retryBackoff; + } - public void setRetryBackoff(Duration retryBackoff) { - this.retryBackoff = retryBackoff; - } + public void setRetryBackoff(Duration retryBackoff) { + this.retryBackoff = retryBackoff; + } - @Override - public Duration getRequestTimeout() { - return requestTimeout; - } + @Override + public Duration getRequestTimeout() { + return requestTimeout; + } - public void setRequestTimeout(Duration requestTimeout) { - this.requestTimeout = requestTimeout; - } + public void setRequestTimeout(Duration requestTimeout) { + this.requestTimeout = requestTimeout; + } - @Override - public int getBatchSize() { - return batchSize; - } + @Override + public int getBatchSize() { + return batchSize; + } - public void setBatchSize(int batchSize) { - this.batchSize = batchSize; - } + public void setBatchSize(int batchSize) { + this.batchSize = batchSize; + } - @Override - public int getTcpSendBuffer() { - return tcpSendBuffer; - } + @Override + public int getTcpSendBuffer() { + return tcpSendBuffer; + } - public void setTcpSendBuffer(int tcpSendBuffer) { - this.tcpSendBuffer = tcpSendBuffer; - } + public void setTcpSendBuffer(int tcpSendBuffer) { + this.tcpSendBuffer = tcpSendBuffer; + } - @Override - public int getMaxRequestSize() { - return maxRequestSize; - } + @Override + public int getMaxRequestSize() { + return maxRequestSize; + } - public void setMaxRequestSize(int maxRequestSize) { - this.maxRequestSize = maxRequestSize; - } + public void setMaxRequestSize(int maxRequestSize) { + this.maxRequestSize = maxRequestSize; + } - @Override - public Duration getLinger() { - return linger; - } + @Override + public Duration getLinger() { + return linger; + } - public void setLinger(Duration linger) { - this.linger = linger; - } - - @Override - public Duration getMetricsSampleWindow() { - return metricsSampleWindow; - } - - public void setMetricsSampleWindow(Duration metricsSampleWindow) { - this.metricsSampleWindow = metricsSampleWindow; - } - - @Override - public int getMaxInflightRequestsPerConnection() { - return maxInflightRequestsPerConnection; - } - - public void setMaxInflightRequestsPerConnection(int maxInflightRequestsPerConnection) { - this.maxInflightRequestsPerConnection = maxInflightRequestsPerConnection; - } - - @Override - public boolean isReportNodeMetricsEnabled() { - return reportNodeMetricsEnabled; - } - - public void setReportNodeMetricsEnabled(boolean reportNodeMetricsEnabled) { - this.reportNodeMetricsEnabled = reportNodeMetricsEnabled; - } - - @Override - public Duration getDeliveryTimeout() { - return deliveryTimeout; - } - - public void setDeliveryTimeout(Duration deliveryTimeout) { - this.deliveryTimeout = deliveryTimeout; - } - - public boolean isIdempotenceEnabled() { - return idempotenceEnabled; - } - - public void setIdempotenceEnabled(boolean idempotenceEnabled) { - this.idempotenceEnabled = idempotenceEnabled; - } + public void setLinger(Duration linger) { + this.linger = linger; + } + + @Override + public Duration getMetricsSampleWindow() { + return metricsSampleWindow; + } + + public void setMetricsSampleWindow(Duration metricsSampleWindow) { + this.metricsSampleWindow = metricsSampleWindow; + } + + @Override + public int getMaxInflightRequestsPerConnection() { + return maxInflightRequestsPerConnection; + } + + public void setMaxInflightRequestsPerConnection(int maxInflightRequestsPerConnection) { + this.maxInflightRequestsPerConnection = maxInflightRequestsPerConnection; + } + + @Override + public boolean isReportNodeMetricsEnabled() { + return reportNodeMetricsEnabled; + } + + public void setReportNodeMetricsEnabled(boolean reportNodeMetricsEnabled) { + this.reportNodeMetricsEnabled = reportNodeMetricsEnabled; + } + + @Override + public Duration getDeliveryTimeout() { + return deliveryTimeout; + } + + public void setDeliveryTimeout(Duration deliveryTimeout) { + this.deliveryTimeout = deliveryTimeout; + } + + public boolean isIdempotenceEnabled() { + return idempotenceEnabled; + } + + public void setIdempotenceEnabled(boolean idempotenceEnabled) { + this.idempotenceEnabled = idempotenceEnabled; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FailFastRemoteKafkaProducerProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FailFastRemoteKafkaProducerProperties.java index b051486460..316c89b521 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FailFastRemoteKafkaProducerProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FailFastRemoteKafkaProducerProperties.java @@ -1,176 +1,175 @@ package pl.allegro.tech.hermes.frontend.config; -import pl.allegro.tech.hermes.frontend.producer.kafka.KafkaProducerParameters; - import java.time.Duration; +import pl.allegro.tech.hermes.frontend.producer.kafka.KafkaProducerParameters; /** - * See {@link pl.allegro.tech.hermes.frontend.config.FailFastLocalKafkaProducerProperties} - * for the explanation of default values used. + * See {@link pl.allegro.tech.hermes.frontend.config.FailFastLocalKafkaProducerProperties} for the + * explanation of default values used. */ public class FailFastRemoteKafkaProducerProperties implements KafkaProducerParameters { - private Duration maxBlock = Duration.ofMillis(250); + private Duration maxBlock = Duration.ofMillis(250); - private Duration metadataMaxAge = Duration.ofMinutes(5); + private Duration metadataMaxAge = Duration.ofMinutes(5); - private String compressionCodec = "none"; + private String compressionCodec = "none"; - private int retries = Integer.MAX_VALUE; + private int retries = Integer.MAX_VALUE; - private Duration retryBackoff = Duration.ofMillis(50); + private Duration retryBackoff = Duration.ofMillis(50); - private Duration requestTimeout = Duration.ofSeconds(30); + private Duration requestTimeout = Duration.ofSeconds(30); - private Duration deliveryTimeout = Duration.ofSeconds(30); + private Duration deliveryTimeout = Duration.ofSeconds(30); - private int batchSize = 16 * 1024; + private int batchSize = 16 * 1024; - private int tcpSendBuffer = 128 * 1024; + private int tcpSendBuffer = 128 * 1024; - private int maxRequestSize = 1024 * 1024; + private int maxRequestSize = 1024 * 1024; - private Duration linger = Duration.ofMillis(0); + private Duration linger = Duration.ofMillis(0); - private Duration metricsSampleWindow = Duration.ofSeconds(30); + private Duration metricsSampleWindow = Duration.ofSeconds(30); - private int maxInflightRequestsPerConnection = 5; + private int maxInflightRequestsPerConnection = 5; - private boolean reportNodeMetricsEnabled = false; + private boolean reportNodeMetricsEnabled = false; - private boolean idempotenceEnabled = false; + private boolean idempotenceEnabled = false; - @Override - public Duration getMaxBlock() { - return maxBlock; - } + @Override + public Duration getMaxBlock() { + return maxBlock; + } - public void setMaxBlock(Duration maxBlock) { - this.maxBlock = maxBlock; - } + public void setMaxBlock(Duration maxBlock) { + this.maxBlock = maxBlock; + } - @Override - public Duration getMetadataMaxAge() { - return metadataMaxAge; - } + @Override + public Duration getMetadataMaxAge() { + return metadataMaxAge; + } - public void setMetadataMaxAge(Duration metadataMaxAge) { - this.metadataMaxAge = metadataMaxAge; - } + public void setMetadataMaxAge(Duration metadataMaxAge) { + this.metadataMaxAge = metadataMaxAge; + } - @Override - public String getCompressionCodec() { - return compressionCodec; - } + @Override + public String getCompressionCodec() { + return compressionCodec; + } - public void setCompressionCodec(String compressionCodec) { - this.compressionCodec = compressionCodec; - } + public void setCompressionCodec(String compressionCodec) { + this.compressionCodec = compressionCodec; + } - @Override - public int getRetries() { - return retries; - } + @Override + public int getRetries() { + return retries; + } - public void setRetries(int retries) { - this.retries = retries; - } + public void setRetries(int retries) { + this.retries = retries; + } - @Override - public Duration getRetryBackoff() { - return retryBackoff; - } + @Override + public Duration getRetryBackoff() { + return retryBackoff; + } - public void setRetryBackoff(Duration retryBackoff) { - this.retryBackoff = retryBackoff; - } + public void setRetryBackoff(Duration retryBackoff) { + this.retryBackoff = retryBackoff; + } - @Override - public Duration getRequestTimeout() { - return requestTimeout; - } + @Override + public Duration getRequestTimeout() { + return requestTimeout; + } - public void setRequestTimeout(Duration requestTimeout) { - this.requestTimeout = requestTimeout; - } + public void setRequestTimeout(Duration requestTimeout) { + this.requestTimeout = requestTimeout; + } - @Override - public int getBatchSize() { - return batchSize; - } + @Override + public int getBatchSize() { + return batchSize; + } - public void setBatchSize(int batchSize) { - this.batchSize = batchSize; - } + public void setBatchSize(int batchSize) { + this.batchSize = batchSize; + } - @Override - public int getTcpSendBuffer() { - return tcpSendBuffer; - } + @Override + public int getTcpSendBuffer() { + return tcpSendBuffer; + } - public void setTcpSendBuffer(int tcpSendBuffer) { - this.tcpSendBuffer = tcpSendBuffer; - } + public void setTcpSendBuffer(int tcpSendBuffer) { + this.tcpSendBuffer = tcpSendBuffer; + } - @Override - public int getMaxRequestSize() { - return maxRequestSize; - } + @Override + public int getMaxRequestSize() { + return maxRequestSize; + } - public void setMaxRequestSize(int maxRequestSize) { - this.maxRequestSize = maxRequestSize; - } + public void setMaxRequestSize(int maxRequestSize) { + this.maxRequestSize = maxRequestSize; + } - @Override - public Duration getLinger() { - return linger; - } + @Override + public Duration getLinger() { + return linger; + } - public void setLinger(Duration linger) { - this.linger = linger; - } - - @Override - public Duration getMetricsSampleWindow() { - return metricsSampleWindow; - } - - public void setMetricsSampleWindow(Duration metricsSampleWindow) { - this.metricsSampleWindow = metricsSampleWindow; - } - - @Override - public int getMaxInflightRequestsPerConnection() { - return maxInflightRequestsPerConnection; - } - - public void setMaxInflightRequestsPerConnection(int maxInflightRequestsPerConnection) { - this.maxInflightRequestsPerConnection = maxInflightRequestsPerConnection; - } - - @Override - public boolean isReportNodeMetricsEnabled() { - return reportNodeMetricsEnabled; - } - - public void setReportNodeMetricsEnabled(boolean reportNodeMetricsEnabled) { - this.reportNodeMetricsEnabled = reportNodeMetricsEnabled; - } - - @Override - public Duration getDeliveryTimeout() { - return deliveryTimeout; - } - - public void setDeliveryTimeout(Duration deliveryTimeout) { - this.deliveryTimeout = deliveryTimeout; - } - - @Override - public boolean isIdempotenceEnabled() { - return idempotenceEnabled; - } - - public void setIdempotenceEnabled(boolean idempotenceEnabled) { - this.idempotenceEnabled = idempotenceEnabled; - } + public void setLinger(Duration linger) { + this.linger = linger; + } + + @Override + public Duration getMetricsSampleWindow() { + return metricsSampleWindow; + } + + public void setMetricsSampleWindow(Duration metricsSampleWindow) { + this.metricsSampleWindow = metricsSampleWindow; + } + + @Override + public int getMaxInflightRequestsPerConnection() { + return maxInflightRequestsPerConnection; + } + + public void setMaxInflightRequestsPerConnection(int maxInflightRequestsPerConnection) { + this.maxInflightRequestsPerConnection = maxInflightRequestsPerConnection; + } + + @Override + public boolean isReportNodeMetricsEnabled() { + return reportNodeMetricsEnabled; + } + + public void setReportNodeMetricsEnabled(boolean reportNodeMetricsEnabled) { + this.reportNodeMetricsEnabled = reportNodeMetricsEnabled; + } + + @Override + public Duration getDeliveryTimeout() { + return deliveryTimeout; + } + + public void setDeliveryTimeout(Duration deliveryTimeout) { + this.deliveryTimeout = deliveryTimeout; + } + + @Override + public boolean isIdempotenceEnabled() { + return idempotenceEnabled; + } + + public void setIdempotenceEnabled(boolean idempotenceEnabled) { + this.idempotenceEnabled = idempotenceEnabled; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FrontendConfiguration.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FrontendConfiguration.java index a11e42a9a0..f8c78508bc 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FrontendConfiguration.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FrontendConfiguration.java @@ -1,6 +1,8 @@ package pl.allegro.tech.hermes.frontend.config; import jakarta.inject.Named; +import java.time.Clock; +import java.util.List; import org.apache.curator.framework.CuratorFramework; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; @@ -25,73 +27,78 @@ import pl.allegro.tech.hermes.schema.SchemaRepository; import pl.allegro.tech.hermes.tracker.frontend.Trackers; -import java.time.Clock; -import java.util.List; - @Configuration @EnableConfigurationProperties(LocalMessageStorageProperties.class) public class FrontendConfiguration { - @Bean - public MessageValidators messageValidators(List messageValidators) { - return new MessageValidators(messageValidators); - } + @Bean + public MessageValidators messageValidators(List messageValidators) { + return new MessageValidators(messageValidators); + } - @Bean(initMethod = "start") - public TopicsCache notificationBasedTopicsCache(InternalNotificationsBus internalNotificationsBus, - GroupRepository groupRepository, - TopicRepository topicRepository, - MetricsFacade metricsFacade, - ThroughputRegistry throughputRegistry, - KafkaNamesMapper kafkaNamesMapper, - BlacklistZookeeperNotifyingCache blacklistZookeeperNotifyingCache) { + @Bean(initMethod = "start") + public TopicsCache notificationBasedTopicsCache( + InternalNotificationsBus internalNotificationsBus, + GroupRepository groupRepository, + TopicRepository topicRepository, + MetricsFacade metricsFacade, + ThroughputRegistry throughputRegistry, + KafkaNamesMapper kafkaNamesMapper, + BlacklistZookeeperNotifyingCache blacklistZookeeperNotifyingCache) { - return new NotificationBasedTopicsCache(internalNotificationsBus, blacklistZookeeperNotifyingCache, - groupRepository, topicRepository, metricsFacade, throughputRegistry, kafkaNamesMapper); - } + return new NotificationBasedTopicsCache( + internalNotificationsBus, + blacklistZookeeperNotifyingCache, + groupRepository, + topicRepository, + metricsFacade, + throughputRegistry, + kafkaNamesMapper); + } - @Bean - public BackupMessagesLoader backupMessagesLoader(@Named("localDatacenterBrokerProducer") BrokerMessageProducer brokerMessageProducer, - BrokerListeners brokerListeners, - TopicsCache topicsCache, - SchemaRepository schemaRepository, - Trackers trackers, - LocalMessageStorageProperties localMessageStorageProperties) { - return new BackupMessagesLoader( - brokerMessageProducer, - brokerMessageProducer, - brokerListeners, - topicsCache, - schemaRepository, - new SchemaExistenceEnsurer(schemaRepository), - trackers, - localMessageStorageProperties - ); - } + @Bean + public BackupMessagesLoader backupMessagesLoader( + @Named("localDatacenterBrokerProducer") BrokerMessageProducer brokerMessageProducer, + BrokerListeners brokerListeners, + TopicsCache topicsCache, + SchemaRepository schemaRepository, + Trackers trackers, + LocalMessageStorageProperties localMessageStorageProperties) { + return new BackupMessagesLoader( + brokerMessageProducer, + brokerMessageProducer, + brokerListeners, + topicsCache, + schemaRepository, + new SchemaExistenceEnsurer(schemaRepository), + trackers, + localMessageStorageProperties); + } - @Bean(initMethod = "extend") - public PersistentBufferExtension persistentBufferExtension(LocalMessageStorageProperties localMessageStorageProperties, - Clock clock, - BrokerListeners listeners, - BackupMessagesLoader backupMessagesLoader, - MetricsFacade metricsFacade) { - return new PersistentBufferExtension(localMessageStorageProperties, clock, listeners, backupMessagesLoader, - metricsFacade); - } + @Bean(initMethod = "extend") + public PersistentBufferExtension persistentBufferExtension( + LocalMessageStorageProperties localMessageStorageProperties, + Clock clock, + BrokerListeners listeners, + BackupMessagesLoader backupMessagesLoader, + MetricsFacade metricsFacade) { + return new PersistentBufferExtension( + localMessageStorageProperties, clock, listeners, backupMessagesLoader, metricsFacade); + } - @Bean(initMethod = "startup") - public BlacklistZookeeperNotifyingCache blacklistZookeeperNotifyingCache(CuratorFramework curator, - ZookeeperPaths zookeeperPaths) { - return new BlacklistZookeeperNotifyingCache(curator, zookeeperPaths); - } + @Bean(initMethod = "startup") + public BlacklistZookeeperNotifyingCache blacklistZookeeperNotifyingCache( + CuratorFramework curator, ZookeeperPaths zookeeperPaths) { + return new BlacklistZookeeperNotifyingCache(curator, zookeeperPaths); + } - @Bean - public BrokerListeners defaultBrokerListeners() { - return new BrokerListeners(); - } + @Bean + public BrokerListeners defaultBrokerListeners() { + return new BrokerListeners(); + } - @Bean - public String moduleName() { - return "producer"; - } + @Bean + public String moduleName() { + return "producer"; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FrontendProducerConfiguration.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FrontendProducerConfiguration.java index fe70b502c2..124578aac4 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FrontendProducerConfiguration.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FrontendProducerConfiguration.java @@ -1,6 +1,16 @@ package pl.allegro.tech.hermes.frontend.config; +import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG; +import static org.apache.kafka.clients.CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL; +import static org.apache.kafka.clients.CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG; +import static org.apache.kafka.clients.CommonClientConfigs.SECURITY_PROTOCOL_CONFIG; +import static org.apache.kafka.common.config.SaslConfigs.SASL_JAAS_CONFIG; +import static org.apache.kafka.common.config.SaslConfigs.SASL_MECHANISM; + import jakarta.inject.Named; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.ScheduledExecutorService; import org.apache.kafka.clients.admin.AdminClient; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; @@ -24,156 +34,151 @@ import pl.allegro.tech.hermes.frontend.readiness.AdminReadinessService; import pl.allegro.tech.hermes.infrastructure.dc.DatacenterNameProvider; -import java.util.List; -import java.util.Properties; -import java.util.concurrent.ScheduledExecutorService; - -import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG; -import static org.apache.kafka.clients.CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL; -import static org.apache.kafka.clients.CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG; -import static org.apache.kafka.clients.CommonClientConfigs.SECURITY_PROTOCOL_CONFIG; -import static org.apache.kafka.common.config.SaslConfigs.SASL_JAAS_CONFIG; -import static org.apache.kafka.common.config.SaslConfigs.SASL_MECHANISM; - @Configuration @EnableConfigurationProperties({ - LocalMessageStorageProperties.class, - SchemaProperties.class, - KafkaHeaderNameProperties.class, - KafkaProducerProperties.class, - FailFastKafkaProducerProperties.class, - KafkaChaosProperties.class, - KafkaClustersProperties.class, - HTTPHeadersProperties.class + LocalMessageStorageProperties.class, + SchemaProperties.class, + KafkaHeaderNameProperties.class, + KafkaProducerProperties.class, + FailFastKafkaProducerProperties.class, + KafkaChaosProperties.class, + KafkaClustersProperties.class, + HTTPHeadersProperties.class }) public class FrontendProducerConfiguration { - @Bean - public BrokerMessageProducer kafkaBrokerMessageProducer(@Named("localDatacenterBrokerProducer") BrokerMessageProducer localDatacenterBrokerProducer, - @Named("multiDatacenterBrokerProducer") BrokerMessageProducer multiDatacenterBrokerProducer) { - return new FallbackToRemoteDatacenterAwareMessageProducer( - localDatacenterBrokerProducer, - multiDatacenterBrokerProducer - ); - } - - @Bean - public BrokerMessageProducer localDatacenterBrokerProducer(@Named("kafkaMessageSenders") KafkaMessageSenders kafkaMessageSenders, - MessageToKafkaProducerRecordConverter messageConverter) { - return new LocalDatacenterMessageProducer(kafkaMessageSenders, messageConverter); - } - - @Bean - public BrokerMessageProducer multiDatacenterBrokerProducer(@Named("failFastKafkaMessageSenders") KafkaMessageSenders kafkaMessageSenders, - MessageToKafkaProducerRecordConverter messageConverter, - FailFastKafkaProducerProperties kafkaProducerProperties, - AdminReadinessService adminReadinessService, - InstrumentedExecutorServiceFactory executorServiceFactory) { - FallbackSchedulerProperties fallbackSchedulerProperties = kafkaProducerProperties.getFallbackScheduler(); - ScheduledExecutorService fallbackScheduler = executorServiceFactory.scheduledExecutorBuilder( - "fallback-to-remote", - fallbackSchedulerProperties.getThreadPoolSize() - ) - .withMonitoringEnabled(fallbackSchedulerProperties.isThreadPoolMonitoringEnabled()) - .withRemoveOnCancel(true) - .create(); - return new MultiDatacenterMessageProducer( - kafkaMessageSenders, - adminReadinessService, - messageConverter, - kafkaProducerProperties.getSpeculativeSendDelay(), - fallbackScheduler - ); - } - - @Bean - public KafkaHeaderFactory kafkaHeaderFactory(KafkaHeaderNameProperties kafkaHeaderNameProperties, - HTTPHeadersProperties httpHeadersProperties) { - return new KafkaHeaderFactory(kafkaHeaderNameProperties, httpHeadersProperties.getPropagationAsKafkaHeaders()); - } - - @Bean(destroyMethod = "close") - public KafkaMessageSenders kafkaMessageSenders(KafkaProducerProperties kafkaProducerProperties, - KafkaMessageSendersFactory kafkaMessageSendersFactory) { - return kafkaMessageSendersFactory.provide(kafkaProducerProperties, "default"); - } - - @Bean(destroyMethod = "close") - public KafkaMessageSenders failFastKafkaMessageSenders(FailFastKafkaProducerProperties kafkaProducerProperties, - KafkaMessageSendersFactory kafkaMessageSendersFactory) { - return kafkaMessageSendersFactory.provide( - kafkaProducerProperties.getLocal(), - kafkaProducerProperties.getRemote(), - "failFast"); - } - - @Bean - public ScheduledExecutorService chaosScheduler(KafkaChaosProperties chaosProperties, InstrumentedExecutorServiceFactory executorServiceFactory) { - KafkaChaosProperties.ChaosSchedulerProperties chaosSchedulerProperties = chaosProperties.getChaosScheduler(); - return executorServiceFactory.scheduledExecutorBuilder( - "chaos", - chaosSchedulerProperties.getThreadPoolSize() - ) - .withMonitoringEnabled(chaosSchedulerProperties.isThreadPoolMonitoringEnabled()) - .create(); - } - - @Bean(destroyMethod = "close") - public KafkaMessageSendersFactory kafkaMessageSendersFactory(KafkaClustersProperties kafkaClustersProperties, - KafkaProducerProperties kafkaProducerProperties, - TopicLoadingProperties topicLoadingProperties, - TopicsCache topicsCache, - LocalMessageStorageProperties localMessageStorageProperties, - DatacenterNameProvider datacenterNameProvider, - BrokerLatencyReporter brokerLatencyReporter, - MetricsFacade metricsFacade, - @Named("chaosScheduler") ScheduledExecutorService chaosScheduler - ) { - KafkaProperties kafkaProperties = kafkaClustersProperties.toKafkaProperties(datacenterNameProvider); - List remoteKafkaProperties = kafkaClustersProperties.toRemoteKafkaProperties(datacenterNameProvider); - - return new KafkaMessageSendersFactory( - kafkaProperties, - remoteKafkaProperties, - brokerLatencyReporter, - metricsFacade, - createAdminClient(kafkaProperties), - topicsCache, - topicLoadingProperties.getMetadata().getRetryCount(), - topicLoadingProperties.getMetadata().getRetryInterval(), - topicLoadingProperties.getMetadata().getThreadPoolSize(), - localMessageStorageProperties.getBufferedSizeBytes(), - kafkaProducerProperties.getMetadataMaxAge(), - chaosScheduler - ); - } - - private static AdminClient createAdminClient(KafkaProperties kafkaProperties) { - Properties props = new Properties(); - props.put(BOOTSTRAP_SERVERS_CONFIG, kafkaProperties.getBrokerList()); - props.put(SECURITY_PROTOCOL_CONFIG, DEFAULT_SECURITY_PROTOCOL); - props.put(REQUEST_TIMEOUT_MS_CONFIG, (int) kafkaProperties.getAdminRequestTimeout().toMillis()); - if (kafkaProperties.isAuthenticationEnabled()) { - props.put(SASL_MECHANISM, kafkaProperties.getAuthenticationMechanism()); - props.put(SECURITY_PROTOCOL_CONFIG, kafkaProperties.getAuthenticationProtocol()); - props.put(SASL_JAAS_CONFIG, kafkaProperties.getJaasConfig()); - } - return AdminClient.create(props); - } - - @Bean(initMethod = "start", destroyMethod = "stop") - public ProducerMetadataLoadingJob producerMetadataLoadingJob(List kafkaMessageSendersList, - TopicLoadingProperties topicLoadingProperties) { - return new ProducerMetadataLoadingJob( - kafkaMessageSendersList, - topicLoadingProperties.getMetadataRefreshJob().isEnabled(), - topicLoadingProperties.getMetadataRefreshJob().getInterval() - ); - } - - @Bean - public MessageToKafkaProducerRecordConverter messageToKafkaProducerRecordConverter(KafkaHeaderFactory kafkaHeaderFactory, - SchemaProperties schemaProperties) { - return new MessageToKafkaProducerRecordConverter(kafkaHeaderFactory, schemaProperties.isIdHeaderEnabled()); + @Bean + public BrokerMessageProducer kafkaBrokerMessageProducer( + @Named("localDatacenterBrokerProducer") BrokerMessageProducer localDatacenterBrokerProducer, + @Named("multiDatacenterBrokerProducer") BrokerMessageProducer multiDatacenterBrokerProducer) { + return new FallbackToRemoteDatacenterAwareMessageProducer( + localDatacenterBrokerProducer, multiDatacenterBrokerProducer); + } + + @Bean + public BrokerMessageProducer localDatacenterBrokerProducer( + @Named("kafkaMessageSenders") KafkaMessageSenders kafkaMessageSenders, + MessageToKafkaProducerRecordConverter messageConverter) { + return new LocalDatacenterMessageProducer(kafkaMessageSenders, messageConverter); + } + + @Bean + public BrokerMessageProducer multiDatacenterBrokerProducer( + @Named("failFastKafkaMessageSenders") KafkaMessageSenders kafkaMessageSenders, + MessageToKafkaProducerRecordConverter messageConverter, + FailFastKafkaProducerProperties kafkaProducerProperties, + AdminReadinessService adminReadinessService, + InstrumentedExecutorServiceFactory executorServiceFactory) { + FallbackSchedulerProperties fallbackSchedulerProperties = + kafkaProducerProperties.getFallbackScheduler(); + ScheduledExecutorService fallbackScheduler = + executorServiceFactory + .scheduledExecutorBuilder( + "fallback-to-remote", fallbackSchedulerProperties.getThreadPoolSize()) + .withMonitoringEnabled(fallbackSchedulerProperties.isThreadPoolMonitoringEnabled()) + .withRemoveOnCancel(true) + .create(); + return new MultiDatacenterMessageProducer( + kafkaMessageSenders, + adminReadinessService, + messageConverter, + kafkaProducerProperties.getSpeculativeSendDelay(), + fallbackScheduler); + } + + @Bean + public KafkaHeaderFactory kafkaHeaderFactory( + KafkaHeaderNameProperties kafkaHeaderNameProperties, + HTTPHeadersProperties httpHeadersProperties) { + return new KafkaHeaderFactory( + kafkaHeaderNameProperties, httpHeadersProperties.getPropagationAsKafkaHeaders()); + } + + @Bean(destroyMethod = "close") + public KafkaMessageSenders kafkaMessageSenders( + KafkaProducerProperties kafkaProducerProperties, + KafkaMessageSendersFactory kafkaMessageSendersFactory) { + return kafkaMessageSendersFactory.provide(kafkaProducerProperties, "default"); + } + + @Bean(destroyMethod = "close") + public KafkaMessageSenders failFastKafkaMessageSenders( + FailFastKafkaProducerProperties kafkaProducerProperties, + KafkaMessageSendersFactory kafkaMessageSendersFactory) { + return kafkaMessageSendersFactory.provide( + kafkaProducerProperties.getLocal(), kafkaProducerProperties.getRemote(), "failFast"); + } + + @Bean + public ScheduledExecutorService chaosScheduler( + KafkaChaosProperties chaosProperties, + InstrumentedExecutorServiceFactory executorServiceFactory) { + KafkaChaosProperties.ChaosSchedulerProperties chaosSchedulerProperties = + chaosProperties.getChaosScheduler(); + return executorServiceFactory + .scheduledExecutorBuilder("chaos", chaosSchedulerProperties.getThreadPoolSize()) + .withMonitoringEnabled(chaosSchedulerProperties.isThreadPoolMonitoringEnabled()) + .create(); + } + + @Bean(destroyMethod = "close") + public KafkaMessageSendersFactory kafkaMessageSendersFactory( + KafkaClustersProperties kafkaClustersProperties, + KafkaProducerProperties kafkaProducerProperties, + TopicLoadingProperties topicLoadingProperties, + TopicsCache topicsCache, + LocalMessageStorageProperties localMessageStorageProperties, + DatacenterNameProvider datacenterNameProvider, + BrokerLatencyReporter brokerLatencyReporter, + MetricsFacade metricsFacade, + @Named("chaosScheduler") ScheduledExecutorService chaosScheduler) { + KafkaProperties kafkaProperties = + kafkaClustersProperties.toKafkaProperties(datacenterNameProvider); + List remoteKafkaProperties = + kafkaClustersProperties.toRemoteKafkaProperties(datacenterNameProvider); + + return new KafkaMessageSendersFactory( + kafkaProperties, + remoteKafkaProperties, + brokerLatencyReporter, + metricsFacade, + createAdminClient(kafkaProperties), + topicsCache, + topicLoadingProperties.getMetadata().getRetryCount(), + topicLoadingProperties.getMetadata().getRetryInterval(), + topicLoadingProperties.getMetadata().getThreadPoolSize(), + localMessageStorageProperties.getBufferedSizeBytes(), + kafkaProducerProperties.getMetadataMaxAge(), + chaosScheduler); + } + + private static AdminClient createAdminClient(KafkaProperties kafkaProperties) { + Properties props = new Properties(); + props.put(BOOTSTRAP_SERVERS_CONFIG, kafkaProperties.getBrokerList()); + props.put(SECURITY_PROTOCOL_CONFIG, DEFAULT_SECURITY_PROTOCOL); + props.put(REQUEST_TIMEOUT_MS_CONFIG, (int) kafkaProperties.getAdminRequestTimeout().toMillis()); + if (kafkaProperties.isAuthenticationEnabled()) { + props.put(SASL_MECHANISM, kafkaProperties.getAuthenticationMechanism()); + props.put(SECURITY_PROTOCOL_CONFIG, kafkaProperties.getAuthenticationProtocol()); + props.put(SASL_JAAS_CONFIG, kafkaProperties.getJaasConfig()); } + return AdminClient.create(props); + } + + @Bean(initMethod = "start", destroyMethod = "stop") + public ProducerMetadataLoadingJob producerMetadataLoadingJob( + List kafkaMessageSendersList, + TopicLoadingProperties topicLoadingProperties) { + return new ProducerMetadataLoadingJob( + kafkaMessageSendersList, + topicLoadingProperties.getMetadataRefreshJob().isEnabled(), + topicLoadingProperties.getMetadataRefreshJob().getInterval()); + } + + @Bean + public MessageToKafkaProducerRecordConverter messageToKafkaProducerRecordConverter( + KafkaHeaderFactory kafkaHeaderFactory, SchemaProperties schemaProperties) { + return new MessageToKafkaProducerRecordConverter( + kafkaHeaderFactory, schemaProperties.isIdHeaderEnabled()); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FrontendPublishingConfiguration.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FrontendPublishingConfiguration.java index cbb9920840..1f9e8cd262 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FrontendPublishingConfiguration.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FrontendPublishingConfiguration.java @@ -4,6 +4,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.undertow.server.HttpHandler; import jakarta.inject.Named; +import java.time.Clock; +import java.util.Optional; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -34,94 +36,124 @@ import pl.allegro.tech.hermes.schema.SchemaRepository; import pl.allegro.tech.hermes.tracker.frontend.Trackers; -import java.time.Clock; -import java.util.Optional; - @Configuration @EnableConfigurationProperties({ - ThroughputProperties.class, - MessagePreviewProperties.class, - HandlersChainProperties.class, - SchemaProperties.class + ThroughputProperties.class, + MessagePreviewProperties.class, + HandlersChainProperties.class, + SchemaProperties.class }) public class FrontendPublishingConfiguration { - @Bean - public HttpHandler httpHandler(TopicsCache topicsCache, MessageErrorProcessor messageErrorProcessor, - MessageEndProcessor messageEndProcessor, MessageFactory messageFactory, - @Named("kafkaBrokerMessageProducer") BrokerMessageProducer brokerMessageProducer, MessagePreviewLog messagePreviewLog, - ThroughputLimiter throughputLimiter, Optional authConfig, - MessagePreviewProperties messagePreviewProperties, HandlersChainProperties handlersChainProperties) { - return new HandlersChainFactory(topicsCache, messageErrorProcessor, messageEndProcessor, messageFactory, - brokerMessageProducer, messagePreviewLog, throughputLimiter, authConfig, messagePreviewProperties.isEnabled(), - handlersChainProperties).provide(); - } + @Bean + public HttpHandler httpHandler( + TopicsCache topicsCache, + MessageErrorProcessor messageErrorProcessor, + MessageEndProcessor messageEndProcessor, + MessageFactory messageFactory, + @Named("kafkaBrokerMessageProducer") BrokerMessageProducer brokerMessageProducer, + MessagePreviewLog messagePreviewLog, + ThroughputLimiter throughputLimiter, + Optional authConfig, + MessagePreviewProperties messagePreviewProperties, + HandlersChainProperties handlersChainProperties) { + return new HandlersChainFactory( + topicsCache, + messageErrorProcessor, + messageEndProcessor, + messageFactory, + brokerMessageProducer, + messagePreviewLog, + throughputLimiter, + authConfig, + messagePreviewProperties.isEnabled(), + handlersChainProperties) + .provide(); + } - @Bean - public ThroughputRegistry throughputRegistry(MetricsFacade metricsFacade) { - return new ThroughputRegistry(metricsFacade, new MetricRegistry()); - } + @Bean + public ThroughputRegistry throughputRegistry(MetricsFacade metricsFacade) { + return new ThroughputRegistry(metricsFacade, new MetricRegistry()); + } - @Bean - public ThroughputLimiter throughputLimiter(ThroughputProperties throughputProperties, ThroughputRegistry throughputRegistry) { - return new ThroughputLimiterFactory(throughputProperties, throughputRegistry).provide(); - } + @Bean + public ThroughputLimiter throughputLimiter( + ThroughputProperties throughputProperties, ThroughputRegistry throughputRegistry) { + return new ThroughputLimiterFactory(throughputProperties, throughputRegistry).provide(); + } - @Bean - public MessageEndProcessor messageEndProcessor(Trackers trackers, BrokerListeners brokerListeners, - TrackingHeadersExtractor trackingHeadersExtractor) { - return new MessageEndProcessor(trackers, brokerListeners, trackingHeadersExtractor); - } + @Bean + public MessageEndProcessor messageEndProcessor( + Trackers trackers, + BrokerListeners brokerListeners, + TrackingHeadersExtractor trackingHeadersExtractor) { + return new MessageEndProcessor(trackers, brokerListeners, trackingHeadersExtractor); + } - @Bean - public TrackingHeadersExtractor extraHeadersExtractor() { - return new DefaultTrackingHeaderExtractor(); - } + @Bean + public TrackingHeadersExtractor extraHeadersExtractor() { + return new DefaultTrackingHeaderExtractor(); + } - @Bean - public MessageErrorProcessor messageErrorProcessor(ObjectMapper objectMapper, Trackers trackers, - TrackingHeadersExtractor trackingHeadersExtractor) { - return new MessageErrorProcessor(objectMapper, trackers, trackingHeadersExtractor); - } + @Bean + public MessageErrorProcessor messageErrorProcessor( + ObjectMapper objectMapper, + Trackers trackers, + TrackingHeadersExtractor trackingHeadersExtractor) { + return new MessageErrorProcessor(objectMapper, trackers, trackingHeadersExtractor); + } - @Bean - public AvroEnforcer messageContentTypeEnforcer() { - return new MessageContentTypeEnforcer(); - } + @Bean + public AvroEnforcer messageContentTypeEnforcer() { + return new MessageContentTypeEnforcer(); + } - @Bean - public MessageFactory messageFactory(MessageValidators validators, - AvroEnforcer enforcer, - SchemaRepository schemaRepository, - HeadersPropagator headersPropagator, - CompositeMessageContentWrapper compositeMessageContentWrapper, - Clock clock, - SchemaProperties schemaProperties) { - return new MessageFactory(validators, enforcer, schemaRepository, headersPropagator, compositeMessageContentWrapper, - clock, schemaProperties.isIdHeaderEnabled()); - } + @Bean + public MessageFactory messageFactory( + MessageValidators validators, + AvroEnforcer enforcer, + SchemaRepository schemaRepository, + HeadersPropagator headersPropagator, + CompositeMessageContentWrapper compositeMessageContentWrapper, + Clock clock, + SchemaProperties schemaProperties) { + return new MessageFactory( + validators, + enforcer, + schemaRepository, + headersPropagator, + compositeMessageContentWrapper, + clock, + schemaProperties.isIdHeaderEnabled()); + } - @Bean - public HeadersPropagator defaultHeadersPropagator(HTTPHeadersProperties httpHeadersProperties) { - return new DefaultHeadersPropagator(httpHeadersProperties); - } + @Bean + public HeadersPropagator defaultHeadersPropagator(HTTPHeadersProperties httpHeadersProperties) { + return new DefaultHeadersPropagator(httpHeadersProperties); + } - @Bean - public MessagePreviewFactory messagePreviewFactory(MessagePreviewProperties messagePreviewProperties) { - return new MessagePreviewFactory(messagePreviewProperties.getMaxSizeKb()); - } + @Bean + public MessagePreviewFactory messagePreviewFactory( + MessagePreviewProperties messagePreviewProperties) { + return new MessagePreviewFactory(messagePreviewProperties.getMaxSizeKb()); + } - @Bean - public MessagePreviewLog messagePreviewLog(MessagePreviewFactory messagePreviewFactory, - MessagePreviewProperties messagePreviewProperties) { - return new MessagePreviewLog(messagePreviewFactory, messagePreviewProperties.getSize()); - } + @Bean + public MessagePreviewLog messagePreviewLog( + MessagePreviewFactory messagePreviewFactory, + MessagePreviewProperties messagePreviewProperties) { + return new MessagePreviewLog(messagePreviewFactory, messagePreviewProperties.getSize()); + } - @Bean - public DefaultMessagePreviewPersister messagePreviewPersister(MessagePreviewLog messagePreviewLog, - MessagePreviewRepository repository, - MessagePreviewProperties messagePreviewProperties) { - return new DefaultMessagePreviewPersister(messagePreviewLog, repository, messagePreviewProperties.getLogPersistPeriod(), - messagePreviewProperties.isEnabled()); - } + @Bean + public DefaultMessagePreviewPersister messagePreviewPersister( + MessagePreviewLog messagePreviewLog, + MessagePreviewRepository repository, + MessagePreviewProperties messagePreviewProperties) { + return new DefaultMessagePreviewPersister( + messagePreviewLog, + repository, + messagePreviewProperties.getLogPersistPeriod(), + messagePreviewProperties.isEnabled()); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FrontendServerConfiguration.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FrontendServerConfiguration.java index a324ea7f50..8f707fbab8 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FrontendServerConfiguration.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FrontendServerConfiguration.java @@ -2,6 +2,7 @@ import io.micrometer.prometheus.PrometheusMeterRegistry; import io.undertow.server.HttpHandler; +import java.util.Optional; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -17,55 +18,56 @@ import pl.allegro.tech.hermes.frontend.server.TopicSchemaLoadingStartupHook; import pl.allegro.tech.hermes.schema.SchemaRepository; -import java.util.Optional; - @Configuration @EnableConfigurationProperties({ - TopicLoadingProperties.class, - ReadinessCheckProperties.class, - SslProperties.class, - HermesServerProperties.class + TopicLoadingProperties.class, + ReadinessCheckProperties.class, + SslProperties.class, + HermesServerProperties.class }) public class FrontendServerConfiguration { - @Bean(initMethod = "start", destroyMethod = "stop") - public HermesServer hermesServer(HermesServerProperties hermesServerProperties, - SslProperties sslProperties, - MetricsFacade metricsFacade, - HttpHandler publishingHandler, - HealthCheckService healthCheckService, - ReadinessChecker readinessChecker, - DefaultMessagePreviewPersister defaultMessagePreviewPersister, - ThroughputLimiter throughputLimiter, - SslContextFactoryProvider sslContextFactoryProvider, - PrometheusMeterRegistry prometheusMeterRegistry) { - return new HermesServer( - sslProperties, - hermesServerProperties, - metricsFacade, - publishingHandler, - healthCheckService, - readinessChecker, - defaultMessagePreviewPersister, - throughputLimiter, - sslContextFactoryProvider, - prometheusMeterRegistry - ); - } + @Bean(initMethod = "start", destroyMethod = "stop") + public HermesServer hermesServer( + HermesServerProperties hermesServerProperties, + SslProperties sslProperties, + MetricsFacade metricsFacade, + HttpHandler publishingHandler, + HealthCheckService healthCheckService, + ReadinessChecker readinessChecker, + DefaultMessagePreviewPersister defaultMessagePreviewPersister, + ThroughputLimiter throughputLimiter, + SslContextFactoryProvider sslContextFactoryProvider, + PrometheusMeterRegistry prometheusMeterRegistry) { + return new HermesServer( + sslProperties, + hermesServerProperties, + metricsFacade, + publishingHandler, + healthCheckService, + readinessChecker, + defaultMessagePreviewPersister, + throughputLimiter, + sslContextFactoryProvider, + prometheusMeterRegistry); + } - @Bean - public SslContextFactoryProvider sslContextFactoryProvider(Optional sslContextFactory, - SslProperties sslProperties) { - return new SslContextFactoryProvider(sslContextFactory.orElse(null), sslProperties); - } + @Bean + public SslContextFactoryProvider sslContextFactoryProvider( + Optional sslContextFactory, SslProperties sslProperties) { + return new SslContextFactoryProvider(sslContextFactory.orElse(null), sslProperties); + } - @Bean(initMethod = "run") - public TopicSchemaLoadingStartupHook topicSchemaLoadingStartupHook(TopicsCache topicsCache, - SchemaRepository schemaRepository, - TopicLoadingProperties topicLoadingProperties) { - return new TopicSchemaLoadingStartupHook(topicsCache, schemaRepository, - topicLoadingProperties.getSchema().getRetryCount(), - topicLoadingProperties.getSchema().getThreadPoolSize(), - topicLoadingProperties.getSchema().isEnabled()); - } + @Bean(initMethod = "run") + public TopicSchemaLoadingStartupHook topicSchemaLoadingStartupHook( + TopicsCache topicsCache, + SchemaRepository schemaRepository, + TopicLoadingProperties topicLoadingProperties) { + return new TopicSchemaLoadingStartupHook( + topicsCache, + schemaRepository, + topicLoadingProperties.getSchema().getRetryCount(), + topicLoadingProperties.getSchema().getThreadPoolSize(), + topicLoadingProperties.getSchema().isEnabled()); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FrontendTrackerConfiguration.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FrontendTrackerConfiguration.java index 2350ca197d..dbb733c9fc 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FrontendTrackerConfiguration.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/FrontendTrackerConfiguration.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.frontend.config; +import java.time.Clock; +import java.util.List; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import pl.allegro.tech.hermes.tracker.frontend.LogRepository; @@ -7,24 +9,22 @@ import pl.allegro.tech.hermes.tracker.frontend.PublishingMessageTracker; import pl.allegro.tech.hermes.tracker.frontend.Trackers; -import java.time.Clock; -import java.util.List; - @Configuration public class FrontendTrackerConfiguration { - @Bean - public PublishingMessageTracker publishingMessageTracker(List repositories, Clock clock) { - return new PublishingMessageTracker(repositories, clock); - } + @Bean + public PublishingMessageTracker publishingMessageTracker( + List repositories, Clock clock) { + return new PublishingMessageTracker(repositories, clock); + } - @Bean - public NoOperationPublishingTracker noOperationPublishingTracker() { - return new NoOperationPublishingTracker(); - } + @Bean + public NoOperationPublishingTracker noOperationPublishingTracker() { + return new NoOperationPublishingTracker(); + } - @Bean(destroyMethod = "close") - public Trackers trackers(List repositories) { - return new Trackers(repositories); - } + @Bean(destroyMethod = "close") + public Trackers trackers(List repositories) { + return new Trackers(repositories); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/HTTPHeadersProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/HTTPHeadersProperties.java index af89220b78..8e765b8011 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/HTTPHeadersProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/HTTPHeadersProperties.java @@ -1,74 +1,75 @@ package pl.allegro.tech.hermes.frontend.config; -import org.springframework.boot.context.properties.ConfigurationProperties; -import pl.allegro.tech.hermes.common.kafka.HTTPHeadersPropagationAsKafkaHeadersProperties; - import java.util.HashSet; import java.util.Set; +import org.springframework.boot.context.properties.ConfigurationProperties; +import pl.allegro.tech.hermes.common.kafka.HTTPHeadersPropagationAsKafkaHeadersProperties; @ConfigurationProperties(prefix = "frontend.http.headers") public class HTTPHeadersProperties { - public boolean propagationEnabled = false; - public Set allowedSet = new HashSet<>(); - public Set additionalAllowedSetToLog = new HashSet<>(); - public PropagationAsKafkaHeadersProperties propagationAsKafkaHeaders = new PropagationAsKafkaHeadersProperties(); - - public static class PropagationAsKafkaHeadersProperties implements HTTPHeadersPropagationAsKafkaHeadersProperties { - - public boolean enabled = true; - - public String prefix = "h-"; + public boolean propagationEnabled = false; + public Set allowedSet = new HashSet<>(); + public Set additionalAllowedSetToLog = new HashSet<>(); + public PropagationAsKafkaHeadersProperties propagationAsKafkaHeaders = + new PropagationAsKafkaHeadersProperties(); - @Override - public boolean isEnabled() { - return enabled; - } + public static class PropagationAsKafkaHeadersProperties + implements HTTPHeadersPropagationAsKafkaHeadersProperties { - @Override - public String getPrefix() { - return prefix; - } + public boolean enabled = true; - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } + public String prefix = "h-"; - public void setPrefix(String prefix) { - this.prefix = prefix; - } + @Override + public boolean isEnabled() { + return enabled; } - public boolean isPropagationEnabled() { - return propagationEnabled; + @Override + public String getPrefix() { + return prefix; } - public void setPropagationEnabled(boolean propagationEnabled) { - this.propagationEnabled = propagationEnabled; + public void setEnabled(boolean enabled) { + this.enabled = enabled; } - public PropagationAsKafkaHeadersProperties getPropagationAsKafkaHeaders() { - return propagationAsKafkaHeaders; + public void setPrefix(String prefix) { + this.prefix = prefix; } + } - public void setPropagationAsKafkaHeaders( - PropagationAsKafkaHeadersProperties propagationAsKafkaHeaders) { - this.propagationAsKafkaHeaders = propagationAsKafkaHeaders; - } + public boolean isPropagationEnabled() { + return propagationEnabled; + } - public Set getAllowedSet() { - return allowedSet; - } + public void setPropagationEnabled(boolean propagationEnabled) { + this.propagationEnabled = propagationEnabled; + } - public void setAllowedSet(Set allowedSet) { - this.allowedSet = allowedSet; - } + public PropagationAsKafkaHeadersProperties getPropagationAsKafkaHeaders() { + return propagationAsKafkaHeaders; + } - public Set getAdditionalAllowedSetToLog() { - return additionalAllowedSetToLog; - } + public void setPropagationAsKafkaHeaders( + PropagationAsKafkaHeadersProperties propagationAsKafkaHeaders) { + this.propagationAsKafkaHeaders = propagationAsKafkaHeaders; + } - public void setAdditionalAllowedSetToLog(Set additionalAllowedSetToLog) { - this.additionalAllowedSetToLog = additionalAllowedSetToLog; - } + public Set getAllowedSet() { + return allowedSet; + } + + public void setAllowedSet(Set allowedSet) { + this.allowedSet = allowedSet; + } + + public Set getAdditionalAllowedSetToLog() { + return additionalAllowedSetToLog; + } + + public void setAdditionalAllowedSetToLog(Set additionalAllowedSetToLog) { + this.additionalAllowedSetToLog = additionalAllowedSetToLog; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/HandlersChainProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/HandlersChainProperties.java index 451227270e..13920d2b6c 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/HandlersChainProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/HandlersChainProperties.java @@ -1,140 +1,139 @@ package pl.allegro.tech.hermes.frontend.config; +import java.time.Duration; import org.springframework.boot.context.properties.ConfigurationProperties; import pl.allegro.tech.hermes.frontend.publishing.handlers.HandlersChainParameters; -import java.time.Duration; - @ConfigurationProperties(prefix = "frontend.handlers") public class HandlersChainProperties implements HandlersChainParameters { - private Duration idleTimeout = Duration.ofMillis(65); + private Duration idleTimeout = Duration.ofMillis(65); - private Duration longIdleTimeout = Duration.ofMillis(400); + private Duration longIdleTimeout = Duration.ofMillis(400); - private Duration maxPublishRequestDuration = Duration.ofMillis(500); + private Duration maxPublishRequestDuration = Duration.ofMillis(500); - private boolean forceTopicMaxMessageSize = false; + private boolean forceTopicMaxMessageSize = false; - @Override - public Duration getIdleTimeout() { - return idleTimeout; - } + @Override + public Duration getIdleTimeout() { + return idleTimeout; + } - public void setIdleTimeout(Duration idleTimeout) { - this.idleTimeout = idleTimeout; - } + public void setIdleTimeout(Duration idleTimeout) { + this.idleTimeout = idleTimeout; + } - @Override - public Duration getLongIdleTimeout() { - return longIdleTimeout; - } + @Override + public Duration getLongIdleTimeout() { + return longIdleTimeout; + } - public void setLongIdleTimeout(Duration longIdleTimeout) { - this.longIdleTimeout = longIdleTimeout; - } + public void setLongIdleTimeout(Duration longIdleTimeout) { + this.longIdleTimeout = longIdleTimeout; + } - @Override - public boolean isForceTopicMaxMessageSize() { - return forceTopicMaxMessageSize; - } + @Override + public boolean isForceTopicMaxMessageSize() { + return forceTopicMaxMessageSize; + } - @Override - public boolean isKeepAliveHeaderEnabled() { - return keepAliveHeader.enabled; - } + @Override + public boolean isKeepAliveHeaderEnabled() { + return keepAliveHeader.enabled; + } - @Override - public Duration getKeepAliveHeaderTimeout() { - return keepAliveHeader.timeout; - } + @Override + public Duration getKeepAliveHeaderTimeout() { + return keepAliveHeader.timeout; + } - @Override - public boolean isAuthenticationEnabled() { - return authentication.enabled; - } + @Override + public boolean isAuthenticationEnabled() { + return authentication.enabled; + } - @Override - public String getAuthenticationMode() { - return authentication.mode; - } + @Override + public String getAuthenticationMode() { + return authentication.mode; + } - public void setForceTopicMaxMessageSize(boolean forceTopicMaxMessageSize) { - this.forceTopicMaxMessageSize = forceTopicMaxMessageSize; - } + public void setForceTopicMaxMessageSize(boolean forceTopicMaxMessageSize) { + this.forceTopicMaxMessageSize = forceTopicMaxMessageSize; + } - @Override - public Duration getMaxPublishRequestDuration() { - return maxPublishRequestDuration; - } + @Override + public Duration getMaxPublishRequestDuration() { + return maxPublishRequestDuration; + } - public void setMaxPublishRequestDuration(Duration maxPublishRequestDuration) { - this.maxPublishRequestDuration = maxPublishRequestDuration; - } + public void setMaxPublishRequestDuration(Duration maxPublishRequestDuration) { + this.maxPublishRequestDuration = maxPublishRequestDuration; + } - private AuthenticationProperties authentication = new AuthenticationProperties(); + private AuthenticationProperties authentication = new AuthenticationProperties(); - private KeepAliveHeaderProperties keepAliveHeader = new KeepAliveHeaderProperties(); + private KeepAliveHeaderProperties keepAliveHeader = new KeepAliveHeaderProperties(); - public AuthenticationProperties getAuthentication() { - return authentication; - } + public AuthenticationProperties getAuthentication() { + return authentication; + } - public void setAuthentication(AuthenticationProperties authentication) { - this.authentication = authentication; - } + public void setAuthentication(AuthenticationProperties authentication) { + this.authentication = authentication; + } - public KeepAliveHeaderProperties getKeepAliveHeader() { - return keepAliveHeader; - } + public KeepAliveHeaderProperties getKeepAliveHeader() { + return keepAliveHeader; + } - public void setKeepAliveHeader(KeepAliveHeaderProperties keepAliveHeader) { - this.keepAliveHeader = keepAliveHeader; - } + public void setKeepAliveHeader(KeepAliveHeaderProperties keepAliveHeader) { + this.keepAliveHeader = keepAliveHeader; + } - public static class AuthenticationProperties { + public static class AuthenticationProperties { - private boolean enabled = false; + private boolean enabled = false; - private String mode = "constraint_driven"; + private String mode = "constraint_driven"; - public boolean isEnabled() { - return enabled; - } + public boolean isEnabled() { + return enabled; + } - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } - public String getMode() { - return mode; - } + public String getMode() { + return mode; + } - public void setMode(String mode) { - this.mode = mode; - } + public void setMode(String mode) { + this.mode = mode; } + } - public static class KeepAliveHeaderProperties { + public static class KeepAliveHeaderProperties { - private boolean enabled = false; + private boolean enabled = false; - private Duration timeout = Duration.ofSeconds(1); + private Duration timeout = Duration.ofSeconds(1); - public boolean isEnabled() { - return enabled; - } + public boolean isEnabled() { + return enabled; + } - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } - public Duration getTimeout() { - return timeout; - } + public Duration getTimeout() { + return timeout; + } - public void setTimeout(Duration timeout) { - this.timeout = timeout; - } + public void setTimeout(Duration timeout) { + this.timeout = timeout; } + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/HermesServerProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/HermesServerProperties.java index eda985ef3d..849f79ebaa 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/HermesServerProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/HermesServerProperties.java @@ -1,197 +1,196 @@ package pl.allegro.tech.hermes.frontend.config; +import java.time.Duration; import org.springframework.boot.context.properties.ConfigurationProperties; import pl.allegro.tech.hermes.frontend.server.HermesServerParameters; -import java.time.Duration; - @ConfigurationProperties(prefix = "frontend.server") public class HermesServerProperties implements HermesServerParameters { - private int port = 8080; + private int port = 8080; - private String host = "0.0.0.0"; + private String host = "0.0.0.0"; - private Duration readTimeout = Duration.ofMillis(2000); + private Duration readTimeout = Duration.ofMillis(2000); - private Duration requestParseTimeout = Duration.ofMillis(5000); + private Duration requestParseTimeout = Duration.ofMillis(5000); - private int maxHeaders = 20; + private int maxHeaders = 20; - private int maxParameters = 10; + private int maxParameters = 10; - private int maxCookies = 10; + private int maxCookies = 10; - private int backlogSize = 10000; + private int backlogSize = 10000; - private int ioThreadsCount = Runtime.getRuntime().availableProcessors() * 2; + private int ioThreadsCount = Runtime.getRuntime().availableProcessors() * 2; - private int workerThreadCount = 200; + private int workerThreadCount = 200; - private boolean alwaysKeepAlive = false; + private boolean alwaysKeepAlive = false; - private boolean keepAlive = false; + private boolean keepAlive = false; - private boolean requestDumperEnabled = false; + private boolean requestDumperEnabled = false; - private int bufferSize = 16384; + private int bufferSize = 16384; - private boolean gracefulShutdownEnabled = true; + private boolean gracefulShutdownEnabled = true; - private Duration gracefulShutdownInitialWait = Duration.ofSeconds(10); + private Duration gracefulShutdownInitialWait = Duration.ofSeconds(10); - private boolean http2Enabled = false; + private boolean http2Enabled = false; - @Override - public int getPort() { - return port; - } + @Override + public int getPort() { + return port; + } - public void setPort(int port) { - this.port = port; - } + public void setPort(int port) { + this.port = port; + } - @Override - public String getHost() { - return host; - } + @Override + public String getHost() { + return host; + } - public void setHost(String host) { - this.host = host; - } + public void setHost(String host) { + this.host = host; + } - @Override - public Duration getReadTimeout() { - return readTimeout; - } + @Override + public Duration getReadTimeout() { + return readTimeout; + } - public void setReadTimeout(Duration readTimeout) { - this.readTimeout = readTimeout; - } + public void setReadTimeout(Duration readTimeout) { + this.readTimeout = readTimeout; + } - @Override - public Duration getRequestParseTimeout() { - return requestParseTimeout; - } + @Override + public Duration getRequestParseTimeout() { + return requestParseTimeout; + } - public void setRequestParseTimeout(Duration requestParseTimeout) { - this.requestParseTimeout = requestParseTimeout; - } + public void setRequestParseTimeout(Duration requestParseTimeout) { + this.requestParseTimeout = requestParseTimeout; + } - @Override - public int getMaxHeaders() { - return maxHeaders; - } + @Override + public int getMaxHeaders() { + return maxHeaders; + } - public void setMaxHeaders(int maxHeaders) { - this.maxHeaders = maxHeaders; - } + public void setMaxHeaders(int maxHeaders) { + this.maxHeaders = maxHeaders; + } - @Override - public int getMaxParameters() { - return maxParameters; - } + @Override + public int getMaxParameters() { + return maxParameters; + } - public void setMaxParameters(int maxParameters) { - this.maxParameters = maxParameters; - } + public void setMaxParameters(int maxParameters) { + this.maxParameters = maxParameters; + } - @Override - public int getMaxCookies() { - return maxCookies; - } + @Override + public int getMaxCookies() { + return maxCookies; + } - public void setMaxCookies(int maxCookies) { - this.maxCookies = maxCookies; - } + public void setMaxCookies(int maxCookies) { + this.maxCookies = maxCookies; + } - @Override - public int getBacklogSize() { - return backlogSize; - } + @Override + public int getBacklogSize() { + return backlogSize; + } - public void setBacklogSize(int backlogSize) { - this.backlogSize = backlogSize; - } + public void setBacklogSize(int backlogSize) { + this.backlogSize = backlogSize; + } - @Override - public int getIoThreadsCount() { - return ioThreadsCount; - } + @Override + public int getIoThreadsCount() { + return ioThreadsCount; + } - public void setIoThreadsCount(int ioThreadsCount) { - this.ioThreadsCount = ioThreadsCount; - } + public void setIoThreadsCount(int ioThreadsCount) { + this.ioThreadsCount = ioThreadsCount; + } - @Override - public int getWorkerThreadCount() { - return workerThreadCount; - } + @Override + public int getWorkerThreadCount() { + return workerThreadCount; + } - public void setWorkerThreadCount(int workerThreadCount) { - this.workerThreadCount = workerThreadCount; - } - - @Override - public boolean isAlwaysKeepAlive() { - return alwaysKeepAlive; - } - - public void setAlwaysKeepAlive(boolean alwaysKeepAlive) { - this.alwaysKeepAlive = alwaysKeepAlive; - } - - @Override - public boolean isKeepAlive() { - return keepAlive; - } - - public void setKeepAlive(boolean keepAlive) { - this.keepAlive = keepAlive; - } - - @Override - public boolean isRequestDumperEnabled() { - return requestDumperEnabled; - } - - public void setRequestDumperEnabled(boolean requestDumperEnabled) { - this.requestDumperEnabled = requestDumperEnabled; - } - - @Override - public int getBufferSize() { - return bufferSize; - } - - public void setBufferSize(int bufferSize) { - this.bufferSize = bufferSize; - } - - @Override - public boolean isGracefulShutdownEnabled() { - return gracefulShutdownEnabled; - } - - public void setGracefulShutdownEnabled(boolean gracefulShutdownEnabled) { - this.gracefulShutdownEnabled = gracefulShutdownEnabled; - } - - @Override - public Duration getGracefulShutdownInitialWait() { - return gracefulShutdownInitialWait; - } - - public void setGracefulShutdownInitialWait(Duration gracefulShutdownInitialWait) { - this.gracefulShutdownInitialWait = gracefulShutdownInitialWait; - } - - @Override - public boolean isHttp2Enabled() { - return http2Enabled; - } - - public void setHttp2Enabled(boolean http2Enabled) { - this.http2Enabled = http2Enabled; - } + public void setWorkerThreadCount(int workerThreadCount) { + this.workerThreadCount = workerThreadCount; + } + + @Override + public boolean isAlwaysKeepAlive() { + return alwaysKeepAlive; + } + + public void setAlwaysKeepAlive(boolean alwaysKeepAlive) { + this.alwaysKeepAlive = alwaysKeepAlive; + } + + @Override + public boolean isKeepAlive() { + return keepAlive; + } + + public void setKeepAlive(boolean keepAlive) { + this.keepAlive = keepAlive; + } + + @Override + public boolean isRequestDumperEnabled() { + return requestDumperEnabled; + } + + public void setRequestDumperEnabled(boolean requestDumperEnabled) { + this.requestDumperEnabled = requestDumperEnabled; + } + + @Override + public int getBufferSize() { + return bufferSize; + } + + public void setBufferSize(int bufferSize) { + this.bufferSize = bufferSize; + } + + @Override + public boolean isGracefulShutdownEnabled() { + return gracefulShutdownEnabled; + } + + public void setGracefulShutdownEnabled(boolean gracefulShutdownEnabled) { + this.gracefulShutdownEnabled = gracefulShutdownEnabled; + } + + @Override + public Duration getGracefulShutdownInitialWait() { + return gracefulShutdownInitialWait; + } + + public void setGracefulShutdownInitialWait(Duration gracefulShutdownInitialWait) { + this.gracefulShutdownInitialWait = gracefulShutdownInitialWait; + } + + @Override + public boolean isHttp2Enabled() { + return http2Enabled; + } + + public void setHttp2Enabled(boolean http2Enabled) { + this.http2Enabled = http2Enabled; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/KafkaClustersProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/KafkaClustersProperties.java index ae803b23d3..60170d2395 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/KafkaClustersProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/KafkaClustersProperties.java @@ -1,60 +1,63 @@ package pl.allegro.tech.hermes.frontend.config; -import org.springframework.boot.context.properties.ConfigurationProperties; -import pl.allegro.tech.hermes.common.kafka.KafkaParameters; -import pl.allegro.tech.hermes.infrastructure.dc.DatacenterNameProvider; - import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import org.springframework.boot.context.properties.ConfigurationProperties; +import pl.allegro.tech.hermes.common.kafka.KafkaParameters; +import pl.allegro.tech.hermes.infrastructure.dc.DatacenterNameProvider; @ConfigurationProperties(prefix = "frontend.kafka") public class KafkaClustersProperties { - private List clusters = new ArrayList<>(); - - private String namespace = ""; - - private String namespaceSeparator = "_"; - - public List getClusters() { - return clusters; - } - - public void setClusters(List clusters) { - this.clusters = clusters; - } - - public String getNamespace() { - return namespace; - } - - public void setNamespace(String namespace) { - this.namespace = namespace; - } - - public String getNamespaceSeparator() { - return namespaceSeparator; - } - - public void setNamespaceSeparator(String namespaceSeparator) { - this.namespaceSeparator = namespaceSeparator; - } - - public KafkaProperties toKafkaProperties(DatacenterNameProvider datacenterNameProvider) { - return this.clusters - .stream() - .filter(cluster -> cluster.getDatacenter().equals(datacenterNameProvider.getDatacenterName())) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException( - "No properties for datacenter: " + datacenterNameProvider.getDatacenterName() + " defined.")); - } - - public List toRemoteKafkaProperties(DatacenterNameProvider datacenterNameProvider) { - return this.clusters - .stream() - .filter(cluster -> !cluster.getDatacenter().equals(datacenterNameProvider.getDatacenterName())) - .collect(Collectors.toList()); - } + private List clusters = new ArrayList<>(); + + private String namespace = ""; + + private String namespaceSeparator = "_"; + + public List getClusters() { + return clusters; + } + + public void setClusters(List clusters) { + this.clusters = clusters; + } + + public String getNamespace() { + return namespace; + } + + public void setNamespace(String namespace) { + this.namespace = namespace; + } + + public String getNamespaceSeparator() { + return namespaceSeparator; + } + + public void setNamespaceSeparator(String namespaceSeparator) { + this.namespaceSeparator = namespaceSeparator; + } + + public KafkaProperties toKafkaProperties(DatacenterNameProvider datacenterNameProvider) { + return this.clusters.stream() + .filter( + cluster -> cluster.getDatacenter().equals(datacenterNameProvider.getDatacenterName())) + .findFirst() + .orElseThrow( + () -> + new IllegalArgumentException( + "No properties for datacenter: " + + datacenterNameProvider.getDatacenterName() + + " defined.")); + } + + public List toRemoteKafkaProperties( + DatacenterNameProvider datacenterNameProvider) { + return this.clusters.stream() + .filter( + cluster -> !cluster.getDatacenter().equals(datacenterNameProvider.getDatacenterName())) + .collect(Collectors.toList()); + } } - diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/KafkaHeaderNameProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/KafkaHeaderNameProperties.java index f75b168d1d..14633fde74 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/KafkaHeaderNameProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/KafkaHeaderNameProperties.java @@ -6,36 +6,36 @@ @ConfigurationProperties(prefix = "frontend.kafka.header.name") public class KafkaHeaderNameProperties implements KafkaHeaderNameParameters { - private String messageId = "id"; + private String messageId = "id"; - private String schemaVersion = "sv"; + private String schemaVersion = "sv"; - private String schemaId = "sid"; + private String schemaId = "sid"; - @Override - public String getMessageId() { - return messageId; - } + @Override + public String getMessageId() { + return messageId; + } - public void setMessageId(String messageId) { - this.messageId = messageId; - } + public void setMessageId(String messageId) { + this.messageId = messageId; + } - @Override - public String getSchemaVersion() { - return schemaVersion; - } + @Override + public String getSchemaVersion() { + return schemaVersion; + } - public void setSchemaVersion(String schemaVersion) { - this.schemaVersion = schemaVersion; - } + public void setSchemaVersion(String schemaVersion) { + this.schemaVersion = schemaVersion; + } - @Override - public String getSchemaId() { - return schemaId; - } + @Override + public String getSchemaId() { + return schemaId; + } - public void setSchemaId(String schemaId) { - this.schemaId = schemaId; - } + public void setSchemaId(String schemaId) { + this.schemaId = schemaId; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/KafkaProducerProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/KafkaProducerProperties.java index 34e97143e3..cb00b3afab 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/KafkaProducerProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/KafkaProducerProperties.java @@ -1,174 +1,173 @@ package pl.allegro.tech.hermes.frontend.config; +import java.time.Duration; import org.springframework.boot.context.properties.ConfigurationProperties; import pl.allegro.tech.hermes.frontend.producer.kafka.KafkaProducerParameters; -import java.time.Duration; - @ConfigurationProperties(prefix = "frontend.kafka.producer") public class KafkaProducerProperties implements KafkaProducerParameters { - private Duration maxBlock = Duration.ofMillis(500); + private Duration maxBlock = Duration.ofMillis(500); - private Duration metadataMaxAge = Duration.ofMinutes(5); + private Duration metadataMaxAge = Duration.ofMinutes(5); - private String compressionCodec = "none"; + private String compressionCodec = "none"; - private int retries = Integer.MAX_VALUE; + private int retries = Integer.MAX_VALUE; - private Duration retryBackoff = Duration.ofMillis(256); + private Duration retryBackoff = Duration.ofMillis(256); - private Duration requestTimeout = Duration.ofMinutes(30); + private Duration requestTimeout = Duration.ofMinutes(30); - private Duration deliveryTimeout = Duration.ofMinutes(30); + private Duration deliveryTimeout = Duration.ofMinutes(30); - private int batchSize = 16 * 1024; + private int batchSize = 16 * 1024; - private int tcpSendBuffer = 128 * 1024; + private int tcpSendBuffer = 128 * 1024; - private int maxRequestSize = 1024 * 1024; + private int maxRequestSize = 1024 * 1024; - private Duration linger = Duration.ofMillis(0); + private Duration linger = Duration.ofMillis(0); - private Duration metricsSampleWindow = Duration.ofSeconds(30); + private Duration metricsSampleWindow = Duration.ofSeconds(30); - private int maxInflightRequestsPerConnection = 5; + private int maxInflightRequestsPerConnection = 5; - private boolean reportNodeMetricsEnabled = false; + private boolean reportNodeMetricsEnabled = false; - private boolean idempotenceEnabled = false; + private boolean idempotenceEnabled = false; - @Override - public Duration getMaxBlock() { - return maxBlock; - } + @Override + public Duration getMaxBlock() { + return maxBlock; + } - public void setMaxBlock(Duration maxBlock) { - this.maxBlock = maxBlock; - } + public void setMaxBlock(Duration maxBlock) { + this.maxBlock = maxBlock; + } - @Override - public Duration getMetadataMaxAge() { - return metadataMaxAge; - } + @Override + public Duration getMetadataMaxAge() { + return metadataMaxAge; + } - public void setMetadataMaxAge(Duration metadataMaxAge) { - this.metadataMaxAge = metadataMaxAge; - } + public void setMetadataMaxAge(Duration metadataMaxAge) { + this.metadataMaxAge = metadataMaxAge; + } - @Override - public String getCompressionCodec() { - return compressionCodec; - } + @Override + public String getCompressionCodec() { + return compressionCodec; + } - public void setCompressionCodec(String compressionCodec) { - this.compressionCodec = compressionCodec; - } + public void setCompressionCodec(String compressionCodec) { + this.compressionCodec = compressionCodec; + } - @Override - public int getRetries() { - return retries; - } + @Override + public int getRetries() { + return retries; + } - public void setRetries(int retries) { - this.retries = retries; - } + public void setRetries(int retries) { + this.retries = retries; + } - @Override - public Duration getRetryBackoff() { - return retryBackoff; - } + @Override + public Duration getRetryBackoff() { + return retryBackoff; + } - public void setRetryBackoff(Duration retryBackoff) { - this.retryBackoff = retryBackoff; - } + public void setRetryBackoff(Duration retryBackoff) { + this.retryBackoff = retryBackoff; + } - @Override - public Duration getRequestTimeout() { - return requestTimeout; - } + @Override + public Duration getRequestTimeout() { + return requestTimeout; + } - public void setRequestTimeout(Duration requestTimeout) { - this.requestTimeout = requestTimeout; - } + public void setRequestTimeout(Duration requestTimeout) { + this.requestTimeout = requestTimeout; + } - @Override - public int getBatchSize() { - return batchSize; - } + @Override + public int getBatchSize() { + return batchSize; + } - public void setBatchSize(int batchSize) { - this.batchSize = batchSize; - } + public void setBatchSize(int batchSize) { + this.batchSize = batchSize; + } - @Override - public int getTcpSendBuffer() { - return tcpSendBuffer; - } + @Override + public int getTcpSendBuffer() { + return tcpSendBuffer; + } - public void setTcpSendBuffer(int tcpSendBuffer) { - this.tcpSendBuffer = tcpSendBuffer; - } + public void setTcpSendBuffer(int tcpSendBuffer) { + this.tcpSendBuffer = tcpSendBuffer; + } - @Override - public int getMaxRequestSize() { - return maxRequestSize; - } + @Override + public int getMaxRequestSize() { + return maxRequestSize; + } - public void setMaxRequestSize(int maxRequestSize) { - this.maxRequestSize = maxRequestSize; - } + public void setMaxRequestSize(int maxRequestSize) { + this.maxRequestSize = maxRequestSize; + } - @Override - public Duration getLinger() { - return linger; - } + @Override + public Duration getLinger() { + return linger; + } - public void setLinger(Duration linger) { - this.linger = linger; - } - - @Override - public Duration getMetricsSampleWindow() { - return metricsSampleWindow; - } - - public void setMetricsSampleWindow(Duration metricsSampleWindow) { - this.metricsSampleWindow = metricsSampleWindow; - } - - @Override - public int getMaxInflightRequestsPerConnection() { - return maxInflightRequestsPerConnection; - } - - public void setMaxInflightRequestsPerConnection(int maxInflightRequestsPerConnection) { - this.maxInflightRequestsPerConnection = maxInflightRequestsPerConnection; - } - - @Override - public boolean isReportNodeMetricsEnabled() { - return reportNodeMetricsEnabled; - } - - public void setReportNodeMetricsEnabled(boolean reportNodeMetricsEnabled) { - this.reportNodeMetricsEnabled = reportNodeMetricsEnabled; - } - - @Override - public Duration getDeliveryTimeout() { - return deliveryTimeout; - } - - public void setDeliveryTimeout(Duration deliveryTimeout) { - this.deliveryTimeout = deliveryTimeout; - } - - public boolean isIdempotenceEnabled() { - return idempotenceEnabled; - } - - public void setIdempotenceEnabled(boolean idempotenceEnabled) { - this.idempotenceEnabled = idempotenceEnabled; - } + public void setLinger(Duration linger) { + this.linger = linger; + } + + @Override + public Duration getMetricsSampleWindow() { + return metricsSampleWindow; + } + + public void setMetricsSampleWindow(Duration metricsSampleWindow) { + this.metricsSampleWindow = metricsSampleWindow; + } + + @Override + public int getMaxInflightRequestsPerConnection() { + return maxInflightRequestsPerConnection; + } + + public void setMaxInflightRequestsPerConnection(int maxInflightRequestsPerConnection) { + this.maxInflightRequestsPerConnection = maxInflightRequestsPerConnection; + } + + @Override + public boolean isReportNodeMetricsEnabled() { + return reportNodeMetricsEnabled; + } + + public void setReportNodeMetricsEnabled(boolean reportNodeMetricsEnabled) { + this.reportNodeMetricsEnabled = reportNodeMetricsEnabled; + } + + @Override + public Duration getDeliveryTimeout() { + return deliveryTimeout; + } + + public void setDeliveryTimeout(Duration deliveryTimeout) { + this.deliveryTimeout = deliveryTimeout; + } + + public boolean isIdempotenceEnabled() { + return idempotenceEnabled; + } + + public void setIdempotenceEnabled(boolean idempotenceEnabled) { + this.idempotenceEnabled = idempotenceEnabled; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/KafkaProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/KafkaProperties.java index 57ab634b89..795a22ba5f 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/KafkaProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/KafkaProperties.java @@ -1,71 +1,69 @@ package pl.allegro.tech.hermes.frontend.config; +import java.time.Duration; import pl.allegro.tech.hermes.common.config.KafkaAuthenticationProperties; import pl.allegro.tech.hermes.common.kafka.KafkaParameters; -import java.time.Duration; - public class KafkaProperties implements KafkaParameters { - private KafkaAuthenticationProperties authentication = new KafkaAuthenticationProperties(); - - private String datacenter = "dc"; + private KafkaAuthenticationProperties authentication = new KafkaAuthenticationProperties(); - private String brokerList = "localhost:9092"; + private String datacenter = "dc"; - private Duration adminRequestTimeout = Duration.ofMinutes(5); + private String brokerList = "localhost:9092"; - @Deprecated - public void setAuthorization(KafkaAuthenticationProperties authorization) { - this.authentication = authorization; - } + private Duration adminRequestTimeout = Duration.ofMinutes(5); - public void setAuthentication(KafkaAuthenticationProperties authorization) { - this.authentication = authorization; - } + @Deprecated + public void setAuthorization(KafkaAuthenticationProperties authorization) { + this.authentication = authorization; + } - public String getDatacenter() { - return datacenter; - } + public void setAuthentication(KafkaAuthenticationProperties authorization) { + this.authentication = authorization; + } - public void setDatacenter(String datacenter) { - this.datacenter = datacenter; - } + public String getDatacenter() { + return datacenter; + } - @Override - public boolean isAuthenticationEnabled() { - return authentication.isEnabled(); - } + public void setDatacenter(String datacenter) { + this.datacenter = datacenter; + } - @Override - public String getAuthenticationMechanism() { - return authentication.getMechanism(); - } + @Override + public boolean isAuthenticationEnabled() { + return authentication.isEnabled(); + } - @Override - public String getAuthenticationProtocol() { - return authentication.getProtocol(); - } + @Override + public String getAuthenticationMechanism() { + return authentication.getMechanism(); + } - public String getBrokerList() { - return brokerList; - } + @Override + public String getAuthenticationProtocol() { + return authentication.getProtocol(); + } - public void setBrokerList(String brokerList) { - this.brokerList = brokerList; - } + public String getBrokerList() { + return brokerList; + } - public Duration getAdminRequestTimeout() { - return adminRequestTimeout; - } + public void setBrokerList(String brokerList) { + this.brokerList = brokerList; + } - public void setAdminRequestTimeout(Duration adminRequestTimeout) { - this.adminRequestTimeout = adminRequestTimeout; - } + public Duration getAdminRequestTimeout() { + return adminRequestTimeout; + } + public void setAdminRequestTimeout(Duration adminRequestTimeout) { + this.adminRequestTimeout = adminRequestTimeout; + } - @Override - public String getJaasConfig() { - return authentication.getJaasConfig(); - } + @Override + public String getJaasConfig() { + return authentication.getJaasConfig(); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/LocalMessageStorageProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/LocalMessageStorageProperties.java index 61c21cdd00..9186792bc9 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/LocalMessageStorageProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/LocalMessageStorageProperties.java @@ -1,133 +1,133 @@ package pl.allegro.tech.hermes.frontend.config; import com.google.common.io.Files; +import java.time.Duration; import org.springframework.boot.context.properties.ConfigurationProperties; import pl.allegro.tech.hermes.frontend.buffer.BackupMessagesLoaderParameters; import pl.allegro.tech.hermes.frontend.buffer.PersistentBufferExtensionParameters; -import java.time.Duration; - @ConfigurationProperties(prefix = "frontend.messages.local.storage") -public class LocalMessageStorageProperties implements BackupMessagesLoaderParameters, PersistentBufferExtensionParameters { +public class LocalMessageStorageProperties + implements BackupMessagesLoaderParameters, PersistentBufferExtensionParameters { - private long bufferedSizeBytes = 256 * 1024 * 1024L; + private long bufferedSizeBytes = 256 * 1024 * 1024L; - private boolean v2MigrationEnabled = true; + private boolean v2MigrationEnabled = true; - private boolean enabled = false; + private boolean enabled = false; - private String directory = Files.createTempDir().getAbsolutePath(); + private String directory = Files.createTempDir().getAbsolutePath(); - private String temporaryDirectory = Files.createTempDir().getAbsolutePath(); + private String temporaryDirectory = Files.createTempDir().getAbsolutePath(); - private int averageMessageSize = 600; + private int averageMessageSize = 600; - private Duration maxAge = Duration.ofHours(72); + private Duration maxAge = Duration.ofHours(72); - private int maxResendRetries = 5; + private int maxResendRetries = 5; - private Duration loadingPauseBetweenResend = Duration.ofMillis(30); + private Duration loadingPauseBetweenResend = Duration.ofMillis(30); - private Duration loadingWaitForBrokerTopicInfo = Duration.ofSeconds(5); + private Duration loadingWaitForBrokerTopicInfo = Duration.ofSeconds(5); - private boolean sizeReportingEnabled = true; + private boolean sizeReportingEnabled = true; - @Override - public long getBufferedSizeBytes() { - return bufferedSizeBytes; - } + @Override + public long getBufferedSizeBytes() { + return bufferedSizeBytes; + } - public void setBufferedSizeBytes(long bufferedSizeBytes) { - this.bufferedSizeBytes = bufferedSizeBytes; - } + public void setBufferedSizeBytes(long bufferedSizeBytes) { + this.bufferedSizeBytes = bufferedSizeBytes; + } - @Override - public boolean isV2MigrationEnabled() { - return v2MigrationEnabled; - } + @Override + public boolean isV2MigrationEnabled() { + return v2MigrationEnabled; + } - public void setV2MigrationEnabled(boolean v2MigrationEnabled) { - this.v2MigrationEnabled = v2MigrationEnabled; - } + public void setV2MigrationEnabled(boolean v2MigrationEnabled) { + this.v2MigrationEnabled = v2MigrationEnabled; + } - @Override - public boolean isEnabled() { - return enabled; - } + @Override + public boolean isEnabled() { + return enabled; + } - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } - @Override - public String getDirectory() { - return directory; - } + @Override + public String getDirectory() { + return directory; + } - public void setDirectory(String directory) { - this.directory = directory; - } + public void setDirectory(String directory) { + this.directory = directory; + } - @Override - public String getTemporaryDirectory() { - return temporaryDirectory; - } + @Override + public String getTemporaryDirectory() { + return temporaryDirectory; + } - public void setTemporaryDirectory(String temporaryDirectory) { - this.temporaryDirectory = temporaryDirectory; - } + public void setTemporaryDirectory(String temporaryDirectory) { + this.temporaryDirectory = temporaryDirectory; + } - @Override - public int getAverageMessageSize() { - return averageMessageSize; - } + @Override + public int getAverageMessageSize() { + return averageMessageSize; + } - public void setAverageMessageSize(int averageMessageSize) { - this.averageMessageSize = averageMessageSize; - } + public void setAverageMessageSize(int averageMessageSize) { + this.averageMessageSize = averageMessageSize; + } - @Override - public Duration getMaxAge() { - return maxAge; - } + @Override + public Duration getMaxAge() { + return maxAge; + } - public void setMaxAge(Duration maxAge) { - this.maxAge = maxAge; - } + public void setMaxAge(Duration maxAge) { + this.maxAge = maxAge; + } - @Override - public int getMaxResendRetries() { - return maxResendRetries; - } + @Override + public int getMaxResendRetries() { + return maxResendRetries; + } - public void setMaxResendRetries(int maxResendRetries) { - this.maxResendRetries = maxResendRetries; - } + public void setMaxResendRetries(int maxResendRetries) { + this.maxResendRetries = maxResendRetries; + } - @Override - public Duration getLoadingPauseBetweenResend() { - return loadingPauseBetweenResend; - } + @Override + public Duration getLoadingPauseBetweenResend() { + return loadingPauseBetweenResend; + } - public void setLoadingPauseBetweenResend(Duration loadingPauseBetweenResend) { - this.loadingPauseBetweenResend = loadingPauseBetweenResend; - } + public void setLoadingPauseBetweenResend(Duration loadingPauseBetweenResend) { + this.loadingPauseBetweenResend = loadingPauseBetweenResend; + } - @Override - public Duration getLoadingWaitForBrokerTopicInfo() { - return loadingWaitForBrokerTopicInfo; - } + @Override + public Duration getLoadingWaitForBrokerTopicInfo() { + return loadingWaitForBrokerTopicInfo; + } - public void setLoadingWaitForBrokerTopicInfo(Duration loadingWaitForBrokerTopicInfo) { - this.loadingWaitForBrokerTopicInfo = loadingWaitForBrokerTopicInfo; - } + public void setLoadingWaitForBrokerTopicInfo(Duration loadingWaitForBrokerTopicInfo) { + this.loadingWaitForBrokerTopicInfo = loadingWaitForBrokerTopicInfo; + } - @Override - public boolean isSizeReportingEnabled() { - return sizeReportingEnabled; - } - - public void setSizeReportingEnabled(boolean sizeReportingEnabled) { - this.sizeReportingEnabled = sizeReportingEnabled; - } + @Override + public boolean isSizeReportingEnabled() { + return sizeReportingEnabled; + } + + public void setSizeReportingEnabled(boolean sizeReportingEnabled) { + this.sizeReportingEnabled = sizeReportingEnabled; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/MessagePreviewProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/MessagePreviewProperties.java index c7f2752207..9e6ae5155a 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/MessagePreviewProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/MessagePreviewProperties.java @@ -1,49 +1,48 @@ package pl.allegro.tech.hermes.frontend.config; -import org.springframework.boot.context.properties.ConfigurationProperties; - import java.time.Duration; +import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties(prefix = "frontend.message.preview") public class MessagePreviewProperties { - private boolean enabled = false; + private boolean enabled = false; - private int maxSizeKb = 10; + private int maxSizeKb = 10; - private int size = 3; + private int size = 3; - private Duration logPersistPeriod = Duration.ofSeconds(30); + private Duration logPersistPeriod = Duration.ofSeconds(30); - public boolean isEnabled() { - return enabled; - } + public boolean isEnabled() { + return enabled; + } - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } - public int getMaxSizeKb() { - return maxSizeKb; - } + public int getMaxSizeKb() { + return maxSizeKb; + } - public void setMaxSizeKb(int maxSizeKb) { - this.maxSizeKb = maxSizeKb; - } + public void setMaxSizeKb(int maxSizeKb) { + this.maxSizeKb = maxSizeKb; + } - public int getSize() { - return size; - } + public int getSize() { + return size; + } - public void setSize(int size) { - this.size = size; - } + public void setSize(int size) { + this.size = size; + } - public Duration getLogPersistPeriod() { - return logPersistPeriod; - } + public Duration getLogPersistPeriod() { + return logPersistPeriod; + } - public void setLogPersistPeriod(Duration logPersistPeriod) { - this.logPersistPeriod = logPersistPeriod; - } + public void setLogPersistPeriod(Duration logPersistPeriod) { + this.logPersistPeriod = logPersistPeriod; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/MetricRegistryProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/MetricRegistryProperties.java index 3e5e71d80f..0f2a91d5e4 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/MetricRegistryProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/MetricRegistryProperties.java @@ -1,19 +1,18 @@ package pl.allegro.tech.hermes.frontend.config; -import org.springframework.boot.context.properties.ConfigurationProperties; - import java.time.Duration; +import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties(prefix = "frontend.metrics.metric-registry") public class MetricRegistryProperties { - private Duration counterExpireAfterAccess = Duration.ofHours(72); + private Duration counterExpireAfterAccess = Duration.ofHours(72); - public Duration getCounterExpireAfterAccess() { - return counterExpireAfterAccess; - } + public Duration getCounterExpireAfterAccess() { + return counterExpireAfterAccess; + } - public void setCounterExpireAfterAccess(Duration counterExpireAfterAccess) { - this.counterExpireAfterAccess = counterExpireAfterAccess; - } + public void setCounterExpireAfterAccess(Duration counterExpireAfterAccess) { + this.counterExpireAfterAccess = counterExpireAfterAccess; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/MicrometerRegistryProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/MicrometerRegistryProperties.java index 8a99c812bf..9aae6c401d 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/MicrometerRegistryProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/MicrometerRegistryProperties.java @@ -1,51 +1,49 @@ package pl.allegro.tech.hermes.frontend.config; -import org.springframework.boot.context.properties.ConfigurationProperties; -import pl.allegro.tech.hermes.common.di.factories.MicrometerRegistryParameters; - import java.time.Duration; import java.util.List; - +import org.springframework.boot.context.properties.ConfigurationProperties; +import pl.allegro.tech.hermes.common.di.factories.MicrometerRegistryParameters; @ConfigurationProperties(prefix = "frontend.metrics.micrometer") public class MicrometerRegistryProperties implements MicrometerRegistryParameters { - private List percentiles = List.of(0.5, 0.99, 0.999); - private boolean zookeeperReporterEnabled = true; - private Duration reportPeriod = Duration.ofSeconds(20); + private List percentiles = List.of(0.5, 0.99, 0.999); + private boolean zookeeperReporterEnabled = true; + private Duration reportPeriod = Duration.ofSeconds(20); - @Override - public List getPercentiles() { - return percentiles; - } + @Override + public List getPercentiles() { + return percentiles; + } - @Override - public boolean zookeeperReporterEnabled() { - return zookeeperReporterEnabled; - } + @Override + public boolean zookeeperReporterEnabled() { + return zookeeperReporterEnabled; + } - @Override - public Duration zookeeperReportPeriod() { - return reportPeriod; - } + @Override + public Duration zookeeperReportPeriod() { + return reportPeriod; + } - public void setPercentiles(List percentiles) { - this.percentiles = percentiles; - } + public void setPercentiles(List percentiles) { + this.percentiles = percentiles; + } - public boolean isZookeeperReporterEnabled() { - return zookeeperReporterEnabled; - } + public boolean isZookeeperReporterEnabled() { + return zookeeperReporterEnabled; + } - public void setZookeeperReporterEnabled(boolean zookeeperReporterEnabled) { - this.zookeeperReporterEnabled = zookeeperReporterEnabled; - } + public void setZookeeperReporterEnabled(boolean zookeeperReporterEnabled) { + this.zookeeperReporterEnabled = zookeeperReporterEnabled; + } - public Duration getReportPeriod() { - return reportPeriod; - } + public Duration getReportPeriod() { + return reportPeriod; + } - public void setReportPeriod(Duration reportPeriod) { - this.reportPeriod = reportPeriod; - } + public void setReportPeriod(Duration reportPeriod) { + this.reportPeriod = reportPeriod; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/PrometheusConfigAdapter.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/PrometheusConfigAdapter.java index 71478a9da8..cc57060c15 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/PrometheusConfigAdapter.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/PrometheusConfigAdapter.java @@ -1,32 +1,29 @@ package pl.allegro.tech.hermes.frontend.config; import io.micrometer.prometheus.PrometheusConfig; - import java.time.Duration; public class PrometheusConfigAdapter implements PrometheusConfig { - private final PrometheusProperties prometheusReporterProperties; + private final PrometheusProperties prometheusReporterProperties; - public PrometheusConfigAdapter(PrometheusProperties prometheusReporterProperties) { - this.prometheusReporterProperties = prometheusReporterProperties; - } + public PrometheusConfigAdapter(PrometheusProperties prometheusReporterProperties) { + this.prometheusReporterProperties = prometheusReporterProperties; + } - @Override - public boolean descriptions() { - return prometheusReporterProperties.isDescriptions(); - } + @Override + public boolean descriptions() { + return prometheusReporterProperties.isDescriptions(); + } - @Override - public Duration step() { - return prometheusReporterProperties.getStep(); - } + @Override + public Duration step() { + return prometheusReporterProperties.getStep(); + } - /** - * Returning null is fine since we override all the methods from PrometheusConfig. - */ - @Override - public String get(String key) { - return null; // Nothing to see here, move along. - } + /** Returning null is fine since we override all the methods from PrometheusConfig. */ + @Override + public String get(String key) { + return null; // Nothing to see here, move along. + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/PrometheusProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/PrometheusProperties.java index f7b3edf584..b220bea282 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/PrometheusProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/PrometheusProperties.java @@ -1,28 +1,27 @@ package pl.allegro.tech.hermes.frontend.config; -import org.springframework.boot.context.properties.ConfigurationProperties; - import java.time.Duration; +import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties(prefix = "frontend.prometheus") public class PrometheusProperties { - private Duration step = Duration.ofMinutes(1); - private boolean descriptions = true; + private Duration step = Duration.ofMinutes(1); + private boolean descriptions = true; - public Duration getStep() { - return this.step; - } + public Duration getStep() { + return this.step; + } - public void setStep(Duration step) { - this.step = step; - } + public void setStep(Duration step) { + this.step = step; + } - public boolean isDescriptions() { - return this.descriptions; - } + public boolean isDescriptions() { + return this.descriptions; + } - public void setDescriptions(boolean descriptions) { - this.descriptions = descriptions; - } + public void setDescriptions(boolean descriptions) { + this.descriptions = descriptions; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/ReadinessCheckProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/ReadinessCheckProperties.java index dce2d5be68..c0775a6356 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/ReadinessCheckProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/ReadinessCheckProperties.java @@ -1,39 +1,38 @@ package pl.allegro.tech.hermes.frontend.config; -import org.springframework.boot.context.properties.ConfigurationProperties; - import java.time.Duration; +import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties(prefix = "frontend.readiness.check") public class ReadinessCheckProperties { - private boolean enabled = false; + private boolean enabled = false; - private boolean kafkaCheckEnabled = false; + private boolean kafkaCheckEnabled = false; - private Duration interval = Duration.ofSeconds(1); + private Duration interval = Duration.ofSeconds(1); - public boolean isEnabled() { - return enabled; - } + public boolean isEnabled() { + return enabled; + } - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } - public Duration getInterval() { - return interval; - } + public Duration getInterval() { + return interval; + } - public void setIntervalSeconds(Duration intervalSeconds) { - this.interval = intervalSeconds; - } + public void setIntervalSeconds(Duration intervalSeconds) { + this.interval = intervalSeconds; + } - public boolean isKafkaCheckEnabled() { - return kafkaCheckEnabled; - } + public boolean isKafkaCheckEnabled() { + return kafkaCheckEnabled; + } - public void setKafkaCheckEnabled(boolean kafkaCheckEnabled) { - this.kafkaCheckEnabled = kafkaCheckEnabled; - } + public void setKafkaCheckEnabled(boolean kafkaCheckEnabled) { + this.kafkaCheckEnabled = kafkaCheckEnabled; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/ReadinessConfiguration.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/ReadinessConfiguration.java index de4e563123..c581189e3f 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/ReadinessConfiguration.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/ReadinessConfiguration.java @@ -17,30 +17,32 @@ @EnableConfigurationProperties({ReadinessCheckProperties.class}) public class ReadinessConfiguration { - @Bean - public DefaultReadinessChecker readinessChecker(ReadinessCheckProperties readinessCheckProperties, - @Named("localDatacenterBrokerProducer") BrokerTopicAvailabilityChecker brokerTopicAvailabilityChecker, - AdminReadinessService adminReadinessService) { - return new DefaultReadinessChecker( - brokerTopicAvailabilityChecker, - adminReadinessService, - readinessCheckProperties.isEnabled(), - readinessCheckProperties.isKafkaCheckEnabled(), - readinessCheckProperties.getInterval() - ); - } + @Bean + public DefaultReadinessChecker readinessChecker( + ReadinessCheckProperties readinessCheckProperties, + @Named("localDatacenterBrokerProducer") + BrokerTopicAvailabilityChecker brokerTopicAvailabilityChecker, + AdminReadinessService adminReadinessService) { + return new DefaultReadinessChecker( + brokerTopicAvailabilityChecker, + adminReadinessService, + readinessCheckProperties.isEnabled(), + readinessCheckProperties.isKafkaCheckEnabled(), + readinessCheckProperties.getInterval()); + } - @Bean(initMethod = "start", destroyMethod = "stop") - public AdminReadinessService adminReadinessService(ObjectMapper mapper, - CuratorFramework zookeeper, - ZookeeperPaths paths, - DatacenterNameProvider datacenterNameProvider) { - String localDatacenterName = datacenterNameProvider.getDatacenterName(); - return new AdminReadinessService(mapper, zookeeper, paths, localDatacenterName); - } + @Bean(initMethod = "start", destroyMethod = "stop") + public AdminReadinessService adminReadinessService( + ObjectMapper mapper, + CuratorFramework zookeeper, + ZookeeperPaths paths, + DatacenterNameProvider datacenterNameProvider) { + String localDatacenterName = datacenterNameProvider.getDatacenterName(); + return new AdminReadinessService(mapper, zookeeper, paths, localDatacenterName); + } - @Bean(initMethod = "startup") - public HealthCheckService healthCheckService() { - return new HealthCheckService(); - } + @Bean(initMethod = "startup") + public HealthCheckService healthCheckService() { + return new HealthCheckService(); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/SchemaConfiguration.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/SchemaConfiguration.java index 59f6dd5163..d32e7211d2 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/SchemaConfiguration.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/SchemaConfiguration.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.jakarta.rs.json.JacksonJsonProvider; import jakarta.ws.rs.client.Client; import jakarta.ws.rs.client.ClientBuilder; +import java.net.URI; import org.apache.avro.Schema; import org.glassfish.jersey.client.ClientConfig; import org.glassfish.jersey.client.ClientProperties; @@ -23,69 +24,80 @@ import pl.allegro.tech.hermes.schema.resolver.DefaultSchemaRepositoryInstanceResolver; import pl.allegro.tech.hermes.schema.resolver.SchemaRepositoryInstanceResolver; -import java.net.URI; - @Configuration -@EnableConfigurationProperties({ - SchemaProperties.class, - KafkaClustersProperties.class -}) +@EnableConfigurationProperties({SchemaProperties.class, KafkaClustersProperties.class}) public class SchemaConfiguration { - @Bean - public SchemaRepository schemaRepository(SchemaVersionsRepository schemaVersionsRepository, - CompiledSchemaRepository compiledAvroSchemaRepository) { - return new SchemaRepositoryFactory(schemaVersionsRepository, compiledAvroSchemaRepository).provide(); - } + @Bean + public SchemaRepository schemaRepository( + SchemaVersionsRepository schemaVersionsRepository, + CompiledSchemaRepository compiledAvroSchemaRepository) { + return new SchemaRepositoryFactory(schemaVersionsRepository, compiledAvroSchemaRepository) + .provide(); + } - @Bean - public CompiledSchemaRepository avroCompiledSchemaRepository(RawSchemaClient rawSchemaClient, - SchemaProperties schemaProperties) { - return new AvroCompiledSchemaRepositoryFactory( - rawSchemaClient, schemaProperties.getCache().getCompiledMaximumSize(), - schemaProperties.getCache().getCompiledExpireAfterAccess(), schemaProperties.getCache().isEnabled() - ).provide(); - } + @Bean + public CompiledSchemaRepository avroCompiledSchemaRepository( + RawSchemaClient rawSchemaClient, SchemaProperties schemaProperties) { + return new AvroCompiledSchemaRepositoryFactory( + rawSchemaClient, + schemaProperties.getCache().getCompiledMaximumSize(), + schemaProperties.getCache().getCompiledExpireAfterAccess(), + schemaProperties.getCache().isEnabled()) + .provide(); + } - @Bean - public RawSchemaClient rawSchemaClient(KafkaClustersProperties kafkaClustersProperties, - MetricsFacade metricsFacade, - ObjectMapper objectMapper, - SchemaRepositoryInstanceResolver resolver, - SchemaProperties schemaProperties) { - return new RawSchemaClientFactory( - kafkaClustersProperties.getNamespace(), - kafkaClustersProperties.getNamespaceSeparator(), - metricsFacade, - objectMapper, - resolver, - schemaProperties.getRepository().isSubjectSuffixEnabled(), - schemaProperties.getRepository().isSubjectNamespaceEnabled() - ).provide(); - } + @Bean + public RawSchemaClient rawSchemaClient( + KafkaClustersProperties kafkaClustersProperties, + MetricsFacade metricsFacade, + ObjectMapper objectMapper, + SchemaRepositoryInstanceResolver resolver, + SchemaProperties schemaProperties) { + return new RawSchemaClientFactory( + kafkaClustersProperties.getNamespace(), + kafkaClustersProperties.getNamespaceSeparator(), + metricsFacade, + objectMapper, + resolver, + schemaProperties.getRepository().isSubjectSuffixEnabled(), + schemaProperties.getRepository().isSubjectNamespaceEnabled()) + .provide(); + } - @Bean - public SchemaRepositoryInstanceResolver schemaRepositoryInstanceResolver(SchemaProperties schemaProperties, Client client) { - URI schemaRepositoryServerUri = URI.create(schemaProperties.getRepository().getServerUrl()); - return new DefaultSchemaRepositoryInstanceResolver(client, schemaRepositoryServerUri); - } + @Bean + public SchemaRepositoryInstanceResolver schemaRepositoryInstanceResolver( + SchemaProperties schemaProperties, Client client) { + URI schemaRepositoryServerUri = URI.create(schemaProperties.getRepository().getServerUrl()); + return new DefaultSchemaRepositoryInstanceResolver(client, schemaRepositoryServerUri); + } - @Bean - public Client schemaRepositoryClient(ObjectMapper mapper, SchemaProperties schemaProperties) { - ClientConfig config = new ClientConfig() - .property(ClientProperties.READ_TIMEOUT, (int) schemaProperties.getRepository().getHttpReadTimeout().toMillis()) - .property(ClientProperties.CONNECT_TIMEOUT, (int) schemaProperties.getRepository().getHttpConnectTimeout().toMillis()) - .register(new JacksonJsonProvider(mapper)); + @Bean + public Client schemaRepositoryClient(ObjectMapper mapper, SchemaProperties schemaProperties) { + ClientConfig config = + new ClientConfig() + .property( + ClientProperties.READ_TIMEOUT, + (int) schemaProperties.getRepository().getHttpReadTimeout().toMillis()) + .property( + ClientProperties.CONNECT_TIMEOUT, + (int) schemaProperties.getRepository().getHttpConnectTimeout().toMillis()) + .register(new JacksonJsonProvider(mapper)); - return ClientBuilder.newClient(config); - } + return ClientBuilder.newClient(config); + } - @Bean - public SchemaVersionsRepository schemaVersionsRepositoryFactory(RawSchemaClient rawSchemaClient, - SchemaProperties schemaProperties, - InternalNotificationsBus notificationsBus, - CompiledSchemaRepository compiledSchemaRepository) { - return new SchemaVersionsRepositoryFactory(rawSchemaClient, schemaProperties.getCache(), notificationsBus, compiledSchemaRepository) - .provide(); - } + @Bean + public SchemaVersionsRepository schemaVersionsRepositoryFactory( + RawSchemaClient rawSchemaClient, + SchemaProperties schemaProperties, + InternalNotificationsBus notificationsBus, + CompiledSchemaRepository compiledSchemaRepository) { + return new SchemaVersionsRepositoryFactory( + rawSchemaClient, + schemaProperties.getCache(), + notificationsBus, + compiledSchemaRepository) + .provide(); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/SchemaProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/SchemaProperties.java index 3c71e2ce53..a022fdce49 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/SchemaProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/SchemaProperties.java @@ -7,53 +7,53 @@ @ConfigurationProperties(prefix = "frontend.schema") public class SchemaProperties { - private SchemaCacheProperties cache = new SchemaCacheProperties(); + private SchemaCacheProperties cache = new SchemaCacheProperties(); - private SchemaRepositoryProperties repository = new SchemaRepositoryProperties(); + private SchemaRepositoryProperties repository = new SchemaRepositoryProperties(); - private boolean idHeaderEnabled = false; + private boolean idHeaderEnabled = false; - private boolean idSerializationEnabled = false; + private boolean idSerializationEnabled = false; - private boolean versionTruncationEnabled = false; + private boolean versionTruncationEnabled = false; - public SchemaCacheProperties getCache() { - return cache; - } + public SchemaCacheProperties getCache() { + return cache; + } - public void setCache(SchemaCacheProperties cache) { - this.cache = cache; - } + public void setCache(SchemaCacheProperties cache) { + this.cache = cache; + } - public SchemaRepositoryProperties getRepository() { - return repository; - } + public SchemaRepositoryProperties getRepository() { + return repository; + } - public void setRepository(SchemaRepositoryProperties repository) { - this.repository = repository; - } + public void setRepository(SchemaRepositoryProperties repository) { + this.repository = repository; + } - public boolean isIdHeaderEnabled() { - return idHeaderEnabled; - } + public boolean isIdHeaderEnabled() { + return idHeaderEnabled; + } - public void setIdHeaderEnabled(boolean idHeaderEnabled) { - this.idHeaderEnabled = idHeaderEnabled; - } + public void setIdHeaderEnabled(boolean idHeaderEnabled) { + this.idHeaderEnabled = idHeaderEnabled; + } - public boolean isIdSerializationEnabled() { - return idSerializationEnabled; - } + public boolean isIdSerializationEnabled() { + return idSerializationEnabled; + } - public void setIdSerializationEnabled(boolean idSerializationEnabled) { - this.idSerializationEnabled = idSerializationEnabled; - } + public void setIdSerializationEnabled(boolean idSerializationEnabled) { + this.idSerializationEnabled = idSerializationEnabled; + } - public boolean isVersionTruncationEnabled() { - return versionTruncationEnabled; - } + public boolean isVersionTruncationEnabled() { + return versionTruncationEnabled; + } - public void setVersionTruncationEnabled(boolean versionTruncationEnabled) { - this.versionTruncationEnabled = versionTruncationEnabled; - } + public void setVersionTruncationEnabled(boolean versionTruncationEnabled) { + this.versionTruncationEnabled = versionTruncationEnabled; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/SslProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/SslProperties.java index 925a507eea..2f449d68de 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/SslProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/SslProperties.java @@ -6,136 +6,135 @@ @ConfigurationProperties(prefix = "frontend.ssl") public class SslProperties implements SslParameters { - private boolean enabled = false; + private boolean enabled = false; - private int port = 8443; + private int port = 8443; - private String clientAuthMode = "not_requested"; + private String clientAuthMode = "not_requested"; - private String protocol = "TLS"; + private String protocol = "TLS"; - private String keystoreSource = "jre"; + private String keystoreSource = "jre"; - private String keystoreLocation = "classpath:server.keystore"; + private String keystoreLocation = "classpath:server.keystore"; - private String keystorePassword = "password"; + private String keystorePassword = "password"; - private String keystoreFormat = "JKS"; + private String keystoreFormat = "JKS"; - private String truststoreSource = "jre"; + private String truststoreSource = "jre"; - private String truststoreLocation = "classpath:server.truststore"; + private String truststoreLocation = "classpath:server.truststore"; - private String truststorePassword = "password"; + private String truststorePassword = "password"; - private String truststoreFormat = "JKS"; + private String truststoreFormat = "JKS"; - @Override - public boolean isEnabled() { - return enabled; - } + @Override + public boolean isEnabled() { + return enabled; + } - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } - @Override - public int getPort() { - return port; - } + @Override + public int getPort() { + return port; + } - public void setPort(int port) { - this.port = port; - } + public void setPort(int port) { + this.port = port; + } - @Override - public String getClientAuthMode() { - return clientAuthMode; - } + @Override + public String getClientAuthMode() { + return clientAuthMode; + } - public void setClientAuthMode(String clientAuthMode) { - this.clientAuthMode = clientAuthMode; - } + public void setClientAuthMode(String clientAuthMode) { + this.clientAuthMode = clientAuthMode; + } - @Override - public String getProtocol() { - return protocol; - } + @Override + public String getProtocol() { + return protocol; + } - public void setProtocol(String protocol) { - this.protocol = protocol; - } + public void setProtocol(String protocol) { + this.protocol = protocol; + } - @Override - public String getKeystoreSource() { - return keystoreSource; - } + @Override + public String getKeystoreSource() { + return keystoreSource; + } - public void setKeystoreSource(String keystoreSource) { - this.keystoreSource = keystoreSource; - } + public void setKeystoreSource(String keystoreSource) { + this.keystoreSource = keystoreSource; + } - @Override - public String getKeystoreLocation() { - return keystoreLocation; - } + @Override + public String getKeystoreLocation() { + return keystoreLocation; + } - public void setKeystoreLocation(String keystoreLocation) { - this.keystoreLocation = keystoreLocation; - } + public void setKeystoreLocation(String keystoreLocation) { + this.keystoreLocation = keystoreLocation; + } - @Override - public String getKeystorePassword() { - return keystorePassword; - } + @Override + public String getKeystorePassword() { + return keystorePassword; + } - public void setKeystorePassword(String keystorePassword) { - this.keystorePassword = keystorePassword; - } + public void setKeystorePassword(String keystorePassword) { + this.keystorePassword = keystorePassword; + } - @Override - public String getKeystoreFormat() { - return keystoreFormat; - } + @Override + public String getKeystoreFormat() { + return keystoreFormat; + } - public void setKeystoreFormat(String keystoreFormat) { - this.keystoreFormat = keystoreFormat; - } + public void setKeystoreFormat(String keystoreFormat) { + this.keystoreFormat = keystoreFormat; + } - @Override - public String getTruststoreSource() { - return truststoreSource; - } + @Override + public String getTruststoreSource() { + return truststoreSource; + } - public void setTruststoreSource(String truststoreSource) { - this.truststoreSource = truststoreSource; - } + public void setTruststoreSource(String truststoreSource) { + this.truststoreSource = truststoreSource; + } - @Override - public String getTruststoreLocation() { - return truststoreLocation; - } + @Override + public String getTruststoreLocation() { + return truststoreLocation; + } - public void setTruststoreLocation(String truststoreLocation) { - this.truststoreLocation = truststoreLocation; - } + public void setTruststoreLocation(String truststoreLocation) { + this.truststoreLocation = truststoreLocation; + } - @Override - public String getTruststorePassword() { - return truststorePassword; - } - - public void setTruststorePassword(String truststorePassword) { - this.truststorePassword = truststorePassword; - } - - @Override - public String getTruststoreFormat() { - return truststoreFormat; - } - - public void setTruststoreFormat(String truststoreFormat) { - this.truststoreFormat = truststoreFormat; - } + @Override + public String getTruststorePassword() { + return truststorePassword; + } + + public void setTruststorePassword(String truststorePassword) { + this.truststorePassword = truststorePassword; + } + + @Override + public String getTruststoreFormat() { + return truststoreFormat; + } + + public void setTruststoreFormat(String truststoreFormat) { + this.truststoreFormat = truststoreFormat; + } } - diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/ThroughputProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/ThroughputProperties.java index 6d7387d5e5..ce76846ba9 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/ThroughputProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/ThroughputProperties.java @@ -1,87 +1,86 @@ package pl.allegro.tech.hermes.frontend.config; +import java.time.Duration; import org.springframework.boot.context.properties.ConfigurationProperties; import pl.allegro.tech.hermes.frontend.publishing.handlers.ThroughputParameters; -import java.time.Duration; - @ConfigurationProperties(prefix = "frontend.throughput") public class ThroughputProperties implements ThroughputParameters { - private String type = "unlimited"; + private String type = "unlimited"; - private long fixedMax = Long.MAX_VALUE; + private long fixedMax = Long.MAX_VALUE; - private long dynamicMax = Long.MAX_VALUE; + private long dynamicMax = Long.MAX_VALUE; - private long dynamicThreshold = Long.MAX_VALUE; + private long dynamicThreshold = Long.MAX_VALUE; - private long dynamicDesired = Long.MAX_VALUE; + private long dynamicDesired = Long.MAX_VALUE; - private double dynamicIdle = 0.5; + private double dynamicIdle = 0.5; - private Duration dynamicCheckInterval = Duration.ofSeconds(30); + private Duration dynamicCheckInterval = Duration.ofSeconds(30); - @Override - public String getType() { - return type; - } + @Override + public String getType() { + return type; + } - public void setType(String type) { - this.type = type; - } + public void setType(String type) { + this.type = type; + } - @Override - public long getFixedMax() { - return fixedMax; - } + @Override + public long getFixedMax() { + return fixedMax; + } - public void setFixedMax(long fixedMax) { - this.fixedMax = fixedMax; - } + public void setFixedMax(long fixedMax) { + this.fixedMax = fixedMax; + } - @Override - public long getDynamicMax() { - return dynamicMax; - } + @Override + public long getDynamicMax() { + return dynamicMax; + } - public void setDynamicMax(long dynamicMax) { - this.dynamicMax = dynamicMax; - } + public void setDynamicMax(long dynamicMax) { + this.dynamicMax = dynamicMax; + } - @Override - public long getDynamicThreshold() { - return dynamicThreshold; - } + @Override + public long getDynamicThreshold() { + return dynamicThreshold; + } - public void setDynamicThreshold(long dynamicThreshold) { - this.dynamicThreshold = dynamicThreshold; - } + public void setDynamicThreshold(long dynamicThreshold) { + this.dynamicThreshold = dynamicThreshold; + } - @Override - public long getDynamicDesired() { - return dynamicDesired; - } + @Override + public long getDynamicDesired() { + return dynamicDesired; + } - public void setDynamicDesired(long dynamicDesired) { - this.dynamicDesired = dynamicDesired; - } + public void setDynamicDesired(long dynamicDesired) { + this.dynamicDesired = dynamicDesired; + } - @Override - public double getDynamicIdle() { - return dynamicIdle; - } + @Override + public double getDynamicIdle() { + return dynamicIdle; + } - public void setDynamicIdle(double dynamicIdle) { - this.dynamicIdle = dynamicIdle; - } + public void setDynamicIdle(double dynamicIdle) { + this.dynamicIdle = dynamicIdle; + } - @Override - public Duration getDynamicCheckInterval() { - return dynamicCheckInterval; - } + @Override + public Duration getDynamicCheckInterval() { + return dynamicCheckInterval; + } - public void setDynamicCheckInterval(Duration dynamicCheckInterval) { - this.dynamicCheckInterval = dynamicCheckInterval; - } + public void setDynamicCheckInterval(Duration dynamicCheckInterval) { + this.dynamicCheckInterval = dynamicCheckInterval; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/TopicDefaultsProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/TopicDefaultsProperties.java index ac3692840a..cd7e9bd5ad 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/TopicDefaultsProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/TopicDefaultsProperties.java @@ -4,13 +4,13 @@ @ConfigurationProperties(prefix = "frontend.topic.defaults") public class TopicDefaultsProperties { - private boolean fallbackToRemoteDatacenterEnabled = false; + private boolean fallbackToRemoteDatacenterEnabled = false; - public boolean isFallbackToRemoteDatacenterEnabled() { - return fallbackToRemoteDatacenterEnabled; - } + public boolean isFallbackToRemoteDatacenterEnabled() { + return fallbackToRemoteDatacenterEnabled; + } - public void setFallbackToRemoteDatacenterEnabled(boolean fallbackToRemoteDatacenterEnabled) { - this.fallbackToRemoteDatacenterEnabled = fallbackToRemoteDatacenterEnabled; - } + public void setFallbackToRemoteDatacenterEnabled(boolean fallbackToRemoteDatacenterEnabled) { + this.fallbackToRemoteDatacenterEnabled = fallbackToRemoteDatacenterEnabled; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/TopicLoadingProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/TopicLoadingProperties.java index 87b1acd1a5..c009d5ee74 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/TopicLoadingProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/TopicLoadingProperties.java @@ -1,128 +1,127 @@ package pl.allegro.tech.hermes.frontend.config; -import org.springframework.boot.context.properties.ConfigurationProperties; - import java.time.Duration; +import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties(prefix = "frontend.startup.topic.loading") public class TopicLoadingProperties { - private MetadataLoadingProperties metadata = new MetadataLoadingProperties(); - - private SchemaLoadingProperties schema = new SchemaLoadingProperties(); - - private MetadataRefreshJobProperties metadataRefreshJob = new MetadataRefreshJobProperties(); - - public static class MetadataLoadingProperties { + private MetadataLoadingProperties metadata = new MetadataLoadingProperties(); - private Duration retryInterval = Duration.ofSeconds(1); + private SchemaLoadingProperties schema = new SchemaLoadingProperties(); - private int retryCount = 5; + private MetadataRefreshJobProperties metadataRefreshJob = new MetadataRefreshJobProperties(); - private int threadPoolSize = 16; + public static class MetadataLoadingProperties { - public Duration getRetryInterval() { - return retryInterval; - } + private Duration retryInterval = Duration.ofSeconds(1); - public void setRetryInterval(Duration retryInterval) { - this.retryInterval = retryInterval; - } + private int retryCount = 5; - public int getRetryCount() { - return retryCount; - } + private int threadPoolSize = 16; - public void setRetryCount(int retryCount) { - this.retryCount = retryCount; - } + public Duration getRetryInterval() { + return retryInterval; + } - public int getThreadPoolSize() { - return threadPoolSize; - } + public void setRetryInterval(Duration retryInterval) { + this.retryInterval = retryInterval; + } - public void setThreadPoolSize(int threadPoolSize) { - this.threadPoolSize = threadPoolSize; - } + public int getRetryCount() { + return retryCount; } - public static class SchemaLoadingProperties { + public void setRetryCount(int retryCount) { + this.retryCount = retryCount; + } - private boolean enabled = false; + public int getThreadPoolSize() { + return threadPoolSize; + } - private int retryCount = 3; + public void setThreadPoolSize(int threadPoolSize) { + this.threadPoolSize = threadPoolSize; + } + } - private int threadPoolSize = 16; + public static class SchemaLoadingProperties { - public boolean isEnabled() { - return enabled; - } + private boolean enabled = false; - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } + private int retryCount = 3; - public int getRetryCount() { - return retryCount; - } + private int threadPoolSize = 16; - public void setRetryCount(int retryCount) { - this.retryCount = retryCount; - } + public boolean isEnabled() { + return enabled; + } - public int getThreadPoolSize() { - return threadPoolSize; - } + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } - public void setThreadPoolSize(int threadPoolSize) { - this.threadPoolSize = threadPoolSize; - } + public int getRetryCount() { + return retryCount; } - public static class MetadataRefreshJobProperties { + public void setRetryCount(int retryCount) { + this.retryCount = retryCount; + } - private boolean enabled = true; + public int getThreadPoolSize() { + return threadPoolSize; + } - private Duration interval = Duration.ofSeconds(60); + public void setThreadPoolSize(int threadPoolSize) { + this.threadPoolSize = threadPoolSize; + } + } - public boolean isEnabled() { - return enabled; - } + public static class MetadataRefreshJobProperties { - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } + private boolean enabled = true; - public Duration getInterval() { - return interval; - } + private Duration interval = Duration.ofSeconds(60); - public void setInterval(Duration interval) { - this.interval = interval; - } + public boolean isEnabled() { + return enabled; } - public MetadataLoadingProperties getMetadata() { - return metadata; + public void setEnabled(boolean enabled) { + this.enabled = enabled; } - public void setMetadata(MetadataLoadingProperties metadata) { - this.metadata = metadata; + public Duration getInterval() { + return interval; } - public SchemaLoadingProperties getSchema() { - return schema; + public void setInterval(Duration interval) { + this.interval = interval; } + } - public void setSchema(SchemaLoadingProperties schema) { - this.schema = schema; - } + public MetadataLoadingProperties getMetadata() { + return metadata; + } - public MetadataRefreshJobProperties getMetadataRefreshJob() { - return metadataRefreshJob; - } + public void setMetadata(MetadataLoadingProperties metadata) { + this.metadata = metadata; + } - public void setMetadataRefreshJob(MetadataRefreshJobProperties metadataRefreshJob) { - this.metadataRefreshJob = metadataRefreshJob; - } + public SchemaLoadingProperties getSchema() { + return schema; + } + + public void setSchema(SchemaLoadingProperties schema) { + this.schema = schema; + } + + public MetadataRefreshJobProperties getMetadataRefreshJob() { + return metadataRefreshJob; + } + + public void setMetadataRefreshJob(MetadataRefreshJobProperties metadataRefreshJob) { + this.metadataRefreshJob = metadataRefreshJob; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/ZookeeperClustersProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/ZookeeperClustersProperties.java index 13b372c240..2469f8a491 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/ZookeeperClustersProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/ZookeeperClustersProperties.java @@ -1,30 +1,33 @@ package pl.allegro.tech.hermes.frontend.config; -import org.springframework.boot.context.properties.ConfigurationProperties; -import pl.allegro.tech.hermes.infrastructure.dc.DatacenterNameProvider; - import java.util.ArrayList; import java.util.List; +import org.springframework.boot.context.properties.ConfigurationProperties; +import pl.allegro.tech.hermes.infrastructure.dc.DatacenterNameProvider; @ConfigurationProperties(prefix = "frontend.zookeeper") public class ZookeeperClustersProperties { - private List clusters = new ArrayList<>(); + private List clusters = new ArrayList<>(); - public List getClusters() { - return clusters; - } + public List getClusters() { + return clusters; + } - public void setClusters(List clusters) { - this.clusters = clusters; - } + public void setClusters(List clusters) { + this.clusters = clusters; + } - public ZookeeperProperties toZookeeperProperties(DatacenterNameProvider datacenterNameProvider) { - return this.clusters - .stream() - .filter(cluster -> cluster.getDatacenter().equals(datacenterNameProvider.getDatacenterName())) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException( - "No properties for datacenter: " + datacenterNameProvider.getDatacenterName() + " defined.")); - } + public ZookeeperProperties toZookeeperProperties(DatacenterNameProvider datacenterNameProvider) { + return this.clusters.stream() + .filter( + cluster -> cluster.getDatacenter().equals(datacenterNameProvider.getDatacenterName())) + .findFirst() + .orElseThrow( + () -> + new IllegalArgumentException( + "No properties for datacenter: " + + datacenterNameProvider.getDatacenterName() + + " defined.")); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/ZookeeperProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/ZookeeperProperties.java index 0732144ee2..5ea2c5ba40 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/ZookeeperProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/config/ZookeeperProperties.java @@ -1,180 +1,178 @@ package pl.allegro.tech.hermes.frontend.config; -import pl.allegro.tech.hermes.common.di.factories.ZookeeperParameters; - import java.time.Duration; +import pl.allegro.tech.hermes.common.di.factories.ZookeeperParameters; public class ZookeeperProperties implements ZookeeperParameters { - private String connectionString = "localhost:2181"; + private String connectionString = "localhost:2181"; - private String datacenter = "dc"; + private String datacenter = "dc"; - private Duration baseSleepTime = Duration.ofMillis(1000); + private Duration baseSleepTime = Duration.ofMillis(1000); - private Duration maxSleepTime = Duration.ofSeconds(30); + private Duration maxSleepTime = Duration.ofSeconds(30); - private int maxRetries = 29; + private int maxRetries = 29; - private Duration connectionTimeout = Duration.ofSeconds(10); + private Duration connectionTimeout = Duration.ofSeconds(10); - private Duration sessionTimeout = Duration.ofSeconds(10_000); + private Duration sessionTimeout = Duration.ofSeconds(10_000); - private String root = "/hermes"; + private String root = "/hermes"; - private int processingThreadPoolSize = 5; + private int processingThreadPoolSize = 5; - private ZookeeperAuthorizationProperties authorization = new ZookeeperAuthorizationProperties(); + private ZookeeperAuthorizationProperties authorization = new ZookeeperAuthorizationProperties(); - @Override - public String getConnectionString() { - return connectionString; - } + @Override + public String getConnectionString() { + return connectionString; + } - public void setConnectionString(String connectionString) { - this.connectionString = connectionString; - } + public void setConnectionString(String connectionString) { + this.connectionString = connectionString; + } - public String getDatacenter() { - return datacenter; - } + public String getDatacenter() { + return datacenter; + } - public void setDatacenter(String datacenter) { - this.datacenter = datacenter; - } + public void setDatacenter(String datacenter) { + this.datacenter = datacenter; + } - @Override - public Duration getBaseSleepTime() { - return baseSleepTime; - } + @Override + public Duration getBaseSleepTime() { + return baseSleepTime; + } - public void setBaseSleepTime(Duration baseSleepTime) { - this.baseSleepTime = baseSleepTime; - } + public void setBaseSleepTime(Duration baseSleepTime) { + this.baseSleepTime = baseSleepTime; + } - @Override - public Duration getMaxSleepTime() { - return maxSleepTime; - } + @Override + public Duration getMaxSleepTime() { + return maxSleepTime; + } - public void setMaxSleepTime(Duration maxSleepTime) { - this.maxSleepTime = maxSleepTime; - } + public void setMaxSleepTime(Duration maxSleepTime) { + this.maxSleepTime = maxSleepTime; + } - @Override - public int getMaxRetries() { - return maxRetries; - } + @Override + public int getMaxRetries() { + return maxRetries; + } - public void setMaxRetries(int maxRetries) { - this.maxRetries = maxRetries; - } + public void setMaxRetries(int maxRetries) { + this.maxRetries = maxRetries; + } - @Override - public Duration getConnectionTimeout() { - return connectionTimeout; - } + @Override + public Duration getConnectionTimeout() { + return connectionTimeout; + } - public void setConnectionTimeout(Duration connectionTimeout) { - this.connectionTimeout = connectionTimeout; - } + public void setConnectionTimeout(Duration connectionTimeout) { + this.connectionTimeout = connectionTimeout; + } - @Override - public Duration getSessionTimeout() { - return sessionTimeout; - } + @Override + public Duration getSessionTimeout() { + return sessionTimeout; + } - public void setSessionTimeout(Duration sessionTimeout) { - this.sessionTimeout = sessionTimeout; - } + public void setSessionTimeout(Duration sessionTimeout) { + this.sessionTimeout = sessionTimeout; + } - @Override - public String getRoot() { - return root; - } + @Override + public String getRoot() { + return root; + } - public void setRoot(String root) { - this.root = root; - } + public void setRoot(String root) { + this.root = root; + } - @Override - public int getProcessingThreadPoolSize() { - return processingThreadPoolSize; - } + @Override + public int getProcessingThreadPoolSize() { + return processingThreadPoolSize; + } - @Override - public boolean isAuthorizationEnabled() { - return authorization.enabled; - } + @Override + public boolean isAuthorizationEnabled() { + return authorization.enabled; + } - @Override - public String getScheme() { - return authorization.scheme; - } + @Override + public String getScheme() { + return authorization.scheme; + } - @Override - public String getUser() { - return authorization.user; - } + @Override + public String getUser() { + return authorization.user; + } - @Override - public String getPassword() { - return authorization.password; - } + @Override + public String getPassword() { + return authorization.password; + } - public void setProcessingThreadPoolSize(int processingThreadPoolSize) { - this.processingThreadPoolSize = processingThreadPoolSize; - } + public void setProcessingThreadPoolSize(int processingThreadPoolSize) { + this.processingThreadPoolSize = processingThreadPoolSize; + } - public ZookeeperAuthorizationProperties getAuthorization() { - return authorization; - } + public ZookeeperAuthorizationProperties getAuthorization() { + return authorization; + } - public void setAuthorization(ZookeeperAuthorizationProperties authorization) { - this.authorization = authorization; - } + public void setAuthorization(ZookeeperAuthorizationProperties authorization) { + this.authorization = authorization; + } - public static class ZookeeperAuthorizationProperties { + public static class ZookeeperAuthorizationProperties { - private boolean enabled = false; + private boolean enabled = false; - private String scheme = "digest"; + private String scheme = "digest"; - private String user = "user"; + private String user = "user"; - private String password = "password"; + private String password = "password"; - public boolean isEnabled() { - return enabled; - } + public boolean isEnabled() { + return enabled; + } - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } - public String getScheme() { - return scheme; - } + public String getScheme() { + return scheme; + } - public void setScheme(String scheme) { - this.scheme = scheme; - } + public void setScheme(String scheme) { + this.scheme = scheme; + } - public String getUser() { - return user; - } + public String getUser() { + return user; + } - public void setUser(String user) { - this.user = user; - } + public void setUser(String user) { + this.user = user; + } - public String getPassword() { - return password; - } + public String getPassword() { + return password; + } - public void setPassword(String password) { - this.password = password; - } + public void setPassword(String password) { + this.password = password; } + } } - diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/listeners/BrokerAcknowledgeListener.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/listeners/BrokerAcknowledgeListener.java index 50da07f33a..fbb09bf479 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/listeners/BrokerAcknowledgeListener.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/listeners/BrokerAcknowledgeListener.java @@ -1,10 +1,9 @@ package pl.allegro.tech.hermes.frontend.listeners; - import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.frontend.publishing.message.Message; public interface BrokerAcknowledgeListener { - void onAcknowledge(Message message, Topic topic); + void onAcknowledge(Message message, Topic topic); } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/listeners/BrokerErrorListener.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/listeners/BrokerErrorListener.java index 0760682323..f52b60992f 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/listeners/BrokerErrorListener.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/listeners/BrokerErrorListener.java @@ -5,5 +5,5 @@ public interface BrokerErrorListener { - void onError(Message message, Topic topic, Exception ex); + void onError(Message message, Topic topic, Exception ex); } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/listeners/BrokerListeners.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/listeners/BrokerListeners.java index efe2be5aff..76b3ba1191 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/listeners/BrokerListeners.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/listeners/BrokerListeners.java @@ -1,38 +1,37 @@ package pl.allegro.tech.hermes.frontend.listeners; -import pl.allegro.tech.hermes.api.Topic; -import pl.allegro.tech.hermes.frontend.publishing.message.Message; - import java.util.ArrayList; import java.util.List; +import pl.allegro.tech.hermes.api.Topic; +import pl.allegro.tech.hermes.frontend.publishing.message.Message; public class BrokerListeners { - private final List timeoutListeners = new ArrayList<>(); - private final List acknowledgeListeners = new ArrayList<>(); - private final List errorListeners = new ArrayList<>(); + private final List timeoutListeners = new ArrayList<>(); + private final List acknowledgeListeners = new ArrayList<>(); + private final List errorListeners = new ArrayList<>(); - public void addTimeoutListener(BrokerTimeoutListener brokerTimeoutListener) { - timeoutListeners.add(brokerTimeoutListener); - } + public void addTimeoutListener(BrokerTimeoutListener brokerTimeoutListener) { + timeoutListeners.add(brokerTimeoutListener); + } - public void addAcknowledgeListener(BrokerAcknowledgeListener brokerAcknowledgeListener) { - acknowledgeListeners.add(brokerAcknowledgeListener); - } + public void addAcknowledgeListener(BrokerAcknowledgeListener brokerAcknowledgeListener) { + acknowledgeListeners.add(brokerAcknowledgeListener); + } - public void addErrorListener(BrokerErrorListener brokerErrorListener) { - errorListeners.add(brokerErrorListener); - } + public void addErrorListener(BrokerErrorListener brokerErrorListener) { + errorListeners.add(brokerErrorListener); + } - public void onAcknowledge(Message message, Topic topic) { - acknowledgeListeners.forEach(l -> l.onAcknowledge(message, topic)); - } + public void onAcknowledge(Message message, Topic topic) { + acknowledgeListeners.forEach(l -> l.onAcknowledge(message, topic)); + } - public void onTimeout(Message message, Topic topic) { - timeoutListeners.forEach(l -> l.onTimeout(message, topic)); - } + public void onTimeout(Message message, Topic topic) { + timeoutListeners.forEach(l -> l.onTimeout(message, topic)); + } - public void onError(Message message, Topic topic, Exception ex) { - errorListeners.forEach(l -> l.onError(message, topic, ex)); - } + public void onError(Message message, Topic topic, Exception ex) { + errorListeners.forEach(l -> l.onError(message, topic, ex)); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/listeners/BrokerTimeoutListener.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/listeners/BrokerTimeoutListener.java index ffea94c748..d628db0c10 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/listeners/BrokerTimeoutListener.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/listeners/BrokerTimeoutListener.java @@ -5,5 +5,5 @@ public interface BrokerTimeoutListener { - void onTimeout(Message message, Topic topic); + void onTimeout(Message message, Topic topic); } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/metric/CachedTopic.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/metric/CachedTopic.java index 9ffb726071..854832f89f 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/metric/CachedTopic.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/metric/CachedTopic.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.frontend.metric; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.api.TopicName; import pl.allegro.tech.hermes.common.kafka.KafkaTopics; @@ -11,144 +13,149 @@ import pl.allegro.tech.hermes.metrics.HermesTimer; import pl.allegro.tech.hermes.metrics.HermesTimerContext; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - public class CachedTopic { - private final Topic topic; - private final KafkaTopics kafkaTopics; - private final MetricsFacade metricsFacade; - private final boolean blacklisted; + private final Topic topic; + private final KafkaTopics kafkaTopics; + private final MetricsFacade metricsFacade; + private final boolean blacklisted; - private final HermesTimer topicProducerLatencyTimer; - private final HermesTimer globalProducerLatencyTimer; + private final HermesTimer topicProducerLatencyTimer; + private final HermesTimer globalProducerLatencyTimer; - private final HermesTimer topicBrokerLatencyTimer; + private final HermesTimer topicBrokerLatencyTimer; - private final HermesCounter globalRequestMeter; - private final HermesCounter topicRequestMeter; + private final HermesCounter globalRequestMeter; + private final HermesCounter topicRequestMeter; - private final HermesCounter globalDelayedProcessingMeter; - private final HermesCounter topicDelayedProcessingMeter; + private final HermesCounter globalDelayedProcessingMeter; + private final HermesCounter topicDelayedProcessingMeter; - private final HermesHistogram topicMessageContentSize; - private final HermesHistogram globalMessageContentSize; + private final HermesHistogram topicMessageContentSize; + private final HermesHistogram globalMessageContentSize; - private final ThroughputMeter throughputMeter; + private final ThroughputMeter throughputMeter; - private final HermesCounter topicDuplicatedMessageCounter; + private final HermesCounter topicDuplicatedMessageCounter; - private final Map published = new ConcurrentHashMap<>(); + private final Map published = new ConcurrentHashMap<>(); - private final Map httpStatusCodesMeters = new ConcurrentHashMap<>(); + private final Map httpStatusCodesMeters = new ConcurrentHashMap<>(); - public CachedTopic(Topic topic, - MetricsFacade metricsFacade, - ThroughputRegistry throughputRegistry, - KafkaTopics kafkaTopics) { - this(topic, metricsFacade, throughputRegistry, kafkaTopics, false); - } + public CachedTopic( + Topic topic, + MetricsFacade metricsFacade, + ThroughputRegistry throughputRegistry, + KafkaTopics kafkaTopics) { + this(topic, metricsFacade, throughputRegistry, kafkaTopics, false); + } - public CachedTopic(Topic topic, - MetricsFacade metricsFacade, - ThroughputRegistry throughputRegistry, - KafkaTopics kafkaTopics, - boolean blacklisted) { - this.topic = topic; - this.kafkaTopics = kafkaTopics; - this.metricsFacade = metricsFacade; - this.blacklisted = blacklisted; - - globalRequestMeter = metricsFacade.topics().topicGlobalRequestCounter(); - topicRequestMeter = metricsFacade.topics().topicRequestCounter(topic.getName()); - - globalDelayedProcessingMeter = metricsFacade.topics().topicGlobalDelayedProcessingCounter(); - topicDelayedProcessingMeter = metricsFacade.topics().topicDelayedProcessingCounter(topic.getName()); - - globalMessageContentSize = metricsFacade.topics().topicGlobalMessageContentSizeHistogram(); - topicMessageContentSize = metricsFacade.topics().topicMessageContentSizeHistogram(topic.getName()); - - throughputMeter = throughputRegistry.forTopic(topic.getName()); - - if (Topic.Ack.ALL.equals(topic.getAck())) { - globalProducerLatencyTimer = metricsFacade.topics().ackAllGlobalLatency(); - topicProducerLatencyTimer = metricsFacade.topics().ackAllTopicLatency(topic.getName()); - topicBrokerLatencyTimer = metricsFacade.topics().ackAllBrokerLatency(); - } else { - globalProducerLatencyTimer = metricsFacade.topics().ackLeaderGlobalLatency(); - topicProducerLatencyTimer = metricsFacade.topics().ackLeaderTopicLatency(topic.getName()); - topicBrokerLatencyTimer = metricsFacade.topics().ackLeaderBrokerLatency(); - } - - topicDuplicatedMessageCounter = metricsFacade.topics().topicDuplicatedMessageCounter(topic.getName()); - } + public CachedTopic( + Topic topic, + MetricsFacade metricsFacade, + ThroughputRegistry throughputRegistry, + KafkaTopics kafkaTopics, + boolean blacklisted) { + this.topic = topic; + this.kafkaTopics = kafkaTopics; + this.metricsFacade = metricsFacade; + this.blacklisted = blacklisted; - public Topic getTopic() { - return topic; - } + globalRequestMeter = metricsFacade.topics().topicGlobalRequestCounter(); + topicRequestMeter = metricsFacade.topics().topicRequestCounter(topic.getName()); - public TopicName getTopicName() { - return topic.getName(); - } + globalDelayedProcessingMeter = metricsFacade.topics().topicGlobalDelayedProcessingCounter(); + topicDelayedProcessingMeter = + metricsFacade.topics().topicDelayedProcessingCounter(topic.getName()); - public String getQualifiedName() { - return topic.getName().qualifiedName(); - } + globalMessageContentSize = metricsFacade.topics().topicGlobalMessageContentSizeHistogram(); + topicMessageContentSize = + metricsFacade.topics().topicMessageContentSizeHistogram(topic.getName()); - public KafkaTopics getKafkaTopics() { - return kafkaTopics; - } + throughputMeter = throughputRegistry.forTopic(topic.getName()); - public boolean isBlacklisted() { - return blacklisted; + if (Topic.Ack.ALL.equals(topic.getAck())) { + globalProducerLatencyTimer = metricsFacade.topics().ackAllGlobalLatency(); + topicProducerLatencyTimer = metricsFacade.topics().ackAllTopicLatency(topic.getName()); + topicBrokerLatencyTimer = metricsFacade.topics().ackAllBrokerLatency(); + } else { + globalProducerLatencyTimer = metricsFacade.topics().ackLeaderGlobalLatency(); + topicProducerLatencyTimer = metricsFacade.topics().ackLeaderTopicLatency(topic.getName()); + topicBrokerLatencyTimer = metricsFacade.topics().ackLeaderBrokerLatency(); } - public StartedTimersPair startProducerLatencyTimers() { - return new StartedTimersPair(topicProducerLatencyTimer.time(), globalProducerLatencyTimer.time()); - } + topicDuplicatedMessageCounter = + metricsFacade.topics().topicDuplicatedMessageCounter(topic.getName()); + } - public void markStatusCodeMeter(int status) { - httpStatusCodesMeters.computeIfAbsent( - status, - code -> new MetersPair( - metricsFacade.topics().topicGlobalHttpStatusCodeCounter(status), - metricsFacade.topics().topicHttpStatusCodeCounter(topic.getName(), status)) - ).mark(); - } + public Topic getTopic() { + return topic; + } - public void markRequestMeter() { - globalRequestMeter.increment(1L); - topicRequestMeter.increment(1L); - } + public TopicName getTopicName() { + return topic.getName(); + } - public HermesTimerContext startBrokerLatencyTimer() { - return topicBrokerLatencyTimer.time(); - } + public String getQualifiedName() { + return topic.getName().qualifiedName(); + } - public void incrementPublished(String datacenter) { - published.computeIfAbsent( - datacenter, - dc -> metricsFacade.topics().topicPublished(topic.getName(), datacenter) - ).increment(); - } + public KafkaTopics getKafkaTopics() { + return kafkaTopics; + } - public void reportMessageContentSize(int size) { - topicMessageContentSize.record(size); - globalMessageContentSize.record(size); - throughputMeter.increment(size); - } + public boolean isBlacklisted() { + return blacklisted; + } - public void markDelayedProcessing() { - topicDelayedProcessingMeter.increment(1L); - globalDelayedProcessingMeter.increment(1L); - } - - public HermesRateMeter getThroughput() { - return throughputMeter; - } + public StartedTimersPair startProducerLatencyTimers() { + return new StartedTimersPair( + topicProducerLatencyTimer.time(), globalProducerLatencyTimer.time()); + } - public void markMessageDuplicated() { - topicDuplicatedMessageCounter.increment(); - } + public void markStatusCodeMeter(int status) { + httpStatusCodesMeters + .computeIfAbsent( + status, + code -> + new MetersPair( + metricsFacade.topics().topicGlobalHttpStatusCodeCounter(status), + metricsFacade.topics().topicHttpStatusCodeCounter(topic.getName(), status))) + .mark(); + } + + public void markRequestMeter() { + globalRequestMeter.increment(1L); + topicRequestMeter.increment(1L); + } + + public HermesTimerContext startBrokerLatencyTimer() { + return topicBrokerLatencyTimer.time(); + } + + public void incrementPublished(String datacenter) { + published + .computeIfAbsent( + datacenter, dc -> metricsFacade.topics().topicPublished(topic.getName(), datacenter)) + .increment(); + } + + public void reportMessageContentSize(int size) { + topicMessageContentSize.record(size); + globalMessageContentSize.record(size); + throughputMeter.increment(size); + } + + public void markDelayedProcessing() { + topicDelayedProcessingMeter.increment(1L); + globalDelayedProcessingMeter.increment(1L); + } + + public HermesRateMeter getThroughput() { + return throughputMeter; + } + + public void markMessageDuplicated() { + topicDuplicatedMessageCounter.increment(); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/metric/MetersPair.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/metric/MetersPair.java index 324cbe47e5..fa1c776149 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/metric/MetersPair.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/metric/MetersPair.java @@ -3,16 +3,16 @@ import pl.allegro.tech.hermes.metrics.HermesCounter; public class MetersPair { - private final HermesCounter meter1; - private final HermesCounter meter2; + private final HermesCounter meter1; + private final HermesCounter meter2; - public MetersPair(HermesCounter meter1, HermesCounter meter2) { - this.meter1 = meter1; - this.meter2 = meter2; - } + public MetersPair(HermesCounter meter1, HermesCounter meter2) { + this.meter1 = meter1; + this.meter2 = meter2; + } - void mark() { - meter1.increment(1L); - meter2.increment(1L); - } -} \ No newline at end of file + void mark() { + meter1.increment(1L); + meter2.increment(1L); + } +} diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/metric/ThroughputMeter.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/metric/ThroughputMeter.java index 3f225b87e0..31c1d985d5 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/metric/ThroughputMeter.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/metric/ThroughputMeter.java @@ -6,31 +6,31 @@ public class ThroughputMeter implements HermesRateMeter { - private final HermesCounter topicThroughputMetric; - private final HermesCounter globalThroughputMetric; - private final Meter topicMeter; - private final Meter globalMeter; + private final HermesCounter topicThroughputMetric; + private final HermesCounter globalThroughputMetric; + private final Meter topicMeter; + private final Meter globalMeter; - public ThroughputMeter( - HermesCounter topicThroughputMetric, - HermesCounter globalThroughputMetric, - Meter topicMeter, - Meter globalMeter) { - this.topicThroughputMetric = topicThroughputMetric; - this.globalThroughputMetric = globalThroughputMetric; - this.topicMeter = topicMeter; - this.globalMeter = globalMeter; - } + public ThroughputMeter( + HermesCounter topicThroughputMetric, + HermesCounter globalThroughputMetric, + Meter topicMeter, + Meter globalMeter) { + this.topicThroughputMetric = topicThroughputMetric; + this.globalThroughputMetric = globalThroughputMetric; + this.topicMeter = topicMeter; + this.globalMeter = globalMeter; + } - @Override - public double getOneMinuteRate() { - return topicMeter.getOneMinuteRate(); - } + @Override + public double getOneMinuteRate() { + return topicMeter.getOneMinuteRate(); + } - public void increment(long size) { - this.topicMeter.mark(size); - this.globalMeter.mark(size); - this.topicThroughputMetric.increment(size); - this.globalThroughputMetric.increment(size); - } + public void increment(long size) { + this.topicMeter.mark(size); + this.globalMeter.mark(size); + this.topicThroughputMetric.increment(size); + this.globalThroughputMetric.increment(size); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/metric/ThroughputRegistry.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/metric/ThroughputRegistry.java index ac01b899b6..01db082470 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/metric/ThroughputRegistry.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/metric/ThroughputRegistry.java @@ -7,26 +7,25 @@ public class ThroughputRegistry { - private final MetricsFacade metricsFacade; - private final MetricRegistry metricRegistry; - private final Meter globalThroughputMeter; + private final MetricsFacade metricsFacade; + private final MetricRegistry metricRegistry; + private final Meter globalThroughputMeter; - public ThroughputRegistry(MetricsFacade metricsFacade, MetricRegistry metricRegistry) { - this.metricsFacade = metricsFacade; - this.metricRegistry = metricRegistry; - this.globalThroughputMeter = metricRegistry.meter("globalThroughputMeter"); - } + public ThroughputRegistry(MetricsFacade metricsFacade, MetricRegistry metricRegistry) { + this.metricsFacade = metricsFacade; + this.metricRegistry = metricRegistry; + this.globalThroughputMeter = metricRegistry.meter("globalThroughputMeter"); + } - public double getGlobalThroughputOneMinuteRate() { - return globalThroughputMeter.getOneMinuteRate(); - } + public double getGlobalThroughputOneMinuteRate() { + return globalThroughputMeter.getOneMinuteRate(); + } - public ThroughputMeter forTopic(TopicName topic) { - return new ThroughputMeter( - metricsFacade.topics().topicThroughputBytes(topic), - metricsFacade.topics().topicGlobalThroughputBytes(), - metricRegistry.meter(topic.qualifiedName() + "Throughput"), - globalThroughputMeter - ); - } + public ThroughputMeter forTopic(TopicName topic) { + return new ThroughputMeter( + metricsFacade.topics().topicThroughputBytes(topic), + metricsFacade.topics().topicGlobalThroughputBytes(), + metricRegistry.meter(topic.qualifiedName() + "Throughput"), + globalThroughputMeter); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/BrokerLatencyReporter.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/BrokerLatencyReporter.java index 53be7793d5..a0ce01aa1c 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/BrokerLatencyReporter.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/BrokerLatencyReporter.java @@ -1,5 +1,9 @@ package pl.allegro.tech.hermes.frontend.producer; +import java.time.Duration; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.Topic; @@ -8,57 +12,60 @@ import pl.allegro.tech.hermes.frontend.publishing.metadata.ProduceMetadata; import pl.allegro.tech.hermes.metrics.HermesTimerContext; -import java.time.Duration; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.RejectedExecutionException; -import java.util.function.Supplier; - public class BrokerLatencyReporter { - private static final Logger logger = LoggerFactory.getLogger(BrokerLatencyReporter.class); - - private final boolean perBrokerLatencyEnabled; - private final MetricsFacade metricsFacade; - private final Duration slowResponseThreshold; - private final ExecutorService reporterExecutorService; + private static final Logger logger = LoggerFactory.getLogger(BrokerLatencyReporter.class); - public BrokerLatencyReporter(boolean perBrokerLatencyEnabled, - MetricsFacade metricsFacade, - Duration slowResponseThreshold, - ExecutorService reporterExecutorService) { - this.perBrokerLatencyEnabled = perBrokerLatencyEnabled; - this.metricsFacade = metricsFacade; - this.slowResponseThreshold = slowResponseThreshold; - this.reporterExecutorService = reporterExecutorService; - } + private final boolean perBrokerLatencyEnabled; + private final MetricsFacade metricsFacade; + private final Duration slowResponseThreshold; + private final ExecutorService reporterExecutorService; - public void report(HermesTimerContext timerContext, - Message message, - Topic.Ack ack, - Supplier produceMetadata) { - Duration duration = timerContext.closeAndGet(); - if (perBrokerLatencyEnabled) { - try { - reporterExecutorService.submit(() -> doReport(duration, message.getId(), ack, produceMetadata)); - } catch (RejectedExecutionException ignored) { - // don't propagate the exception - allow metrics to be dropped if executor is overloaded - // executor service should already be instrumented to meter rejected executions so no action is needed - } - } + public BrokerLatencyReporter( + boolean perBrokerLatencyEnabled, + MetricsFacade metricsFacade, + Duration slowResponseThreshold, + ExecutorService reporterExecutorService) { + this.perBrokerLatencyEnabled = perBrokerLatencyEnabled; + this.metricsFacade = metricsFacade; + this.slowResponseThreshold = slowResponseThreshold; + this.reporterExecutorService = reporterExecutorService; + } + public void report( + HermesTimerContext timerContext, + Message message, + Topic.Ack ack, + Supplier produceMetadata) { + Duration duration = timerContext.closeAndGet(); + if (perBrokerLatencyEnabled) { + try { + reporterExecutorService.submit( + () -> doReport(duration, message.getId(), ack, produceMetadata)); + } catch (RejectedExecutionException ignored) { + // don't propagate the exception - allow metrics to be dropped if executor is overloaded + // executor service should already be instrumented to meter rejected executions so no action + // is needed + } } + } - private void doReport(Duration duration, - String messageId, - Topic.Ack ack, - Supplier produceMetadata) { - String broker = produceMetadata.get().getBroker().orElse("unknown"); - - if (duration.compareTo(slowResponseThreshold) > 0) { - logger.debug("Slow produce request, broker response time: {} ms, ackLevel: {}, messageId: {}, broker: {}", - duration.toMillis(), ack, messageId, broker); - } + private void doReport( + Duration duration, + String messageId, + Topic.Ack ack, + Supplier produceMetadata) { + String broker = produceMetadata.get().getBroker().orElse("unknown"); - metricsFacade.broker().recordBrokerLatency(broker, ack, duration); + if (duration.compareTo(slowResponseThreshold) > 0) { + logger.debug( + "Slow produce request, broker response time: {} ms, ackLevel: {}, messageId: {}, broker: {}", + duration.toMillis(), + ack, + messageId, + broker); } + + metricsFacade.broker().recordBrokerLatency(broker, ack, duration); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/BrokerMessageProducer.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/BrokerMessageProducer.java index f09d29545b..1e9afdaf76 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/BrokerMessageProducer.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/BrokerMessageProducer.java @@ -6,5 +6,5 @@ public interface BrokerMessageProducer extends BrokerTopicAvailabilityChecker { - void send(Message message, CachedTopic topic, PublishingCallback callback); + void send(Message message, CachedTopic topic, PublishingCallback callback); } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/BrokerTopicAvailabilityChecker.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/BrokerTopicAvailabilityChecker.java index 4dd6b0b49a..635070bf84 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/BrokerTopicAvailabilityChecker.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/BrokerTopicAvailabilityChecker.java @@ -4,7 +4,7 @@ public interface BrokerTopicAvailabilityChecker { - boolean areAllTopicsAvailable(); + boolean areAllTopicsAvailable(); - boolean isTopicAvailable(CachedTopic cachedTopic); + boolean isTopicAvailable(CachedTopic cachedTopic); } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/ChaosException.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/ChaosException.java index c2b1a27e7e..6d0ba17f40 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/ChaosException.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/ChaosException.java @@ -2,7 +2,14 @@ public class ChaosException extends RuntimeException { - public ChaosException(String datacenter, long delayMs, String messageId) { - super("Scheduled failure occurred for datacenter: " + datacenter + ", messageId: " + messageId + " after " + delayMs + "ms"); - } + public ChaosException(String datacenter, long delayMs, String messageId) { + super( + "Scheduled failure occurred for datacenter: " + + datacenter + + ", messageId: " + + messageId + + " after " + + delayMs + + "ms"); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/FallbackToRemoteDatacenterAwareMessageProducer.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/FallbackToRemoteDatacenterAwareMessageProducer.java index f6cddafab9..af61cf4ee5 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/FallbackToRemoteDatacenterAwareMessageProducer.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/FallbackToRemoteDatacenterAwareMessageProducer.java @@ -7,31 +7,32 @@ public class FallbackToRemoteDatacenterAwareMessageProducer implements BrokerMessageProducer { - private final BrokerMessageProducer localDatacenterMessageProducer; - private final BrokerMessageProducer multiDatacenterMessageProducer; + private final BrokerMessageProducer localDatacenterMessageProducer; + private final BrokerMessageProducer multiDatacenterMessageProducer; - public FallbackToRemoteDatacenterAwareMessageProducer(BrokerMessageProducer localDatacenterMessageProducer, - BrokerMessageProducer multiDatacenterMessageProducer) { - this.localDatacenterMessageProducer = localDatacenterMessageProducer; - this.multiDatacenterMessageProducer = multiDatacenterMessageProducer; - } + public FallbackToRemoteDatacenterAwareMessageProducer( + BrokerMessageProducer localDatacenterMessageProducer, + BrokerMessageProducer multiDatacenterMessageProducer) { + this.localDatacenterMessageProducer = localDatacenterMessageProducer; + this.multiDatacenterMessageProducer = multiDatacenterMessageProducer; + } - @Override - public void send(Message message, CachedTopic topic, PublishingCallback callback) { - if (topic.getTopic().isFallbackToRemoteDatacenterEnabled()) { - this.multiDatacenterMessageProducer.send(message, topic, callback); - } else { - this.localDatacenterMessageProducer.send(message, topic, callback); - } + @Override + public void send(Message message, CachedTopic topic, PublishingCallback callback) { + if (topic.getTopic().isFallbackToRemoteDatacenterEnabled()) { + this.multiDatacenterMessageProducer.send(message, topic, callback); + } else { + this.localDatacenterMessageProducer.send(message, topic, callback); } + } - @Override - public boolean areAllTopicsAvailable() { - return localDatacenterMessageProducer.areAllTopicsAvailable(); - } + @Override + public boolean areAllTopicsAvailable() { + return localDatacenterMessageProducer.areAllTopicsAvailable(); + } - @Override - public boolean isTopicAvailable(CachedTopic topic) { - return localDatacenterMessageProducer.isTopicAvailable(topic); - } + @Override + public boolean isTopicAvailable(CachedTopic topic) { + return localDatacenterMessageProducer.isTopicAvailable(topic); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/KafkaChaosProperties.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/KafkaChaosProperties.java index 9d2ebbdd7e..f578072e98 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/KafkaChaosProperties.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/KafkaChaosProperties.java @@ -4,37 +4,36 @@ @ConfigurationProperties(prefix = "frontend.kafka.chaos") public class KafkaChaosProperties { - private ChaosSchedulerProperties chaosScheduler = new ChaosSchedulerProperties(); + private ChaosSchedulerProperties chaosScheduler = new ChaosSchedulerProperties(); - public ChaosSchedulerProperties getChaosScheduler() { - return chaosScheduler; - } - - public void setChaosScheduler(ChaosSchedulerProperties chaosScheduler) { - this.chaosScheduler = chaosScheduler; - } + public ChaosSchedulerProperties getChaosScheduler() { + return chaosScheduler; + } - public static class ChaosSchedulerProperties { + public void setChaosScheduler(ChaosSchedulerProperties chaosScheduler) { + this.chaosScheduler = chaosScheduler; + } - private int threadPoolSize = 16; + public static class ChaosSchedulerProperties { - private boolean threadPoolMonitoringEnabled = false; + private int threadPoolSize = 16; - public int getThreadPoolSize() { - return threadPoolSize; - } + private boolean threadPoolMonitoringEnabled = false; - public void setThreadPoolSize(int threadPoolSize) { - this.threadPoolSize = threadPoolSize; - } + public int getThreadPoolSize() { + return threadPoolSize; + } - public boolean isThreadPoolMonitoringEnabled() { - return threadPoolMonitoringEnabled; - } + public void setThreadPoolSize(int threadPoolSize) { + this.threadPoolSize = threadPoolSize; + } - public void setThreadPoolMonitoringEnabled(boolean threadPoolMonitoringEnabled) { - this.threadPoolMonitoringEnabled = threadPoolMonitoringEnabled; - } + public boolean isThreadPoolMonitoringEnabled() { + return threadPoolMonitoringEnabled; } + public void setThreadPoolMonitoringEnabled(boolean threadPoolMonitoringEnabled) { + this.threadPoolMonitoringEnabled = threadPoolMonitoringEnabled; + } + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/KafkaHeaderFactory.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/KafkaHeaderFactory.java index 3b21b3549d..2f5396f141 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/KafkaHeaderFactory.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/KafkaHeaderFactory.java @@ -1,47 +1,50 @@ package pl.allegro.tech.hermes.frontend.producer.kafka; import com.google.common.primitives.Ints; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; import org.apache.kafka.common.header.Header; import org.apache.kafka.common.header.internals.RecordHeader; import pl.allegro.tech.hermes.common.kafka.HTTPHeadersPropagationAsKafkaHeadersProperties; import pl.allegro.tech.hermes.common.kafka.KafkaHeaderNameParameters; -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.Map; - public class KafkaHeaderFactory { - private final KafkaHeaderNameParameters kafkaHeaderNameParameters; - private final boolean isHTTPHeadersPropagationAsKafkaHeadersEnabled; - private final String httpHeaderPrefix; - - public KafkaHeaderFactory(KafkaHeaderNameParameters kafkaHeaderNameParameters, - HTTPHeadersPropagationAsKafkaHeadersProperties httpHeadersPropagationAsKafkaHeadersProperties) { - this.kafkaHeaderNameParameters = kafkaHeaderNameParameters; - this.isHTTPHeadersPropagationAsKafkaHeadersEnabled = httpHeadersPropagationAsKafkaHeadersProperties.isEnabled(); - this.httpHeaderPrefix = httpHeadersPropagationAsKafkaHeadersProperties.getPrefix(); + private final KafkaHeaderNameParameters kafkaHeaderNameParameters; + private final boolean isHTTPHeadersPropagationAsKafkaHeadersEnabled; + private final String httpHeaderPrefix; + + public KafkaHeaderFactory( + KafkaHeaderNameParameters kafkaHeaderNameParameters, + HTTPHeadersPropagationAsKafkaHeadersProperties + httpHeadersPropagationAsKafkaHeadersProperties) { + this.kafkaHeaderNameParameters = kafkaHeaderNameParameters; + this.isHTTPHeadersPropagationAsKafkaHeadersEnabled = + httpHeadersPropagationAsKafkaHeadersProperties.isEnabled(); + this.httpHeaderPrefix = httpHeadersPropagationAsKafkaHeadersProperties.getPrefix(); + } + + Header messageId(String messageId) { + return new RecordHeader(kafkaHeaderNameParameters.getMessageId(), messageId.getBytes()); + } + + Header schemaVersion(int schemaVersion) { + return new RecordHeader( + kafkaHeaderNameParameters.getSchemaVersion(), Ints.toByteArray(schemaVersion)); + } + + Header schemaId(int schemaId) { + return new RecordHeader(kafkaHeaderNameParameters.getSchemaId(), Ints.toByteArray(schemaId)); + } + + void setHTTPHeadersIfEnabled(List

headers, Map httpHeaders) { + if (isHTTPHeadersPropagationAsKafkaHeadersEnabled) { + httpHeaders.forEach((name, value) -> headers.add(createHttpHeader(name, value))); } + } - Header messageId(String messageId) { - return new RecordHeader(kafkaHeaderNameParameters.getMessageId(), messageId.getBytes()); - } - - Header schemaVersion(int schemaVersion) { - return new RecordHeader(kafkaHeaderNameParameters.getSchemaVersion(), Ints.toByteArray(schemaVersion)); - } - - Header schemaId(int schemaId) { - return new RecordHeader(kafkaHeaderNameParameters.getSchemaId(), Ints.toByteArray(schemaId)); - } - - void setHTTPHeadersIfEnabled(List
headers, Map httpHeaders) { - if (isHTTPHeadersPropagationAsKafkaHeadersEnabled) { - httpHeaders.forEach((name, value) -> headers.add(createHttpHeader(name, value))); - } - } - - private Header createHttpHeader(String name, String value) { - return new RecordHeader(httpHeaderPrefix + name, value.getBytes(StandardCharsets.UTF_8)); - } + private Header createHttpHeader(String name, String value) { + return new RecordHeader(httpHeaderPrefix + name, value.getBytes(StandardCharsets.UTF_8)); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/KafkaMessageSender.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/KafkaMessageSender.java index 457b372a34..8b50b6b886 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/KafkaMessageSender.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/KafkaMessageSender.java @@ -1,5 +1,15 @@ package pl.allegro.tech.hermes.frontend.producer.kafka; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.function.ToDoubleFunction; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerRecord; @@ -21,185 +31,252 @@ import pl.allegro.tech.hermes.frontend.publishing.metadata.ProduceMetadata; import pl.allegro.tech.hermes.metrics.HermesTimerContext; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.function.Predicate; -import java.util.function.Supplier; -import java.util.function.ToDoubleFunction; - public class KafkaMessageSender { - private static final Logger logger = LoggerFactory.getLogger(KafkaMessageSender.class); - - private final Producer producer; - private final BrokerLatencyReporter brokerLatencyReporter; - private final MetricsFacade metricsFacade; - private final String datacenter; - private final ScheduledExecutorService chaosScheduler; - - KafkaMessageSender(Producer kafkaProducer, - BrokerLatencyReporter brokerLatencyReporter, - MetricsFacade metricsFacade, - String datacenter, - ScheduledExecutorService chaosScheduler) { - this.producer = kafkaProducer; - this.brokerLatencyReporter = brokerLatencyReporter; - this.metricsFacade = metricsFacade; - this.datacenter = datacenter; - this.chaosScheduler = chaosScheduler; - } + private static final Logger logger = LoggerFactory.getLogger(KafkaMessageSender.class); - public String getDatacenter() { - return datacenter; - } + private final Producer producer; + private final BrokerLatencyReporter brokerLatencyReporter; + private final MetricsFacade metricsFacade; + private final String datacenter; + private final ScheduledExecutorService chaosScheduler; - public void send(ProducerRecord producerRecord, - CachedTopic cachedTopic, - Message message, - Callback callback, - MultiDatacenterMessageProducer.ChaosExperiment experiment) { - if (experiment.enabled()) { - try { - chaosScheduler.schedule(() -> { - if (experiment.completeWithError()) { - var exception = new ChaosException(datacenter, experiment.delayInMillis(), message.getId()); - callback.onCompletion(exceptionalRecordMetadata(cachedTopic), exception); - } else { - send(producerRecord, cachedTopic, message, callback); - } - }, experiment.delayInMillis(), TimeUnit.MILLISECONDS); - } catch (RejectedExecutionException e) { - logger.warn("Failed while scheduling chaos experiment. Sending message to Kafka.", e); + KafkaMessageSender( + Producer kafkaProducer, + BrokerLatencyReporter brokerLatencyReporter, + MetricsFacade metricsFacade, + String datacenter, + ScheduledExecutorService chaosScheduler) { + this.producer = kafkaProducer; + this.brokerLatencyReporter = brokerLatencyReporter; + this.metricsFacade = metricsFacade; + this.datacenter = datacenter; + this.chaosScheduler = chaosScheduler; + } + + public String getDatacenter() { + return datacenter; + } + + public void send( + ProducerRecord producerRecord, + CachedTopic cachedTopic, + Message message, + Callback callback, + MultiDatacenterMessageProducer.ChaosExperiment experiment) { + if (experiment.enabled()) { + try { + chaosScheduler.schedule( + () -> { + if (experiment.completeWithError()) { + var exception = + new ChaosException(datacenter, experiment.delayInMillis(), message.getId()); + callback.onCompletion(exceptionalRecordMetadata(cachedTopic), exception); + } else { send(producerRecord, cachedTopic, message, callback); - } - } else { - send(producerRecord, cachedTopic, message, callback); - } + } + }, + experiment.delayInMillis(), + TimeUnit.MILLISECONDS); + } catch (RejectedExecutionException e) { + logger.warn("Failed while scheduling chaos experiment. Sending message to Kafka.", e); + send(producerRecord, cachedTopic, message, callback); + } + } else { + send(producerRecord, cachedTopic, message, callback); } + } - public void send(ProducerRecord producerRecord, - CachedTopic cachedTopic, - Message message, - Callback callback) { - HermesTimerContext timer = cachedTopic.startBrokerLatencyTimer(); - Callback meteredCallback = new MeteredCallback(timer, message, cachedTopic, callback); - try { - producer.send(producerRecord, meteredCallback); - } catch (Exception e) { - callback.onCompletion(exceptionalRecordMetadata(cachedTopic), e); - } + public void send( + ProducerRecord producerRecord, + CachedTopic cachedTopic, + Message message, + Callback callback) { + HermesTimerContext timer = cachedTopic.startBrokerLatencyTimer(); + Callback meteredCallback = new MeteredCallback(timer, message, cachedTopic, callback); + try { + producer.send(producerRecord, meteredCallback); + } catch (Exception e) { + callback.onCompletion(exceptionalRecordMetadata(cachedTopic), e); } + } - private static RecordMetadata exceptionalRecordMetadata(CachedTopic cachedTopic) { - var tp = new TopicPartition(cachedTopic.getKafkaTopics().getPrimary().name().asString(), RecordMetadata.UNKNOWN_PARTITION); - return new RecordMetadata(tp, -1, -1, RecordBatch.NO_TIMESTAMP, -1L, -1, -1); - } + private static RecordMetadata exceptionalRecordMetadata(CachedTopic cachedTopic) { + var tp = + new TopicPartition( + cachedTopic.getKafkaTopics().getPrimary().name().asString(), + RecordMetadata.UNKNOWN_PARTITION); + return new RecordMetadata(tp, -1, -1, RecordBatch.NO_TIMESTAMP, -1L, -1, -1); + } - List loadPartitionMetadataFor(String topic) { - return producer.partitionsFor(topic); - } + List loadPartitionMetadataFor(String topic) { + return producer.partitionsFor(topic); + } - public void close() { - producer.close(); - } + public void close() { + producer.close(); + } + + private Supplier produceMetadataSupplier(RecordMetadata recordMetadata) { + return () -> { + String kafkaTopicName = recordMetadata.topic(); + try { + List topicPartitions = producer.partitionsFor(kafkaTopicName); + + Optional partitionInfo = + topicPartitions.stream() + .filter(p -> p.partition() == recordMetadata.partition()) + .findFirst(); + + return partitionInfo + .flatMap(partition -> Optional.ofNullable(partition.leader())) + .map(Node::host) + .map(ProduceMetadata::new) + .orElse(ProduceMetadata.empty()); + } catch (InterruptException e) { + Thread.currentThread().interrupt(); + } catch (Exception e) { + logger.warn( + "Could not read information about partitions for topic {}. {}", + kafkaTopicName, + e.getMessage()); + } + return ProduceMetadata.empty(); + }; + } + + private class MeteredCallback implements Callback { - private Supplier produceMetadataSupplier(RecordMetadata recordMetadata) { - return () -> { - String kafkaTopicName = recordMetadata.topic(); - try { - List topicPartitions = producer.partitionsFor(kafkaTopicName); - - Optional partitionInfo = topicPartitions.stream() - .filter(p -> p.partition() == recordMetadata.partition()) - .findFirst(); - - return partitionInfo.flatMap(partition -> Optional.ofNullable(partition.leader())) - .map(Node::host) - .map(ProduceMetadata::new) - .orElse(ProduceMetadata.empty()); - } catch (InterruptException e) { - Thread.currentThread().interrupt(); - } catch (Exception e) { - logger.warn("Could not read information about partitions for topic {}. {}", kafkaTopicName, e.getMessage()); - } - return ProduceMetadata.empty(); - }; + private final HermesTimerContext hermesTimerContext; + private final Message message; + private final CachedTopic cachedTopic; + private final Callback callback; + + public MeteredCallback( + HermesTimerContext hermesTimerContext, + Message message, + CachedTopic cachedTopic, + Callback callback) { + this.hermesTimerContext = hermesTimerContext; + this.message = message; + this.cachedTopic = cachedTopic; + this.callback = callback; } - private class MeteredCallback implements Callback { - - private final HermesTimerContext hermesTimerContext; - private final Message message; - private final CachedTopic cachedTopic; - private final Callback callback; - - public MeteredCallback(HermesTimerContext hermesTimerContext, Message message, CachedTopic cachedTopic, Callback callback) { - this.hermesTimerContext = hermesTimerContext; - this.message = message; - this.cachedTopic = cachedTopic; - this.callback = callback; - } - - @Override - public void onCompletion(RecordMetadata metadata, Exception exception) { - callback.onCompletion(metadata, exception); - Supplier produceMetadataSupplier = produceMetadataSupplier(metadata); - brokerLatencyReporter.report(hermesTimerContext, message, cachedTopic.getTopic().getAck(), produceMetadataSupplier); - } + @Override + public void onCompletion(RecordMetadata metadata, Exception exception) { + callback.onCompletion(metadata, exception); + Supplier produceMetadataSupplier = produceMetadataSupplier(metadata); + brokerLatencyReporter.report( + hermesTimerContext, message, cachedTopic.getTopic().getAck(), produceMetadataSupplier); } + } + + public void registerGauges(Topic.Ack ack, String sender) { + MetricName bufferTotalBytes = + producerMetricMame("buffer-total-bytes", "producer-metrics", "buffer total bytes"); + MetricName bufferAvailableBytes = + producerMetricMame("buffer-available-bytes", "producer-metrics", "buffer available bytes"); + MetricName compressionRate = + producerMetricMame("compression-rate-avg", "producer-metrics", "average compression rate"); + MetricName failedBatches = + producerMetricMame("record-error-total", "producer-metrics", "failed publishing batches"); + MetricName metadataAge = + producerMetricMame("metadata-age", "producer-metrics", "age [s] of metadata"); + MetricName queueTimeMax = + producerMetricMame( + "record-queue-time-max", + "producer-metrics", + "maximum time [ms] that batch spent in the send buffer"); + MetricName recordSendTotal = + producerMetricMame( + "record-send-total", + "producer-metrics", + "total number of records sent - including retries"); - public void registerGauges(Topic.Ack ack, String sender) { - MetricName bufferTotalBytes = producerMetricMame("buffer-total-bytes", "producer-metrics", "buffer total bytes"); - MetricName bufferAvailableBytes = producerMetricMame("buffer-available-bytes", "producer-metrics", "buffer available bytes"); - MetricName compressionRate = producerMetricMame("compression-rate-avg", "producer-metrics", "average compression rate"); - MetricName failedBatches = producerMetricMame("record-error-total", "producer-metrics", "failed publishing batches"); - MetricName metadataAge = producerMetricMame("metadata-age", "producer-metrics", "age [s] of metadata"); - MetricName queueTimeMax = producerMetricMame("record-queue-time-max", "producer-metrics", "maximum time [ms] that batch spent in the send buffer"); - MetricName recordSendTotal = producerMetricMame("record-send-total", "producer-metrics", "total number of records sent - including retries"); - - if (ack == Topic.Ack.ALL) { - metricsFacade.producer().registerAckAllTotalBytesGauge(producer, producerMetric(bufferTotalBytes), sender, datacenter); - metricsFacade.producer().registerAckAllAvailableBytesGauge(producer, producerMetric(bufferAvailableBytes), sender, datacenter); - metricsFacade.producer().registerAckAllCompressionRateGauge(producer, producerMetric(compressionRate), sender, datacenter); - metricsFacade.producer().registerAckAllFailedBatchesGauge(producer, producerMetric(failedBatches), sender, datacenter); - metricsFacade.producer().registerAckAllMetadataAgeGauge(producer, producerMetric(metadataAge), sender, datacenter); - metricsFacade.producer().registerAckAllRecordQueueTimeMaxGauge(producer, producerMetric(queueTimeMax), sender, datacenter); - metricsFacade.producer().registerAckAllRecordSendCounter(producer, producerMetric(recordSendTotal), sender, datacenter); - } else if (ack == Topic.Ack.LEADER) { - metricsFacade.producer().registerAckLeaderTotalBytesGauge(producer, producerMetric(bufferTotalBytes), sender, datacenter); - metricsFacade.producer().registerAckLeaderAvailableBytesGauge(producer, producerMetric(bufferAvailableBytes), sender, datacenter); - metricsFacade.producer().registerAckLeaderCompressionRateGauge(producer, producerMetric(compressionRate), sender, datacenter); - metricsFacade.producer().registerAckLeaderFailedBatchesGauge(producer, producerMetric(failedBatches), sender, datacenter); - metricsFacade.producer().registerAckLeaderMetadataAgeGauge(producer, producerMetric(metadataAge), sender, datacenter); - metricsFacade.producer().registerAckLeaderRecordQueueTimeMaxGauge(producer, producerMetric(queueTimeMax), sender, datacenter); - metricsFacade.producer().registerAckLeaderRecordSendCounter(producer, producerMetric(recordSendTotal), sender, datacenter); - } + if (ack == Topic.Ack.ALL) { + metricsFacade + .producer() + .registerAckAllTotalBytesGauge( + producer, producerMetric(bufferTotalBytes), sender, datacenter); + metricsFacade + .producer() + .registerAckAllAvailableBytesGauge( + producer, producerMetric(bufferAvailableBytes), sender, datacenter); + metricsFacade + .producer() + .registerAckAllCompressionRateGauge( + producer, producerMetric(compressionRate), sender, datacenter); + metricsFacade + .producer() + .registerAckAllFailedBatchesGauge( + producer, producerMetric(failedBatches), sender, datacenter); + metricsFacade + .producer() + .registerAckAllMetadataAgeGauge( + producer, producerMetric(metadataAge), sender, datacenter); + metricsFacade + .producer() + .registerAckAllRecordQueueTimeMaxGauge( + producer, producerMetric(queueTimeMax), sender, datacenter); + metricsFacade + .producer() + .registerAckAllRecordSendCounter( + producer, producerMetric(recordSendTotal), sender, datacenter); + } else if (ack == Topic.Ack.LEADER) { + metricsFacade + .producer() + .registerAckLeaderTotalBytesGauge( + producer, producerMetric(bufferTotalBytes), sender, datacenter); + metricsFacade + .producer() + .registerAckLeaderAvailableBytesGauge( + producer, producerMetric(bufferAvailableBytes), sender, datacenter); + metricsFacade + .producer() + .registerAckLeaderCompressionRateGauge( + producer, producerMetric(compressionRate), sender, datacenter); + metricsFacade + .producer() + .registerAckLeaderFailedBatchesGauge( + producer, producerMetric(failedBatches), sender, datacenter); + metricsFacade + .producer() + .registerAckLeaderMetadataAgeGauge( + producer, producerMetric(metadataAge), sender, datacenter); + metricsFacade + .producer() + .registerAckLeaderRecordQueueTimeMaxGauge( + producer, producerMetric(queueTimeMax), sender, datacenter); + metricsFacade + .producer() + .registerAckLeaderRecordSendCounter( + producer, producerMetric(recordSendTotal), sender, datacenter); } + } - private double findProducerMetric(Producer producer, - Predicate> predicate) { - Optional> first = - producer.metrics().entrySet().stream().filter(predicate).findFirst(); - Object value = first.map(metricNameEntry -> metricNameEntry.getValue().metricValue()).orElse(0.0d); - if (value instanceof Number) { - return ((Number) value).doubleValue(); - } else { - return 0.0; - } + private double findProducerMetric( + Producer producer, Predicate> predicate) { + Optional> first = + producer.metrics().entrySet().stream().filter(predicate).findFirst(); + Object value = + first.map(metricNameEntry -> metricNameEntry.getValue().metricValue()).orElse(0.0d); + if (value instanceof Number) { + return ((Number) value).doubleValue(); + } else { + return 0.0; } + } - private ToDoubleFunction> producerMetric(MetricName producerMetricName) { - Predicate> predicate = entry -> entry.getKey().group().equals(producerMetricName.group()) + private ToDoubleFunction> producerMetric(MetricName producerMetricName) { + Predicate> predicate = + entry -> + entry.getKey().group().equals(producerMetricName.group()) && entry.getKey().name().equals(producerMetricName.name()); - return producer -> findProducerMetric(producer, predicate); - } + return producer -> findProducerMetric(producer, predicate); + } - private static MetricName producerMetricMame(String name, String group, String description) { - return new MetricName(name, group, description, Collections.emptyMap()); - } + private static MetricName producerMetricMame(String name, String group, String description) { + return new MetricName(name, group, description, Collections.emptyMap()); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/KafkaMessageSenders.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/KafkaMessageSenders.java index ddd707657e..66cea1384a 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/KafkaMessageSenders.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/KafkaMessageSenders.java @@ -1,170 +1,178 @@ package pl.allegro.tech.hermes.frontend.producer.kafka; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.kafka.common.PartitionInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.frontend.metric.CachedTopic; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -// exposes kafka producer metrics, see: https://docs.confluent.io/platform/current/kafka/monitoring.html#producer-metrics +// exposes kafka producer metrics, see: +// https://docs.confluent.io/platform/current/kafka/monitoring.html#producer-metrics public class KafkaMessageSenders { - private static final Logger logger = LoggerFactory.getLogger(KafkaMessageSenders.class); - - private final KafkaMessageSender ackLeader; - private final KafkaMessageSender ackAll; - - private final List> remoteAckLeader; - private final List> remoteAckAll; - - private final MinInSyncReplicasLoader localMinInSyncReplicasLoader; - private final TopicMetadataLoadingExecutor topicMetadataLoadingExecutor; - private final List localDatacenterTopicMetadataLoaders; - private final List kafkaProducerMetadataRefreshers; - private final List datacenters; - - KafkaMessageSenders(TopicMetadataLoadingExecutor topicMetadataLoadingExecutor, - MinInSyncReplicasLoader localMinInSyncReplicasLoader, - Tuple localSenders, - List remoteSenders) { - this.topicMetadataLoadingExecutor = topicMetadataLoadingExecutor; - this.localMinInSyncReplicasLoader = localMinInSyncReplicasLoader; - this.ackLeader = localSenders.ackLeader; - this.ackAll = localSenders.ackAll; - this.remoteAckLeader = remoteSenders.stream().map(it -> it.ackLeader).collect(Collectors.toList()); - this.remoteAckAll = remoteSenders.stream().map(it -> it.ackAll).collect(Collectors.toList()); - this.localDatacenterTopicMetadataLoaders = List.of( - new LocalDatacenterTopicAvailabilityChecker() - ); - this.kafkaProducerMetadataRefreshers = Stream.concat(Stream.of(localSenders), remoteSenders.stream()) - .map(KafkaProducerMetadataRefresher::new) - .collect(Collectors.toList()); - this.datacenters = Stream.concat(Stream.of(localSenders), remoteSenders.stream()) - .map(tuple -> tuple.ackAll) - .map(KafkaMessageSender::getDatacenter) - .toList(); - } - - KafkaMessageSender get(Topic topic) { - return topic.isReplicationConfirmRequired() ? ackAll : ackLeader; - } - - List> getRemote(Topic topic) { - return topic.isReplicationConfirmRequired() ? remoteAckAll : remoteAckLeader; + private static final Logger logger = LoggerFactory.getLogger(KafkaMessageSenders.class); + + private final KafkaMessageSender ackLeader; + private final KafkaMessageSender ackAll; + + private final List> remoteAckLeader; + private final List> remoteAckAll; + + private final MinInSyncReplicasLoader localMinInSyncReplicasLoader; + private final TopicMetadataLoadingExecutor topicMetadataLoadingExecutor; + private final List localDatacenterTopicMetadataLoaders; + private final List kafkaProducerMetadataRefreshers; + private final List datacenters; + + KafkaMessageSenders( + TopicMetadataLoadingExecutor topicMetadataLoadingExecutor, + MinInSyncReplicasLoader localMinInSyncReplicasLoader, + Tuple localSenders, + List remoteSenders) { + this.topicMetadataLoadingExecutor = topicMetadataLoadingExecutor; + this.localMinInSyncReplicasLoader = localMinInSyncReplicasLoader; + this.ackLeader = localSenders.ackLeader; + this.ackAll = localSenders.ackAll; + this.remoteAckLeader = + remoteSenders.stream().map(it -> it.ackLeader).collect(Collectors.toList()); + this.remoteAckAll = remoteSenders.stream().map(it -> it.ackAll).collect(Collectors.toList()); + this.localDatacenterTopicMetadataLoaders = + List.of(new LocalDatacenterTopicAvailabilityChecker()); + this.kafkaProducerMetadataRefreshers = + Stream.concat(Stream.of(localSenders), remoteSenders.stream()) + .map(KafkaProducerMetadataRefresher::new) + .collect(Collectors.toList()); + this.datacenters = + Stream.concat(Stream.of(localSenders), remoteSenders.stream()) + .map(tuple -> tuple.ackAll) + .map(KafkaMessageSender::getDatacenter) + .toList(); + } + + KafkaMessageSender get(Topic topic) { + return topic.isReplicationConfirmRequired() ? ackAll : ackLeader; + } + + List> getRemote(Topic topic) { + return topic.isReplicationConfirmRequired() ? remoteAckAll : remoteAckLeader; + } + + List getDatacenters() { + return datacenters; + } + + void refreshTopicMetadata() { + topicMetadataLoadingExecutor.execute(kafkaProducerMetadataRefreshers); + } + + boolean areAllTopicsAvailable() { + return topicMetadataLoadingExecutor.execute(localDatacenterTopicMetadataLoaders); + } + + boolean isTopicAvailable(CachedTopic cachedTopic) { + String kafkaTopicName = cachedTopic.getKafkaTopics().getPrimary().name().asString(); + + try { + List partitionInfos = + get(cachedTopic.getTopic()).loadPartitionMetadataFor(kafkaTopicName); + if (anyPartitionWithoutLeader(partitionInfos)) { + logger.warn("Topic {} has partitions without a leader.", kafkaTopicName); + return false; + } + if (anyUnderReplicatedPartition(partitionInfos, kafkaTopicName)) { + logger.warn("Topic {} has under replicated partitions.", kafkaTopicName); + return false; + } + if (!partitionInfos.isEmpty()) { + return true; + } + } catch (Exception e) { + logger.warn( + "Could not read information about partitions for topic {}. {}", + kafkaTopicName, + e.getMessage()); + return false; } - List getDatacenters() { - return datacenters; - } + logger.warn("No information about partitions for topic {}", kafkaTopicName); + return false; + } - void refreshTopicMetadata() { - topicMetadataLoadingExecutor.execute(kafkaProducerMetadataRefreshers); - } + private boolean anyPartitionWithoutLeader(List partitionInfos) { + return partitionInfos.stream().anyMatch(p -> p.leader() == null); + } - boolean areAllTopicsAvailable() { - return topicMetadataLoadingExecutor.execute(localDatacenterTopicMetadataLoaders); - } + private boolean anyUnderReplicatedPartition( + List partitionInfos, String kafkaTopicName) throws Exception { + int minInSyncReplicas = localMinInSyncReplicasLoader.get(kafkaTopicName); + return partitionInfos.stream().anyMatch(p -> p.inSyncReplicas().length < minInSyncReplicas); + } - boolean isTopicAvailable(CachedTopic cachedTopic) { - String kafkaTopicName = cachedTopic.getKafkaTopics().getPrimary().name().asString(); - - try { - List partitionInfos = get(cachedTopic.getTopic()).loadPartitionMetadataFor(kafkaTopicName); - if (anyPartitionWithoutLeader(partitionInfos)) { - logger.warn("Topic {} has partitions without a leader.", kafkaTopicName); - return false; - } - if (anyUnderReplicatedPartition(partitionInfos, kafkaTopicName)) { - logger.warn("Topic {} has under replicated partitions.", kafkaTopicName); - return false; - } - if (!partitionInfos.isEmpty()) { - return true; - } - } catch (Exception e) { - logger.warn("Could not read information about partitions for topic {}. {}", kafkaTopicName, e.getMessage()); - return false; - } + public void registerSenderMetrics(String name) { + ackLeader.registerGauges(Topic.Ack.LEADER, name); + ackAll.registerGauges(Topic.Ack.ALL, name); + remoteAckLeader.forEach(sender -> sender.registerGauges(Topic.Ack.LEADER, name)); + remoteAckAll.forEach(sender -> sender.registerGauges(Topic.Ack.ALL, name)); + } - logger.warn("No information about partitions for topic {}", kafkaTopicName); - return false; - } + static class Tuple { + private final KafkaMessageSender ackLeader; + private final KafkaMessageSender ackAll; - private boolean anyPartitionWithoutLeader(List partitionInfos) { - return partitionInfos.stream().anyMatch(p -> p.leader() == null); + Tuple(KafkaMessageSender ackLeader, KafkaMessageSender ackAll) { + this.ackLeader = ackLeader; + this.ackAll = ackAll; } + } - private boolean anyUnderReplicatedPartition(List partitionInfos, String kafkaTopicName) throws Exception { - int minInSyncReplicas = localMinInSyncReplicasLoader.get(kafkaTopicName); - return partitionInfos.stream().anyMatch(p -> p.inSyncReplicas().length < minInSyncReplicas); - } + public void close() { + ackAll.close(); + ackLeader.close(); + } - public void registerSenderMetrics(String name) { - ackLeader.registerGauges(Topic.Ack.LEADER, name); - ackAll.registerGauges(Topic.Ack.ALL, name); - remoteAckLeader.forEach(sender -> sender.registerGauges(Topic.Ack.LEADER, name)); - remoteAckAll.forEach(sender -> sender.registerGauges(Topic.Ack.ALL, name)); - } + private class KafkaProducerMetadataRefresher implements TopicMetadataLoader { - static class Tuple { - private final KafkaMessageSender ackLeader; - private final KafkaMessageSender ackAll; + private final KafkaMessageSender ackLeader; + private final KafkaMessageSender ackAll; - Tuple(KafkaMessageSender ackLeader, KafkaMessageSender ackAll) { - this.ackLeader = ackLeader; - this.ackAll = ackAll; - } + KafkaProducerMetadataRefresher(Tuple tuple) { + this.ackLeader = tuple.ackLeader; + this.ackAll = tuple.ackAll; } - public void close() { - ackAll.close(); - ackLeader.close(); + @Override + public MetadataLoadingResult load(CachedTopic cachedTopic) { + String kafkaTopicName = cachedTopic.getKafkaTopics().getPrimary().name().asString(); + var sender = getSender(cachedTopic.getTopic()); + var partitionInfos = sender.loadPartitionMetadataFor(kafkaTopicName); + if (anyPartitionWithoutLeader(partitionInfos)) { + logger.warn("Topic {} has partitions without a leader.", kafkaTopicName); + return MetadataLoadingResult.failure(cachedTopic.getTopicName(), sender.getDatacenter()); + } + if (partitionInfos.isEmpty()) { + logger.warn("No information about partitions for topic {}", kafkaTopicName); + return MetadataLoadingResult.failure(cachedTopic.getTopicName(), sender.getDatacenter()); + } + return MetadataLoadingResult.success(cachedTopic.getTopicName(), sender.getDatacenter()); } - private class KafkaProducerMetadataRefresher implements TopicMetadataLoader { - - private final KafkaMessageSender ackLeader; - private final KafkaMessageSender ackAll; - - KafkaProducerMetadataRefresher(Tuple tuple) { - this.ackLeader = tuple.ackLeader; - this.ackAll = tuple.ackAll; - } - - @Override - public MetadataLoadingResult load(CachedTopic cachedTopic) { - String kafkaTopicName = cachedTopic.getKafkaTopics().getPrimary().name().asString(); - var sender = getSender(cachedTopic.getTopic()); - var partitionInfos = sender.loadPartitionMetadataFor(kafkaTopicName); - if (anyPartitionWithoutLeader(partitionInfos)) { - logger.warn("Topic {} has partitions without a leader.", kafkaTopicName); - return MetadataLoadingResult.failure(cachedTopic.getTopicName(), sender.getDatacenter()); - } - if (partitionInfos.isEmpty()) { - logger.warn("No information about partitions for topic {}", kafkaTopicName); - return MetadataLoadingResult.failure(cachedTopic.getTopicName(), sender.getDatacenter()); - } - return MetadataLoadingResult.success(cachedTopic.getTopicName(), sender.getDatacenter()); - } - - private KafkaMessageSender getSender(Topic topic) { - return topic.isReplicationConfirmRequired() ? ackAll : ackLeader; - } + private KafkaMessageSender getSender(Topic topic) { + return topic.isReplicationConfirmRequired() ? ackAll : ackLeader; } + } - private class LocalDatacenterTopicAvailabilityChecker implements TopicMetadataLoader { + private class LocalDatacenterTopicAvailabilityChecker implements TopicMetadataLoader { - @Override - public MetadataLoadingResult load(CachedTopic topic) { - String datacenter = get(topic.getTopic()).getDatacenter(); - if (isTopicAvailable(topic)) { - return MetadataLoadingResult.success(topic.getTopicName(), datacenter); - } - return MetadataLoadingResult.failure(topic.getTopicName(), datacenter); - } + @Override + public MetadataLoadingResult load(CachedTopic topic) { + String datacenter = get(topic.getTopic()).getDatacenter(); + if (isTopicAvailable(topic)) { + return MetadataLoadingResult.success(topic.getTopicName(), datacenter); + } + return MetadataLoadingResult.failure(topic.getTopicName(), datacenter); } + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/KafkaMessageSendersFactory.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/KafkaMessageSendersFactory.java index cd10168f3c..391dbbc827 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/KafkaMessageSendersFactory.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/KafkaMessageSendersFactory.java @@ -1,17 +1,5 @@ package pl.allegro.tech.hermes.frontend.producer.kafka; -import org.apache.kafka.clients.admin.AdminClient; -import pl.allegro.tech.hermes.common.kafka.KafkaParameters; -import pl.allegro.tech.hermes.common.metric.MetricsFacade; -import pl.allegro.tech.hermes.frontend.cache.topic.TopicsCache; -import pl.allegro.tech.hermes.frontend.producer.BrokerLatencyReporter; - -import java.time.Duration; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ScheduledExecutorService; - import static org.apache.kafka.clients.CommonClientConfigs.SECURITY_PROTOCOL_CONFIG; import static org.apache.kafka.clients.producer.ProducerConfig.ACKS_CONFIG; import static org.apache.kafka.clients.producer.ProducerConfig.BATCH_SIZE_CONFIG; @@ -35,110 +23,138 @@ import static org.apache.kafka.common.config.SaslConfigs.SASL_JAAS_CONFIG; import static org.apache.kafka.common.config.SaslConfigs.SASL_MECHANISM; -public class KafkaMessageSendersFactory { +import java.time.Duration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ScheduledExecutorService; +import org.apache.kafka.clients.admin.AdminClient; +import pl.allegro.tech.hermes.common.kafka.KafkaParameters; +import pl.allegro.tech.hermes.common.metric.MetricsFacade; +import pl.allegro.tech.hermes.frontend.cache.topic.TopicsCache; +import pl.allegro.tech.hermes.frontend.producer.BrokerLatencyReporter; - private static final String ACK_ALL = "-1"; - private static final String ACK_LEADER = "1"; +public class KafkaMessageSendersFactory { - private final TopicMetadataLoadingExecutor topicMetadataLoadingExecutor; - private final MinInSyncReplicasLoader localMinInSyncReplicasLoader; - private final KafkaParameters kafkaParameters; - private final List remoteKafkaParameters; - private final BrokerLatencyReporter brokerLatencyReporter; - private final MetricsFacade metricsFacade; - private final long bufferedSizeBytes; - private final ScheduledExecutorService chaosScheduler; + private static final String ACK_ALL = "-1"; + private static final String ACK_LEADER = "1"; - public KafkaMessageSendersFactory(KafkaParameters kafkaParameters, - List remoteKafkaParameters, - BrokerLatencyReporter brokerLatencyReporter, - MetricsFacade metricsFacade, - AdminClient localAdminClient, - TopicsCache topicsCache, - int retryCount, - Duration retryInterval, - int threadPoolSize, - long bufferedSizeBytes, - Duration metadataMaxAge, - ScheduledExecutorService chaosScheduler) { - this.topicMetadataLoadingExecutor = new TopicMetadataLoadingExecutor(topicsCache, retryCount, retryInterval, threadPoolSize); - this.localMinInSyncReplicasLoader = new MinInSyncReplicasLoader(localAdminClient, metadataMaxAge); - this.bufferedSizeBytes = bufferedSizeBytes; - this.kafkaParameters = kafkaParameters; - this.remoteKafkaParameters = remoteKafkaParameters; - this.metricsFacade = metricsFacade; - this.brokerLatencyReporter = brokerLatencyReporter; - this.chaosScheduler = chaosScheduler; - } + private final TopicMetadataLoadingExecutor topicMetadataLoadingExecutor; + private final MinInSyncReplicasLoader localMinInSyncReplicasLoader; + private final KafkaParameters kafkaParameters; + private final List remoteKafkaParameters; + private final BrokerLatencyReporter brokerLatencyReporter; + private final MetricsFacade metricsFacade; + private final long bufferedSizeBytes; + private final ScheduledExecutorService chaosScheduler; + public KafkaMessageSendersFactory( + KafkaParameters kafkaParameters, + List remoteKafkaParameters, + BrokerLatencyReporter brokerLatencyReporter, + MetricsFacade metricsFacade, + AdminClient localAdminClient, + TopicsCache topicsCache, + int retryCount, + Duration retryInterval, + int threadPoolSize, + long bufferedSizeBytes, + Duration metadataMaxAge, + ScheduledExecutorService chaosScheduler) { + this.topicMetadataLoadingExecutor = + new TopicMetadataLoadingExecutor(topicsCache, retryCount, retryInterval, threadPoolSize); + this.localMinInSyncReplicasLoader = + new MinInSyncReplicasLoader(localAdminClient, metadataMaxAge); + this.bufferedSizeBytes = bufferedSizeBytes; + this.kafkaParameters = kafkaParameters; + this.remoteKafkaParameters = remoteKafkaParameters; + this.metricsFacade = metricsFacade; + this.brokerLatencyReporter = brokerLatencyReporter; + this.chaosScheduler = chaosScheduler; + } - public KafkaMessageSenders provide(KafkaProducerParameters kafkaProducerParameters, String senderName) { - return provide(kafkaProducerParameters, kafkaProducerParameters, senderName); - } + public KafkaMessageSenders provide( + KafkaProducerParameters kafkaProducerParameters, String senderName) { + return provide(kafkaProducerParameters, kafkaProducerParameters, senderName); + } - public KafkaMessageSenders provide(KafkaProducerParameters localKafkaProducerParameters, - KafkaProducerParameters remoteKafkaProducerParameters, - String senderName) { - KafkaMessageSenders.Tuple localProducers = new KafkaMessageSenders.Tuple( - sender(kafkaParameters, localKafkaProducerParameters, ACK_LEADER), - sender(kafkaParameters, localKafkaProducerParameters, ACK_ALL) - ); + public KafkaMessageSenders provide( + KafkaProducerParameters localKafkaProducerParameters, + KafkaProducerParameters remoteKafkaProducerParameters, + String senderName) { + KafkaMessageSenders.Tuple localProducers = + new KafkaMessageSenders.Tuple( + sender(kafkaParameters, localKafkaProducerParameters, ACK_LEADER), + sender(kafkaParameters, localKafkaProducerParameters, ACK_ALL)); - List remoteProducers = remoteKafkaParameters.stream().map( - kafkaProperties -> new KafkaMessageSenders.Tuple( + List remoteProducers = + remoteKafkaParameters.stream() + .map( + kafkaProperties -> + new KafkaMessageSenders.Tuple( sender(kafkaProperties, remoteKafkaProducerParameters, ACK_LEADER), - sender(kafkaProperties, remoteKafkaProducerParameters, ACK_ALL))).toList(); - KafkaMessageSenders senders = new KafkaMessageSenders( - topicMetadataLoadingExecutor, - localMinInSyncReplicasLoader, - localProducers, - remoteProducers - ); - senders.registerSenderMetrics(senderName); - return senders; - } + sender(kafkaProperties, remoteKafkaProducerParameters, ACK_ALL))) + .toList(); + KafkaMessageSenders senders = + new KafkaMessageSenders( + topicMetadataLoadingExecutor, + localMinInSyncReplicasLoader, + localProducers, + remoteProducers); + senders.registerSenderMetrics(senderName); + return senders; + } - private KafkaMessageSender sender(KafkaParameters kafkaParameters, - KafkaProducerParameters kafkaProducerParameters, - String acks) { - Map props = new HashMap<>(); - props.put(BOOTSTRAP_SERVERS_CONFIG, kafkaParameters.getBrokerList()); - props.put(MAX_BLOCK_MS_CONFIG, (int) kafkaProducerParameters.getMaxBlock().toMillis()); - props.put(COMPRESSION_TYPE_CONFIG, kafkaProducerParameters.getCompressionCodec()); - props.put(BUFFER_MEMORY_CONFIG, bufferedSizeBytes); - props.put(REQUEST_TIMEOUT_MS_CONFIG, (int) kafkaProducerParameters.getRequestTimeout().toMillis()); - props.put(DELIVERY_TIMEOUT_MS_CONFIG, (int) kafkaProducerParameters.getDeliveryTimeout().toMillis()); - props.put(BATCH_SIZE_CONFIG, kafkaProducerParameters.getBatchSize()); - props.put(SEND_BUFFER_CONFIG, kafkaProducerParameters.getTcpSendBuffer()); - props.put(RETRIES_CONFIG, kafkaProducerParameters.getRetries()); - props.put(RETRY_BACKOFF_MS_CONFIG, (int) kafkaProducerParameters.getRetryBackoff().toMillis()); - props.put(METADATA_MAX_AGE_CONFIG, (int) kafkaProducerParameters.getMetadataMaxAge().toMillis()); - props.put(KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); - props.put(VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); - props.put(MAX_REQUEST_SIZE_CONFIG, kafkaProducerParameters.getMaxRequestSize()); - props.put(LINGER_MS_CONFIG, (int) kafkaProducerParameters.getLinger().toMillis()); - props.put(METRICS_SAMPLE_WINDOW_MS_CONFIG, (int) kafkaProducerParameters.getMetricsSampleWindow().toMillis()); - props.put(MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, kafkaProducerParameters.getMaxInflightRequestsPerConnection()); - props.put(ENABLE_IDEMPOTENCE_CONFIG, kafkaProducerParameters.isIdempotenceEnabled()); - props.put(ACKS_CONFIG, acks); + private KafkaMessageSender sender( + KafkaParameters kafkaParameters, + KafkaProducerParameters kafkaProducerParameters, + String acks) { + Map props = new HashMap<>(); + props.put(BOOTSTRAP_SERVERS_CONFIG, kafkaParameters.getBrokerList()); + props.put(MAX_BLOCK_MS_CONFIG, (int) kafkaProducerParameters.getMaxBlock().toMillis()); + props.put(COMPRESSION_TYPE_CONFIG, kafkaProducerParameters.getCompressionCodec()); + props.put(BUFFER_MEMORY_CONFIG, bufferedSizeBytes); + props.put( + REQUEST_TIMEOUT_MS_CONFIG, (int) kafkaProducerParameters.getRequestTimeout().toMillis()); + props.put( + DELIVERY_TIMEOUT_MS_CONFIG, (int) kafkaProducerParameters.getDeliveryTimeout().toMillis()); + props.put(BATCH_SIZE_CONFIG, kafkaProducerParameters.getBatchSize()); + props.put(SEND_BUFFER_CONFIG, kafkaProducerParameters.getTcpSendBuffer()); + props.put(RETRIES_CONFIG, kafkaProducerParameters.getRetries()); + props.put(RETRY_BACKOFF_MS_CONFIG, (int) kafkaProducerParameters.getRetryBackoff().toMillis()); + props.put( + METADATA_MAX_AGE_CONFIG, (int) kafkaProducerParameters.getMetadataMaxAge().toMillis()); + props.put( + KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); + props.put( + VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); + props.put(MAX_REQUEST_SIZE_CONFIG, kafkaProducerParameters.getMaxRequestSize()); + props.put(LINGER_MS_CONFIG, (int) kafkaProducerParameters.getLinger().toMillis()); + props.put( + METRICS_SAMPLE_WINDOW_MS_CONFIG, + (int) kafkaProducerParameters.getMetricsSampleWindow().toMillis()); + props.put( + MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, + kafkaProducerParameters.getMaxInflightRequestsPerConnection()); + props.put(ENABLE_IDEMPOTENCE_CONFIG, kafkaProducerParameters.isIdempotenceEnabled()); + props.put(ACKS_CONFIG, acks); - if (kafkaParameters.isAuthenticationEnabled()) { - props.put(SASL_MECHANISM, kafkaParameters.getAuthenticationMechanism()); - props.put(SECURITY_PROTOCOL_CONFIG, kafkaParameters.getAuthenticationProtocol()); - props.put(SASL_JAAS_CONFIG, kafkaParameters.getJaasConfig()); - } - - return new KafkaMessageSender<>( - new org.apache.kafka.clients.producer.KafkaProducer<>(props), - brokerLatencyReporter, - metricsFacade, - kafkaParameters.getDatacenter(), - chaosScheduler - ); + if (kafkaParameters.isAuthenticationEnabled()) { + props.put(SASL_MECHANISM, kafkaParameters.getAuthenticationMechanism()); + props.put(SECURITY_PROTOCOL_CONFIG, kafkaParameters.getAuthenticationProtocol()); + props.put(SASL_JAAS_CONFIG, kafkaParameters.getJaasConfig()); } - public void close() throws Exception { - topicMetadataLoadingExecutor.close(); - localMinInSyncReplicasLoader.close(); - } + return new KafkaMessageSender<>( + new org.apache.kafka.clients.producer.KafkaProducer<>(props), + brokerLatencyReporter, + metricsFacade, + kafkaParameters.getDatacenter(), + chaosScheduler); + } + + public void close() throws Exception { + topicMetadataLoadingExecutor.close(); + localMinInSyncReplicasLoader.close(); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/KafkaProducerParameters.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/KafkaProducerParameters.java index c0caa3913c..e8c5a5cc49 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/KafkaProducerParameters.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/KafkaProducerParameters.java @@ -4,33 +4,33 @@ public interface KafkaProducerParameters { - Duration getMaxBlock(); + Duration getMaxBlock(); - Duration getMetadataMaxAge(); + Duration getMetadataMaxAge(); - String getCompressionCodec(); + String getCompressionCodec(); - int getRetries(); + int getRetries(); - Duration getRetryBackoff(); + Duration getRetryBackoff(); - Duration getRequestTimeout(); + Duration getRequestTimeout(); - int getBatchSize(); + int getBatchSize(); - int getTcpSendBuffer(); + int getTcpSendBuffer(); - int getMaxRequestSize(); + int getMaxRequestSize(); - Duration getDeliveryTimeout(); + Duration getDeliveryTimeout(); - Duration getLinger(); + Duration getLinger(); - Duration getMetricsSampleWindow(); + Duration getMetricsSampleWindow(); - int getMaxInflightRequestsPerConnection(); + int getMaxInflightRequestsPerConnection(); - boolean isReportNodeMetricsEnabled(); + boolean isReportNodeMetricsEnabled(); - boolean isIdempotenceEnabled(); + boolean isIdempotenceEnabled(); } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/LocalDatacenterMessageProducer.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/LocalDatacenterMessageProducer.java index 2fc42890c4..f23df903e2 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/LocalDatacenterMessageProducer.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/LocalDatacenterMessageProducer.java @@ -12,57 +12,61 @@ @Singleton public class LocalDatacenterMessageProducer implements BrokerMessageProducer { - private final KafkaMessageSenders kafkaMessageSenders; - private final MessageToKafkaProducerRecordConverter messageConverter; + private final KafkaMessageSenders kafkaMessageSenders; + private final MessageToKafkaProducerRecordConverter messageConverter; - public LocalDatacenterMessageProducer(KafkaMessageSenders kafkaMessageSenders, - MessageToKafkaProducerRecordConverter messageConverter) { - this.kafkaMessageSenders = kafkaMessageSenders; - this.messageConverter = messageConverter; - } + public LocalDatacenterMessageProducer( + KafkaMessageSenders kafkaMessageSenders, + MessageToKafkaProducerRecordConverter messageConverter) { + this.kafkaMessageSenders = kafkaMessageSenders; + this.messageConverter = messageConverter; + } - @Override - public void send(Message message, CachedTopic cachedTopic, final PublishingCallback callback) { - ProducerRecord producerRecord = - messageConverter.convertToProducerRecord(message, cachedTopic.getKafkaTopics().getPrimary().name()); + @Override + public void send(Message message, CachedTopic cachedTopic, final PublishingCallback callback) { + ProducerRecord producerRecord = + messageConverter.convertToProducerRecord( + message, cachedTopic.getKafkaTopics().getPrimary().name()); - var producer = kafkaMessageSenders.get(cachedTopic.getTopic()); - Callback wrappedCallback = new SendCallback(message, cachedTopic, callback, producer.getDatacenter()); - producer.send(producerRecord, cachedTopic, message, wrappedCallback); - } + var producer = kafkaMessageSenders.get(cachedTopic.getTopic()); + Callback wrappedCallback = + new SendCallback(message, cachedTopic, callback, producer.getDatacenter()); + producer.send(producerRecord, cachedTopic, message, wrappedCallback); + } - @Override - public boolean areAllTopicsAvailable() { - return kafkaMessageSenders.areAllTopicsAvailable(); - } + @Override + public boolean areAllTopicsAvailable() { + return kafkaMessageSenders.areAllTopicsAvailable(); + } - @Override - public boolean isTopicAvailable(CachedTopic cachedTopic) { - return kafkaMessageSenders.isTopicAvailable(cachedTopic); - } + @Override + public boolean isTopicAvailable(CachedTopic cachedTopic) { + return kafkaMessageSenders.isTopicAvailable(cachedTopic); + } - private static class SendCallback implements org.apache.kafka.clients.producer.Callback { + private static class SendCallback implements org.apache.kafka.clients.producer.Callback { - private final Message message; - private final CachedTopic topic; - private final PublishingCallback callback; - private final String datacenter; + private final Message message; + private final CachedTopic topic; + private final PublishingCallback callback; + private final String datacenter; - public SendCallback(Message message, CachedTopic topic, PublishingCallback callback, String datacenter) { - this.message = message; - this.topic = topic; - this.callback = callback; - this.datacenter = datacenter; - } + public SendCallback( + Message message, CachedTopic topic, PublishingCallback callback, String datacenter) { + this.message = message; + this.topic = topic; + this.callback = callback; + this.datacenter = datacenter; + } - @Override - public void onCompletion(RecordMetadata recordMetadata, Exception e) { - if (e == null) { - callback.onEachPublished(message, topic.getTopic(), datacenter); - callback.onPublished(message, topic.getTopic()); - } else { - callback.onUnpublished(message, topic.getTopic(), e); - } - } + @Override + public void onCompletion(RecordMetadata recordMetadata, Exception e) { + if (e == null) { + callback.onEachPublished(message, topic.getTopic(), datacenter); + callback.onPublished(message, topic.getTopic()); + } else { + callback.onUnpublished(message, topic.getTopic(), e); + } } + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/MessageToKafkaProducerRecordConverter.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/MessageToKafkaProducerRecordConverter.java index 51683a03f8..b7fcc267b7 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/MessageToKafkaProducerRecordConverter.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/MessageToKafkaProducerRecordConverter.java @@ -1,50 +1,56 @@ package pl.allegro.tech.hermes.frontend.producer.kafka; +import static java.util.Optional.ofNullable; + import com.google.common.collect.Lists; +import java.util.List; import org.apache.avro.Schema; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.header.Header; import pl.allegro.tech.hermes.common.kafka.KafkaTopicName; import pl.allegro.tech.hermes.frontend.publishing.message.Message; -import java.util.List; - -import static java.util.Optional.ofNullable; - public class MessageToKafkaProducerRecordConverter { - private final KafkaHeaderFactory kafkaHeaderFactory; - private final boolean schemaIdHeaderEnabled; - - public MessageToKafkaProducerRecordConverter(KafkaHeaderFactory kafkaHeaderFactory, - boolean schemaIdHeaderEnabled) { - this.kafkaHeaderFactory = kafkaHeaderFactory; - this.schemaIdHeaderEnabled = schemaIdHeaderEnabled; - } - - public ProducerRecord convertToProducerRecord(Message message, KafkaTopicName kafkaTopicName) { - Iterable
headers = createRecordHeaders(message); - byte[] partitionKey = ofNullable(message.getPartitionKey()).map(String::getBytes).orElse(null); - - return new ProducerRecord<>(kafkaTopicName.asString(), null, message.getTimestamp(), - partitionKey, message.getData(), headers); - } - - private Iterable
createRecordHeaders(Message message) { - List
headers = Lists.newArrayList( - kafkaHeaderFactory.messageId(message.getId()) - ); - - message.getCompiledSchema().ifPresent(compiledSchemaVersion -> { - headers.add(kafkaHeaderFactory.schemaVersion(compiledSchemaVersion.getVersion().value())); - if (schemaIdHeaderEnabled) { + private final KafkaHeaderFactory kafkaHeaderFactory; + private final boolean schemaIdHeaderEnabled; + + public MessageToKafkaProducerRecordConverter( + KafkaHeaderFactory kafkaHeaderFactory, boolean schemaIdHeaderEnabled) { + this.kafkaHeaderFactory = kafkaHeaderFactory; + this.schemaIdHeaderEnabled = schemaIdHeaderEnabled; + } + + public ProducerRecord convertToProducerRecord( + Message message, KafkaTopicName kafkaTopicName) { + Iterable
headers = createRecordHeaders(message); + byte[] partitionKey = ofNullable(message.getPartitionKey()).map(String::getBytes).orElse(null); + + return new ProducerRecord<>( + kafkaTopicName.asString(), + null, + message.getTimestamp(), + partitionKey, + message.getData(), + headers); + } + + private Iterable
createRecordHeaders(Message message) { + List
headers = Lists.newArrayList(kafkaHeaderFactory.messageId(message.getId())); + + message + .getCompiledSchema() + .ifPresent( + compiledSchemaVersion -> { + headers.add( + kafkaHeaderFactory.schemaVersion(compiledSchemaVersion.getVersion().value())); + if (schemaIdHeaderEnabled) { headers.add(kafkaHeaderFactory.schemaId(compiledSchemaVersion.getId().value())); - } - }); - - kafkaHeaderFactory.setHTTPHeadersIfEnabled(headers, message.getHTTPHeaders()); + } + }); - return headers; - } + kafkaHeaderFactory.setHTTPHeadersIfEnabled(headers, message.getHTTPHeaders()); + return headers; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/MinInSyncReplicasLoader.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/MinInSyncReplicasLoader.java index e3d693d221..9462b9317b 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/MinInSyncReplicasLoader.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/MinInSyncReplicasLoader.java @@ -1,54 +1,55 @@ package pl.allegro.tech.hermes.frontend.producer.kafka; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.apache.kafka.common.config.ConfigResource.Type.TOPIC; +import static org.apache.kafka.common.config.TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG; + import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableList; +import java.time.Duration; +import java.util.Map; +import java.util.concurrent.ExecutionException; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.Config; import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.clients.admin.DescribeConfigsResult; import org.apache.kafka.common.config.ConfigResource; -import java.time.Duration; -import java.util.Map; -import java.util.concurrent.ExecutionException; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static org.apache.kafka.common.config.ConfigResource.Type.TOPIC; -import static org.apache.kafka.common.config.TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG; - class MinInSyncReplicasLoader { - private final AdminClient adminClient; - private final LoadingCache minInSyncReplicasCache; - - MinInSyncReplicasLoader(AdminClient adminClient, Duration metadataMaxAge) { - this.adminClient = adminClient; - this.minInSyncReplicasCache = CacheBuilder.newBuilder() - .expireAfterWrite(metadataMaxAge.toMillis(), MILLISECONDS) - .build(new MinInSyncReplicasCacheLoader()); - } - - int get(String kafkaTopicName) throws ExecutionException { - return minInSyncReplicasCache.get(kafkaTopicName); - } - - void close() { - adminClient.close(); - } - - private class MinInSyncReplicasCacheLoader extends CacheLoader { - - @Override - public Integer load(String kafkaTopicName) throws Exception { - ConfigResource resource = new ConfigResource(TOPIC, kafkaTopicName); - DescribeConfigsResult describeTopicsResult = adminClient.describeConfigs(ImmutableList.of(resource)); - Map configMap = describeTopicsResult.all().get(); - Config config = configMap.get(resource); - ConfigEntry configEntry = config.get(MIN_IN_SYNC_REPLICAS_CONFIG); - String value = configEntry.value(); - return Integer.parseInt(value); - } + private final AdminClient adminClient; + private final LoadingCache minInSyncReplicasCache; + + MinInSyncReplicasLoader(AdminClient adminClient, Duration metadataMaxAge) { + this.adminClient = adminClient; + this.minInSyncReplicasCache = + CacheBuilder.newBuilder() + .expireAfterWrite(metadataMaxAge.toMillis(), MILLISECONDS) + .build(new MinInSyncReplicasCacheLoader()); + } + + int get(String kafkaTopicName) throws ExecutionException { + return minInSyncReplicasCache.get(kafkaTopicName); + } + + void close() { + adminClient.close(); + } + + private class MinInSyncReplicasCacheLoader extends CacheLoader { + + @Override + public Integer load(String kafkaTopicName) throws Exception { + ConfigResource resource = new ConfigResource(TOPIC, kafkaTopicName); + DescribeConfigsResult describeTopicsResult = + adminClient.describeConfigs(ImmutableList.of(resource)); + Map configMap = describeTopicsResult.all().get(); + Config config = configMap.get(resource); + ConfigEntry configEntry = config.get(MIN_IN_SYNC_REPLICAS_CONFIG); + String value = configEntry.value(); + return Integer.parseInt(value); } + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/MultiDatacenterMessageProducer.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/MultiDatacenterMessageProducer.java index 349380023c..8ba8b27c93 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/MultiDatacenterMessageProducer.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/MultiDatacenterMessageProducer.java @@ -1,18 +1,6 @@ package pl.allegro.tech.hermes.frontend.producer.kafka; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import pl.allegro.tech.hermes.api.PublishingChaosPolicy; -import pl.allegro.tech.hermes.api.PublishingChaosPolicy.ChaosPolicy; -import pl.allegro.tech.hermes.api.Topic; -import pl.allegro.tech.hermes.frontend.metric.CachedTopic; -import pl.allegro.tech.hermes.frontend.producer.BrokerMessageProducer; -import pl.allegro.tech.hermes.frontend.publishing.PublishingCallback; -import pl.allegro.tech.hermes.frontend.publishing.message.Message; -import pl.allegro.tech.hermes.frontend.readiness.AdminReadinessService; +import static org.apache.commons.lang3.exception.ExceptionUtils.getRootCauseMessage; import java.time.Duration; import java.util.HashMap; @@ -27,357 +15,391 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; - -import static org.apache.commons.lang3.exception.ExceptionUtils.getRootCauseMessage; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import pl.allegro.tech.hermes.api.PublishingChaosPolicy; +import pl.allegro.tech.hermes.api.PublishingChaosPolicy.ChaosPolicy; +import pl.allegro.tech.hermes.api.Topic; +import pl.allegro.tech.hermes.frontend.metric.CachedTopic; +import pl.allegro.tech.hermes.frontend.producer.BrokerMessageProducer; +import pl.allegro.tech.hermes.frontend.publishing.PublishingCallback; +import pl.allegro.tech.hermes.frontend.publishing.message.Message; +import pl.allegro.tech.hermes.frontend.readiness.AdminReadinessService; public class MultiDatacenterMessageProducer implements BrokerMessageProducer { - private static final Logger logger = LoggerFactory.getLogger(MultiDatacenterMessageProducer.class); - - private final KafkaMessageSenders kafkaMessageSenders; - private final MessageToKafkaProducerRecordConverter messageConverter; - private final Duration speculativeSendDelay; - private final AdminReadinessService adminReadinessService; - private final ScheduledExecutorService fallbackScheduler; - - public MultiDatacenterMessageProducer(KafkaMessageSenders kafkaMessageSenders, - AdminReadinessService adminReadinessService, - MessageToKafkaProducerRecordConverter messageConverter, - Duration speculativeSendDelay, - ScheduledExecutorService fallbackScheduler) { - this.messageConverter = messageConverter; - this.kafkaMessageSenders = kafkaMessageSenders; - this.speculativeSendDelay = speculativeSendDelay; - this.adminReadinessService = adminReadinessService; - this.fallbackScheduler = fallbackScheduler; + private static final Logger logger = + LoggerFactory.getLogger(MultiDatacenterMessageProducer.class); + + private final KafkaMessageSenders kafkaMessageSenders; + private final MessageToKafkaProducerRecordConverter messageConverter; + private final Duration speculativeSendDelay; + private final AdminReadinessService adminReadinessService; + private final ScheduledExecutorService fallbackScheduler; + + public MultiDatacenterMessageProducer( + KafkaMessageSenders kafkaMessageSenders, + AdminReadinessService adminReadinessService, + MessageToKafkaProducerRecordConverter messageConverter, + Duration speculativeSendDelay, + ScheduledExecutorService fallbackScheduler) { + this.messageConverter = messageConverter; + this.kafkaMessageSenders = kafkaMessageSenders; + this.speculativeSendDelay = speculativeSendDelay; + this.adminReadinessService = adminReadinessService; + this.fallbackScheduler = fallbackScheduler; + } + + @Override + public void send(Message message, CachedTopic cachedTopic, PublishingCallback callback) { + var producerRecord = + messageConverter.convertToProducerRecord( + message, cachedTopic.getKafkaTopics().getPrimary().name()); + + KafkaMessageSender localSender = + kafkaMessageSenders.get(cachedTopic.getTopic()); + Optional> remoteSender = getRemoteSender(cachedTopic); + + Map experiments = + createChaosExperimentsPerDatacenter(cachedTopic.getTopic()); + + if (remoteSender.isPresent()) { + sendWithFallback( + localSender, + remoteSender.get(), + producerRecord, + cachedTopic, + message, + experiments, + callback); + } else { + sendWithoutFallback(localSender, producerRecord, cachedTopic, message, callback); } + } - @Override - public void send(Message message, CachedTopic cachedTopic, PublishingCallback callback) { - var producerRecord = messageConverter.convertToProducerRecord(message, cachedTopic.getKafkaTopics().getPrimary().name()); - - KafkaMessageSender localSender = kafkaMessageSenders.get(cachedTopic.getTopic()); - Optional> remoteSender = getRemoteSender(cachedTopic); - - Map experiments = createChaosExperimentsPerDatacenter(cachedTopic.getTopic()); - - if (remoteSender.isPresent()) { - sendWithFallback( - localSender, - remoteSender.get(), - producerRecord, - cachedTopic, - message, - experiments, - callback - ); - } else { - sendWithoutFallback( - localSender, - producerRecord, - cachedTopic, - message, - callback - ); - } - } - - private static class SendWithFallbackExecutionContext { - - private final AtomicBoolean fallbackExecuted = new AtomicBoolean(false); - private final AtomicBoolean sent = new AtomicBoolean(false); - private final AtomicInteger tries; - private final ConcurrentHashMap errors; - - private SendWithFallbackExecutionContext() { - this.tries = new AtomicInteger(2); - this.errors = new ConcurrentHashMap<>(2); - } - - public boolean tryTransitionToFallback() { - return fallbackExecuted.compareAndSet(false, true) && !sent.get(); - } + private static class SendWithFallbackExecutionContext { - boolean tryTransitionToUnpublished(String datacenter, Exception exception) { - errors.put(datacenter, exception); - return tries.decrementAndGet() == 0; - } + private final AtomicBoolean fallbackExecuted = new AtomicBoolean(false); + private final AtomicBoolean sent = new AtomicBoolean(false); + private final AtomicInteger tries; + private final ConcurrentHashMap errors; - public boolean tryTransitionToFirstSent() { - return sent.compareAndSet(false, true); - } + private SendWithFallbackExecutionContext() { + this.tries = new AtomicInteger(2); + this.errors = new ConcurrentHashMap<>(2); } - /* - We first try to send message to local DC. If the local send fails we perform 'immediate' fallback to remote DC. - - Additionally, we schedule a 'speculative' fallback to remote DC to execute after 'speculativeSendDelay' elapses. - Speculative fallback decreases publication latency but may result in messages being duplicated across DCs. - - If local DC send succeeds or fails before 'speculativeSendDelay' elapses we try to cancel the 'speculative' fallback if - it has not been executed yet. We guarantee that only one fallback executes - either 'immediate' or 'speculative'. - */ - private void sendWithFallback(KafkaMessageSender localSender, - KafkaMessageSender remoteSender, - ProducerRecord producerRecord, - CachedTopic cachedTopic, - Message message, - Map experiments, - PublishingCallback publishingCallback) { - - SendWithFallbackExecutionContext context = new SendWithFallbackExecutionContext(); - - FallbackRunnable fallback = new FallbackRunnable( - remoteSender, - producerRecord, - cachedTopic, - message, - experiments.getOrDefault(remoteSender.getDatacenter(), ChaosExperiment.DISABLED), - publishingCallback, - context - ); - - Future speculativeFallback; - try { - speculativeFallback = fallbackScheduler.schedule(fallback, speculativeSendDelay.toMillis(), TimeUnit.MILLISECONDS); - } catch (RejectedExecutionException rejectedExecutionException) { - logger.warn("Failed to run schedule fallback for message: {}, topic: {}", message, cachedTopic.getQualifiedName(), rejectedExecutionException); - speculativeFallback = CompletableFuture.completedFuture(null); - } - - localSender.send( - producerRecord, - cachedTopic, - message, - new FallbackAwareLocalSendCallback( - message, cachedTopic, localSender.getDatacenter(), - context, publishingCallback, fallback, speculativeFallback - ), - experiments.getOrDefault(localSender.getDatacenter(), ChaosExperiment.DISABLED) - ); + public boolean tryTransitionToFallback() { + return fallbackExecuted.compareAndSet(false, true) && !sent.get(); } - private void sendWithoutFallback(KafkaMessageSender sender, - ProducerRecord producerRecord, - CachedTopic cachedTopic, - Message message, - PublishingCallback callback) { - sender.send( - producerRecord, - cachedTopic, - message, - new LocalSendCallback(message, cachedTopic, sender.getDatacenter(), callback) - ); + boolean tryTransitionToUnpublished(String datacenter, Exception exception) { + errors.put(datacenter, exception); + return tries.decrementAndGet() == 0; } - private Map createChaosExperimentsPerDatacenter(Topic topic) { - PublishingChaosPolicy chaos = topic.getChaos(); - return switch (chaos.mode()) { - case DISABLED -> Map.of(); - case GLOBAL -> { - Map experiments = new HashMap<>(); - ChaosPolicy policy = chaos.globalPolicy(); - boolean enabled = computeIfShouldBeEnabled(policy); - for (String datacenter : kafkaMessageSenders.getDatacenters()) { - experiments.put(datacenter, createChaosExperimentForDatacenter(policy, enabled)); - } - yield experiments; - } - case DATACENTER -> { - Map experiments = new HashMap<>(); - Map policies = chaos.datacenterPolicies(); - for (String datacenter : kafkaMessageSenders.getDatacenters()) { - ChaosPolicy policy = policies.get(datacenter); - boolean enabled = computeIfShouldBeEnabled(policy); - experiments.put(datacenter, createChaosExperimentForDatacenter(policy, enabled)); - } - yield experiments; - } - }; + public boolean tryTransitionToFirstSent() { + return sent.compareAndSet(false, true); } - - private boolean computeIfShouldBeEnabled(ChaosPolicy policy) { - if (policy == null) { - return false; - } - return ThreadLocalRandom.current().nextInt(100) < policy.probability(); + } + + /* + We first try to send message to local DC. If the local send fails we perform 'immediate' fallback to remote DC. + + Additionally, we schedule a 'speculative' fallback to remote DC to execute after 'speculativeSendDelay' elapses. + Speculative fallback decreases publication latency but may result in messages being duplicated across DCs. + + If local DC send succeeds or fails before 'speculativeSendDelay' elapses we try to cancel the 'speculative' fallback if + it has not been executed yet. We guarantee that only one fallback executes - either 'immediate' or 'speculative'. + */ + private void sendWithFallback( + KafkaMessageSender localSender, + KafkaMessageSender remoteSender, + ProducerRecord producerRecord, + CachedTopic cachedTopic, + Message message, + Map experiments, + PublishingCallback publishingCallback) { + + SendWithFallbackExecutionContext context = new SendWithFallbackExecutionContext(); + + FallbackRunnable fallback = + new FallbackRunnable( + remoteSender, + producerRecord, + cachedTopic, + message, + experiments.getOrDefault(remoteSender.getDatacenter(), ChaosExperiment.DISABLED), + publishingCallback, + context); + + Future speculativeFallback; + try { + speculativeFallback = + fallbackScheduler.schedule( + fallback, speculativeSendDelay.toMillis(), TimeUnit.MILLISECONDS); + } catch (RejectedExecutionException rejectedExecutionException) { + logger.warn( + "Failed to run schedule fallback for message: {}, topic: {}", + message, + cachedTopic.getQualifiedName(), + rejectedExecutionException); + speculativeFallback = CompletableFuture.completedFuture(null); } - private ChaosExperiment createChaosExperimentForDatacenter(ChaosPolicy policy, boolean enabled) { - if (!enabled) { - return ChaosExperiment.DISABLED; + localSender.send( + producerRecord, + cachedTopic, + message, + new FallbackAwareLocalSendCallback( + message, + cachedTopic, + localSender.getDatacenter(), + context, + publishingCallback, + fallback, + speculativeFallback), + experiments.getOrDefault(localSender.getDatacenter(), ChaosExperiment.DISABLED)); + } + + private void sendWithoutFallback( + KafkaMessageSender sender, + ProducerRecord producerRecord, + CachedTopic cachedTopic, + Message message, + PublishingCallback callback) { + sender.send( + producerRecord, + cachedTopic, + message, + new LocalSendCallback(message, cachedTopic, sender.getDatacenter(), callback)); + } + + private Map createChaosExperimentsPerDatacenter(Topic topic) { + PublishingChaosPolicy chaos = topic.getChaos(); + return switch (chaos.mode()) { + case DISABLED -> Map.of(); + case GLOBAL -> { + Map experiments = new HashMap<>(); + ChaosPolicy policy = chaos.globalPolicy(); + boolean enabled = computeIfShouldBeEnabled(policy); + for (String datacenter : kafkaMessageSenders.getDatacenters()) { + experiments.put(datacenter, createChaosExperimentForDatacenter(policy, enabled)); } - long delayMillisFrom = policy.delayFrom(); - long delayMillisTo = policy.delayTo(); - long delayMillis = ThreadLocalRandom.current().nextLong(delayMillisTo - delayMillisFrom) + delayMillisFrom; - return new ChaosExperiment(true, policy.completeWithError(), delayMillis); - } - - public record ChaosExperiment(boolean enabled, boolean completeWithError, long delayInMillis) { - - private static final ChaosExperiment DISABLED = new ChaosExperiment(false, false, 0); - + yield experiments; + } + case DATACENTER -> { + Map experiments = new HashMap<>(); + Map policies = chaos.datacenterPolicies(); + for (String datacenter : kafkaMessageSenders.getDatacenters()) { + ChaosPolicy policy = policies.get(datacenter); + boolean enabled = computeIfShouldBeEnabled(policy); + experiments.put(datacenter, createChaosExperimentForDatacenter(policy, enabled)); + } + yield experiments; + } + }; + } + + private boolean computeIfShouldBeEnabled(ChaosPolicy policy) { + if (policy == null) { + return false; } + return ThreadLocalRandom.current().nextInt(100) < policy.probability(); + } - private Optional> getRemoteSender(CachedTopic cachedTopic) { - return kafkaMessageSenders.getRemote(cachedTopic.getTopic()) - .stream() - .filter(producer -> adminReadinessService.isDatacenterReady(producer.getDatacenter())) - .findFirst(); + private ChaosExperiment createChaosExperimentForDatacenter(ChaosPolicy policy, boolean enabled) { + if (!enabled) { + return ChaosExperiment.DISABLED; } + long delayMillisFrom = policy.delayFrom(); + long delayMillisTo = policy.delayTo(); + long delayMillis = + ThreadLocalRandom.current().nextLong(delayMillisTo - delayMillisFrom) + delayMillisFrom; + return new ChaosExperiment(true, policy.completeWithError(), delayMillis); + } + + public record ChaosExperiment(boolean enabled, boolean completeWithError, long delayInMillis) { + + private static final ChaosExperiment DISABLED = new ChaosExperiment(false, false, 0); + } + + private Optional> getRemoteSender(CachedTopic cachedTopic) { + return kafkaMessageSenders.getRemote(cachedTopic.getTopic()).stream() + .filter(producer -> adminReadinessService.isDatacenterReady(producer.getDatacenter())) + .findFirst(); + } + + private record RemoteSendCallback( + Message message, + CachedTopic cachedTopic, + String datacenter, + PublishingCallback callback, + SendWithFallbackExecutionContext state) + implements Callback { - private record RemoteSendCallback(Message message, CachedTopic cachedTopic, - String datacenter, PublishingCallback callback, - SendWithFallbackExecutionContext state) implements Callback { - - @Override - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (exception == null) { - callback.onEachPublished(message, cachedTopic.getTopic(), datacenter); - if (state.tryTransitionToFirstSent()) { - callback.onPublished(message, cachedTopic.getTopic()); - } else { - cachedTopic.markMessageDuplicated(); - } - } else { - if (state.tryTransitionToUnpublished(datacenter, exception)) { - callback.onUnpublished(message, cachedTopic.getTopic(), new MultiDCPublishException(state.errors)); - } - } + @Override + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (exception == null) { + callback.onEachPublished(message, cachedTopic.getTopic(), datacenter); + if (state.tryTransitionToFirstSent()) { + callback.onPublished(message, cachedTopic.getTopic()); + } else { + cachedTopic.markMessageDuplicated(); } - } - - private class FallbackAwareLocalSendCallback implements Callback { - - private final Message message; - private final CachedTopic cachedTopic; - private final String datacenter; - private final PublishingCallback callback; - private final FallbackRunnable fallback; - private final Future speculativeFallback; - private final SendWithFallbackExecutionContext state; - - private FallbackAwareLocalSendCallback(Message message, CachedTopic cachedTopic, String datacenter, - SendWithFallbackExecutionContext state, - PublishingCallback callback, - FallbackRunnable fallback, Future speculativeFallback) { - this.message = message; - this.cachedTopic = cachedTopic; - this.datacenter = datacenter; - this.callback = callback; - this.fallback = fallback; - this.speculativeFallback = speculativeFallback; - this.state = state; + } else { + if (state.tryTransitionToUnpublished(datacenter, exception)) { + callback.onUnpublished( + message, cachedTopic.getTopic(), new MultiDCPublishException(state.errors)); } + } + } + } + + private class FallbackAwareLocalSendCallback implements Callback { + + private final Message message; + private final CachedTopic cachedTopic; + private final String datacenter; + private final PublishingCallback callback; + private final FallbackRunnable fallback; + private final Future speculativeFallback; + private final SendWithFallbackExecutionContext state; + + private FallbackAwareLocalSendCallback( + Message message, + CachedTopic cachedTopic, + String datacenter, + SendWithFallbackExecutionContext state, + PublishingCallback callback, + FallbackRunnable fallback, + Future speculativeFallback) { + this.message = message; + this.cachedTopic = cachedTopic; + this.datacenter = datacenter; + this.callback = callback; + this.fallback = fallback; + this.speculativeFallback = speculativeFallback; + this.state = state; + } - @Override - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (exception == null) { - cancelSpeculativeFallback(); - callback.onEachPublished(message, cachedTopic.getTopic(), datacenter); - if (state.tryTransitionToFirstSent()) { - callback.onPublished(message, cachedTopic.getTopic()); - } else { - cachedTopic.markMessageDuplicated(); - } - } else { - if (state.tryTransitionToUnpublished(datacenter, exception)) { - callback.onUnpublished(message, cachedTopic.getTopic(), new MultiDCPublishException(state.errors)); - } else { - fallback(); - } - } + @Override + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (exception == null) { + cancelSpeculativeFallback(); + callback.onEachPublished(message, cachedTopic.getTopic(), datacenter); + if (state.tryTransitionToFirstSent()) { + callback.onPublished(message, cachedTopic.getTopic()); + } else { + cachedTopic.markMessageDuplicated(); } - - private void fallback() { - try { - speculativeFallback.cancel(false); - fallbackScheduler.execute(fallback); - } catch (RejectedExecutionException rejectedExecutionException) { - logger.warn("Failed to run immediate fallback for message: {}, topic: {}", message, cachedTopic.getQualifiedName(), rejectedExecutionException); - } + } else { + if (state.tryTransitionToUnpublished(datacenter, exception)) { + callback.onUnpublished( + message, cachedTopic.getTopic(), new MultiDCPublishException(state.errors)); + } else { + fallback(); } + } + } - private void cancelSpeculativeFallback() { - speculativeFallback.cancel(false); - } + private void fallback() { + try { + speculativeFallback.cancel(false); + fallbackScheduler.execute(fallback); + } catch (RejectedExecutionException rejectedExecutionException) { + logger.warn( + "Failed to run immediate fallback for message: {}, topic: {}", + message, + cachedTopic.getQualifiedName(), + rejectedExecutionException); + } } - private record LocalSendCallback(Message message, CachedTopic cachedTopic, String datacenter, - PublishingCallback callback) implements Callback { - - @Override - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (exception != null) { - callback.onUnpublished(message, cachedTopic.getTopic(), exception); - } else { - callback.onEachPublished(message, cachedTopic.getTopic(), datacenter); - callback.onPublished(message, cachedTopic.getTopic()); - } - } + private void cancelSpeculativeFallback() { + speculativeFallback.cancel(false); } + } - private class FallbackRunnable implements Runnable { - private final KafkaMessageSender remoteSender; - private final ProducerRecord producerRecord; - private final CachedTopic cachedTopic; - private final Message message; - private final ChaosExperiment experiment; - private final PublishingCallback callback; - private final SendWithFallbackExecutionContext context; - - public FallbackRunnable(KafkaMessageSender remoteSender, - ProducerRecord producerRecord, - CachedTopic cachedTopic, - Message message, - ChaosExperiment experiment, - PublishingCallback callback, - SendWithFallbackExecutionContext context - ) { - this.remoteSender = remoteSender; - this.producerRecord = producerRecord; - this.cachedTopic = cachedTopic; - this.message = message; - this.experiment = experiment; - this.callback = callback; - this.context = context; - } + private record LocalSendCallback( + Message message, CachedTopic cachedTopic, String datacenter, PublishingCallback callback) + implements Callback { - public void run() { - if (context.tryTransitionToFallback()) { - remoteSender.send( - producerRecord, - cachedTopic, - message, - new RemoteSendCallback(message, cachedTopic, remoteSender.getDatacenter(), callback, context), - experiment - ); - } - } + @Override + public void onCompletion(RecordMetadata metadata, Exception exception) { + if (exception != null) { + callback.onUnpublished(message, cachedTopic.getTopic(), exception); + } else { + callback.onEachPublished(message, cachedTopic.getTopic(), datacenter); + callback.onPublished(message, cachedTopic.getTopic()); + } + } + } + + private class FallbackRunnable implements Runnable { + private final KafkaMessageSender remoteSender; + private final ProducerRecord producerRecord; + private final CachedTopic cachedTopic; + private final Message message; + private final ChaosExperiment experiment; + private final PublishingCallback callback; + private final SendWithFallbackExecutionContext context; + + public FallbackRunnable( + KafkaMessageSender remoteSender, + ProducerRecord producerRecord, + CachedTopic cachedTopic, + Message message, + ChaosExperiment experiment, + PublishingCallback callback, + SendWithFallbackExecutionContext context) { + this.remoteSender = remoteSender; + this.producerRecord = producerRecord; + this.cachedTopic = cachedTopic; + this.message = message; + this.experiment = experiment; + this.callback = callback; + this.context = context; } - public static class MultiDCPublishException extends RuntimeException { + public void run() { + if (context.tryTransitionToFallback()) { + remoteSender.send( + producerRecord, + cachedTopic, + message, + new RemoteSendCallback( + message, cachedTopic, remoteSender.getDatacenter(), callback, context), + experiment); + } + } + } - public MultiDCPublishException(Map exceptionsPerDC) { - super(errorMessage(exceptionsPerDC)); - } + public static class MultiDCPublishException extends RuntimeException { - private static String errorMessage(Map exceptionsPerDC) { - StringBuilder builder = new StringBuilder(); - exceptionsPerDC.forEach( - (dc, exception) -> builder.append(String.format("[%s]: %s, ", dc, getRootCauseMessage(exception))) - ); - return builder.toString(); - } + public MultiDCPublishException(Map exceptionsPerDC) { + super(errorMessage(exceptionsPerDC)); } - @Override - public boolean areAllTopicsAvailable() { - return kafkaMessageSenders.areAllTopicsAvailable(); + private static String errorMessage(Map exceptionsPerDC) { + StringBuilder builder = new StringBuilder(); + exceptionsPerDC.forEach( + (dc, exception) -> + builder.append(String.format("[%s]: %s, ", dc, getRootCauseMessage(exception)))); + return builder.toString(); } + } - @Override - public boolean isTopicAvailable(CachedTopic topic) { - return kafkaMessageSenders.isTopicAvailable(topic); - } + @Override + public boolean areAllTopicsAvailable() { + return kafkaMessageSenders.areAllTopicsAvailable(); + } + + @Override + public boolean isTopicAvailable(CachedTopic topic) { + return kafkaMessageSenders.isTopicAvailable(topic); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/ProducerBrokerNodeReader.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/ProducerBrokerNodeReader.java index e7bbc994c5..a0d3ffa8ff 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/ProducerBrokerNodeReader.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/ProducerBrokerNodeReader.java @@ -1,5 +1,9 @@ package pl.allegro.tech.hermes.frontend.producer.kafka; +import static java.util.Collections.singletonList; + +import java.lang.reflect.Field; +import java.util.List; import org.apache.kafka.clients.Metadata; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.Producer; @@ -7,26 +11,21 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.lang.reflect.Field; -import java.util.List; - -import static java.util.Collections.singletonList; - public class ProducerBrokerNodeReader { - private static final Logger LOGGER = LoggerFactory.getLogger(ProducerBrokerNodeReader.class); + private static final Logger LOGGER = LoggerFactory.getLogger(ProducerBrokerNodeReader.class); - public static List read(Producer producer) { - if (producer instanceof KafkaProducer) { - KafkaProducer kafkaProducer = (KafkaProducer) producer; - try { - Field field = KafkaProducer.class.getDeclaredField("metadata"); - field.setAccessible(true); - Metadata metadata = (Metadata) field.get(kafkaProducer); - return metadata.fetch().nodes(); - } catch (Exception e) { - LOGGER.error("Could not read broker node list from producer.", e); - } - } - return singletonList(new Node(0, "none", 0)); + public static List read(Producer producer) { + if (producer instanceof KafkaProducer) { + KafkaProducer kafkaProducer = (KafkaProducer) producer; + try { + Field field = KafkaProducer.class.getDeclaredField("metadata"); + field.setAccessible(true); + Metadata metadata = (Metadata) field.get(kafkaProducer); + return metadata.fetch().nodes(); + } catch (Exception e) { + LOGGER.error("Could not read broker node list from producer.", e); + } } + return singletonList(new Node(0, "none", 0)); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/ProducerMetadataLoadingJob.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/ProducerMetadataLoadingJob.java index 8e868afa98..3e9d45722c 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/ProducerMetadataLoadingJob.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/ProducerMetadataLoadingJob.java @@ -1,7 +1,6 @@ package pl.allegro.tech.hermes.frontend.producer.kafka; import com.google.common.util.concurrent.ThreadFactoryBuilder; - import java.time.Duration; import java.util.List; import java.util.concurrent.Executors; @@ -11,57 +10,58 @@ import java.util.concurrent.TimeUnit; /** - * The purpose of this job is to periodically refresh the cache in - * {@link org.apache.kafka.clients.producer.KafkaProducer} that stores topic metadata. - * This is especially important to avoid a cold start, i.e. when a new hermes-frontend - * instance is launched with the cache being empty. Since the producer relies on topic - * metadata to send produce requests to Kafka, if the cache is empty, the producer must - * load the metadata before sending the produce request. Fetching the metadata might be - * costly, therefore we want to avoid passing on this cost to the Hermes client. + * The purpose of this job is to periodically refresh the cache in {@link + * org.apache.kafka.clients.producer.KafkaProducer} that stores topic metadata. This is especially + * important to avoid a cold start, i.e. when a new hermes-frontend instance is launched with the + * cache being empty. Since the producer relies on topic metadata to send produce requests to Kafka, + * if the cache is empty, the producer must load the metadata before sending the produce request. + * Fetching the metadata might be costly, therefore we want to avoid passing on this cost to the + * Hermes client. */ public class ProducerMetadataLoadingJob implements Runnable { - private final List kafkaMessageSendersList; - private final ScheduledExecutorService executorService; - private final boolean enabled; - private final Duration interval; + private final List kafkaMessageSendersList; + private final ScheduledExecutorService executorService; + private final boolean enabled; + private final Duration interval; - private ScheduledFuture job; + private ScheduledFuture job; - public ProducerMetadataLoadingJob(List kafkaMessageSendersList, - boolean enabled, - Duration interval) { - this.kafkaMessageSendersList = kafkaMessageSendersList; - this.enabled = enabled; - this.interval = interval; - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setNameFormat("TopicMetadataLoadingJob-%d").build(); - this.executorService = Executors.newSingleThreadScheduledExecutor(threadFactory); - } + public ProducerMetadataLoadingJob( + List kafkaMessageSendersList, boolean enabled, Duration interval) { + this.kafkaMessageSendersList = kafkaMessageSendersList; + this.enabled = enabled; + this.interval = interval; + ThreadFactory threadFactory = + new ThreadFactoryBuilder().setNameFormat("TopicMetadataLoadingJob-%d").build(); + this.executorService = Executors.newSingleThreadScheduledExecutor(threadFactory); + } - @Override - public void run() { - refreshTopicMetadata(); - } + @Override + public void run() { + refreshTopicMetadata(); + } - public void start() { - if (enabled) { - refreshTopicMetadata(); - job = executorService.scheduleAtFixedRate(this, interval.toSeconds(), interval.toSeconds(), TimeUnit.SECONDS); - } + public void start() { + if (enabled) { + refreshTopicMetadata(); + job = + executorService.scheduleAtFixedRate( + this, interval.toSeconds(), interval.toSeconds(), TimeUnit.SECONDS); } + } - public void stop() throws InterruptedException { - if (enabled) { - job.cancel(false); - executorService.shutdown(); - executorService.awaitTermination(1, TimeUnit.MINUTES); - } + public void stop() throws InterruptedException { + if (enabled) { + job.cancel(false); + executorService.shutdown(); + executorService.awaitTermination(1, TimeUnit.MINUTES); } + } - private void refreshTopicMetadata() { - for (KafkaMessageSenders kafkaMessageSenders : kafkaMessageSendersList) { - kafkaMessageSenders.refreshTopicMetadata(); - } + private void refreshTopicMetadata() { + for (KafkaMessageSenders kafkaMessageSenders : kafkaMessageSendersList) { + kafkaMessageSenders.refreshTopicMetadata(); } + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/TopicMetadataLoader.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/TopicMetadataLoader.java index 565b4a3c47..506e65f0da 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/TopicMetadataLoader.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/TopicMetadataLoader.java @@ -5,25 +5,25 @@ interface TopicMetadataLoader { - MetadataLoadingResult load(CachedTopic cachedTopic); + MetadataLoadingResult load(CachedTopic cachedTopic); - record MetadataLoadingResult(Type type, TopicName topicName, String datacenter) { + record MetadataLoadingResult(Type type, TopicName topicName, String datacenter) { - static MetadataLoadingResult success(TopicName topicName, String datacenter) { - return new MetadataLoadingResult(Type.SUCCESS, topicName, datacenter); - } - - static MetadataLoadingResult failure(TopicName topicName, String datacenter) { - return new MetadataLoadingResult(Type.FAILURE, topicName, datacenter); - } + static MetadataLoadingResult success(TopicName topicName, String datacenter) { + return new MetadataLoadingResult(Type.SUCCESS, topicName, datacenter); + } - boolean isFailure() { - return Type.FAILURE == type; - } + static MetadataLoadingResult failure(TopicName topicName, String datacenter) { + return new MetadataLoadingResult(Type.FAILURE, topicName, datacenter); } - enum Type { - SUCCESS, - FAILURE + boolean isFailure() { + return Type.FAILURE == type; } + } + + enum Type { + SUCCESS, + FAILURE + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/TopicMetadataLoadingExecutor.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/TopicMetadataLoadingExecutor.java index 32318cf43c..a52e8564eb 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/TopicMetadataLoadingExecutor.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/producer/kafka/TopicMetadataLoadingExecutor.java @@ -1,14 +1,11 @@ package pl.allegro.tech.hermes.frontend.producer.kafka; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import net.jodah.failsafe.Failsafe; -import net.jodah.failsafe.RetryPolicy; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import pl.allegro.tech.hermes.api.TopicName; -import pl.allegro.tech.hermes.frontend.cache.topic.TopicsCache; -import pl.allegro.tech.hermes.frontend.metric.CachedTopic; +import static java.util.concurrent.CompletableFuture.completedFuture; +import static pl.allegro.tech.hermes.frontend.producer.kafka.TopicMetadataLoader.MetadataLoadingResult; +import static pl.allegro.tech.hermes.frontend.producer.kafka.TopicMetadataLoader.Type; +import static pl.allegro.tech.hermes.frontend.utils.CompletableFuturesHelper.allComplete; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.time.Duration; import java.util.ArrayList; import java.util.List; @@ -20,89 +17,108 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; - -import static java.util.concurrent.CompletableFuture.completedFuture; -import static pl.allegro.tech.hermes.frontend.producer.kafka.TopicMetadataLoader.MetadataLoadingResult; -import static pl.allegro.tech.hermes.frontend.producer.kafka.TopicMetadataLoader.Type; -import static pl.allegro.tech.hermes.frontend.utils.CompletableFuturesHelper.allComplete; +import net.jodah.failsafe.Failsafe; +import net.jodah.failsafe.RetryPolicy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import pl.allegro.tech.hermes.api.TopicName; +import pl.allegro.tech.hermes.frontend.cache.topic.TopicsCache; +import pl.allegro.tech.hermes.frontend.metric.CachedTopic; class TopicMetadataLoadingExecutor { - private static final Logger logger = LoggerFactory.getLogger(TopicMetadataLoadingExecutor.class); - - private final ScheduledExecutorService scheduler; - private final RetryPolicy retryPolicy; - private final TopicsCache topicsCache; - - TopicMetadataLoadingExecutor(TopicsCache topicsCache, int retryCount, Duration retryInterval, int threadPoolSize) { - this.topicsCache = topicsCache; - ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat("topic-metadata-loader-%d").build(); - this.scheduler = Executors.newScheduledThreadPool(threadPoolSize, threadFactory); - this.retryPolicy = new RetryPolicy() - .withMaxRetries(retryCount) - .withDelay(retryInterval) - .handleIf((resp, cause) -> resp.isFailure()); - } - - boolean execute(List loaders) { - try { - long start = System.currentTimeMillis(); - logger.info("Loading topic metadata"); - List topics = topicsCache.getTopics(); - List allResults = loadMetadataForTopics(topics, loaders); - logger.info("Finished loading topic metadata in {}ms", System.currentTimeMillis() - start); - logResultInfo(allResults); - return allResults.stream().noneMatch(MetadataLoadingResult::isFailure); - } catch (Exception e) { - logger.error("An error occurred while refreshing topic metadata", e); - return false; - } - } - - private List loadMetadataForTopics(List topics, List loaders) { - List> completableFutures = new ArrayList<>(); - for (CachedTopic topic : topics) { - for (TopicMetadataLoader loader : loaders) { - completableFutures.add(loadTopicMetadata(topic, loader)); - } - } - return allComplete(completableFutures).join(); + private static final Logger logger = LoggerFactory.getLogger(TopicMetadataLoadingExecutor.class); + + private final ScheduledExecutorService scheduler; + private final RetryPolicy retryPolicy; + private final TopicsCache topicsCache; + + TopicMetadataLoadingExecutor( + TopicsCache topicsCache, int retryCount, Duration retryInterval, int threadPoolSize) { + this.topicsCache = topicsCache; + ThreadFactory threadFactory = + new ThreadFactoryBuilder().setNameFormat("topic-metadata-loader-%d").build(); + this.scheduler = Executors.newScheduledThreadPool(threadPoolSize, threadFactory); + this.retryPolicy = + new RetryPolicy() + .withMaxRetries(retryCount) + .withDelay(retryInterval) + .handleIf((resp, cause) -> resp.isFailure()); + } + + boolean execute(List loaders) { + try { + long start = System.currentTimeMillis(); + logger.info("Loading topic metadata"); + List topics = topicsCache.getTopics(); + List allResults = loadMetadataForTopics(topics, loaders); + logger.info("Finished loading topic metadata in {}ms", System.currentTimeMillis() - start); + logResultInfo(allResults); + return allResults.stream().noneMatch(MetadataLoadingResult::isFailure); + } catch (Exception e) { + logger.error("An error occurred while refreshing topic metadata", e); + return false; } - - private CompletableFuture loadTopicMetadata(CachedTopic topic, TopicMetadataLoader loader) { - return Failsafe.with(retryPolicy).with(scheduler) - .getStageAsync((context) -> completedFuture(loader.load(topic))); - } - - private void logResultInfo(List allResults) { - Map> resultsPerDatacenter = allResults.stream() - .collect(Collectors.groupingBy(MetadataLoadingResult::datacenter, Collectors.toList())); - - for (Map.Entry> datacenterResults : resultsPerDatacenter.entrySet()) { - Map> groupedResults = getGroupedResults(datacenterResults.getValue()); - Optional> successes = Optional.ofNullable(groupedResults.get(Type.SUCCESS)); - Optional> failures = Optional.ofNullable(groupedResults.get(Type.FAILURE)); - - logger.info("Results of loading topic metadata from datacenter {}: successfully loaded {} topics, failed for {} topics{}", - datacenterResults.getKey(), - successes.map(List::size).orElse(0), - failures.map(List::size).orElse(0), - failures.map(results -> String.format("Failed topics: [%s].", topicsOfResults(results))).orElse("") - ); - } + } + + private List loadMetadataForTopics( + List topics, List loaders) { + List> completableFutures = new ArrayList<>(); + for (CachedTopic topic : topics) { + for (TopicMetadataLoader loader : loaders) { + completableFutures.add(loadTopicMetadata(topic, loader)); + } } - - private Map> getGroupedResults(List allResults) { - return allResults.stream().collect(Collectors.groupingBy(MetadataLoadingResult::type, Collectors.toList())); - } - - private String topicsOfResults(List results) { - return results.stream().map(MetadataLoadingResult::topicName).map(TopicName::qualifiedName) - .collect(Collectors.joining(", ")); - } - - void close() throws Exception { - scheduler.shutdown(); - scheduler.awaitTermination(1, TimeUnit.SECONDS); + return allComplete(completableFutures).join(); + } + + private CompletableFuture loadTopicMetadata( + CachedTopic topic, TopicMetadataLoader loader) { + return Failsafe.with(retryPolicy) + .with(scheduler) + .getStageAsync((context) -> completedFuture(loader.load(topic))); + } + + private void logResultInfo(List allResults) { + Map> resultsPerDatacenter = + allResults.stream() + .collect(Collectors.groupingBy(MetadataLoadingResult::datacenter, Collectors.toList())); + + for (Map.Entry> datacenterResults : + resultsPerDatacenter.entrySet()) { + Map> groupedResults = + getGroupedResults(datacenterResults.getValue()); + Optional> successes = + Optional.ofNullable(groupedResults.get(Type.SUCCESS)); + Optional> failures = + Optional.ofNullable(groupedResults.get(Type.FAILURE)); + + logger.info( + "Results of loading topic metadata from datacenter {}: successfully loaded {} topics, failed for {} topics{}", + datacenterResults.getKey(), + successes.map(List::size).orElse(0), + failures.map(List::size).orElse(0), + failures + .map(results -> String.format("Failed topics: [%s].", topicsOfResults(results))) + .orElse("")); } + } + + private Map> getGroupedResults( + List allResults) { + return allResults.stream() + .collect(Collectors.groupingBy(MetadataLoadingResult::type, Collectors.toList())); + } + + private String topicsOfResults(List results) { + return results.stream() + .map(MetadataLoadingResult::topicName) + .map(TopicName::qualifiedName) + .collect(Collectors.joining(", ")); + } + + void close() throws Exception { + scheduler.shutdown(); + scheduler.awaitTermination(1, TimeUnit.SECONDS); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/PublishingCallback.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/PublishingCallback.java index d175b1895e..6af0ccff48 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/PublishingCallback.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/PublishingCallback.java @@ -5,23 +5,19 @@ public interface PublishingCallback { - /** - * Invoked when publishing to the broker fails and the message won't be delivered. - */ - void onUnpublished(Message message, Topic topic, Exception exception); + /** Invoked when publishing to the broker fails and the message won't be delivered. */ + void onUnpublished(Message message, Topic topic, Exception exception); - /** - * Invoked the first time the message is successfully published to the broker. - */ - void onPublished(Message message, Topic topic); + /** Invoked the first time the message is successfully published to the broker. */ + void onPublished(Message message, Topic topic); - /** - * Invoked every time the message is successfully published to the broker. - * Could be invoked one or many times depending on the underlying implementation. - * - * @param message the delivered message - * @param topic the topic that the message was delivered to - * @param datacenter the datacenter that the messages was delivered to - */ - void onEachPublished(Message message, Topic topic, String datacenter); + /** + * Invoked every time the message is successfully published to the broker. Could be invoked one or + * many times depending on the underlying implementation. + * + * @param message the delivered message + * @param topic the topic that the message was delivered to + * @param datacenter the datacenter that the messages was delivered to + */ + void onEachPublished(Message message, Topic topic, String datacenter); } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/avro/AvroMessage.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/avro/AvroMessage.java index 4a8dbf6f3d..7bf5f9e4e9 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/avro/AvroMessage.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/avro/AvroMessage.java @@ -1,73 +1,73 @@ package pl.allegro.tech.hermes.frontend.publishing.avro; +import java.util.Map; +import java.util.Optional; import org.apache.avro.Schema; import pl.allegro.tech.hermes.api.ContentType; import pl.allegro.tech.hermes.frontend.publishing.message.Message; import pl.allegro.tech.hermes.schema.CompiledSchema; -import java.util.Map; -import java.util.Optional; - public class AvroMessage implements Message { - private final String id; - private final byte[] data; - private final long timestamp; - private final CompiledSchema schema; - private final String partitionKey; - private final Map propagatedHTTPHeaders; + private final String id; + private final byte[] data; + private final long timestamp; + private final CompiledSchema schema; + private final String partitionKey; + private final Map propagatedHTTPHeaders; - public AvroMessage(String id, - byte[] data, - long timestamp, - CompiledSchema schema, - String partitionKey, - Map propagatedHTTPHeaders) { - this.id = id; - this.data = data; - this.timestamp = timestamp; - this.schema = schema; - this.partitionKey = partitionKey; - this.propagatedHTTPHeaders = propagatedHTTPHeaders; - } + public AvroMessage( + String id, + byte[] data, + long timestamp, + CompiledSchema schema, + String partitionKey, + Map propagatedHTTPHeaders) { + this.id = id; + this.data = data; + this.timestamp = timestamp; + this.schema = schema; + this.partitionKey = partitionKey; + this.propagatedHTTPHeaders = propagatedHTTPHeaders; + } - @Override - public String getId() { - return id; - } + @Override + public String getId() { + return id; + } - @Override - public byte[] getData() { - return data; - } + @Override + public byte[] getData() { + return data; + } - @Override - public long getTimestamp() { - return timestamp; - } + @Override + public long getTimestamp() { + return timestamp; + } - @Override - public ContentType getContentType() { - return ContentType.AVRO; - } + @Override + public ContentType getContentType() { + return ContentType.AVRO; + } - @Override - public String getPartitionKey() { - return partitionKey; - } + @Override + public String getPartitionKey() { + return partitionKey; + } - @SuppressWarnings("unchecked") - @Override - public Optional> getCompiledSchema() { - return Optional.of((CompiledSchema) schema); - } + @SuppressWarnings("unchecked") + @Override + public Optional> getCompiledSchema() { + return Optional.of((CompiledSchema) schema); + } - @Override - public Map getHTTPHeaders() { - return propagatedHTTPHeaders; - } + @Override + public Map getHTTPHeaders() { + return propagatedHTTPHeaders; + } - public AvroMessage withDataReplaced(byte[] newData) { - return new AvroMessage(id, newData, timestamp, schema, partitionKey, propagatedHTTPHeaders); - } + public AvroMessage withDataReplaced(byte[] newData) { + return new AvroMessage(id, newData, timestamp, schema, partitionKey, propagatedHTTPHeaders); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/AttachmentContent.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/AttachmentContent.java index d3af2f8fa7..84ce8240ba 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/AttachmentContent.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/AttachmentContent.java @@ -8,71 +8,72 @@ public class AttachmentContent { - public static final AttachmentKey KEY = AttachmentKey.create(AttachmentContent.class); - - private final CachedTopic cachedTopic; - private final MessageState messageState; - private final String messageId; - private byte[] messageContent; - private volatile TimeoutHolder timeoutHolder; - private volatile Message message; - private volatile boolean responseReady; - - public Message getMessage() { - return message; - } - - AttachmentContent(CachedTopic cachedTopic, MessageState messageState, String messageId) { - this.cachedTopic = cachedTopic; - this.messageState = messageState; - this.messageId = messageId; - } - - public MessageState getMessageState() { - return messageState; - } - - public TimeoutHolder getTimeoutHolder() { - return timeoutHolder; - } - - public void removeTimeout() { - timeoutHolder.remove(); - } - - public void setTimeoutHolder(TimeoutHolder timeoutHolder) { - this.timeoutHolder = timeoutHolder; - } - - public Topic getTopic() { - return cachedTopic.getTopic(); - } - - public CachedTopic getCachedTopic() { - return cachedTopic; - } - - public void setMessageContent(byte[] messageContent) { - this.messageContent = messageContent; - } - - public String getMessageId() { - return messageId; - } - - public byte[] getMessageContent() { - return messageContent; - } - - public void setMessage(Message message) { - this.message = message; - } - - public boolean isResponseReady() { - return responseReady; - } - - public void markResponseAsReady() { - responseReady = true; - } + public static final AttachmentKey KEY = + AttachmentKey.create(AttachmentContent.class); + + private final CachedTopic cachedTopic; + private final MessageState messageState; + private final String messageId; + private byte[] messageContent; + private volatile TimeoutHolder timeoutHolder; + private volatile Message message; + private volatile boolean responseReady; + + public Message getMessage() { + return message; + } + + AttachmentContent(CachedTopic cachedTopic, MessageState messageState, String messageId) { + this.cachedTopic = cachedTopic; + this.messageState = messageState; + this.messageId = messageId; + } + + public MessageState getMessageState() { + return messageState; + } + + public TimeoutHolder getTimeoutHolder() { + return timeoutHolder; + } + + public void removeTimeout() { + timeoutHolder.remove(); + } + + public void setTimeoutHolder(TimeoutHolder timeoutHolder) { + this.timeoutHolder = timeoutHolder; + } + + public Topic getTopic() { + return cachedTopic.getTopic(); + } + + public CachedTopic getCachedTopic() { + return cachedTopic; + } + + public void setMessageContent(byte[] messageContent) { + this.messageContent = messageContent; + } + + public String getMessageId() { + return messageId; + } + + public byte[] getMessageContent() { + return messageContent; + } + + public void setMessage(Message message) { + this.message = message; + } + + public boolean isResponseReady() { + return responseReady; + } + + public void markResponseAsReady() { + responseReady = true; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/ContentLengthChecker.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/ContentLengthChecker.java index 4881fa0abe..805349fc39 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/ContentLengthChecker.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/ContentLengthChecker.java @@ -1,58 +1,65 @@ package pl.allegro.tech.hermes.frontend.publishing.handlers; +import static java.lang.String.format; + import io.undertow.server.HttpServerExchange; import io.undertow.util.HeaderMap; import io.undertow.util.HeaderValues; import io.undertow.util.Headers; +import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; - -import static java.lang.String.format; - final class ContentLengthChecker { - private static final Logger logger = LoggerFactory.getLogger(ContentLengthChecker.class); + private static final Logger logger = LoggerFactory.getLogger(ContentLengthChecker.class); - private final boolean forceMaxMessageSizePerTopic; + private final boolean forceMaxMessageSizePerTopic; - ContentLengthChecker(boolean forceMaxMessageSizePerTopic) { - this.forceMaxMessageSizePerTopic = forceMaxMessageSizePerTopic; - } + ContentLengthChecker(boolean forceMaxMessageSizePerTopic) { + this.forceMaxMessageSizePerTopic = forceMaxMessageSizePerTopic; + } - void check(HttpServerExchange exchange, int contentLength, AttachmentContent attachment) - throws InvalidContentLengthException, ContentTooLargeException { - - int max = attachment.getCachedTopic().getTopic().getMaxMessageSize(); - long expected = exchange.getRequestContentLength(); - if (expected != contentLength && !isChunked(exchange.getRequestHeaders(), expected)) { - throw new InvalidContentLengthException(expected, contentLength); - } else if (contentLength > max) { - if (forceMaxMessageSizePerTopic) { - throw new ContentTooLargeException(contentLength, max); - } else { - logger.warn("Content-Length is larger than max on this topic [length:{}, max:{}, topic: {}]", - contentLength, max, attachment.getCachedTopic().getQualifiedName()); - } - } - } + void check(HttpServerExchange exchange, int contentLength, AttachmentContent attachment) + throws InvalidContentLengthException, ContentTooLargeException { - private static boolean isChunked(HeaderMap headerMap, long requestContentLength) { - HeaderValues headerValue = headerMap.get(Headers.TRANSFER_ENCODING); - return headerValue != null && ("chunked".equals(headerValue.getFirst()) && requestContentLength < 0); + int max = attachment.getCachedTopic().getTopic().getMaxMessageSize(); + long expected = exchange.getRequestContentLength(); + if (expected != contentLength && !isChunked(exchange.getRequestHeaders(), expected)) { + throw new InvalidContentLengthException(expected, contentLength); + } else if (contentLength > max) { + if (forceMaxMessageSizePerTopic) { + throw new ContentTooLargeException(contentLength, max); + } else { + logger.warn( + "Content-Length is larger than max on this topic [length:{}, max:{}, topic: {}]", + contentLength, + max, + attachment.getCachedTopic().getQualifiedName()); + } } + } + + private static boolean isChunked(HeaderMap headerMap, long requestContentLength) { + HeaderValues headerValue = headerMap.get(Headers.TRANSFER_ENCODING); + return headerValue != null + && ("chunked".equals(headerValue.getFirst()) && requestContentLength < 0); + } - public static final class InvalidContentLengthException extends IOException { - InvalidContentLengthException(long expected, int contentLength) { - super(format("Content-Length does not match the header [header:%s, actual:%s].", - expected, contentLength)); - } + public static final class InvalidContentLengthException extends IOException { + InvalidContentLengthException(long expected, int contentLength) { + super( + format( + "Content-Length does not match the header [header:%s, actual:%s].", + expected, contentLength)); } + } - public static final class ContentTooLargeException extends IOException { - ContentTooLargeException(long contentLength, int max) { - super(format("Content-Length is larger than max on this topic [length:%s, max:%s].", - contentLength, max)); - } + public static final class ContentTooLargeException extends IOException { + ContentTooLargeException(long contentLength, int max) { + super( + format( + "Content-Length is larger than max on this topic [length:%s, max:%s].", + contentLength, max)); } + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/DynamicThroughputLimiter.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/DynamicThroughputLimiter.java index 4473165696..ab3a266e58 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/DynamicThroughputLimiter.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/DynamicThroughputLimiter.java @@ -1,99 +1,101 @@ package pl.allegro.tech.hermes.frontend.publishing.handlers; -import pl.allegro.tech.hermes.api.TopicName; -import pl.allegro.tech.hermes.metrics.HermesRateMeter; +import static pl.allegro.tech.hermes.frontend.publishing.handlers.ThroughputLimiter.QuotaInsight.globalQuotaViolation; +import static pl.allegro.tech.hermes.frontend.publishing.handlers.ThroughputLimiter.QuotaInsight.quotaConfirmed; +import static pl.allegro.tech.hermes.frontend.publishing.handlers.ThroughputLimiter.QuotaInsight.quotaViolation; import java.time.Duration; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; - -import static pl.allegro.tech.hermes.frontend.publishing.handlers.ThroughputLimiter.QuotaInsight.globalQuotaViolation; -import static pl.allegro.tech.hermes.frontend.publishing.handlers.ThroughputLimiter.QuotaInsight.quotaConfirmed; -import static pl.allegro.tech.hermes.frontend.publishing.handlers.ThroughputLimiter.QuotaInsight.quotaViolation; +import pl.allegro.tech.hermes.api.TopicName; +import pl.allegro.tech.hermes.metrics.HermesRateMeter; public class DynamicThroughputLimiter implements ThroughputLimiter, Runnable { - private final long max; - private final long threshold; - private final long desired; - private final double idleThreshold; - private final HermesRateMeter globalThroughputMeter; + private final long max; + private final long threshold; + private final long desired; + private final double idleThreshold; + private final HermesRateMeter globalThroughputMeter; - private final ScheduledExecutorService executor; - private final Duration checkInterval; + private final ScheduledExecutorService executor; + private final Duration checkInterval; - private final ConcurrentHashMap users = new ConcurrentHashMap<>(); + private final ConcurrentHashMap users = new ConcurrentHashMap<>(); - public DynamicThroughputLimiter(long max, - long threshold, - long desired, - double idleThreshold, - Duration checkInterval, - HermesRateMeter globalThroughput, - ScheduledExecutorService executor) { - this.max = max; - this.threshold = threshold; - this.desired = desired; - this.idleThreshold = idleThreshold; - this.globalThroughputMeter = globalThroughput; - this.checkInterval = checkInterval; - this.executor = executor; - } + public DynamicThroughputLimiter( + long max, + long threshold, + long desired, + double idleThreshold, + Duration checkInterval, + HermesRateMeter globalThroughput, + ScheduledExecutorService executor) { + this.max = max; + this.threshold = threshold; + this.desired = desired; + this.idleThreshold = idleThreshold; + this.globalThroughputMeter = globalThroughput; + this.checkInterval = checkInterval; + this.executor = executor; + } - @Override - public QuotaInsight checkQuota(TopicName topic, HermesRateMeter rate) { - Throughput throughput = users.computeIfAbsent(topic, name -> new Throughput(rate, max)); - long value = throughput.getRoundedOneMinuteRate(); - if (value > throughput.max) { - return quotaViolation(value, throughput.max); - } else if (globalThroughputMeter.getOneMinuteRate() > max) { - return globalQuotaViolation(); - } else { - return quotaConfirmed(); - } + @Override + public QuotaInsight checkQuota(TopicName topic, HermesRateMeter rate) { + Throughput throughput = users.computeIfAbsent(topic, name -> new Throughput(rate, max)); + long value = throughput.getRoundedOneMinuteRate(); + if (value > throughput.max) { + return quotaViolation(value, throughput.max); + } else if (globalThroughputMeter.getOneMinuteRate() > max) { + return globalQuotaViolation(); + } else { + return quotaConfirmed(); } + } - @Override - public void start() { - executor.scheduleAtFixedRate(this, checkInterval.toSeconds(), checkInterval.toSeconds(), TimeUnit.SECONDS); - } + @Override + public void start() { + executor.scheduleAtFixedRate( + this, checkInterval.toSeconds(), checkInterval.toSeconds(), TimeUnit.SECONDS); + } - @Override - public void run() { - if (globalThroughputMeter.getOneMinuteRate() > threshold) { - calibrateLimits(); - } + @Override + public void run() { + if (globalThroughputMeter.getOneMinuteRate() > threshold) { + calibrateLimits(); } + } - private void calibrateLimits() { - users.entrySet().removeIf(entry -> entry.getValue().getOneMinuteRate() <= idleThreshold); - int userCount = users.size(); - if (userCount > 0) { - long total = users.reduceValuesToLong(Long.MAX_VALUE, Throughput::getRoundedOneMinuteRate, 0, (Long::sum)); - long mean = total / userCount; - long desiredMean = desired / userCount; - users.entrySet() - .stream() - .filter(entry -> entry.getValue().getRoundedOneMinuteRate() >= mean) - .forEach(entry -> entry.getValue().max = desiredMean); - } + private void calibrateLimits() { + users.entrySet().removeIf(entry -> entry.getValue().getOneMinuteRate() <= idleThreshold); + int userCount = users.size(); + if (userCount > 0) { + long total = + users.reduceValuesToLong( + Long.MAX_VALUE, Throughput::getRoundedOneMinuteRate, 0, (Long::sum)); + long mean = total / userCount; + long desiredMean = desired / userCount; + users.entrySet().stream() + .filter(entry -> entry.getValue().getRoundedOneMinuteRate() >= mean) + .forEach(entry -> entry.getValue().max = desiredMean); } + } - private static class Throughput { - HermesRateMeter current; - volatile long max; + private static class Throughput { + HermesRateMeter current; + volatile long max; - Throughput(HermesRateMeter current, long max) { - this.current = current; - this.max = max; - } + Throughput(HermesRateMeter current, long max) { + this.current = current; + this.max = max; + } - long getRoundedOneMinuteRate() { - return (long) Math.floor(current.getOneMinuteRate()); - } + long getRoundedOneMinuteRate() { + return (long) Math.floor(current.getOneMinuteRate()); + } - double getOneMinuteRate() { - return current.getOneMinuteRate(); - } + double getOneMinuteRate() { + return current.getOneMinuteRate(); } + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/ExchangeMetrics.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/ExchangeMetrics.java index 11bd41b2e3..f3e7578fb4 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/ExchangeMetrics.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/ExchangeMetrics.java @@ -9,26 +9,27 @@ class ExchangeMetrics implements ExchangeCompletionListener { - private static final Logger logger = LoggerFactory.getLogger(ExchangeMetrics.class); + private static final Logger logger = LoggerFactory.getLogger(ExchangeMetrics.class); - private final StartedTimersPair producerLatencyTimers; - private final CachedTopic cachedTopic; + private final StartedTimersPair producerLatencyTimers; + private final CachedTopic cachedTopic; - ExchangeMetrics(CachedTopic cachedTopic) { - this.cachedTopic = cachedTopic; - producerLatencyTimers = cachedTopic.startProducerLatencyTimers(); - } + ExchangeMetrics(CachedTopic cachedTopic) { + this.cachedTopic = cachedTopic; + producerLatencyTimers = cachedTopic.startProducerLatencyTimers(); + } - @Override - public void exchangeEvent(HttpServerExchange exchange, NextListener nextListener) { - try { - cachedTopic.markRequestMeter(); - cachedTopic.markStatusCodeMeter(exchange.getStatusCode()); - producerLatencyTimers.close(); - } catch (RuntimeException e) { - logger.error("Exception while invoking metrics for topic {}", cachedTopic.getQualifiedName(), e); - } finally { - nextListener.proceed(); - } + @Override + public void exchangeEvent(HttpServerExchange exchange, NextListener nextListener) { + try { + cachedTopic.markRequestMeter(); + cachedTopic.markStatusCodeMeter(exchange.getStatusCode()); + producerLatencyTimers.close(); + } catch (RuntimeException e) { + logger.error( + "Exception while invoking metrics for topic {}", cachedTopic.getQualifiedName(), e); + } finally { + nextListener.proceed(); } + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/FixedThroughputLimiter.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/FixedThroughputLimiter.java index 7d210959d4..0acedfea5b 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/FixedThroughputLimiter.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/FixedThroughputLimiter.java @@ -1,22 +1,21 @@ package pl.allegro.tech.hermes.frontend.publishing.handlers; -import pl.allegro.tech.hermes.api.TopicName; -import pl.allegro.tech.hermes.metrics.HermesRateMeter; - import static pl.allegro.tech.hermes.frontend.publishing.handlers.ThroughputLimiter.QuotaInsight.quotaConfirmed; import static pl.allegro.tech.hermes.frontend.publishing.handlers.ThroughputLimiter.QuotaInsight.quotaViolation; -public class FixedThroughputLimiter implements ThroughputLimiter { - private final long limit; +import pl.allegro.tech.hermes.api.TopicName; +import pl.allegro.tech.hermes.metrics.HermesRateMeter; - public FixedThroughputLimiter(long limit) { - this.limit = limit; - } +public class FixedThroughputLimiter implements ThroughputLimiter { + private final long limit; - @Override - public QuotaInsight checkQuota(TopicName topic, HermesRateMeter throughput) { - long rate = (long) Math.floor(throughput.getOneMinuteRate()); - return rate > limit ? quotaViolation(rate, limit) : quotaConfirmed(); - } + public FixedThroughputLimiter(long limit) { + this.limit = limit; + } + @Override + public QuotaInsight checkQuota(TopicName topic, HermesRateMeter throughput) { + long rate = (long) Math.floor(throughput.getOneMinuteRate()); + return rate > limit ? quotaViolation(rate, limit) : quotaConfirmed(); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/HandlersChainFactory.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/HandlersChainFactory.java index f16394a340..d78e37304f 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/HandlersChainFactory.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/HandlersChainFactory.java @@ -5,6 +5,7 @@ import io.undertow.security.handlers.AuthenticationMechanismsHandler; import io.undertow.security.handlers.SecurityInitialHandler; import io.undertow.server.HttpHandler; +import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.frontend.cache.topic.TopicsCache; @@ -16,89 +17,105 @@ import pl.allegro.tech.hermes.frontend.server.auth.AuthenticationConfiguration; import pl.allegro.tech.hermes.frontend.server.auth.AuthenticationPredicateAwareConstraintHandler; -import java.util.Optional; - public class HandlersChainFactory { - private static final Logger logger = LoggerFactory.getLogger(HandlersChainFactory.class); + private static final Logger logger = LoggerFactory.getLogger(HandlersChainFactory.class); - private final TopicsCache topicsCache; - private final MessageErrorProcessor messageErrorProcessor; - private final MessageEndProcessor messageEndProcessor; - private final MessageFactory messageFactory; - private final BrokerMessageProducer brokerMessageProducer; - private final MessagePreviewLog previewLog; - private final boolean previewEnabled; - private final ThroughputLimiter throughputLimiter; - private final Optional authenticationConfiguration; - private final HandlersChainParameters handlersChainParameters; + private final TopicsCache topicsCache; + private final MessageErrorProcessor messageErrorProcessor; + private final MessageEndProcessor messageEndProcessor; + private final MessageFactory messageFactory; + private final BrokerMessageProducer brokerMessageProducer; + private final MessagePreviewLog previewLog; + private final boolean previewEnabled; + private final ThroughputLimiter throughputLimiter; + private final Optional authenticationConfiguration; + private final HandlersChainParameters handlersChainParameters; - public HandlersChainFactory(TopicsCache topicsCache, MessageErrorProcessor messageErrorProcessor, - MessageEndProcessor messageEndProcessor, MessageFactory messageFactory, - BrokerMessageProducer brokerMessageProducer, MessagePreviewLog messagePreviewLog, - ThroughputLimiter throughputLimiter, Optional authenticationConfiguration, - boolean messagePreviewEnabled, HandlersChainParameters handlersChainParameters) { - this.topicsCache = topicsCache; - this.messageErrorProcessor = messageErrorProcessor; - this.messageEndProcessor = messageEndProcessor; - this.messageFactory = messageFactory; - this.brokerMessageProducer = brokerMessageProducer; - this.previewLog = messagePreviewLog; - this.previewEnabled = messagePreviewEnabled; - this.throughputLimiter = throughputLimiter; - this.authenticationConfiguration = authenticationConfiguration; - this.handlersChainParameters = handlersChainParameters; - } + public HandlersChainFactory( + TopicsCache topicsCache, + MessageErrorProcessor messageErrorProcessor, + MessageEndProcessor messageEndProcessor, + MessageFactory messageFactory, + BrokerMessageProducer brokerMessageProducer, + MessagePreviewLog messagePreviewLog, + ThroughputLimiter throughputLimiter, + Optional authenticationConfiguration, + boolean messagePreviewEnabled, + HandlersChainParameters handlersChainParameters) { + this.topicsCache = topicsCache; + this.messageErrorProcessor = messageErrorProcessor; + this.messageEndProcessor = messageEndProcessor; + this.messageFactory = messageFactory; + this.brokerMessageProducer = brokerMessageProducer; + this.previewLog = messagePreviewLog; + this.previewEnabled = messagePreviewEnabled; + this.throughputLimiter = throughputLimiter; + this.authenticationConfiguration = authenticationConfiguration; + this.handlersChainParameters = handlersChainParameters; + } - public HttpHandler provide() { - HttpHandler publishing = new PublishingHandler(brokerMessageProducer, messageErrorProcessor, - messageEndProcessor); - HttpHandler messageCreateHandler = new MessageCreateHandler(publishing, messageFactory, messageErrorProcessor); - HttpHandler timeoutHandler = new TimeoutHandler(messageEndProcessor, messageErrorProcessor); - HttpHandler handlerAfterRead = previewEnabled ? new PreviewHandler(messageCreateHandler, previewLog) : messageCreateHandler; - HttpHandler readHandler = new MessageReadHandler( - handlerAfterRead, - timeoutHandler, - messageErrorProcessor, - throughputLimiter, - handlersChainParameters.isForceTopicMaxMessageSize(), - handlersChainParameters.getIdleTimeout(), - handlersChainParameters.getLongIdleTimeout(), - handlersChainParameters.getMaxPublishRequestDuration()); - TopicHandler topicHandler = new TopicHandler(readHandler, topicsCache, messageErrorProcessor); - boolean keepAliveHeaderEnabled = handlersChainParameters.isKeepAliveHeaderEnabled(); - HttpHandler rootPublishingHandler = keepAliveHeaderEnabled ? withKeepAliveHeaderHandler(topicHandler) : topicHandler; + public HttpHandler provide() { + HttpHandler publishing = + new PublishingHandler(brokerMessageProducer, messageErrorProcessor, messageEndProcessor); + HttpHandler messageCreateHandler = + new MessageCreateHandler(publishing, messageFactory, messageErrorProcessor); + HttpHandler timeoutHandler = new TimeoutHandler(messageEndProcessor, messageErrorProcessor); + HttpHandler handlerAfterRead = + previewEnabled + ? new PreviewHandler(messageCreateHandler, previewLog) + : messageCreateHandler; + HttpHandler readHandler = + new MessageReadHandler( + handlerAfterRead, + timeoutHandler, + messageErrorProcessor, + throughputLimiter, + handlersChainParameters.isForceTopicMaxMessageSize(), + handlersChainParameters.getIdleTimeout(), + handlersChainParameters.getLongIdleTimeout(), + handlersChainParameters.getMaxPublishRequestDuration()); + TopicHandler topicHandler = new TopicHandler(readHandler, topicsCache, messageErrorProcessor); + boolean keepAliveHeaderEnabled = handlersChainParameters.isKeepAliveHeaderEnabled(); + HttpHandler rootPublishingHandler = + keepAliveHeaderEnabled ? withKeepAliveHeaderHandler(topicHandler) : topicHandler; - boolean authenticationEnabled = handlersChainParameters.isAuthenticationEnabled(); - return authenticationEnabled ? withAuthenticationHandlersChain(rootPublishingHandler) : rootPublishingHandler; - } + boolean authenticationEnabled = handlersChainParameters.isAuthenticationEnabled(); + return authenticationEnabled + ? withAuthenticationHandlersChain(rootPublishingHandler) + : rootPublishingHandler; + } - private HttpHandler withKeepAliveHeaderHandler(HttpHandler next) { - return new KeepAliveHeaderHandler(next, (int) handlersChainParameters.getKeepAliveHeaderTimeout().toSeconds()); - } + private HttpHandler withKeepAliveHeaderHandler(HttpHandler next) { + return new KeepAliveHeaderHandler( + next, (int) handlersChainParameters.getKeepAliveHeaderTimeout().toSeconds()); + } - private HttpHandler withAuthenticationHandlersChain(HttpHandler next) { - AuthenticationConfiguration authConfig = authenticationConfiguration - .orElseThrow(() -> new IllegalStateException("AuthenticationConfiguration was not provided")); - try { - return createAuthenticationHandlersChain(next, authConfig); - } catch (Exception e) { - logger.error("Could not create authentication handlers chain", e); - throw e; - } + private HttpHandler withAuthenticationHandlersChain(HttpHandler next) { + AuthenticationConfiguration authConfig = + authenticationConfiguration.orElseThrow( + () -> new IllegalStateException("AuthenticationConfiguration was not provided")); + try { + return createAuthenticationHandlersChain(next, authConfig); + } catch (Exception e) { + logger.error("Could not create authentication handlers chain", e); + throw e; } + } - private HttpHandler createAuthenticationHandlersChain(HttpHandler next, AuthenticationConfiguration authConfig) { - HttpHandler authenticationCallHandler = new AuthenticationCallHandler(next); - HttpHandler constraintHandler = new AuthenticationPredicateAwareConstraintHandler( - authenticationCallHandler, authConfig.getIsAuthenticationRequiredPredicate()); + private HttpHandler createAuthenticationHandlersChain( + HttpHandler next, AuthenticationConfiguration authConfig) { + HttpHandler authenticationCallHandler = new AuthenticationCallHandler(next); + HttpHandler constraintHandler = + new AuthenticationPredicateAwareConstraintHandler( + authenticationCallHandler, authConfig.getIsAuthenticationRequiredPredicate()); - HttpHandler mechanismsHandler = new AuthenticationMechanismsHandler(constraintHandler, - authConfig.getAuthMechanisms()); - AuthenticationMode authenticationMode = AuthenticationMode.valueOf( - handlersChainParameters.getAuthenticationMode().toUpperCase() - ); + HttpHandler mechanismsHandler = + new AuthenticationMechanismsHandler(constraintHandler, authConfig.getAuthMechanisms()); + AuthenticationMode authenticationMode = + AuthenticationMode.valueOf(handlersChainParameters.getAuthenticationMode().toUpperCase()); - return new SecurityInitialHandler(authenticationMode, authConfig.getIdentityManager(), mechanismsHandler); - } + return new SecurityInitialHandler( + authenticationMode, authConfig.getIdentityManager(), mechanismsHandler); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/HandlersChainParameters.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/HandlersChainParameters.java index 98a0a37522..83648dddef 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/HandlersChainParameters.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/HandlersChainParameters.java @@ -4,34 +4,34 @@ public interface HandlersChainParameters { - /** - * Defines the time allowed for processing a publish request to a topic with ack=leader. - * After this time elapses a TimeoutHandler will be run which, with PersistentBufferExtension enabled, - * would result in message being sent to local buffer. - */ - Duration getIdleTimeout(); + /** + * Defines the time allowed for processing a publish request to a topic with ack=leader. After + * this time elapses a TimeoutHandler will be run which, with PersistentBufferExtension enabled, + * would result in message being sent to local buffer. + */ + Duration getIdleTimeout(); - /** - * Defines the time allowed for processing a publish request to a topic with ack=all. - * After this time elapses a TimeoutHandler will be run which, with PersistentBufferExtension enabled, - * would result in message being sent to local buffer. - */ - Duration getLongIdleTimeout(); + /** + * Defines the time allowed for processing a publish request to a topic with ack=all. After this + * time elapses a TimeoutHandler will be run which, with PersistentBufferExtension enabled, would + * result in message being sent to local buffer. + */ + Duration getLongIdleTimeout(); - /** - * Defines the time allowed for processing a publish request to a topic with - * fallbackToRemoteDatacenterEnabled=true. After this time elapses a TimeoutHandler will be run which - * would result in error being returned to a client. - */ - Duration getMaxPublishRequestDuration(); + /** + * Defines the time allowed for processing a publish request to a topic with + * fallbackToRemoteDatacenterEnabled=true. After this time elapses a TimeoutHandler will be run + * which would result in error being returned to a client. + */ + Duration getMaxPublishRequestDuration(); - boolean isForceTopicMaxMessageSize(); + boolean isForceTopicMaxMessageSize(); - boolean isKeepAliveHeaderEnabled(); + boolean isKeepAliveHeaderEnabled(); - Duration getKeepAliveHeaderTimeout(); + Duration getKeepAliveHeaderTimeout(); - boolean isAuthenticationEnabled(); + boolean isAuthenticationEnabled(); - String getAuthenticationMode(); + String getAuthenticationMode(); } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/KeepAliveHeaderHandler.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/KeepAliveHeaderHandler.java index 9d19ed0c3e..7237b38e13 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/KeepAliveHeaderHandler.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/KeepAliveHeaderHandler.java @@ -5,7 +5,7 @@ class KeepAliveHeaderHandler extends SetHeaderHandler { - KeepAliveHeaderHandler(HttpHandler next, int keepAliveTimeoutSec) { - super(next, "Keep-Alive", "timeout=" + keepAliveTimeoutSec); - } + KeepAliveHeaderHandler(HttpHandler next, int keepAliveTimeoutSec) { + super(next, "Keep-Alive", "timeout=" + keepAliveTimeoutSec); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/MessageCreateHandler.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/MessageCreateHandler.java index 244b367054..4718f38c15 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/MessageCreateHandler.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/MessageCreateHandler.java @@ -1,5 +1,13 @@ package pl.allegro.tech.hermes.frontend.publishing.handlers; +import static java.lang.String.format; +import static pl.allegro.tech.hermes.api.ErrorCode.AVRO_SCHEMA_INVALID_METADATA; +import static pl.allegro.tech.hermes.api.ErrorCode.INTERNAL_ERROR; +import static pl.allegro.tech.hermes.api.ErrorCode.SCHEMA_COULD_NOT_BE_LOADED; +import static pl.allegro.tech.hermes.api.ErrorCode.SCHEMA_VERSION_DOES_NOT_EXIST; +import static pl.allegro.tech.hermes.api.ErrorCode.VALIDATION_ERROR; +import static pl.allegro.tech.hermes.api.ErrorDescription.error; + import io.undertow.server.HttpHandler; import io.undertow.server.HttpServerExchange; import pl.allegro.tech.hermes.common.message.wrapper.AvroInvalidMetadataException; @@ -12,77 +20,75 @@ import pl.allegro.tech.hermes.schema.SchemaVersionDoesNotExistException; import tech.allegro.schema.json2avro.converter.AvroConversionException; -import static java.lang.String.format; -import static pl.allegro.tech.hermes.api.ErrorCode.AVRO_SCHEMA_INVALID_METADATA; -import static pl.allegro.tech.hermes.api.ErrorCode.INTERNAL_ERROR; -import static pl.allegro.tech.hermes.api.ErrorCode.SCHEMA_COULD_NOT_BE_LOADED; -import static pl.allegro.tech.hermes.api.ErrorCode.SCHEMA_VERSION_DOES_NOT_EXIST; -import static pl.allegro.tech.hermes.api.ErrorCode.VALIDATION_ERROR; -import static pl.allegro.tech.hermes.api.ErrorDescription.error; - class MessageCreateHandler implements HttpHandler { - private final HttpHandler next; - private final MessageFactory messageFactory; - private final MessageErrorProcessor messageErrorProcessor; + private final HttpHandler next; + private final MessageFactory messageFactory; + private final MessageErrorProcessor messageErrorProcessor; - MessageCreateHandler(HttpHandler next, MessageFactory messageFactory, MessageErrorProcessor messageErrorProcessor) { - this.next = next; - this.messageFactory = messageFactory; - this.messageErrorProcessor = messageErrorProcessor; - } + MessageCreateHandler( + HttpHandler next, + MessageFactory messageFactory, + MessageErrorProcessor messageErrorProcessor) { + this.next = next; + this.messageFactory = messageFactory; + this.messageErrorProcessor = messageErrorProcessor; + } - @Override - public void handleRequest(HttpServerExchange exchange) throws Exception { - AttachmentContent attachment = exchange.getAttachment(AttachmentContent.KEY); + @Override + public void handleRequest(HttpServerExchange exchange) throws Exception { + AttachmentContent attachment = exchange.getAttachment(AttachmentContent.KEY); - try { - attachment.setMessage(messageFactory.create(exchange.getRequestHeaders(), attachment)); - next.handleRequest(exchange); - } catch (InvalidMessageException | AvroConversionException | UnsupportedContentTypeException exception) { - attachment.removeTimeout(); - messageErrorProcessor.sendAndLog( - exchange, - attachment.getTopic(), - attachment.getMessageId(), - error("Invalid message: " + exception.getMessage(), VALIDATION_ERROR), - exception); - } catch (CouldNotLoadSchemaException | SchemaNotFoundException exception) { - attachment.removeTimeout(); - messageErrorProcessor.sendAndLog( - exchange, - attachment.getTopic(), - attachment.getMessageId(), - error("Missing schema", SCHEMA_COULD_NOT_BE_LOADED), - exception); - } catch (SchemaVersionDoesNotExistException exception) { - attachment.removeTimeout(); - messageErrorProcessor.sendAndLog( - exchange, - attachment.getTopic(), - attachment.getMessageId(), - error( - format("Given schema version '%s' does not exist", exception.getSchemaVersion().value()), - SCHEMA_VERSION_DOES_NOT_EXIST), - exception); - } catch (AvroInvalidMetadataException exception) { - attachment.removeTimeout(); - messageErrorProcessor.sendAndLog( - exchange, - attachment.getTopic(), - attachment.getMessageId(), - error("Schema does not contain mandatory __metadata field for Hermes internal metadata. Please fix topic schema.", - AVRO_SCHEMA_INVALID_METADATA), - exception); - } catch (Exception exception) { - attachment.removeTimeout(); - messageErrorProcessor.sendAndLog( - exchange, - attachment.getTopic(), - attachment.getMessageId(), - error("Exception caught while creating message", INTERNAL_ERROR), - exception); - } + try { + attachment.setMessage(messageFactory.create(exchange.getRequestHeaders(), attachment)); + next.handleRequest(exchange); + } catch (InvalidMessageException + | AvroConversionException + | UnsupportedContentTypeException exception) { + attachment.removeTimeout(); + messageErrorProcessor.sendAndLog( + exchange, + attachment.getTopic(), + attachment.getMessageId(), + error("Invalid message: " + exception.getMessage(), VALIDATION_ERROR), + exception); + } catch (CouldNotLoadSchemaException | SchemaNotFoundException exception) { + attachment.removeTimeout(); + messageErrorProcessor.sendAndLog( + exchange, + attachment.getTopic(), + attachment.getMessageId(), + error("Missing schema", SCHEMA_COULD_NOT_BE_LOADED), + exception); + } catch (SchemaVersionDoesNotExistException exception) { + attachment.removeTimeout(); + messageErrorProcessor.sendAndLog( + exchange, + attachment.getTopic(), + attachment.getMessageId(), + error( + format( + "Given schema version '%s' does not exist", exception.getSchemaVersion().value()), + SCHEMA_VERSION_DOES_NOT_EXIST), + exception); + } catch (AvroInvalidMetadataException exception) { + attachment.removeTimeout(); + messageErrorProcessor.sendAndLog( + exchange, + attachment.getTopic(), + attachment.getMessageId(), + error( + "Schema does not contain mandatory __metadata field for Hermes internal metadata. Please fix topic schema.", + AVRO_SCHEMA_INVALID_METADATA), + exception); + } catch (Exception exception) { + attachment.removeTimeout(); + messageErrorProcessor.sendAndLog( + exchange, + attachment.getTopic(), + attachment.getMessageId(), + error("Exception caught while creating message", INTERNAL_ERROR), + exception); } - + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/MessageReadHandler.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/MessageReadHandler.java index 050771f72a..9a53dca60f 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/MessageReadHandler.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/MessageReadHandler.java @@ -1,231 +1,265 @@ package pl.allegro.tech.hermes.frontend.publishing.handlers; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.apache.commons.lang3.exception.ExceptionUtils.getRootCauseMessage; +import static pl.allegro.tech.hermes.api.ErrorCode.INTERNAL_ERROR; +import static pl.allegro.tech.hermes.api.ErrorCode.THROUGHPUT_QUOTA_VIOLATION; +import static pl.allegro.tech.hermes.api.ErrorCode.VALIDATION_ERROR; +import static pl.allegro.tech.hermes.api.ErrorDescription.error; + import io.undertow.io.Receiver; import io.undertow.server.DefaultResponseListener; import io.undertow.server.HttpHandler; import io.undertow.server.HttpServerExchange; +import java.io.ByteArrayOutputStream; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicBoolean; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.frontend.publishing.handlers.end.MessageErrorProcessor; import pl.allegro.tech.hermes.frontend.publishing.message.MessageState; -import java.io.ByteArrayOutputStream; -import java.time.Duration; -import java.util.concurrent.atomic.AtomicBoolean; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static org.apache.commons.lang3.exception.ExceptionUtils.getRootCauseMessage; -import static pl.allegro.tech.hermes.api.ErrorCode.INTERNAL_ERROR; -import static pl.allegro.tech.hermes.api.ErrorCode.THROUGHPUT_QUOTA_VIOLATION; -import static pl.allegro.tech.hermes.api.ErrorCode.VALIDATION_ERROR; -import static pl.allegro.tech.hermes.api.ErrorDescription.error; - class MessageReadHandler implements HttpHandler { - private static final Logger logger = LoggerFactory.getLogger(MessageReadHandler.class); - - private final HttpHandler next; - private final HttpHandler timeoutHandler; - private final MessageErrorProcessor messageErrorProcessor; - private final ContentLengthChecker contentLengthChecker; - private final Duration defaultAsyncTimeout; - private final Duration longAsyncTimeout; - private final Duration maxPublishRequestDuration; - private final ThroughputLimiter throughputLimiter; - - MessageReadHandler(HttpHandler next, HttpHandler timeoutHandler, - MessageErrorProcessor messageErrorProcessor, ThroughputLimiter throughputLimiter, - boolean forceMaxMessageSizePerTopic, Duration idleTime, Duration longIdleTime, Duration maxPublishRequestDuration) { - this.next = next; - this.timeoutHandler = timeoutHandler; - this.messageErrorProcessor = messageErrorProcessor; - this.contentLengthChecker = new ContentLengthChecker(forceMaxMessageSizePerTopic); - this.defaultAsyncTimeout = idleTime; - this.longAsyncTimeout = longIdleTime; - this.throughputLimiter = throughputLimiter; - this.maxPublishRequestDuration = maxPublishRequestDuration; - } + private static final Logger logger = LoggerFactory.getLogger(MessageReadHandler.class); - @Override - public void handleRequest(HttpServerExchange exchange) { - AttachmentContent attachment = exchange.getAttachment(AttachmentContent.KEY); - - Duration timeout = calculateTimeout(attachment.getTopic()); - - attachment.setTimeoutHolder(new TimeoutHolder( - (int) timeout.toMillis(), - exchange.getIoThread().executeAfter( - () -> runTimeoutHandler(exchange, attachment), - timeout.toMillis(), - MILLISECONDS))); - - ThroughputLimiter.QuotaInsight quotaInsight = throughputLimiter.checkQuota( - attachment.getCachedTopic().getTopicName(), - attachment.getCachedTopic().getThroughput()); - if (quotaInsight.hasQuota()) { - readMessage(exchange, attachment); - } else { - respondWithQuotaViolation(exchange, attachment, quotaInsight.getReason()); - } - } + private final HttpHandler next; + private final HttpHandler timeoutHandler; + private final MessageErrorProcessor messageErrorProcessor; + private final ContentLengthChecker contentLengthChecker; + private final Duration defaultAsyncTimeout; + private final Duration longAsyncTimeout; + private final Duration maxPublishRequestDuration; + private final ThroughputLimiter throughputLimiter; - private Duration calculateTimeout(Topic topic) { - if (topic.isFallbackToRemoteDatacenterEnabled()) { - return maxPublishRequestDuration; - } - return topic.isReplicationConfirmRequired() ? longAsyncTimeout : defaultAsyncTimeout; - } + MessageReadHandler( + HttpHandler next, + HttpHandler timeoutHandler, + MessageErrorProcessor messageErrorProcessor, + ThroughputLimiter throughputLimiter, + boolean forceMaxMessageSizePerTopic, + Duration idleTime, + Duration longIdleTime, + Duration maxPublishRequestDuration) { + this.next = next; + this.timeoutHandler = timeoutHandler; + this.messageErrorProcessor = messageErrorProcessor; + this.contentLengthChecker = new ContentLengthChecker(forceMaxMessageSizePerTopic); + this.defaultAsyncTimeout = idleTime; + this.longAsyncTimeout = longIdleTime; + this.throughputLimiter = throughputLimiter; + this.maxPublishRequestDuration = maxPublishRequestDuration; + } - private void runTimeoutHandler(HttpServerExchange exchange, AttachmentContent attachment) { - try { - timeoutHandler.handleRequest(exchange); - } catch (Exception e) { - messageErrorProcessor.sendAndLog(exchange, attachment.getTopic(), attachment.getMessageId(), - error("Error while handling timeout task", INTERNAL_ERROR), e); - } - } + @Override + public void handleRequest(HttpServerExchange exchange) { + AttachmentContent attachment = exchange.getAttachment(AttachmentContent.KEY); - private void readMessage(HttpServerExchange exchange, AttachmentContent attachment) { - ByteArrayOutputStream messageContent = new ByteArrayOutputStream(); - MessageState state = attachment.getMessageState(); + Duration timeout = calculateTimeout(attachment.getTopic()); - Receiver receiver = exchange.getRequestReceiver(); + attachment.setTimeoutHolder( + new TimeoutHolder( + (int) timeout.toMillis(), + exchange + .getIoThread() + .executeAfter( + () -> runTimeoutHandler(exchange, attachment), + timeout.toMillis(), + MILLISECONDS))); - attachment.getTimeoutHolder().onTimeout((Void unused) -> receiver.pause()); - - if (state.setReading()) { - receiver.receivePartialBytes( - partialMessageRead(state, messageContent, attachment), - readingError(state, attachment)); - } else { - messageErrorProcessor.sendAndLog( - exchange, - attachment.getTopic(), - attachment.getMessageId(), - error("Probably context switching problem as timeout elapsed before message reading was started", INTERNAL_ERROR)); - } + ThroughputLimiter.QuotaInsight quotaInsight = + throughputLimiter.checkQuota( + attachment.getCachedTopic().getTopicName(), + attachment.getCachedTopic().getThroughput()); + if (quotaInsight.hasQuota()) { + readMessage(exchange, attachment); + } else { + respondWithQuotaViolation(exchange, attachment, quotaInsight.getReason()); } + } - private Receiver.PartialBytesCallback partialMessageRead(MessageState state, - ByteArrayOutputStream messageContent, - AttachmentContent attachment) { - return (exchange, message, last) -> { - if (state.isReadingTimeout()) { - endWithoutDefaultResponse(exchange); - return; - } - messageContent.write(message, 0, message.length); - - if (last) { - if (state.setFullyRead()) { - messageRead(exchange, messageContent.toByteArray(), attachment); - } else { - endWithoutDefaultResponse(exchange); - } - } - }; + private Duration calculateTimeout(Topic topic) { + if (topic.isFallbackToRemoteDatacenterEnabled()) { + return maxPublishRequestDuration; } + return topic.isReplicationConfirmRequired() ? longAsyncTimeout : defaultAsyncTimeout; + } - private Receiver.ErrorCallback readingError(MessageState state, AttachmentContent attachment) { - return (exchange, exception) -> { - if (state.setReadingError()) { - attachment.removeTimeout(); - messageErrorProcessor.sendAndLog(exchange, attachment.getTopic(), attachment.getMessageId(), - error("Error while reading message. " + getRootCauseMessage(exception), INTERNAL_ERROR), exception); - } else { - messageErrorProcessor.log( - exchange, - "Error while reading message after timeout execution. " + getRootCauseMessage(exception), - exception); - } - }; + private void runTimeoutHandler(HttpServerExchange exchange, AttachmentContent attachment) { + try { + timeoutHandler.handleRequest(exchange); + } catch (Exception e) { + messageErrorProcessor.sendAndLog( + exchange, + attachment.getTopic(), + attachment.getMessageId(), + error("Error while handling timeout task", INTERNAL_ERROR), + e); } + } - private void messageRead(HttpServerExchange exchange, byte[] messageContent, AttachmentContent attachment) { - try { - contentLengthChecker.check(exchange, messageContent.length, attachment); - attachment.getCachedTopic().reportMessageContentSize(messageContent.length); - ThroughputLimiter.QuotaInsight quotaCheck = throughputLimiter.checkQuota( - attachment.getCachedTopic().getTopicName(), - attachment.getCachedTopic().getThroughput()); - if (quotaCheck.hasQuota()) { - finalizeMessageRead(exchange, messageContent, attachment); - } else { - respondWithQuotaViolation(exchange, attachment, quotaCheck.getReason()); - } - } catch (ContentLengthChecker.InvalidContentLengthException | ContentLengthChecker.ContentTooLargeException e) { - attachment.removeTimeout(); - messageErrorProcessor.sendAndLog(exchange, attachment.getTopic(), - attachment.getMessageId(), error(e.getMessage(), VALIDATION_ERROR)); - } catch (Exception e) { - attachment.removeTimeout(); - messageErrorProcessor.sendAndLog(exchange, attachment.getTopic(), attachment.getMessageId(), e); - } + private void readMessage(HttpServerExchange exchange, AttachmentContent attachment) { + ByteArrayOutputStream messageContent = new ByteArrayOutputStream(); + MessageState state = attachment.getMessageState(); + + Receiver receiver = exchange.getRequestReceiver(); + + attachment.getTimeoutHolder().onTimeout((Void unused) -> receiver.pause()); + + if (state.setReading()) { + receiver.receivePartialBytes( + partialMessageRead(state, messageContent, attachment), readingError(state, attachment)); + } else { + messageErrorProcessor.sendAndLog( + exchange, + attachment.getTopic(), + attachment.getMessageId(), + error( + "Probably context switching problem as timeout elapsed before message reading was started", + INTERNAL_ERROR)); } + } - private void finalizeMessageRead(HttpServerExchange exchange, - byte[] messageContent, - AttachmentContent attachment) throws Exception { - attachment.setMessageContent(messageContent); + private Receiver.PartialBytesCallback partialMessageRead( + MessageState state, ByteArrayOutputStream messageContent, AttachmentContent attachment) { + return (exchange, message, last) -> { + if (state.isReadingTimeout()) { endWithoutDefaultResponse(exchange); - if (exchange.isInIoThread()) { - dispatchToWorker(exchange, attachment); + return; + } + messageContent.write(message, 0, message.length); + + if (last) { + if (state.setFullyRead()) { + messageRead(exchange, messageContent.toByteArray(), attachment); } else { - next.handleRequest(exchange); + endWithoutDefaultResponse(exchange); } - } + } + }; + } - private void respondWithQuotaViolation(HttpServerExchange exchange, - AttachmentContent attachment, - String reason) { + private Receiver.ErrorCallback readingError(MessageState state, AttachmentContent attachment) { + return (exchange, exception) -> { + if (state.setReadingError()) { attachment.removeTimeout(); messageErrorProcessor.sendAndLog( - exchange, - attachment.getTopic(), - attachment.getMessageId(), - error(reason, THROUGHPUT_QUOTA_VIOLATION)); + exchange, + attachment.getTopic(), + attachment.getMessageId(), + error("Error while reading message. " + getRootCauseMessage(exception), INTERNAL_ERROR), + exception); + } else { + messageErrorProcessor.log( + exchange, + "Error while reading message after timeout execution. " + + getRootCauseMessage(exception), + exception); + } + }; + } + + private void messageRead( + HttpServerExchange exchange, byte[] messageContent, AttachmentContent attachment) { + try { + contentLengthChecker.check(exchange, messageContent.length, attachment); + attachment.getCachedTopic().reportMessageContentSize(messageContent.length); + ThroughputLimiter.QuotaInsight quotaCheck = + throughputLimiter.checkQuota( + attachment.getCachedTopic().getTopicName(), + attachment.getCachedTopic().getThroughput()); + if (quotaCheck.hasQuota()) { + finalizeMessageRead(exchange, messageContent, attachment); + } else { + respondWithQuotaViolation(exchange, attachment, quotaCheck.getReason()); + } + } catch (ContentLengthChecker.InvalidContentLengthException + | ContentLengthChecker.ContentTooLargeException e) { + attachment.removeTimeout(); + messageErrorProcessor.sendAndLog( + exchange, + attachment.getTopic(), + attachment.getMessageId(), + error(e.getMessage(), VALIDATION_ERROR)); + } catch (Exception e) { + attachment.removeTimeout(); + messageErrorProcessor.sendAndLog( + exchange, attachment.getTopic(), attachment.getMessageId(), e); + } + } + + private void finalizeMessageRead( + HttpServerExchange exchange, byte[] messageContent, AttachmentContent attachment) + throws Exception { + attachment.setMessageContent(messageContent); + endWithoutDefaultResponse(exchange); + if (exchange.isInIoThread()) { + dispatchToWorker(exchange, attachment); + } else { + next.handleRequest(exchange); } + } - private void dispatchToWorker(HttpServerExchange exchange, AttachmentContent attachment) { - // exchange.dispatch(next) is not called here because async io read flag can be still set to true which combined with - // dispatch() leads to an exception - exchange.getConnection().getWorker().execute(() -> { - try { + private void respondWithQuotaViolation( + HttpServerExchange exchange, AttachmentContent attachment, String reason) { + attachment.removeTimeout(); + messageErrorProcessor.sendAndLog( + exchange, + attachment.getTopic(), + attachment.getMessageId(), + error(reason, THROUGHPUT_QUOTA_VIOLATION)); + } + + private void dispatchToWorker(HttpServerExchange exchange, AttachmentContent attachment) { + // exchange.dispatch(next) is not called here because async io read flag can be still set to + // true which combined with + // dispatch() leads to an exception + exchange + .getConnection() + .getWorker() + .execute( + () -> { + try { next.handleRequest(exchange); - } catch (Exception e) { + } catch (Exception e) { attachment.removeTimeout(); - messageErrorProcessor.sendAndLog(exchange, attachment.getTopic(), attachment.getMessageId(), - error("Error while executing next handler after read handler", INTERNAL_ERROR), e); - } - }); - } + messageErrorProcessor.sendAndLog( + exchange, + attachment.getTopic(), + attachment.getMessageId(), + error("Error while executing next handler after read handler", INTERNAL_ERROR), + e); + } + }); + } - private void endWithoutDefaultResponse(HttpServerExchange exchange) { - // when a handler doesn't return a response (for example when is interrupted by timeout) - // then without this listener default response can be returned with 200 status code when the handler finishes - // execution before the other one - exchange.addDefaultResponseListener(new DefaultResponseSimulator()); - } + private void endWithoutDefaultResponse(HttpServerExchange exchange) { + // when a handler doesn't return a response (for example when is interrupted by timeout) + // then without this listener default response can be returned with 200 status code when the + // handler finishes + // execution before the other one + exchange.addDefaultResponseListener(new DefaultResponseSimulator()); + } - static final class DefaultResponseSimulator implements DefaultResponseListener { - - private static final boolean RESPONSE_SIMULATED = true; - private final AtomicBoolean responseNotSimulatedOnlyOnce = new AtomicBoolean(); - - @Override - public boolean handleDefaultResponse(HttpServerExchange exchange) { - if (exchange.getAttachment(AttachmentContent.KEY).isResponseReady()) { - if (exchange.getStatusCode() == 200) { - try { - exchange.setStatusCode(500); - } catch (RuntimeException e) { - logger.error("Exception has been thrown during an exchange status modification", e); - } - } - return !responseNotSimulatedOnlyOnce.compareAndSet(false, true); - } else { - return RESPONSE_SIMULATED; - } + static final class DefaultResponseSimulator implements DefaultResponseListener { + + private static final boolean RESPONSE_SIMULATED = true; + private final AtomicBoolean responseNotSimulatedOnlyOnce = new AtomicBoolean(); + + @Override + public boolean handleDefaultResponse(HttpServerExchange exchange) { + if (exchange.getAttachment(AttachmentContent.KEY).isResponseReady()) { + if (exchange.getStatusCode() == 200) { + try { + exchange.setStatusCode(500); + } catch (RuntimeException e) { + logger.error("Exception has been thrown during an exchange status modification", e); + } } + return !responseNotSimulatedOnlyOnce.compareAndSet(false, true); + } else { + return RESPONSE_SIMULATED; + } } + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/PreviewHandler.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/PreviewHandler.java index f316c541b9..44bd2e632f 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/PreviewHandler.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/PreviewHandler.java @@ -6,22 +6,22 @@ public class PreviewHandler implements HttpHandler { - private final HttpHandler next; - private final MessagePreviewLog messagePreviewLog; + private final HttpHandler next; + private final MessagePreviewLog messagePreviewLog; - public PreviewHandler(HttpHandler next, MessagePreviewLog messagePreviewLog) { - this.next = next; - this.messagePreviewLog = messagePreviewLog; - } + public PreviewHandler(HttpHandler next, MessagePreviewLog messagePreviewLog) { + this.next = next; + this.messagePreviewLog = messagePreviewLog; + } - @Override - public void handleRequest(HttpServerExchange exchange) throws Exception { - AttachmentContent attachment = exchange.getAttachment(AttachmentContent.KEY); + @Override + public void handleRequest(HttpServerExchange exchange) throws Exception { + AttachmentContent attachment = exchange.getAttachment(AttachmentContent.KEY); - try { - next.handleRequest(exchange); - } finally { - messagePreviewLog.add(attachment.getTopic(), attachment.getMessage()); - } + try { + next.handleRequest(exchange); + } finally { + messagePreviewLog.add(attachment.getTopic(), attachment.getMessage()); } + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/PublishingHandler.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/PublishingHandler.java index aa20a45520..3782340af7 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/PublishingHandler.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/PublishingHandler.java @@ -1,5 +1,9 @@ package pl.allegro.tech.hermes.frontend.publishing.handlers; +import static org.apache.commons.lang3.exception.ExceptionUtils.getRootCauseMessage; +import static pl.allegro.tech.hermes.api.ErrorCode.INTERNAL_ERROR; +import static pl.allegro.tech.hermes.api.ErrorDescription.error; + import io.undertow.server.HttpHandler; import io.undertow.server.HttpServerExchange; import pl.allegro.tech.hermes.api.Topic; @@ -10,89 +14,106 @@ import pl.allegro.tech.hermes.frontend.publishing.message.Message; import pl.allegro.tech.hermes.frontend.publishing.message.MessageState; -import static org.apache.commons.lang3.exception.ExceptionUtils.getRootCauseMessage; -import static pl.allegro.tech.hermes.api.ErrorCode.INTERNAL_ERROR; -import static pl.allegro.tech.hermes.api.ErrorDescription.error; - class PublishingHandler implements HttpHandler { - private final BrokerMessageProducer brokerMessageProducer; - private final MessageErrorProcessor messageErrorProcessor; - private final MessageEndProcessor messageEndProcessor; + private final BrokerMessageProducer brokerMessageProducer; + private final MessageErrorProcessor messageErrorProcessor; + private final MessageEndProcessor messageEndProcessor; - PublishingHandler(BrokerMessageProducer brokerMessageProducer, MessageErrorProcessor messageErrorProcessor, - MessageEndProcessor messageEndProcessor) { - this.brokerMessageProducer = brokerMessageProducer; - this.messageErrorProcessor = messageErrorProcessor; - this.messageEndProcessor = messageEndProcessor; - } + PublishingHandler( + BrokerMessageProducer brokerMessageProducer, + MessageErrorProcessor messageErrorProcessor, + MessageEndProcessor messageEndProcessor) { + this.brokerMessageProducer = brokerMessageProducer; + this.messageErrorProcessor = messageErrorProcessor; + this.messageEndProcessor = messageEndProcessor; + } - @Override - public void handleRequest(HttpServerExchange exchange) { - // change state of exchange to dispatched, - // thanks to this call, default response with 200 status code is not returned after handlerRequest() finishes its execution - exchange.dispatch(() -> { - try { - handle(exchange); - } catch (RuntimeException e) { - AttachmentContent attachment = exchange.getAttachment(AttachmentContent.KEY); - MessageState messageState = attachment.getMessageState(); - messageState.setErrorInSendingToKafka(); - messageErrorProcessor.sendAndLog(exchange, "Exception while publishing message to a broker.", e); - } + @Override + public void handleRequest(HttpServerExchange exchange) { + // change state of exchange to dispatched, + // thanks to this call, default response with 200 status code is not returned after + // handlerRequest() finishes its execution + exchange.dispatch( + () -> { + try { + handle(exchange); + } catch (RuntimeException e) { + AttachmentContent attachment = exchange.getAttachment(AttachmentContent.KEY); + MessageState messageState = attachment.getMessageState(); + messageState.setErrorInSendingToKafka(); + messageErrorProcessor.sendAndLog( + exchange, "Exception while publishing message to a broker.", e); + } }); - } + } - private void handle(HttpServerExchange exchange) { - AttachmentContent attachment = exchange.getAttachment(AttachmentContent.KEY); - MessageState messageState = attachment.getMessageState(); + private void handle(HttpServerExchange exchange) { + AttachmentContent attachment = exchange.getAttachment(AttachmentContent.KEY); + MessageState messageState = attachment.getMessageState(); - messageState.setSendingToKafkaProducerQueue(); - brokerMessageProducer.send(attachment.getMessage(), attachment.getCachedTopic(), new PublishingCallback() { + messageState.setSendingToKafkaProducerQueue(); + brokerMessageProducer.send( + attachment.getMessage(), + attachment.getCachedTopic(), + new PublishingCallback() { - @Override - public void onPublished(Message message, Topic topic) { - exchange.getConnection().getWorker().execute(() -> { - if (messageState.setSentToKafka()) { + @Override + public void onPublished(Message message, Topic topic) { + exchange + .getConnection() + .getWorker() + .execute( + () -> { + if (messageState.setSentToKafka()) { attachment.removeTimeout(); messageEndProcessor.sent(exchange, attachment); - } else if (messageState.setDelayedSentToKafka()) { + } else if (messageState.setDelayedSentToKafka()) { messageEndProcessor.delayedSent(attachment.getCachedTopic(), message); - } - }); - } + } + }); + } - @Override - public void onEachPublished(Message message, Topic topic, String datacenter) { - exchange.getConnection().getWorker().execute(() -> { - attachment.getCachedTopic().incrementPublished(datacenter); - messageEndProcessor.eachSent(exchange, attachment, datacenter); - }); - } + @Override + public void onEachPublished(Message message, Topic topic, String datacenter) { + exchange + .getConnection() + .getWorker() + .execute( + () -> { + attachment.getCachedTopic().incrementPublished(datacenter); + messageEndProcessor.eachSent(exchange, attachment, datacenter); + }); + } - @Override - public void onUnpublished(Message message, Topic topic, Exception exception) { - exchange.getConnection().getWorker().execute(() -> { - messageState.setErrorInSendingToKafka(); - attachment.removeTimeout(); - handleNotPublishedMessage(exchange, topic, attachment.getMessageId(), exception); - }); - } + @Override + public void onUnpublished(Message message, Topic topic, Exception exception) { + exchange + .getConnection() + .getWorker() + .execute( + () -> { + messageState.setErrorInSendingToKafka(); + attachment.removeTimeout(); + handleNotPublishedMessage( + exchange, topic, attachment.getMessageId(), exception); + }); + } }); - if (messageState.setSendingToKafka() - && !attachment.getCachedTopic().getTopic().isFallbackToRemoteDatacenterEnabled() - && messageState.setDelayedProcessing()) { - messageEndProcessor.bufferedButDelayedProcessing(exchange, attachment); - } + if (messageState.setSendingToKafka() + && !attachment.getCachedTopic().getTopic().isFallbackToRemoteDatacenterEnabled() + && messageState.setDelayedProcessing()) { + messageEndProcessor.bufferedButDelayedProcessing(exchange, attachment); } + } - - private void handleNotPublishedMessage(HttpServerExchange exchange, Topic topic, String messageId, Exception exception) { - messageErrorProcessor.sendAndLog( - exchange, - topic, - messageId, - error("Message not published. " + getRootCauseMessage(exception), INTERNAL_ERROR)); - } + private void handleNotPublishedMessage( + HttpServerExchange exchange, Topic topic, String messageId, Exception exception) { + messageErrorProcessor.sendAndLog( + exchange, + topic, + messageId, + error("Message not published. " + getRootCauseMessage(exception), INTERNAL_ERROR)); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/ThroughputLimiter.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/ThroughputLimiter.java index 2118c1ac72..2bcd232c80 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/ThroughputLimiter.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/ThroughputLimiter.java @@ -1,56 +1,53 @@ package pl.allegro.tech.hermes.frontend.publishing.handlers; +import static java.lang.String.format; + import pl.allegro.tech.hermes.api.TopicName; import pl.allegro.tech.hermes.metrics.HermesRateMeter; -import static java.lang.String.format; - public interface ThroughputLimiter { - QuotaInsight checkQuota(TopicName topic, HermesRateMeter throughput); - - default void start() { - } + QuotaInsight checkQuota(TopicName topic, HermesRateMeter throughput); - default void stop() { - } + default void start() {} - class QuotaInsight { - private static final QuotaInsight QUOTA_CONFIRMED = new QuotaInsight(); - private static final QuotaInsight GLOBAL_VIOLATION = new QuotaInsight(false, - "Global throughput exceeded. Sorry for the inconvenience."); - private static final String DEFAULT_REASON = "unknown"; + default void stop() {} - private boolean hasQuota = true; - private String reason; + class QuotaInsight { + private static final QuotaInsight QUOTA_CONFIRMED = new QuotaInsight(); + private static final QuotaInsight GLOBAL_VIOLATION = + new QuotaInsight(false, "Global throughput exceeded. Sorry for the inconvenience."); + private static final String DEFAULT_REASON = "unknown"; - private QuotaInsight() { - } + private boolean hasQuota = true; + private String reason; - private QuotaInsight(boolean pass, String reason) { - this.hasQuota = pass; - this.reason = reason; - } + private QuotaInsight() {} - public boolean hasQuota() { - return hasQuota; - } + private QuotaInsight(boolean pass, String reason) { + this.hasQuota = pass; + this.reason = reason; + } - public String getReason() { - return reason != null ? reason : DEFAULT_REASON; - } + public boolean hasQuota() { + return hasQuota; + } - public static QuotaInsight quotaConfirmed() { - return QUOTA_CONFIRMED; - } + public String getReason() { + return reason != null ? reason : DEFAULT_REASON; + } - public static QuotaInsight quotaViolation(long current, long limit) { - return new QuotaInsight(false, - format("Current throughput exceeded limit [current:%s, limit:%s].", current, limit)); - } + public static QuotaInsight quotaConfirmed() { + return QUOTA_CONFIRMED; + } - public static QuotaInsight globalQuotaViolation() { - return GLOBAL_VIOLATION; - } + public static QuotaInsight quotaViolation(long current, long limit) { + return new QuotaInsight( + false, + format("Current throughput exceeded limit [current:%s, limit:%s].", current, limit)); + } + public static QuotaInsight globalQuotaViolation() { + return GLOBAL_VIOLATION; } + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/ThroughputLimiterFactory.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/ThroughputLimiterFactory.java index da0abba8a5..c4252dd92c 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/ThroughputLimiterFactory.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/ThroughputLimiterFactory.java @@ -1,54 +1,61 @@ package pl.allegro.tech.hermes.frontend.publishing.handlers; +import static java.util.concurrent.Executors.newScheduledThreadPool; +import static pl.allegro.tech.hermes.frontend.publishing.handlers.ThroughputLimiter.QuotaInsight.quotaConfirmed; + import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.frontend.metric.ThroughputRegistry; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadFactory; - -import static java.util.concurrent.Executors.newScheduledThreadPool; -import static pl.allegro.tech.hermes.frontend.publishing.handlers.ThroughputLimiter.QuotaInsight.quotaConfirmed; - public class ThroughputLimiterFactory { - private final ThroughputParameters throughputParameters; - - private final ThroughputRegistry throughputRegistry; - - private enum ThroughputLimiterType { UNLIMITED, FIXED, DYNAMIC } - - public ThroughputLimiterFactory(ThroughputParameters throughputParameters, ThroughputRegistry throughputRegistry) { - this.throughputParameters = throughputParameters; - this.throughputRegistry = throughputRegistry; - } - - public ThroughputLimiter provide() { - switch (ThroughputLimiterType.valueOf(throughputParameters.getType().toUpperCase())) { - case UNLIMITED: - return (a, b) -> quotaConfirmed(); - case FIXED: - return new FixedThroughputLimiter(throughputParameters.getFixedMax()); - case DYNAMIC: - return new DynamicThroughputLimiter( - throughputParameters.getDynamicMax(), - throughputParameters.getDynamicThreshold(), - throughputParameters.getDynamicDesired(), - throughputParameters.getDynamicIdle(), - throughputParameters.getDynamicCheckInterval(), - throughputRegistry::getGlobalThroughputOneMinuteRate, - getExecutor()); - default: - throw new IllegalArgumentException("Unknown throughput limiter type."); - } - } - - private ScheduledExecutorService getExecutor() { - Logger logger = LoggerFactory.getLogger(ThroughputLimiterFactory.class); - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setNameFormat("ThroughputLimiterExecutor-%d") - .setUncaughtExceptionHandler((t, e) -> logger.error("ThroughputLimiterExecutor failed {}", t.getName(), e)).build(); - return newScheduledThreadPool(1, threadFactory); + private final ThroughputParameters throughputParameters; + + private final ThroughputRegistry throughputRegistry; + + private enum ThroughputLimiterType { + UNLIMITED, + FIXED, + DYNAMIC + } + + public ThroughputLimiterFactory( + ThroughputParameters throughputParameters, ThroughputRegistry throughputRegistry) { + this.throughputParameters = throughputParameters; + this.throughputRegistry = throughputRegistry; + } + + public ThroughputLimiter provide() { + switch (ThroughputLimiterType.valueOf(throughputParameters.getType().toUpperCase())) { + case UNLIMITED: + return (a, b) -> quotaConfirmed(); + case FIXED: + return new FixedThroughputLimiter(throughputParameters.getFixedMax()); + case DYNAMIC: + return new DynamicThroughputLimiter( + throughputParameters.getDynamicMax(), + throughputParameters.getDynamicThreshold(), + throughputParameters.getDynamicDesired(), + throughputParameters.getDynamicIdle(), + throughputParameters.getDynamicCheckInterval(), + throughputRegistry::getGlobalThroughputOneMinuteRate, + getExecutor()); + default: + throw new IllegalArgumentException("Unknown throughput limiter type."); } + } + + private ScheduledExecutorService getExecutor() { + Logger logger = LoggerFactory.getLogger(ThroughputLimiterFactory.class); + ThreadFactory threadFactory = + new ThreadFactoryBuilder() + .setNameFormat("ThroughputLimiterExecutor-%d") + .setUncaughtExceptionHandler( + (t, e) -> logger.error("ThroughputLimiterExecutor failed {}", t.getName(), e)) + .build(); + return newScheduledThreadPool(1, threadFactory); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/ThroughputParameters.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/ThroughputParameters.java index 2f7ed70eec..d56cba1e2b 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/ThroughputParameters.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/ThroughputParameters.java @@ -4,17 +4,17 @@ public interface ThroughputParameters { - String getType(); + String getType(); - long getFixedMax(); + long getFixedMax(); - long getDynamicMax(); + long getDynamicMax(); - long getDynamicThreshold(); + long getDynamicThreshold(); - long getDynamicDesired(); + long getDynamicDesired(); - double getDynamicIdle(); + double getDynamicIdle(); - Duration getDynamicCheckInterval(); + Duration getDynamicCheckInterval(); } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/TimeoutHandler.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/TimeoutHandler.java index 1dd2774e88..917974591a 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/TimeoutHandler.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/TimeoutHandler.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.frontend.publishing.handlers; +import static pl.allegro.tech.hermes.api.ErrorCode.INTERNAL_ERROR; +import static pl.allegro.tech.hermes.api.ErrorCode.SENDING_TO_KAFKA_TIMEOUT; +import static pl.allegro.tech.hermes.api.ErrorCode.TIMEOUT; +import static pl.allegro.tech.hermes.api.ErrorDescription.error; + import io.undertow.server.HttpHandler; import io.undertow.server.HttpServerExchange; import org.slf4j.Logger; @@ -8,91 +13,111 @@ import pl.allegro.tech.hermes.frontend.publishing.handlers.end.MessageErrorProcessor; import pl.allegro.tech.hermes.frontend.publishing.message.MessageState; -import static pl.allegro.tech.hermes.api.ErrorCode.INTERNAL_ERROR; -import static pl.allegro.tech.hermes.api.ErrorCode.SENDING_TO_KAFKA_TIMEOUT; -import static pl.allegro.tech.hermes.api.ErrorCode.TIMEOUT; -import static pl.allegro.tech.hermes.api.ErrorDescription.error; - class TimeoutHandler implements HttpHandler { - private final MessageErrorProcessor messageErrorProcessor; - private final MessageEndProcessor messageEndProcessor; - private static final Logger logger = LoggerFactory.getLogger(TimeoutHandler.class); + private final MessageErrorProcessor messageErrorProcessor; + private final MessageEndProcessor messageEndProcessor; + private static final Logger logger = LoggerFactory.getLogger(TimeoutHandler.class); - TimeoutHandler(MessageEndProcessor messageEndProcessor, MessageErrorProcessor messageErrorProcessor) { - this.messageErrorProcessor = messageErrorProcessor; - this.messageEndProcessor = messageEndProcessor; - } + TimeoutHandler( + MessageEndProcessor messageEndProcessor, MessageErrorProcessor messageErrorProcessor) { + this.messageErrorProcessor = messageErrorProcessor; + this.messageEndProcessor = messageEndProcessor; + } - @Override - public void handleRequest(HttpServerExchange exchange) throws Exception { - AttachmentContent attachment = exchange.getAttachment(AttachmentContent.KEY); - MessageState state = attachment.getMessageState(); - boolean buffersDisabled = attachment.getCachedTopic().getTopic().isFallbackToRemoteDatacenterEnabled(); + @Override + public void handleRequest(HttpServerExchange exchange) throws Exception { + AttachmentContent attachment = exchange.getAttachment(AttachmentContent.KEY); + MessageState state = attachment.getMessageState(); + boolean buffersDisabled = + attachment.getCachedTopic().getTopic().isFallbackToRemoteDatacenterEnabled(); - state.setTimeoutHasPassed(); - if (state.setReadingTimeout()) { - readingTimeout(exchange, attachment); - } else if (buffersDisabled && state.setTimeoutSendingToKafka()) { - sendingToKafkaTimeout(exchange, attachment); - } else if (state.setDelayedSending()) { - delayedSending(exchange, attachment); - } else { - state.setPrematureTimeout(); - } + state.setTimeoutHasPassed(); + if (state.setReadingTimeout()) { + readingTimeout(exchange, attachment); + } else if (buffersDisabled && state.setTimeoutSendingToKafka()) { + sendingToKafkaTimeout(exchange, attachment); + } else if (state.setDelayedSending()) { + delayedSending(exchange, attachment); + } else { + state.setPrematureTimeout(); } + } - private void delayedSending(HttpServerExchange exchange, AttachmentContent attachment) { - exchange.getConnection().getWorker().execute(() -> { - try { - messageEndProcessor.bufferedButDelayed(exchange, attachment); - } catch (RuntimeException exception) { - messageErrorProcessor.sendAndLog(exchange, "Exception while handling delayed message sending.", exception); - } + private void delayedSending(HttpServerExchange exchange, AttachmentContent attachment) { + exchange + .getConnection() + .getWorker() + .execute( + () -> { + try { + messageEndProcessor.bufferedButDelayed(exchange, attachment); + } catch (RuntimeException exception) { + messageErrorProcessor.sendAndLog( + exchange, "Exception while handling delayed message sending.", exception); + } }); - } + } - private void readingTimeout(HttpServerExchange exchange, AttachmentContent attachment) { - exchange.getConnection().getWorker().execute(() -> { - TimeoutHolder timeoutHolder = attachment.getTimeoutHolder(); + private void readingTimeout(HttpServerExchange exchange, AttachmentContent attachment) { + exchange + .getConnection() + .getWorker() + .execute( + () -> { + TimeoutHolder timeoutHolder = attachment.getTimeoutHolder(); - if (timeoutHolder != null) { + if (timeoutHolder != null) { timeoutHolder.timeout(); messageErrorProcessor.sendAndLog( - exchange, - attachment.getTopic(), - attachment.getMessageId(), - error("Timeout while reading message after " + timeoutHolder.getTimeout() + " milliseconds", TIMEOUT)); - } else { + exchange, + attachment.getTopic(), + attachment.getMessageId(), + error( + "Timeout while reading message after " + + timeoutHolder.getTimeout() + + " milliseconds", + TIMEOUT)); + } else { messageErrorProcessor.sendAndLog( - exchange, - attachment.getTopic(), - attachment.getMessageId(), - error("Probably context switching problem as timeout task was started before it was attached to an exchange", - INTERNAL_ERROR)); - } - }); - } + exchange, + attachment.getTopic(), + attachment.getMessageId(), + error( + "Probably context switching problem as timeout task was started before it was attached to an exchange", + INTERNAL_ERROR)); + } + }); + } - private void sendingToKafkaTimeout(HttpServerExchange exchange, AttachmentContent attachment) { - exchange.getConnection().getWorker().execute(() -> { - TimeoutHolder timeoutHolder = attachment.getTimeoutHolder(); + private void sendingToKafkaTimeout(HttpServerExchange exchange, AttachmentContent attachment) { + exchange + .getConnection() + .getWorker() + .execute( + () -> { + TimeoutHolder timeoutHolder = attachment.getTimeoutHolder(); - if (timeoutHolder != null) { + if (timeoutHolder != null) { timeoutHolder.timeout(); messageErrorProcessor.sendAndLog( - exchange, - attachment.getTopic(), - attachment.getMessageId(), - error("Timeout while sending to kafka message after " + timeoutHolder.getTimeout() + " milliseconds", SENDING_TO_KAFKA_TIMEOUT)); - } else { + exchange, + attachment.getTopic(), + attachment.getMessageId(), + error( + "Timeout while sending to kafka message after " + + timeoutHolder.getTimeout() + + " milliseconds", + SENDING_TO_KAFKA_TIMEOUT)); + } else { messageErrorProcessor.sendAndLog( - exchange, - attachment.getTopic(), - attachment.getMessageId(), - error("Probably context switching problem as timeout task was started before it was attached to an exchange", - INTERNAL_ERROR)); - } - }); - } + exchange, + attachment.getTopic(), + attachment.getMessageId(), + error( + "Probably context switching problem as timeout task was started before it was attached to an exchange", + INTERNAL_ERROR)); + } + }); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/TimeoutHolder.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/TimeoutHolder.java index 97bb617ab2..7161194316 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/TimeoutHolder.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/TimeoutHolder.java @@ -1,34 +1,33 @@ package pl.allegro.tech.hermes.frontend.publishing.handlers; -import org.xnio.XnioExecutor; - import java.util.Optional; import java.util.function.Consumer; +import org.xnio.XnioExecutor; class TimeoutHolder { - private final XnioExecutor.Key timeoutKey; - private final int timeout; - private volatile Optional> timeoutConsumer = Optional.empty(); + private final XnioExecutor.Key timeoutKey; + private final int timeout; + private volatile Optional> timeoutConsumer = Optional.empty(); - TimeoutHolder(int timeout, XnioExecutor.Key timeoutKey) { - this.timeoutKey = timeoutKey; - this.timeout = timeout; - } + TimeoutHolder(int timeout, XnioExecutor.Key timeoutKey) { + this.timeoutKey = timeoutKey; + this.timeout = timeout; + } - public boolean remove() { - return timeoutKey.remove(); - } + public boolean remove() { + return timeoutKey.remove(); + } - public void onTimeout(Consumer timeoutConsumer) { - this.timeoutConsumer = Optional.of(timeoutConsumer); - } + public void onTimeout(Consumer timeoutConsumer) { + this.timeoutConsumer = Optional.of(timeoutConsumer); + } - public void timeout() { - timeoutConsumer.ifPresent(c -> c.accept(null)); - } + public void timeout() { + timeoutConsumer.ifPresent(c -> c.accept(null)); + } - public int getTimeout() { - return timeout; - } + public int getTimeout() { + return timeout; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/TopicHandler.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/TopicHandler.java index 2158892266..baf8faff84 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/TopicHandler.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/TopicHandler.java @@ -1,9 +1,17 @@ package pl.allegro.tech.hermes.frontend.publishing.handlers; +import static io.undertow.util.StatusCodes.INTERNAL_SERVER_ERROR; +import static pl.allegro.tech.hermes.api.ErrorCode.AUTH_ERROR; +import static pl.allegro.tech.hermes.api.ErrorCode.TOPIC_BLACKLISTED; +import static pl.allegro.tech.hermes.api.ErrorCode.TOPIC_NOT_EXISTS; +import static pl.allegro.tech.hermes.api.ErrorDescription.error; + import io.undertow.security.api.SecurityContext; import io.undertow.security.idm.Account; import io.undertow.server.HttpHandler; import io.undertow.server.HttpServerExchange; +import java.util.Optional; +import java.util.function.Consumer; import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.frontend.cache.topic.TopicsCache; import pl.allegro.tech.hermes.frontend.metric.CachedTopic; @@ -12,116 +20,119 @@ import pl.allegro.tech.hermes.frontend.publishing.message.MessageState; import pl.allegro.tech.hermes.frontend.server.auth.Roles; -import java.util.Optional; -import java.util.function.Consumer; - -import static io.undertow.util.StatusCodes.INTERNAL_SERVER_ERROR; -import static pl.allegro.tech.hermes.api.ErrorCode.AUTH_ERROR; -import static pl.allegro.tech.hermes.api.ErrorCode.TOPIC_BLACKLISTED; -import static pl.allegro.tech.hermes.api.ErrorCode.TOPIC_NOT_EXISTS; -import static pl.allegro.tech.hermes.api.ErrorDescription.error; - class TopicHandler implements HttpHandler { - private static final String UNKNOWN_TOPIC_NAME = "unknown"; - - private final HttpHandler next; - private final TopicsCache topicsCache; - private final MessageErrorProcessor messageErrorProcessor; - - TopicHandler(HttpHandler next, TopicsCache topicsCache, MessageErrorProcessor messageErrorProcessor) { - this.next = next; - this.topicsCache = topicsCache; - this.messageErrorProcessor = messageErrorProcessor; + private static final String UNKNOWN_TOPIC_NAME = "unknown"; + + private final HttpHandler next; + private final TopicsCache topicsCache; + private final MessageErrorProcessor messageErrorProcessor; + + TopicHandler( + HttpHandler next, TopicsCache topicsCache, MessageErrorProcessor messageErrorProcessor) { + this.next = next; + this.topicsCache = topicsCache; + this.messageErrorProcessor = messageErrorProcessor; + } + + @Override + public void handleRequest(HttpServerExchange exchange) throws Exception { + if (exchange.isInIoThread()) { + // switch to worker thread + exchange.dispatch(this); + return; } - @Override - public void handleRequest(HttpServerExchange exchange) throws Exception { - if (exchange.isInIoThread()) { - // switch to worker thread - exchange.dispatch(this); - return; - } - - String messageId = MessageIdGenerator.generate(); - - onRequestValid(exchange, messageId, cachedTopic -> { - exchange.addExchangeCompleteListener(new ExchangeMetrics(cachedTopic)); - exchange.putAttachment(AttachmentContent.KEY, new AttachmentContent(cachedTopic, new MessageState(), messageId)); - setDefaultResponseCode(exchange); - try { - next.handleRequest(exchange); - } catch (Exception e) { - messageErrorProcessor.sendAndLog(exchange, cachedTopic.getTopic(), messageId, e); - } + String messageId = MessageIdGenerator.generate(); + + onRequestValid( + exchange, + messageId, + cachedTopic -> { + exchange.addExchangeCompleteListener(new ExchangeMetrics(cachedTopic)); + exchange.putAttachment( + AttachmentContent.KEY, + new AttachmentContent(cachedTopic, new MessageState(), messageId)); + setDefaultResponseCode(exchange); + try { + next.handleRequest(exchange); + } catch (Exception e) { + messageErrorProcessor.sendAndLog(exchange, cachedTopic.getTopic(), messageId, e); + } }); - } - - private void onRequestValid(HttpServerExchange exchange, String messageId, Consumer consumer) { - String topicName = exchange.getQueryParameters().get("qualifiedTopicName").getFirst(); - Optional maybeTopic = topicsCache.getTopic(topicName); - - if (!maybeTopic.isPresent()) { - unknownTopic(exchange, topicName, messageId); - return; - } - - CachedTopic cachedTopic = maybeTopic.get(); - if (cachedTopic.isBlacklisted()) { - blacklistedTopic(exchange, topicName, messageId); - return; - } - - Topic topic = cachedTopic.getTopic(); - if (topic.isAuthEnabled() && !hasPermission(exchange, topic)) { - requestForbidden(exchange, messageId, topicName); - return; - } + } - consumer.accept(cachedTopic); - } - - private boolean hasPermission(HttpServerExchange exchange, Topic topic) { - Optional account = extractAccount(exchange); - return account.map(value -> hasPermission(topic, value)).orElseGet(topic::isUnauthenticatedAccessEnabled); - } - - private boolean hasPermission(Topic topic, Account publisher) { - return publisher.getRoles().contains(Roles.PUBLISHER) && topic.hasPermission(publisher.getPrincipal().getName()); - } - - private Optional extractAccount(HttpServerExchange exchange) { - SecurityContext securityCtx = exchange.getSecurityContext(); - return Optional.ofNullable(securityCtx != null ? securityCtx.getAuthenticatedAccount() : null); - } + private void onRequestValid( + HttpServerExchange exchange, String messageId, Consumer consumer) { + String topicName = exchange.getQueryParameters().get("qualifiedTopicName").getFirst(); + Optional maybeTopic = topicsCache.getTopic(topicName); - private void unknownTopic(HttpServerExchange exchange, String qualifiedTopicName, String messageId) { - messageErrorProcessor.sendQuietly( - exchange, - error("Topic not found: " + qualifiedTopicName, TOPIC_NOT_EXISTS), - messageId, - UNKNOWN_TOPIC_NAME); + if (!maybeTopic.isPresent()) { + unknownTopic(exchange, topicName, messageId); + return; } - private void requestForbidden(HttpServerExchange exchange, String messageId, String qualifiedTopicName) { - messageErrorProcessor.sendQuietly( - exchange, - error("Permission denied.", AUTH_ERROR), - messageId, - qualifiedTopicName); + CachedTopic cachedTopic = maybeTopic.get(); + if (cachedTopic.isBlacklisted()) { + blacklistedTopic(exchange, topicName, messageId); + return; } - private void blacklistedTopic(HttpServerExchange exchange, String qualifiedTopicName, String messageId) { - messageErrorProcessor.sendQuietly(exchange, - error("Topic blacklisted: " + qualifiedTopicName, TOPIC_BLACKLISTED), - messageId, - qualifiedTopicName); + Topic topic = cachedTopic.getTopic(); + if (topic.isAuthEnabled() && !hasPermission(exchange, topic)) { + requestForbidden(exchange, messageId, topicName); + return; } - // Default Undertow's response code (200) was changed in order to avoid situations in which something wrong happens and Hermes-Frontend - // does not publish message but return code 200 - // Since the default code is 500, clients have information that they should retry publishing - private void setDefaultResponseCode(HttpServerExchange exchange) { - exchange.setStatusCode(INTERNAL_SERVER_ERROR); - } + consumer.accept(cachedTopic); + } + + private boolean hasPermission(HttpServerExchange exchange, Topic topic) { + Optional account = extractAccount(exchange); + return account + .map(value -> hasPermission(topic, value)) + .orElseGet(topic::isUnauthenticatedAccessEnabled); + } + + private boolean hasPermission(Topic topic, Account publisher) { + return publisher.getRoles().contains(Roles.PUBLISHER) + && topic.hasPermission(publisher.getPrincipal().getName()); + } + + private Optional extractAccount(HttpServerExchange exchange) { + SecurityContext securityCtx = exchange.getSecurityContext(); + return Optional.ofNullable(securityCtx != null ? securityCtx.getAuthenticatedAccount() : null); + } + + private void unknownTopic( + HttpServerExchange exchange, String qualifiedTopicName, String messageId) { + messageErrorProcessor.sendQuietly( + exchange, + error("Topic not found: " + qualifiedTopicName, TOPIC_NOT_EXISTS), + messageId, + UNKNOWN_TOPIC_NAME); + } + + private void requestForbidden( + HttpServerExchange exchange, String messageId, String qualifiedTopicName) { + messageErrorProcessor.sendQuietly( + exchange, error("Permission denied.", AUTH_ERROR), messageId, qualifiedTopicName); + } + + private void blacklistedTopic( + HttpServerExchange exchange, String qualifiedTopicName, String messageId) { + messageErrorProcessor.sendQuietly( + exchange, + error("Topic blacklisted: " + qualifiedTopicName, TOPIC_BLACKLISTED), + messageId, + qualifiedTopicName); + } + + // Default Undertow's response code (200) was changed in order to avoid situations in which + // something wrong happens and Hermes-Frontend + // does not publish message but return code 200 + // Since the default code is 500, clients have information that they should retry publishing + private void setDefaultResponseCode(HttpServerExchange exchange) { + exchange.setStatusCode(INTERNAL_SERVER_ERROR); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/end/DefaultTrackingHeaderExtractor.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/end/DefaultTrackingHeaderExtractor.java index 61ed5a6f7b..ab0a679e92 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/end/DefaultTrackingHeaderExtractor.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/end/DefaultTrackingHeaderExtractor.java @@ -1,15 +1,13 @@ package pl.allegro.tech.hermes.frontend.publishing.handlers.end; import io.undertow.util.HeaderMap; - import java.util.Collections; import java.util.Map; public class DefaultTrackingHeaderExtractor implements TrackingHeadersExtractor { - @Override - public Map extractHeadersToLog(HeaderMap headers) { - return Collections.emptyMap(); - } - + @Override + public Map extractHeadersToLog(HeaderMap headers) { + return Collections.emptyMap(); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/end/MessageEndProcessor.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/end/MessageEndProcessor.java index dee57afd57..e33028aba2 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/end/MessageEndProcessor.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/end/MessageEndProcessor.java @@ -1,5 +1,8 @@ package pl.allegro.tech.hermes.frontend.publishing.handlers.end; +import static pl.allegro.tech.hermes.common.http.MessageMetadataHeaders.MESSAGE_ID; +import static pl.allegro.tech.hermes.frontend.publishing.handlers.end.RemoteHostReader.readHostAndPort; + import io.undertow.server.HttpServerExchange; import io.undertow.util.HttpString; import io.undertow.util.StatusCodes; @@ -12,83 +15,98 @@ import pl.allegro.tech.hermes.frontend.publishing.message.Message; import pl.allegro.tech.hermes.tracker.frontend.Trackers; -import static pl.allegro.tech.hermes.common.http.MessageMetadataHeaders.MESSAGE_ID; -import static pl.allegro.tech.hermes.frontend.publishing.handlers.end.RemoteHostReader.readHostAndPort; - public class MessageEndProcessor { - private static final Logger logger = LoggerFactory.getLogger(MessageEndProcessor.class); - private static final HttpString messageIdHeader = new HttpString(MESSAGE_ID.getName()); + private static final Logger logger = LoggerFactory.getLogger(MessageEndProcessor.class); + private static final HttpString messageIdHeader = new HttpString(MESSAGE_ID.getName()); - private final Trackers trackers; - private final BrokerListeners brokerListeners; - private final TrackingHeadersExtractor trackingHeadersExtractor; + private final Trackers trackers; + private final BrokerListeners brokerListeners; + private final TrackingHeadersExtractor trackingHeadersExtractor; - public MessageEndProcessor(Trackers trackers, BrokerListeners brokerListeners, TrackingHeadersExtractor trackingHeadersExtractor) { - this.trackers = trackers; - this.brokerListeners = brokerListeners; - this.trackingHeadersExtractor = trackingHeadersExtractor; - } + public MessageEndProcessor( + Trackers trackers, + BrokerListeners brokerListeners, + TrackingHeadersExtractor trackingHeadersExtractor) { + this.trackers = trackers; + this.brokerListeners = brokerListeners; + this.trackingHeadersExtractor = trackingHeadersExtractor; + } - public void eachSent(HttpServerExchange exchange, AttachmentContent attachment, String datacenter) { - trackers.get(attachment.getTopic()).logPublished(attachment.getMessageId(), - attachment.getTopic().getName(), readHostAndPort(exchange), datacenter, - trackingHeadersExtractor.extractHeadersToLog(exchange.getRequestHeaders())); - } + public void eachSent( + HttpServerExchange exchange, AttachmentContent attachment, String datacenter) { + trackers + .get(attachment.getTopic()) + .logPublished( + attachment.getMessageId(), + attachment.getTopic().getName(), + readHostAndPort(exchange), + datacenter, + trackingHeadersExtractor.extractHeadersToLog(exchange.getRequestHeaders())); + } - public void sent(HttpServerExchange exchange, AttachmentContent attachment) { - sendResponse(exchange, attachment, StatusCodes.CREATED); - } + public void sent(HttpServerExchange exchange, AttachmentContent attachment) { + sendResponse(exchange, attachment, StatusCodes.CREATED); + } - public void delayedSent(CachedTopic cachedTopic, Message message) { - brokerListeners.onAcknowledge(message, cachedTopic.getTopic()); - } + public void delayedSent(CachedTopic cachedTopic, Message message) { + brokerListeners.onAcknowledge(message, cachedTopic.getTopic()); + } - public void bufferedButDelayedProcessing(HttpServerExchange exchange, AttachmentContent attachment) { - bufferedButDelayed(exchange, attachment); - attachment.getCachedTopic().markDelayedProcessing(); - } + public void bufferedButDelayedProcessing( + HttpServerExchange exchange, AttachmentContent attachment) { + bufferedButDelayed(exchange, attachment); + attachment.getCachedTopic().markDelayedProcessing(); + } - public void bufferedButDelayed(HttpServerExchange exchange, AttachmentContent attachment) { - Topic topic = attachment.getTopic(); - brokerListeners.onTimeout(attachment.getMessage(), topic); - trackers.get(topic).logInflight(attachment.getMessageId(), topic.getName(), - readHostAndPort(exchange), trackingHeadersExtractor.extractHeadersToLog(exchange.getRequestHeaders())); - handleRaceConditionBetweenAckAndTimeout(attachment, topic); - sendResponse(exchange, attachment, StatusCodes.ACCEPTED); - } + public void bufferedButDelayed(HttpServerExchange exchange, AttachmentContent attachment) { + Topic topic = attachment.getTopic(); + brokerListeners.onTimeout(attachment.getMessage(), topic); + trackers + .get(topic) + .logInflight( + attachment.getMessageId(), + topic.getName(), + readHostAndPort(exchange), + trackingHeadersExtractor.extractHeadersToLog(exchange.getRequestHeaders())); + handleRaceConditionBetweenAckAndTimeout(attachment, topic); + sendResponse(exchange, attachment, StatusCodes.ACCEPTED); + } - private void handleRaceConditionBetweenAckAndTimeout(AttachmentContent attachment, Topic topic) { - if (attachment.getMessageState().isDelayedSentToKafka()) { - brokerListeners.onAcknowledge(attachment.getMessage(), topic); - } + private void handleRaceConditionBetweenAckAndTimeout(AttachmentContent attachment, Topic topic) { + if (attachment.getMessageState().isDelayedSentToKafka()) { + brokerListeners.onAcknowledge(attachment.getMessage(), topic); } + } - private void sendResponse(HttpServerExchange exchange, AttachmentContent attachment, int statusCode) { - if (!exchange.isResponseStarted()) { - exchange.setStatusCode(statusCode); - exchange.getResponseHeaders().add(messageIdHeader, attachment.getMessageId()); - } else { - logger.warn("The response has already been started. Status code set on exchange: {}; Expected status code: {};" - + "Topic: {}; Message id: {}; Remote host {}", - exchange.getStatusCode(), - statusCode, - attachment.getCachedTopic().getQualifiedName(), - attachment.getMessageId(), - readHostAndPort(exchange)); - } - attachment.markResponseAsReady(); - try { - exchange.endExchange(); - } catch (RuntimeException exception) { - logger.error("Exception while ending exchange. Status code set on exchange: {}; Expected status code: {};" - + "Topic: {}; Message id: {}; Remote host {}", - exchange.getStatusCode(), - statusCode, - attachment.getCachedTopic().getQualifiedName(), - attachment.getMessageId(), - readHostAndPort(exchange), - exception); - } + private void sendResponse( + HttpServerExchange exchange, AttachmentContent attachment, int statusCode) { + if (!exchange.isResponseStarted()) { + exchange.setStatusCode(statusCode); + exchange.getResponseHeaders().add(messageIdHeader, attachment.getMessageId()); + } else { + logger.warn( + "The response has already been started. Status code set on exchange: {}; Expected status code: {};" + + "Topic: {}; Message id: {}; Remote host {}", + exchange.getStatusCode(), + statusCode, + attachment.getCachedTopic().getQualifiedName(), + attachment.getMessageId(), + readHostAndPort(exchange)); + } + attachment.markResponseAsReady(); + try { + exchange.endExchange(); + } catch (RuntimeException exception) { + logger.error( + "Exception while ending exchange. Status code set on exchange: {}; Expected status code: {};" + + "Topic: {}; Message id: {}; Remote host {}", + exchange.getStatusCode(), + statusCode, + attachment.getCachedTopic().getQualifiedName(), + attachment.getMessageId(), + readHostAndPort(exchange), + exception); } + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/end/MessageErrorProcessor.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/end/MessageErrorProcessor.java index 5858aefea9..bfaf5748bb 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/end/MessageErrorProcessor.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/end/MessageErrorProcessor.java @@ -1,10 +1,18 @@ package pl.allegro.tech.hermes.frontend.publishing.handlers.end; +import static pl.allegro.tech.hermes.api.ErrorCode.INTERNAL_ERROR; +import static pl.allegro.tech.hermes.api.ErrorDescription.error; +import static pl.allegro.tech.hermes.common.http.MessageMetadataHeaders.MESSAGE_ID; +import static pl.allegro.tech.hermes.frontend.publishing.handlers.end.RemoteHostReader.readHostAndPort; + import com.fasterxml.jackson.databind.ObjectMapper; import io.undertow.server.HttpServerExchange; import io.undertow.util.Headers; import io.undertow.util.HttpString; import jakarta.ws.rs.core.MediaType; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.ErrorDescription; @@ -12,118 +20,168 @@ import pl.allegro.tech.hermes.frontend.publishing.handlers.AttachmentContent; import pl.allegro.tech.hermes.tracker.frontend.Trackers; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.Map; - -import static pl.allegro.tech.hermes.api.ErrorCode.INTERNAL_ERROR; -import static pl.allegro.tech.hermes.api.ErrorDescription.error; -import static pl.allegro.tech.hermes.common.http.MessageMetadataHeaders.MESSAGE_ID; -import static pl.allegro.tech.hermes.frontend.publishing.handlers.end.RemoteHostReader.readHostAndPort; - public class MessageErrorProcessor { - private static final Logger logger = LoggerFactory.getLogger(MessageErrorProcessor.class); - private final ObjectMapper objectMapper; - private final Trackers trackers; - private final HttpString messageIdHeader = new HttpString(MESSAGE_ID.getName()); - private final TrackingHeadersExtractor trackingHeadersExtractor; + private static final Logger logger = LoggerFactory.getLogger(MessageErrorProcessor.class); + private final ObjectMapper objectMapper; + private final Trackers trackers; + private final HttpString messageIdHeader = new HttpString(MESSAGE_ID.getName()); + private final TrackingHeadersExtractor trackingHeadersExtractor; - public MessageErrorProcessor(ObjectMapper objectMapper, Trackers trackers, - TrackingHeadersExtractor trackingHeadersExtractor) { - this.objectMapper = objectMapper; - this.trackers = trackers; - this.trackingHeadersExtractor = trackingHeadersExtractor; - } + public MessageErrorProcessor( + ObjectMapper objectMapper, + Trackers trackers, + TrackingHeadersExtractor trackingHeadersExtractor) { + this.objectMapper = objectMapper; + this.trackers = trackers; + this.trackingHeadersExtractor = trackingHeadersExtractor; + } - public void sendAndLog(HttpServerExchange exchange, Topic topic, String messageId, ErrorDescription error) { - sendQuietly(exchange, error, messageId, topic.getQualifiedName()); - log(error.getMessage(), topic, messageId, readHostAndPort(exchange), - trackingHeadersExtractor.extractHeadersToLog(exchange.getRequestHeaders())); - } + public void sendAndLog( + HttpServerExchange exchange, Topic topic, String messageId, ErrorDescription error) { + sendQuietly(exchange, error, messageId, topic.getQualifiedName()); + log( + error.getMessage(), + topic, + messageId, + readHostAndPort(exchange), + trackingHeadersExtractor.extractHeadersToLog(exchange.getRequestHeaders())); + } - public void sendAndLog(HttpServerExchange exchange, Topic topic, String messageId, ErrorDescription error, Exception exception) { - sendQuietly(exchange, error, messageId, topic.getQualifiedName()); - log(error.getMessage(), topic, messageId, readHostAndPort(exchange), exception, - trackingHeadersExtractor.extractHeadersToLog(exchange.getRequestHeaders())); - } + public void sendAndLog( + HttpServerExchange exchange, + Topic topic, + String messageId, + ErrorDescription error, + Exception exception) { + sendQuietly(exchange, error, messageId, topic.getQualifiedName()); + log( + error.getMessage(), + topic, + messageId, + readHostAndPort(exchange), + exception, + trackingHeadersExtractor.extractHeadersToLog(exchange.getRequestHeaders())); + } - public void sendAndLog(HttpServerExchange exchange, Topic topic, String messageId, Exception e) { - ErrorDescription error = error("Error while handling request.", INTERNAL_ERROR); - sendQuietly(exchange, error, messageId, topic.getQualifiedName()); - log(error.getMessage(), topic, messageId, readHostAndPort(exchange), e, - trackingHeadersExtractor.extractHeadersToLog(exchange.getRequestHeaders())); - } + public void sendAndLog(HttpServerExchange exchange, Topic topic, String messageId, Exception e) { + ErrorDescription error = error("Error while handling request.", INTERNAL_ERROR); + sendQuietly(exchange, error, messageId, topic.getQualifiedName()); + log( + error.getMessage(), + topic, + messageId, + readHostAndPort(exchange), + e, + trackingHeadersExtractor.extractHeadersToLog(exchange.getRequestHeaders())); + } - public void sendAndLog(HttpServerExchange exchange, String errorMessage, Exception e) { - AttachmentContent attachment = exchange.getAttachment(AttachmentContent.KEY); - sendAndLog(exchange, attachment.getTopic(), attachment.getMessageId(), error(errorMessage, INTERNAL_ERROR), e); - } + public void sendAndLog(HttpServerExchange exchange, String errorMessage, Exception e) { + AttachmentContent attachment = exchange.getAttachment(AttachmentContent.KEY); + sendAndLog( + exchange, + attachment.getTopic(), + attachment.getMessageId(), + error(errorMessage, INTERNAL_ERROR), + e); + } - public void sendQuietly(HttpServerExchange exchange, ErrorDescription error, String messageId, String topicName) { - try { - if (exchange.getConnection().isOpen()) { - if (!exchange.isResponseStarted()) { - send(exchange, error, messageId); - } else { - logger.warn("Not sending error message to a client as response has already been started. " - + "Error message: {} Topic: {} MessageId: {} Host: {}", - error.getMessage(), topicName, messageId, readHostAndPort(exchange)); - } - } else { - logger.warn("Connection to a client closed. Can't send error response. " - + "Error message: {} Topic: {} MessageId: {} Host: {}", - error.getMessage(), topicName, messageId, readHostAndPort(exchange)); - exchange.endExchange(); - } - } catch (Exception e) { - logger.warn("Exception in sending error response to a client. {} Topic: {} MessageId: {} Host: {}", - error.getMessage(), topicName, messageId, readHostAndPort(exchange), e); + public void sendQuietly( + HttpServerExchange exchange, ErrorDescription error, String messageId, String topicName) { + try { + if (exchange.getConnection().isOpen()) { + if (!exchange.isResponseStarted()) { + send(exchange, error, messageId); + } else { + logger.warn( + "Not sending error message to a client as response has already been started. " + + "Error message: {} Topic: {} MessageId: {} Host: {}", + error.getMessage(), + topicName, + messageId, + readHostAndPort(exchange)); } + } else { + logger.warn( + "Connection to a client closed. Can't send error response. " + + "Error message: {} Topic: {} MessageId: {} Host: {}", + error.getMessage(), + topicName, + messageId, + readHostAndPort(exchange)); + exchange.endExchange(); + } + } catch (Exception e) { + logger.warn( + "Exception in sending error response to a client. {} Topic: {} MessageId: {} Host: {}", + error.getMessage(), + topicName, + messageId, + readHostAndPort(exchange), + e); } + } - public void log(HttpServerExchange exchange, String errorMessage, Exception exception) { - AttachmentContent attachment = exchange.getAttachment(AttachmentContent.KEY); - log(errorMessage, attachment.getTopic(), attachment.getMessageId(), readHostAndPort(exchange), exception, - trackingHeadersExtractor.extractHeadersToLog(exchange.getRequestHeaders())); - } + public void log(HttpServerExchange exchange, String errorMessage, Exception exception) { + AttachmentContent attachment = exchange.getAttachment(AttachmentContent.KEY); + log( + errorMessage, + attachment.getTopic(), + attachment.getMessageId(), + readHostAndPort(exchange), + exception, + trackingHeadersExtractor.extractHeadersToLog(exchange.getRequestHeaders())); + } - private void log(String errorMessage, - Topic topic, - String messageId, - String hostAndPort, - Map extraRequestHeaders) { - logger.error(errorMessage - + "; publishing on topic: " - + topic.getQualifiedName() - + "; message id: " - + messageId - + "; remote host: " - + hostAndPort); - trackers.get(topic).logError(messageId, topic.getName(), errorMessage, hostAndPort, extraRequestHeaders); - } + private void log( + String errorMessage, + Topic topic, + String messageId, + String hostAndPort, + Map extraRequestHeaders) { + logger.error( + errorMessage + + "; publishing on topic: " + + topic.getQualifiedName() + + "; message id: " + + messageId + + "; remote host: " + + hostAndPort); + trackers + .get(topic) + .logError(messageId, topic.getName(), errorMessage, hostAndPort, extraRequestHeaders); + } - private void log(String errorMessage, - Topic topic, - String messageId, - String hostAndPort, - Exception exception, - Map extraRequestHeaders) { - logger.error(errorMessage - + "; publishing on topic: " - + topic.getQualifiedName() - + "; message id: " - + messageId - + "; remote host: " - + hostAndPort, - exception); - trackers.get(topic).logError(messageId, topic.getName(), errorMessage, hostAndPort, extraRequestHeaders); - } + private void log( + String errorMessage, + Topic topic, + String messageId, + String hostAndPort, + Exception exception, + Map extraRequestHeaders) { + logger.error( + errorMessage + + "; publishing on topic: " + + topic.getQualifiedName() + + "; message id: " + + messageId + + "; remote host: " + + hostAndPort, + exception); + trackers + .get(topic) + .logError(messageId, topic.getName(), errorMessage, hostAndPort, extraRequestHeaders); + } - private void send(HttpServerExchange exchange, ErrorDescription error, String messageId) throws IOException { - exchange.setStatusCode(error.getCode().getHttpCode()); - exchange.getResponseHeaders().add(Headers.CONTENT_TYPE, MediaType.APPLICATION_JSON); - exchange.getResponseHeaders().add(messageIdHeader, messageId); - exchange.getResponseSender().send(objectMapper.writeValueAsString(error), StandardCharsets.UTF_8, - ResponseReadyIoCallback.INSTANCE); - } + private void send(HttpServerExchange exchange, ErrorDescription error, String messageId) + throws IOException { + exchange.setStatusCode(error.getCode().getHttpCode()); + exchange.getResponseHeaders().add(Headers.CONTENT_TYPE, MediaType.APPLICATION_JSON); + exchange.getResponseHeaders().add(messageIdHeader, messageId); + exchange + .getResponseSender() + .send( + objectMapper.writeValueAsString(error), + StandardCharsets.UTF_8, + ResponseReadyIoCallback.INSTANCE); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/end/RemoteHostReader.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/end/RemoteHostReader.java index d43acf7fe9..ccff52c0e2 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/end/RemoteHostReader.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/end/RemoteHostReader.java @@ -1,17 +1,16 @@ package pl.allegro.tech.hermes.frontend.publishing.handlers.end; import io.undertow.server.HttpServerExchange; - import java.net.InetSocketAddress; class RemoteHostReader { - static String readHostAndPort(HttpServerExchange exchange) { - InetSocketAddress sourceAddress = exchange.getSourceAddress(); - if (sourceAddress == null) { - return ""; - } - - return sourceAddress.getHostString() + ":" + sourceAddress.getPort(); + static String readHostAndPort(HttpServerExchange exchange) { + InetSocketAddress sourceAddress = exchange.getSourceAddress(); + if (sourceAddress == null) { + return ""; } + + return sourceAddress.getHostString() + ":" + sourceAddress.getPort(); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/end/ResponseReadyIoCallback.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/end/ResponseReadyIoCallback.java index 52a692b9f9..b35ddadbb0 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/end/ResponseReadyIoCallback.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/end/ResponseReadyIoCallback.java @@ -4,51 +4,53 @@ import io.undertow.io.IoCallback; import io.undertow.io.Sender; import io.undertow.server.HttpServerExchange; +import java.io.IOException; import org.xnio.IoUtils; import pl.allegro.tech.hermes.frontend.publishing.handlers.AttachmentContent; -import java.io.IOException; - /* - Marks the response as ready and ends the exchange - */ + Marks the response as ready and ends the exchange +*/ class ResponseReadyIoCallback implements IoCallback { - static final IoCallback INSTANCE = new ResponseReadyIoCallback(); + static final IoCallback INSTANCE = new ResponseReadyIoCallback(); - private static final IoCallback CALLBACK = new IoCallback() { + private static final IoCallback CALLBACK = + new IoCallback() { @Override public void onComplete(final HttpServerExchange exchange, final Sender sender) { - exchange.endExchange(); + exchange.endExchange(); } @Override - public void onException(final HttpServerExchange exchange, final Sender sender, final IOException exception) { - UndertowLogger.REQUEST_IO_LOGGER.ioException(exception); - exchange.endExchange(); - } - }; - - @Override - public void onComplete(final HttpServerExchange exchange, final Sender sender) { - markResponseAsReady(exchange); - sender.close(CALLBACK); - } - - @Override - public void onException(final HttpServerExchange exchange, final Sender sender, final IOException exception) { - try { - markResponseAsReady(exchange); - exchange.endExchange(); - } finally { - IoUtils.safeClose(exchange.getConnection()); + public void onException( + final HttpServerExchange exchange, final Sender sender, final IOException exception) { + UndertowLogger.REQUEST_IO_LOGGER.ioException(exception); + exchange.endExchange(); } + }; + + @Override + public void onComplete(final HttpServerExchange exchange, final Sender sender) { + markResponseAsReady(exchange); + sender.close(CALLBACK); + } + + @Override + public void onException( + final HttpServerExchange exchange, final Sender sender, final IOException exception) { + try { + markResponseAsReady(exchange); + exchange.endExchange(); + } finally { + IoUtils.safeClose(exchange.getConnection()); } + } - private void markResponseAsReady(HttpServerExchange exchange) { - AttachmentContent attachment = exchange.getAttachment(AttachmentContent.KEY); - if (attachment != null) { - attachment.markResponseAsReady(); - } + private void markResponseAsReady(HttpServerExchange exchange) { + AttachmentContent attachment = exchange.getAttachment(AttachmentContent.KEY); + if (attachment != null) { + attachment.markResponseAsReady(); } + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/end/TrackingHeadersExtractor.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/end/TrackingHeadersExtractor.java index e39c65d4c8..0d81431738 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/end/TrackingHeadersExtractor.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/handlers/end/TrackingHeadersExtractor.java @@ -1,9 +1,8 @@ package pl.allegro.tech.hermes.frontend.publishing.handlers.end; import io.undertow.util.HeaderMap; - import java.util.Map; public interface TrackingHeadersExtractor { - Map extractHeadersToLog(HeaderMap headers); + Map extractHeadersToLog(HeaderMap headers); } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/AvroEncodedJsonAvroConverter.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/AvroEncodedJsonAvroConverter.java index a2a041feca..76f4158317 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/AvroEncodedJsonAvroConverter.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/AvroEncodedJsonAvroConverter.java @@ -1,5 +1,9 @@ package pl.allegro.tech.hermes.frontend.publishing.message; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; import org.apache.avro.AvroRuntimeException; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; @@ -11,36 +15,31 @@ import org.apache.avro.io.EncoderFactory; import tech.allegro.schema.json2avro.converter.AvroConversionException; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; - class AvroEncodedJsonAvroConverter { - byte[] convertToAvro(byte[] bytes, Schema schema) { - try { - return convertToAvro(readJson(bytes, schema), schema); - } catch (IOException e) { - throw new AvroConversionException("Failed to convert to AVRO.", e); - } catch (AvroRuntimeException e) { - throw new AvroConversionException( - String.format("Failed to convert to AVRO: %s.", e.getMessage()), e); - } + byte[] convertToAvro(byte[] bytes, Schema schema) { + try { + return convertToAvro(readJson(bytes, schema), schema); + } catch (IOException e) { + throw new AvroConversionException("Failed to convert to AVRO.", e); + } catch (AvroRuntimeException e) { + throw new AvroConversionException( + String.format("Failed to convert to AVRO: %s.", e.getMessage()), e); } + } - private byte[] convertToAvro(GenericData.Record jsonData, Schema schema) throws IOException { - GenericDatumWriter writer = new GenericDatumWriter<>(schema); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - Encoder encoder = EncoderFactory.get().binaryEncoder(outputStream, null); - writer.write(jsonData, encoder); - encoder.flush(); - return outputStream.toByteArray(); - } + private byte[] convertToAvro(GenericData.Record jsonData, Schema schema) throws IOException { + GenericDatumWriter writer = new GenericDatumWriter<>(schema); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + Encoder encoder = EncoderFactory.get().binaryEncoder(outputStream, null); + writer.write(jsonData, encoder); + encoder.flush(); + return outputStream.toByteArray(); + } - private GenericData.Record readJson(byte[] bytes, Schema schema) throws IOException { - InputStream input = new ByteArrayInputStream(bytes); - Decoder decoder = DecoderFactory.get().jsonDecoder(schema, input); - return new GenericDatumReader(schema).read(null, decoder); - } + private GenericData.Record readJson(byte[] bytes, Schema schema) throws IOException { + InputStream input = new ByteArrayInputStream(bytes); + Decoder decoder = DecoderFactory.get().jsonDecoder(schema, input); + return new GenericDatumReader(schema).read(null, decoder); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/AvroEnforcer.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/AvroEnforcer.java index e22dfd1f40..200ebb646d 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/AvroEnforcer.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/AvroEnforcer.java @@ -5,5 +5,5 @@ public interface AvroEnforcer { - byte[] enforceAvro(String payloadContentType, byte[] data, Schema schema, Topic topic); + byte[] enforceAvro(String payloadContentType, byte[] data, Schema schema, Topic topic); } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/JsonMessage.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/JsonMessage.java index 414df3f62f..1f44f67e68 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/JsonMessage.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/JsonMessage.java @@ -1,57 +1,60 @@ package pl.allegro.tech.hermes.frontend.publishing.message; -import pl.allegro.tech.hermes.api.ContentType; - import java.util.Map; +import pl.allegro.tech.hermes.api.ContentType; public class JsonMessage implements Message { - private final String id; - private final byte[] data; - private final long timestamp; - private final String partitionKey; - private final Map propagatedHTTPHeaders; - - public JsonMessage(String id, byte[] data, long timestamp, String partitionKey, Map propagatedHTTPHeaders) { - this.id = id; - this.data = data; - this.timestamp = timestamp; - this.partitionKey = partitionKey; - this.propagatedHTTPHeaders = propagatedHTTPHeaders; - } - - @Override - public String getId() { - return id; - } - - @Override - public byte[] getData() { - return data; - } - - @Override - public long getTimestamp() { - return timestamp; - } - - @Override - public ContentType getContentType() { - return ContentType.JSON; - } - - @Override - public String getPartitionKey() { - return partitionKey; - } - - @Override - public Map getHTTPHeaders() { - return propagatedHTTPHeaders; - } - - public JsonMessage withDataReplaced(byte[] newData) { - return new JsonMessage(id, newData, timestamp, partitionKey, propagatedHTTPHeaders); - } - + private final String id; + private final byte[] data; + private final long timestamp; + private final String partitionKey; + private final Map propagatedHTTPHeaders; + + public JsonMessage( + String id, + byte[] data, + long timestamp, + String partitionKey, + Map propagatedHTTPHeaders) { + this.id = id; + this.data = data; + this.timestamp = timestamp; + this.partitionKey = partitionKey; + this.propagatedHTTPHeaders = propagatedHTTPHeaders; + } + + @Override + public String getId() { + return id; + } + + @Override + public byte[] getData() { + return data; + } + + @Override + public long getTimestamp() { + return timestamp; + } + + @Override + public ContentType getContentType() { + return ContentType.JSON; + } + + @Override + public String getPartitionKey() { + return partitionKey; + } + + @Override + public Map getHTTPHeaders() { + return propagatedHTTPHeaders; + } + + public JsonMessage withDataReplaced(byte[] newData) { + return new JsonMessage(id, newData, timestamp, partitionKey, propagatedHTTPHeaders); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/Message.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/Message.java index 24c80a1497..0ed5589e99 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/Message.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/Message.java @@ -1,29 +1,28 @@ package pl.allegro.tech.hermes.frontend.publishing.message; -import pl.allegro.tech.hermes.api.ContentType; -import pl.allegro.tech.hermes.schema.CompiledSchema; - import java.util.Map; import java.util.Optional; +import pl.allegro.tech.hermes.api.ContentType; +import pl.allegro.tech.hermes.schema.CompiledSchema; public interface Message { - String getId(); + String getId(); - byte[] getData(); + byte[] getData(); - long getTimestamp(); + long getTimestamp(); - ContentType getContentType(); + ContentType getContentType(); - String getPartitionKey(); + String getPartitionKey(); - default Optional> getCompiledSchema() { - return Optional.empty(); - } + default Optional> getCompiledSchema() { + return Optional.empty(); + } - default T getSchema() { - return this.getCompiledSchema().get().getSchema(); - } + default T getSchema() { + return this.getCompiledSchema().get().getSchema(); + } - Map getHTTPHeaders(); + Map getHTTPHeaders(); } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageContentTypeEnforcer.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageContentTypeEnforcer.java index 72dd78f57d..29382c3192 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageContentTypeEnforcer.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageContentTypeEnforcer.java @@ -1,50 +1,53 @@ package pl.allegro.tech.hermes.frontend.publishing.message; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; +import static pl.allegro.tech.hermes.api.AvroMediaType.AVRO_BINARY; +import static pl.allegro.tech.hermes.api.AvroMediaType.AVRO_JSON; + import org.apache.avro.Schema; import org.apache.commons.lang3.StringUtils; import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.common.message.wrapper.UnsupportedContentTypeException; import tech.allegro.schema.json2avro.converter.JsonAvroConverter; -import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; -import static pl.allegro.tech.hermes.api.AvroMediaType.AVRO_BINARY; -import static pl.allegro.tech.hermes.api.AvroMediaType.AVRO_JSON; - public class MessageContentTypeEnforcer implements AvroEnforcer { - private static final String APPLICATION_JSON_WITH_DELIM = APPLICATION_JSON + ";"; - private static final String AVRO_JSON_WITH_DELIM = AVRO_JSON + ";"; - private static final String AVRO_BINARY_WITH_DELIM = AVRO_BINARY + ";"; - private final JsonAvroConverter defaultJsonAvroConverter = new JsonAvroConverter(); - private final AvroEncodedJsonAvroConverter avroEncodedJsonAvroConverter = new AvroEncodedJsonAvroConverter(); - - @Override - public byte[] enforceAvro(String payloadContentType, byte[] data, Schema schema, Topic topic) { - String contentTypeLowerCase = StringUtils.lowerCase(payloadContentType); - if (isJson(contentTypeLowerCase)) { - return defaultJsonAvroConverter.convertToAvro(data, schema); - } else if (isAvroJson(contentTypeLowerCase)) { - return avroEncodedJsonAvroConverter.convertToAvro(data, schema); - } else if (isAvroBinary(contentTypeLowerCase)) { - return data; - } else { - throw new UnsupportedContentTypeException(payloadContentType, topic); - } + private static final String APPLICATION_JSON_WITH_DELIM = APPLICATION_JSON + ";"; + private static final String AVRO_JSON_WITH_DELIM = AVRO_JSON + ";"; + private static final String AVRO_BINARY_WITH_DELIM = AVRO_BINARY + ";"; + private final JsonAvroConverter defaultJsonAvroConverter = new JsonAvroConverter(); + private final AvroEncodedJsonAvroConverter avroEncodedJsonAvroConverter = + new AvroEncodedJsonAvroConverter(); + + @Override + public byte[] enforceAvro(String payloadContentType, byte[] data, Schema schema, Topic topic) { + String contentTypeLowerCase = StringUtils.lowerCase(payloadContentType); + if (isJson(contentTypeLowerCase)) { + return defaultJsonAvroConverter.convertToAvro(data, schema); + } else if (isAvroJson(contentTypeLowerCase)) { + return avroEncodedJsonAvroConverter.convertToAvro(data, schema); + } else if (isAvroBinary(contentTypeLowerCase)) { + return data; + } else { + throw new UnsupportedContentTypeException(payloadContentType, topic); } + } - private boolean isJson(String contentType) { - return isOfType(contentType, APPLICATION_JSON, APPLICATION_JSON_WITH_DELIM); - } + private boolean isJson(String contentType) { + return isOfType(contentType, APPLICATION_JSON, APPLICATION_JSON_WITH_DELIM); + } - private boolean isAvroJson(String contentType) { - return isOfType(contentType, AVRO_JSON, AVRO_JSON_WITH_DELIM); - } + private boolean isAvroJson(String contentType) { + return isOfType(contentType, AVRO_JSON, AVRO_JSON_WITH_DELIM); + } - private boolean isAvroBinary(String contentType) { - return isOfType(contentType, AVRO_BINARY, AVRO_BINARY_WITH_DELIM); - } + private boolean isAvroBinary(String contentType) { + return isOfType(contentType, AVRO_BINARY, AVRO_BINARY_WITH_DELIM); + } - private boolean isOfType(String contentType, String expectedContentType, String expectedWithDelim) { - return contentType != null && (contentType.equals(expectedContentType) || contentType.startsWith(expectedWithDelim)); - } + private boolean isOfType( + String contentType, String expectedContentType, String expectedWithDelim) { + return contentType != null + && (contentType.equals(expectedContentType) || contentType.startsWith(expectedWithDelim)); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageFactory.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageFactory.java index ff92d5c31c..ea032a6b6a 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageFactory.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageFactory.java @@ -1,7 +1,12 @@ package pl.allegro.tech.hermes.frontend.publishing.message; +import static java.util.Optional.of; + import io.undertow.util.HeaderMap; import io.undertow.util.Headers; +import java.time.Clock; +import java.util.Map; +import java.util.Optional; import org.apache.avro.Schema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,144 +26,164 @@ import pl.allegro.tech.hermes.schema.SchemaVersion; import tech.allegro.schema.json2avro.converter.AvroConversionException; -import java.time.Clock; -import java.util.Map; -import java.util.Optional; - -import static java.util.Optional.of; - public class MessageFactory { - private static final Logger logger = LoggerFactory.getLogger(MessageFactory.class); - - private final MessageValidators validators; - private final AvroEnforcer enforcer; - private final SchemaRepository schemaRepository; - private final HeadersPropagator headersPropagator; - private final MessageContentWrapper messageContentWrapper; - private final Clock clock; - private final boolean schemaIdHeaderEnabled; - - public MessageFactory(MessageValidators validators, - AvroEnforcer enforcer, - SchemaRepository schemaRepository, - HeadersPropagator headersPropagator, - MessageContentWrapper messageContentWrapper, - Clock clock, - boolean schemaIdHeaderEnabled) { - this.validators = validators; - this.enforcer = enforcer; - this.messageContentWrapper = messageContentWrapper; - this.schemaRepository = schemaRepository; - this.headersPropagator = headersPropagator; - this.clock = clock; - this.schemaIdHeaderEnabled = schemaIdHeaderEnabled; - } - - public Message create(HeaderMap headerMap, AttachmentContent attachment) { - return create( - headerMap, - attachment.getTopic(), - attachment.getMessageId(), - attachment.getMessageContent()); - } - - private Message create(HeaderMap headerMap, Topic topic, String messageId, byte[] messageContent) { - long timestamp = clock.millis(); - switch (topic.getContentType()) { - case JSON: { - if (topic.isJsonToAvroDryRunEnabled()) { - try { - createAvroMessage(headerMap, topic, messageId, messageContent, timestamp); - } catch (AvroConversionException exception) { - logger.warn("Unsuccessful message conversion from JSON to AVRO on topic {} in dry run mode", - topic.getQualifiedName(), exception); - } catch (WrappingException | AvroInvalidMetadataException exception) { - logger.warn("Unsuccessful wrapping of AVRO message on topic {} in dry run mode", - topic.getQualifiedName(), exception); - } - } - return createJsonMessage(headerMap, messageId, messageContent, timestamp); + private static final Logger logger = LoggerFactory.getLogger(MessageFactory.class); + + private final MessageValidators validators; + private final AvroEnforcer enforcer; + private final SchemaRepository schemaRepository; + private final HeadersPropagator headersPropagator; + private final MessageContentWrapper messageContentWrapper; + private final Clock clock; + private final boolean schemaIdHeaderEnabled; + + public MessageFactory( + MessageValidators validators, + AvroEnforcer enforcer, + SchemaRepository schemaRepository, + HeadersPropagator headersPropagator, + MessageContentWrapper messageContentWrapper, + Clock clock, + boolean schemaIdHeaderEnabled) { + this.validators = validators; + this.enforcer = enforcer; + this.messageContentWrapper = messageContentWrapper; + this.schemaRepository = schemaRepository; + this.headersPropagator = headersPropagator; + this.clock = clock; + this.schemaIdHeaderEnabled = schemaIdHeaderEnabled; + } + + public Message create(HeaderMap headerMap, AttachmentContent attachment) { + return create( + headerMap, + attachment.getTopic(), + attachment.getMessageId(), + attachment.getMessageContent()); + } + + private Message create( + HeaderMap headerMap, Topic topic, String messageId, byte[] messageContent) { + long timestamp = clock.millis(); + switch (topic.getContentType()) { + case JSON: + { + if (topic.isJsonToAvroDryRunEnabled()) { + try { + createAvroMessage(headerMap, topic, messageId, messageContent, timestamp); + } catch (AvroConversionException exception) { + logger.warn( + "Unsuccessful message conversion from JSON to AVRO on topic {} in dry run mode", + topic.getQualifiedName(), + exception); + } catch (WrappingException | AvroInvalidMetadataException exception) { + logger.warn( + "Unsuccessful wrapping of AVRO message on topic {} in dry run mode", + topic.getQualifiedName(), + exception); } - case AVRO: - return createAvroMessage(headerMap, topic, messageId, messageContent, timestamp); - default: - throw new UnsupportedContentTypeException(topic); + } + return createJsonMessage(headerMap, messageId, messageContent, timestamp); } + case AVRO: + return createAvroMessage(headerMap, topic, messageId, messageContent, timestamp); + default: + throw new UnsupportedContentTypeException(topic); } - - private JsonMessage createJsonMessage(HeaderMap headerMap, String messageId, byte[] messageContent, long timestamp) { - Map extraRequestHeaders = headersPropagator.extract(headerMap); - JsonMessage message = new JsonMessage(messageId, messageContent, timestamp, extractPartitionKey(headerMap), - extraRequestHeaders); - byte[] wrapped = messageContentWrapper - .wrapJson(message.getData(), message.getId(), message.getTimestamp(), extraRequestHeaders); - return message.withDataReplaced(wrapped); - } - - private CompiledSchema getCompiledSchemaBySchemaVersion(HeaderMap headerMap, Topic topic) { - return extractSchemaVersion(headerMap) - .map(version -> schemaRepository.getAvroSchema(topic, version)) - .orElseGet(() -> schemaRepository.getLatestAvroSchema(topic)); + } + + private JsonMessage createJsonMessage( + HeaderMap headerMap, String messageId, byte[] messageContent, long timestamp) { + Map extraRequestHeaders = headersPropagator.extract(headerMap); + JsonMessage message = + new JsonMessage( + messageId, + messageContent, + timestamp, + extractPartitionKey(headerMap), + extraRequestHeaders); + byte[] wrapped = + messageContentWrapper.wrapJson( + message.getData(), message.getId(), message.getTimestamp(), extraRequestHeaders); + return message.withDataReplaced(wrapped); + } + + private CompiledSchema getCompiledSchemaBySchemaVersion( + HeaderMap headerMap, Topic topic) { + return extractSchemaVersion(headerMap) + .map(version -> schemaRepository.getAvroSchema(topic, version)) + .orElseGet(() -> schemaRepository.getLatestAvroSchema(topic)); + } + + private CompiledSchema getCompiledSchema(HeaderMap headerMap, Topic topic) { + return extractSchemaId(headerMap) + .map(id -> schemaRepository.getAvroSchema(topic, id)) + .orElseGet(() -> getCompiledSchemaBySchemaVersion(headerMap, topic)); + } + + private AvroMessage createAvroMessage( + HeaderMap headerMap, Topic topic, String messageId, byte[] messageContent, long timestamp) { + CompiledSchema schema = getCompiledSchema(headerMap, topic); + Map extraRequestHeaders = headersPropagator.extract(headerMap); + + AvroMessage message = + new AvroMessage( + messageId, + enforcer.enforceAvro( + headerMap.getFirst(Headers.CONTENT_TYPE_STRING), + messageContent, + schema.getSchema(), + topic), + timestamp, + schema, + extractPartitionKey(headerMap), + extraRequestHeaders); + + validators.check(topic, message); + byte[] wrapped = + messageContentWrapper.wrapAvro( + message.getData(), + message.getId(), + message.getTimestamp(), + topic, + schema, + extraRequestHeaders); + return message.withDataReplaced(wrapped); + } + + private Optional extractSchemaVersion(HeaderMap headerMap) { + String schemaVersion = headerMap.getFirst(MessageMetadataHeaders.SCHEMA_VERSION.getName()); + + if (schemaVersion == null) { + return Optional.empty(); } - - private CompiledSchema getCompiledSchema(HeaderMap headerMap, Topic topic) { - return extractSchemaId(headerMap) - .map(id -> schemaRepository.getAvroSchema(topic, id)) - .orElseGet(() -> getCompiledSchemaBySchemaVersion(headerMap, topic)); + try { + return of(SchemaVersion.valueOf(Integer.parseInt(schemaVersion))); + } catch (NumberFormatException e) { + return Optional.empty(); } + } - private AvroMessage createAvroMessage(HeaderMap headerMap, Topic topic, String messageId, byte[] messageContent, long timestamp) { - CompiledSchema schema = getCompiledSchema(headerMap, topic); - Map extraRequestHeaders = headersPropagator.extract(headerMap); - - AvroMessage message = new AvroMessage( - messageId, - enforcer.enforceAvro(headerMap.getFirst(Headers.CONTENT_TYPE_STRING), messageContent, schema.getSchema(), topic), - timestamp, - schema, - extractPartitionKey(headerMap), - extraRequestHeaders); - - validators.check(topic, message); - byte[] wrapped = messageContentWrapper.wrapAvro(message.getData(), message.getId(), message.getTimestamp(), - topic, schema, extraRequestHeaders); - return message.withDataReplaced(wrapped); - } - - private Optional extractSchemaVersion(HeaderMap headerMap) { - String schemaVersion = headerMap.getFirst(MessageMetadataHeaders.SCHEMA_VERSION.getName()); - - if (schemaVersion == null) { - return Optional.empty(); - } - try { - return of(SchemaVersion.valueOf(Integer.parseInt(schemaVersion))); - } catch (NumberFormatException e) { - return Optional.empty(); - } + private Optional extractSchemaId(HeaderMap headerMap) { + if (!schemaIdHeaderEnabled) { + return Optional.empty(); } - private Optional extractSchemaId(HeaderMap headerMap) { - if (!schemaIdHeaderEnabled) { - return Optional.empty(); - } - - String schemaId = headerMap.getFirst(MessageMetadataHeaders.SCHEMA_ID.getName()); + String schemaId = headerMap.getFirst(MessageMetadataHeaders.SCHEMA_ID.getName()); - if (schemaId == null) { - return Optional.empty(); - } - - try { - return of(SchemaId.valueOf(Integer.parseInt(schemaId))); - } catch (NumberFormatException e) { - return Optional.empty(); - } + if (schemaId == null) { + return Optional.empty(); } - private String extractPartitionKey(HeaderMap headerMap) { - return headerMap.getFirst(MessageMetadataHeaders.PARTITION_KEY.getName()); + try { + return of(SchemaId.valueOf(Integer.parseInt(schemaId))); + } catch (NumberFormatException e) { + return Optional.empty(); } + } + private String extractPartitionKey(HeaderMap headerMap) { + return headerMap.getFirst(MessageMetadataHeaders.PARTITION_KEY.getName()); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageIdGenerator.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageIdGenerator.java index 4baf14fbe4..08d419e1ee 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageIdGenerator.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageIdGenerator.java @@ -4,7 +4,7 @@ public class MessageIdGenerator { - public static String generate() { - return UUID.randomUUID().toString(); - } + public static String generate() { + return UUID.randomUUID().toString(); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageState.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageState.java index 159e57faac..07b9944a39 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageState.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageState.java @@ -1,7 +1,5 @@ package pl.allegro.tech.hermes.frontend.publishing.message; -import java.util.concurrent.atomic.AtomicReference; - import static pl.allegro.tech.hermes.frontend.publishing.message.MessageState.State.DELAYED_PROCESSING; import static pl.allegro.tech.hermes.frontend.publishing.message.MessageState.State.DELAYED_SENDING; import static pl.allegro.tech.hermes.frontend.publishing.message.MessageState.State.DELAYED_SENT_TO_KAFKA; @@ -17,90 +15,94 @@ import static pl.allegro.tech.hermes.frontend.publishing.message.MessageState.State.SENT_TO_KAFKA; import static pl.allegro.tech.hermes.frontend.publishing.message.MessageState.State.TIMEOUT_SENDING_TO_KAFKA; +import java.util.concurrent.atomic.AtomicReference; + public class MessageState { - enum State { - INIT, - PREMATURE_TIMEOUT, - READING, - READING_TIMEOUT, - READING_ERROR, - FULLY_READ, - SENDING_TO_KAFKA_PRODUCER_QUEUE, - ERROR_IN_SENDING_TO_KAFKA, - SENDING_TO_KAFKA, - SENT_TO_KAFKA, - DELAYED_SENDING, - DELAYED_PROCESSING, - DELAYED_SENT_TO_KAFKA, - TIMEOUT_SENDING_TO_KAFKA, - } - - private volatile boolean timeoutHasPassed = false; - private final AtomicReference state = new AtomicReference<>(State.INIT); - - public boolean setReading() { - return state.compareAndSet(INIT, READING); - } - - public void setPrematureTimeout() { - state.compareAndSet(INIT, PREMATURE_TIMEOUT); - } - - public boolean setFullyRead() { - return state.compareAndSet(READING, FULLY_READ); - } - - public boolean isReadingTimeout() { - return state.get() == READING_TIMEOUT; - } - - public void setSendingToKafkaProducerQueue() { - state.set(SENDING_TO_KAFKA_PRODUCER_QUEUE); - } - - public boolean setSentToKafka() { - return state.compareAndSet(SENDING_TO_KAFKA, SENT_TO_KAFKA) || state.compareAndSet(SENDING_TO_KAFKA_PRODUCER_QUEUE, SENT_TO_KAFKA); - } - - public boolean isDelayedSentToKafka() { - return state.get() == DELAYED_SENT_TO_KAFKA; - } - - public boolean setDelayedSending() { - return state.compareAndSet(SENDING_TO_KAFKA, DELAYED_SENDING); - } - - public boolean setTimeoutSendingToKafka() { - return state.compareAndSet(SENDING_TO_KAFKA_PRODUCER_QUEUE, TIMEOUT_SENDING_TO_KAFKA) || state.compareAndSet(SENDING_TO_KAFKA, TIMEOUT_SENDING_TO_KAFKA); - } - - public boolean setReadingTimeout() { - return state.compareAndSet(READING, READING_TIMEOUT); - } - - public boolean setReadingError() { - return state.compareAndSet(READING, READING_ERROR); - } - - public void setErrorInSendingToKafka() { - state.set(ERROR_IN_SENDING_TO_KAFKA); - } - - public boolean setSendingToKafka() { - return state.compareAndSet(SENDING_TO_KAFKA_PRODUCER_QUEUE, SENDING_TO_KAFKA); - } - - public boolean setDelayedProcessing() { - return timeoutHasPassed && state.compareAndSet(SENDING_TO_KAFKA, DELAYED_PROCESSING); - } - - public boolean setDelayedSentToKafka() { - return state.compareAndSet(DELAYED_SENDING, DELAYED_SENT_TO_KAFKA) - || state.compareAndSet(DELAYED_PROCESSING, DELAYED_SENT_TO_KAFKA); - } - - public void setTimeoutHasPassed() { - timeoutHasPassed = true; - } + enum State { + INIT, + PREMATURE_TIMEOUT, + READING, + READING_TIMEOUT, + READING_ERROR, + FULLY_READ, + SENDING_TO_KAFKA_PRODUCER_QUEUE, + ERROR_IN_SENDING_TO_KAFKA, + SENDING_TO_KAFKA, + SENT_TO_KAFKA, + DELAYED_SENDING, + DELAYED_PROCESSING, + DELAYED_SENT_TO_KAFKA, + TIMEOUT_SENDING_TO_KAFKA, + } + + private volatile boolean timeoutHasPassed = false; + private final AtomicReference state = new AtomicReference<>(State.INIT); + + public boolean setReading() { + return state.compareAndSet(INIT, READING); + } + + public void setPrematureTimeout() { + state.compareAndSet(INIT, PREMATURE_TIMEOUT); + } + + public boolean setFullyRead() { + return state.compareAndSet(READING, FULLY_READ); + } + + public boolean isReadingTimeout() { + return state.get() == READING_TIMEOUT; + } + + public void setSendingToKafkaProducerQueue() { + state.set(SENDING_TO_KAFKA_PRODUCER_QUEUE); + } + + public boolean setSentToKafka() { + return state.compareAndSet(SENDING_TO_KAFKA, SENT_TO_KAFKA) + || state.compareAndSet(SENDING_TO_KAFKA_PRODUCER_QUEUE, SENT_TO_KAFKA); + } + + public boolean isDelayedSentToKafka() { + return state.get() == DELAYED_SENT_TO_KAFKA; + } + + public boolean setDelayedSending() { + return state.compareAndSet(SENDING_TO_KAFKA, DELAYED_SENDING); + } + + public boolean setTimeoutSendingToKafka() { + return state.compareAndSet(SENDING_TO_KAFKA_PRODUCER_QUEUE, TIMEOUT_SENDING_TO_KAFKA) + || state.compareAndSet(SENDING_TO_KAFKA, TIMEOUT_SENDING_TO_KAFKA); + } + + public boolean setReadingTimeout() { + return state.compareAndSet(READING, READING_TIMEOUT); + } + + public boolean setReadingError() { + return state.compareAndSet(READING, READING_ERROR); + } + + public void setErrorInSendingToKafka() { + state.set(ERROR_IN_SENDING_TO_KAFKA); + } + + public boolean setSendingToKafka() { + return state.compareAndSet(SENDING_TO_KAFKA_PRODUCER_QUEUE, SENDING_TO_KAFKA); + } + + public boolean setDelayedProcessing() { + return timeoutHasPassed && state.compareAndSet(SENDING_TO_KAFKA, DELAYED_PROCESSING); + } + + public boolean setDelayedSentToKafka() { + return state.compareAndSet(DELAYED_SENDING, DELAYED_SENT_TO_KAFKA) + || state.compareAndSet(DELAYED_PROCESSING, DELAYED_SENT_TO_KAFKA); + } + + public void setTimeoutHasPassed() { + timeoutHasPassed = true; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageToJsonConverter.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageToJsonConverter.java index 0b4f5332db..7422f60633 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageToJsonConverter.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageToJsonConverter.java @@ -1,28 +1,33 @@ package pl.allegro.tech.hermes.frontend.publishing.message; +import static pl.allegro.tech.hermes.common.message.converter.AvroRecordToBytesConverter.bytesToRecord; + import org.apache.avro.Schema; import pl.allegro.tech.hermes.common.message.wrapper.SchemaAwareSerDe; import pl.allegro.tech.hermes.schema.CompiledSchema; import tech.allegro.schema.json2avro.converter.JsonAvroConverter; -import static pl.allegro.tech.hermes.common.message.converter.AvroRecordToBytesConverter.bytesToRecord; - public class MessageToJsonConverter { - private final JsonAvroConverter converter = new JsonAvroConverter(); - - public byte[] convert(Message message, boolean schemaIdAwareSerializationEnabled) { - try { - return message.getCompiledSchema() - .map(schema -> convertToJson(message.getData(), schema, schemaIdAwareSerializationEnabled)) - .orElseGet(message::getData); - } catch (Exception ignored) { - return message.getData(); - } - } + private final JsonAvroConverter converter = new JsonAvroConverter(); - private byte[] convertToJson(byte[] avro, CompiledSchema schema, boolean schemaIdAwareSerializationEnabled) { - byte[] schemaAwareAvro = schemaIdAwareSerializationEnabled ? SchemaAwareSerDe.trimMagicByteAndSchemaVersion(avro) : avro; - return converter.convertToJson(bytesToRecord(schemaAwareAvro, schema.getSchema())); + public byte[] convert(Message message, boolean schemaIdAwareSerializationEnabled) { + try { + return message + .getCompiledSchema() + .map( + schema -> convertToJson(message.getData(), schema, schemaIdAwareSerializationEnabled)) + .orElseGet(message::getData); + } catch (Exception ignored) { + return message.getData(); } + } + private byte[] convertToJson( + byte[] avro, CompiledSchema schema, boolean schemaIdAwareSerializationEnabled) { + byte[] schemaAwareAvro = + schemaIdAwareSerializationEnabled + ? SchemaAwareSerDe.trimMagicByteAndSchemaVersion(avro) + : avro; + return converter.convertToJson(bytesToRecord(schemaAwareAvro, schema.getSchema())); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/metadata/DefaultHeadersPropagator.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/metadata/DefaultHeadersPropagator.java index 6893e1a46c..7aa6f0238d 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/metadata/DefaultHeadersPropagator.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/metadata/DefaultHeadersPropagator.java @@ -1,68 +1,65 @@ package pl.allegro.tech.hermes.frontend.publishing.metadata; +import static java.util.Spliterators.spliteratorUnknownSize; +import static java.util.stream.Collectors.toMap; +import static java.util.stream.StreamSupport.stream; + import com.google.common.collect.ImmutableMap; import io.undertow.util.HeaderMap; import io.undertow.util.HeaderValues; -import pl.allegro.tech.hermes.frontend.config.HTTPHeadersProperties; -import pl.allegro.tech.hermes.frontend.publishing.handlers.end.TrackingHeadersExtractor; - import java.util.HashSet; import java.util.Map; import java.util.Set; - -import static java.util.Spliterators.spliteratorUnknownSize; -import static java.util.stream.Collectors.toMap; -import static java.util.stream.StreamSupport.stream; +import pl.allegro.tech.hermes.frontend.config.HTTPHeadersProperties; +import pl.allegro.tech.hermes.frontend.publishing.handlers.end.TrackingHeadersExtractor; public class DefaultHeadersPropagator implements HeadersPropagator, TrackingHeadersExtractor { - private final boolean propagate; - private final Set supportedHeaders; - private final Set trackingHeaders; + private final boolean propagate; + private final Set supportedHeaders; + private final Set trackingHeaders; - public DefaultHeadersPropagator(HTTPHeadersProperties httpHeadersProperties) { - propagate = httpHeadersProperties.isPropagationEnabled(); - supportedHeaders = httpHeadersProperties.getAllowedSet(); - trackingHeaders = new HashSet<>() { - { - addAll(httpHeadersProperties.getAllowedSet()); - addAll(httpHeadersProperties.getAdditionalAllowedSetToLog()); - } + public DefaultHeadersPropagator(HTTPHeadersProperties httpHeadersProperties) { + propagate = httpHeadersProperties.isPropagationEnabled(); + supportedHeaders = httpHeadersProperties.getAllowedSet(); + trackingHeaders = + new HashSet<>() { + { + addAll(httpHeadersProperties.getAllowedSet()); + addAll(httpHeadersProperties.getAdditionalAllowedSetToLog()); + } }; - } + } - @Override - public Map extract(HeaderMap headerMap) { - if (propagate) { - Map headers = toHeadersMap(headerMap); - if (supportedHeaders.isEmpty()) { - return ImmutableMap.copyOf(headers); - } + @Override + public Map extract(HeaderMap headerMap) { + if (propagate) { + Map headers = toHeadersMap(headerMap); + if (supportedHeaders.isEmpty()) { + return ImmutableMap.copyOf(headers); + } - return extractHeaders(headers, supportedHeaders); - } else { - return ImmutableMap.of(); - } + return extractHeaders(headers, supportedHeaders); + } else { + return ImmutableMap.of(); } + } - @Override - public Map extractHeadersToLog(HeaderMap headers) { - return extractHeaders(toHeadersMap(headers), trackingHeaders); - } + @Override + public Map extractHeadersToLog(HeaderMap headers) { + return extractHeaders(toHeadersMap(headers), trackingHeaders); + } - private static Map toHeadersMap(HeaderMap headerMap) { - return stream(spliteratorUnknownSize(headerMap.iterator(), 0), false) - .collect(toMap( - h -> h.getHeaderName().toString(), - HeaderValues::getFirst)); - } + private static Map toHeadersMap(HeaderMap headerMap) { + return stream(spliteratorUnknownSize(headerMap.iterator(), 0), false) + .collect(toMap(h -> h.getHeaderName().toString(), HeaderValues::getFirst)); + } - private static Map extractHeaders(Map headers, Set headersToExtract) { - return headers.entrySet().stream() - .filter(headerEntry -> headersToExtract.contains(headerEntry.getKey().toLowerCase())) - .filter(headerEntry -> !headerEntry.getValue().isEmpty()) - .collect(toMap( - Map.Entry::getKey, - Map.Entry::getValue)); - } + private static Map extractHeaders( + Map headers, Set headersToExtract) { + return headers.entrySet().stream() + .filter(headerEntry -> headersToExtract.contains(headerEntry.getKey().toLowerCase())) + .filter(headerEntry -> !headerEntry.getValue().isEmpty()) + .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/metadata/HeadersPropagator.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/metadata/HeadersPropagator.java index c026c69f7f..4b9da6e54f 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/metadata/HeadersPropagator.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/metadata/HeadersPropagator.java @@ -1,11 +1,9 @@ package pl.allegro.tech.hermes.frontend.publishing.metadata; - import io.undertow.util.HeaderMap; - import java.util.Map; public interface HeadersPropagator { - Map extract(HeaderMap headers); + Map extract(HeaderMap headers); } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/metadata/ProduceMetadata.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/metadata/ProduceMetadata.java index f57e581d7d..e02ce7a641 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/metadata/ProduceMetadata.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/metadata/ProduceMetadata.java @@ -1,22 +1,20 @@ package pl.allegro.tech.hermes.frontend.publishing.metadata; import jakarta.annotation.Nullable; - import java.util.Optional; public class ProduceMetadata { - @Nullable - private final String broker; + @Nullable private final String broker; - public ProduceMetadata(@Nullable String broker) { - this.broker = broker; - } + public ProduceMetadata(@Nullable String broker) { + this.broker = broker; + } - public Optional getBroker() { - return Optional.ofNullable(broker); - } + public Optional getBroker() { + return Optional.ofNullable(broker); + } - public static ProduceMetadata empty() { - return new ProduceMetadata(null); - } + public static ProduceMetadata empty() { + return new ProduceMetadata(null); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/preview/DefaultMessagePreviewPersister.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/preview/DefaultMessagePreviewPersister.java index 8cc18c14db..4f884ed0d7 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/preview/DefaultMessagePreviewPersister.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/preview/DefaultMessagePreviewPersister.java @@ -1,50 +1,57 @@ package pl.allegro.tech.hermes.frontend.publishing.preview; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import pl.allegro.tech.hermes.domain.topic.preview.MessagePreviewRepository; - import java.time.Duration; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import pl.allegro.tech.hermes.domain.topic.preview.MessagePreviewRepository; public class DefaultMessagePreviewPersister implements MessagePreviewPersister { - private final Duration period; - - private final MessagePreviewLog messagePreviewLog; - - private final MessagePreviewRepository repository; - - private final Optional scheduledExecutorService; - - public DefaultMessagePreviewPersister(MessagePreviewLog messagePreviewLog, - MessagePreviewRepository repository, - Duration logPersistPeriod, - boolean previewEnabled) { - this.messagePreviewLog = messagePreviewLog; - this.repository = repository; - this.period = logPersistPeriod; - - this.scheduledExecutorService = previewEnabled ? Optional.of(Executors.newSingleThreadScheduledExecutor( - new ThreadFactoryBuilder().setNameFormat("message-preview-persister-%d").build())) : Optional.empty(); - } - - @Override - public void start() { - scheduledExecutorService.ifPresent( - s -> s.scheduleAtFixedRate(this::persist, period.toSeconds(), period.toSeconds(), TimeUnit.SECONDS)); - } - - private void persist() { - repository.persist(messagePreviewLog.snapshotAndClean()); - } - - @Override - public void shutdown() { - scheduledExecutorService.ifPresent(ExecutorService::shutdown); - } - + private final Duration period; + + private final MessagePreviewLog messagePreviewLog; + + private final MessagePreviewRepository repository; + + private final Optional scheduledExecutorService; + + public DefaultMessagePreviewPersister( + MessagePreviewLog messagePreviewLog, + MessagePreviewRepository repository, + Duration logPersistPeriod, + boolean previewEnabled) { + this.messagePreviewLog = messagePreviewLog; + this.repository = repository; + this.period = logPersistPeriod; + + this.scheduledExecutorService = + previewEnabled + ? Optional.of( + Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder() + .setNameFormat("message-preview-persister-%d") + .build())) + : Optional.empty(); + } + + @Override + public void start() { + scheduledExecutorService.ifPresent( + s -> + s.scheduleAtFixedRate( + this::persist, period.toSeconds(), period.toSeconds(), TimeUnit.SECONDS)); + } + + private void persist() { + repository.persist(messagePreviewLog.snapshotAndClean()); + } + + @Override + public void shutdown() { + scheduledExecutorService.ifPresent(ExecutorService::shutdown); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/preview/MessagePreviewFactory.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/preview/MessagePreviewFactory.java index fae6b9205e..3c9bc0b098 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/preview/MessagePreviewFactory.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/preview/MessagePreviewFactory.java @@ -7,22 +7,22 @@ public class MessagePreviewFactory { - private final int maxMessagePreviewLength; - private final MessageToJsonConverter converter; + private final int maxMessagePreviewLength; + private final MessageToJsonConverter converter; - public MessagePreviewFactory(int maxMessagePreviewSizeKb) { - this.maxMessagePreviewLength = maxMessagePreviewSizeKb * 1024; - converter = new MessageToJsonConverter(); - } + public MessagePreviewFactory(int maxMessagePreviewSizeKb) { + this.maxMessagePreviewLength = maxMessagePreviewSizeKb * 1024; + converter = new MessageToJsonConverter(); + } - public MessagePreview create(Message message, boolean schemaIdAwareSerializationEnabled) { - byte[] content = converter.convert(message, schemaIdAwareSerializationEnabled); - final boolean truncated = (content.length > maxMessagePreviewLength); + public MessagePreview create(Message message, boolean schemaIdAwareSerializationEnabled) { + byte[] content = converter.convert(message, schemaIdAwareSerializationEnabled); + final boolean truncated = (content.length > maxMessagePreviewLength); - if (truncated) { - return new MessagePreview(ArrayUtils.subarray(content, 0, maxMessagePreviewLength), true); - } else { - return new MessagePreview(content); - } + if (truncated) { + return new MessagePreview(ArrayUtils.subarray(content, 0, maxMessagePreviewLength), true); + } else { + return new MessagePreview(content); } + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/preview/MessagePreviewLog.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/preview/MessagePreviewLog.java index 5123c5552b..4b3d45ffe6 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/preview/MessagePreviewLog.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/preview/MessagePreviewLog.java @@ -1,62 +1,62 @@ package pl.allegro.tech.hermes.frontend.publishing.preview; import com.google.common.util.concurrent.AtomicLongMap; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentLinkedDeque; import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.api.TopicName; import pl.allegro.tech.hermes.domain.topic.preview.MessagePreview; import pl.allegro.tech.hermes.domain.topic.preview.TopicsMessagesPreview; import pl.allegro.tech.hermes.frontend.publishing.message.Message; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ConcurrentLinkedDeque; - public class MessagePreviewLog { - private final MessagePreviewFactory messagePreviewFactory; + private final MessagePreviewFactory messagePreviewFactory; - private final int previewSizePerTopic; + private final int previewSizePerTopic; - private final AtomicLongMap limiter = AtomicLongMap.create(); + private final AtomicLongMap limiter = AtomicLongMap.create(); - private final ConcurrentLinkedDeque messages = new ConcurrentLinkedDeque<>(); + private final ConcurrentLinkedDeque messages = + new ConcurrentLinkedDeque<>(); - public MessagePreviewLog(MessagePreviewFactory messagePreviewFactory, int previewSizePerTopic) { - this.messagePreviewFactory = messagePreviewFactory; - this.previewSizePerTopic = previewSizePerTopic; - } + public MessagePreviewLog(MessagePreviewFactory messagePreviewFactory, int previewSizePerTopic) { + this.messagePreviewFactory = messagePreviewFactory; + this.previewSizePerTopic = previewSizePerTopic; + } - public void add(Topic topic, Message message) { - long counter = limiter.getAndIncrement(topic.getName()); - if (counter < previewSizePerTopic) { - messages.add( - new MessagePreviewSnapshot( - topic.getName(), - messagePreviewFactory.create(message, topic.isSchemaIdAwareSerializationEnabled()))); - } + public void add(Topic topic, Message message) { + long counter = limiter.getAndIncrement(topic.getName()); + if (counter < previewSizePerTopic) { + messages.add( + new MessagePreviewSnapshot( + topic.getName(), + messagePreviewFactory.create(message, topic.isSchemaIdAwareSerializationEnabled()))); } + } - public TopicsMessagesPreview snapshotAndClean() { - List snapshot = new ArrayList<>(messages); - messages.clear(); - limiter.clear(); + public TopicsMessagesPreview snapshotAndClean() { + List snapshot = new ArrayList<>(messages); + messages.clear(); + limiter.clear(); - TopicsMessagesPreview preview = new TopicsMessagesPreview(); - for (MessagePreviewSnapshot message : snapshot) { - preview.add(message.topicName, message.content); - } - return preview; + TopicsMessagesPreview preview = new TopicsMessagesPreview(); + for (MessagePreviewSnapshot message : snapshot) { + preview.add(message.topicName, message.content); } + return preview; + } - private static class MessagePreviewSnapshot { + private static class MessagePreviewSnapshot { - final TopicName topicName; + final TopicName topicName; - final MessagePreview content; + final MessagePreview content; - MessagePreviewSnapshot(TopicName topicName, MessagePreview content) { - this.topicName = topicName; - this.content = content; - } + MessagePreviewSnapshot(TopicName topicName, MessagePreview content) { + this.topicName = topicName; + this.content = content; } + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/preview/MessagePreviewPersister.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/preview/MessagePreviewPersister.java index ad2cb0d004..2740d294e7 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/preview/MessagePreviewPersister.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/publishing/preview/MessagePreviewPersister.java @@ -1,9 +1,8 @@ package pl.allegro.tech.hermes.frontend.publishing.preview; - public interface MessagePreviewPersister { - void start(); + void start(); - void shutdown(); + void shutdown(); } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/readiness/AdminReadinessService.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/readiness/AdminReadinessService.java index 66377880b4..d7876e718e 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/readiness/AdminReadinessService.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/readiness/AdminReadinessService.java @@ -1,6 +1,11 @@ package pl.allegro.tech.hermes.frontend.readiness; +import static pl.allegro.tech.hermes.api.DatacenterReadiness.ReadinessStatus.READY; + import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Collections; +import java.util.Map; +import java.util.stream.Collectors; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.NodeCache; @@ -12,77 +17,75 @@ import pl.allegro.tech.hermes.domain.readiness.DatacenterReadinessList; import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; -import java.util.Collections; -import java.util.Map; -import java.util.stream.Collectors; - -import static pl.allegro.tech.hermes.api.DatacenterReadiness.ReadinessStatus.READY; - public class AdminReadinessService implements NodeCacheListener { - private static final Logger logger = LoggerFactory.getLogger(AdminReadinessService.class); + private static final Logger logger = LoggerFactory.getLogger(AdminReadinessService.class); - private final NodeCache cache; - private final ObjectMapper mapper; - private final String localDatacenterName; + private final NodeCache cache; + private final ObjectMapper mapper; + private final String localDatacenterName; - private volatile Map readinessPerDatacenter; + private volatile Map readinessPerDatacenter; - public AdminReadinessService(ObjectMapper mapper, - CuratorFramework curator, - ZookeeperPaths paths, - String localDatacenterName) { - this.mapper = mapper; - this.localDatacenterName = localDatacenterName; - this.cache = new NodeCache(curator, paths.datacenterReadinessPath()); - cache.getListenable().addListener(this); - try { - cache.start(true); - } catch (Exception e) { - throw new InternalProcessingException("Readiness cache cannot start.", e); - } + public AdminReadinessService( + ObjectMapper mapper, + CuratorFramework curator, + ZookeeperPaths paths, + String localDatacenterName) { + this.mapper = mapper; + this.localDatacenterName = localDatacenterName; + this.cache = new NodeCache(curator, paths.datacenterReadinessPath()); + cache.getListenable().addListener(this); + try { + cache.start(true); + } catch (Exception e) { + throw new InternalProcessingException("Readiness cache cannot start.", e); } + } - public void start() { - refreshAdminReady(); - logger.info("Initial readiness per datacenter: {}", readinessPerDatacenter); - } + public void start() { + refreshAdminReady(); + logger.info("Initial readiness per datacenter: {}", readinessPerDatacenter); + } - public void stop() { - try { - cache.close(); - } catch (Exception e) { - logger.warn("Failed to stop readiness cache", e); - } + public void stop() { + try { + cache.close(); + } catch (Exception e) { + logger.warn("Failed to stop readiness cache", e); } + } - @Override - public void nodeChanged() { - refreshAdminReady(); - logger.info("Readiness per datacenter changed to: {}", readinessPerDatacenter); - } + @Override + public void nodeChanged() { + refreshAdminReady(); + logger.info("Readiness per datacenter changed to: {}", readinessPerDatacenter); + } - private void refreshAdminReady() { - try { - ChildData nodeData = cache.getCurrentData(); - if (nodeData != null) { - byte[] data = nodeData.getData(); - DatacenterReadinessList readiness = mapper.readValue(data, DatacenterReadinessList.class); - readinessPerDatacenter = readiness.datacenters().stream() - .collect(Collectors.toMap(DatacenterReadiness::getDatacenter, e -> e.getStatus() == READY)); - } else { - readinessPerDatacenter = Collections.emptyMap(); - } - } catch (Exception e) { - logger.error("Failed reloading readiness cache.", e); - } + private void refreshAdminReady() { + try { + ChildData nodeData = cache.getCurrentData(); + if (nodeData != null) { + byte[] data = nodeData.getData(); + DatacenterReadinessList readiness = mapper.readValue(data, DatacenterReadinessList.class); + readinessPerDatacenter = + readiness.datacenters().stream() + .collect( + Collectors.toMap( + DatacenterReadiness::getDatacenter, e -> e.getStatus() == READY)); + } else { + readinessPerDatacenter = Collections.emptyMap(); + } + } catch (Exception e) { + logger.error("Failed reloading readiness cache.", e); } + } - public boolean isLocalDatacenterReady() { - return isDatacenterReady(localDatacenterName); - } + public boolean isLocalDatacenterReady() { + return isDatacenterReady(localDatacenterName); + } - public boolean isDatacenterReady(String datacenter) { - return readinessPerDatacenter != null && readinessPerDatacenter.getOrDefault(datacenter, true); - } + public boolean isDatacenterReady(String datacenter) { + return readinessPerDatacenter != null && readinessPerDatacenter.getOrDefault(datacenter, true); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/readiness/DefaultReadinessChecker.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/readiness/DefaultReadinessChecker.java index ec62d45fae..53ad0b3704 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/readiness/DefaultReadinessChecker.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/readiness/DefaultReadinessChecker.java @@ -1,86 +1,90 @@ package pl.allegro.tech.hermes.frontend.readiness; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import pl.allegro.tech.hermes.frontend.producer.BrokerTopicAvailabilityChecker; - import java.time.Duration; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import pl.allegro.tech.hermes.frontend.producer.BrokerTopicAvailabilityChecker; public class DefaultReadinessChecker implements ReadinessChecker { - private final boolean enabled; - private final boolean topicsCheckEnabled; - private final Duration interval; - private final BrokerTopicAvailabilityChecker brokerTopicAvailabilityChecker; - private final ScheduledExecutorService scheduler; - private final AdminReadinessService adminReadinessService; + private final boolean enabled; + private final boolean topicsCheckEnabled; + private final Duration interval; + private final BrokerTopicAvailabilityChecker brokerTopicAvailabilityChecker; + private final ScheduledExecutorService scheduler; + private final AdminReadinessService adminReadinessService; - private volatile boolean ready = false; + private volatile boolean ready = false; - private static final Logger logger = LoggerFactory.getLogger(DefaultReadinessChecker.class); + private static final Logger logger = LoggerFactory.getLogger(DefaultReadinessChecker.class); - public DefaultReadinessChecker(BrokerTopicAvailabilityChecker brokerTopicAvailabilityChecker, - AdminReadinessService adminReadinessService, - boolean enabled, - boolean topicsCheckEnabled, - Duration interval) { - this.enabled = enabled; - this.topicsCheckEnabled = topicsCheckEnabled; - this.interval = interval; - this.brokerTopicAvailabilityChecker = brokerTopicAvailabilityChecker; - this.adminReadinessService = adminReadinessService; - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setNameFormat("ReadinessChecker-%d").build(); - this.scheduler = Executors.newSingleThreadScheduledExecutor(threadFactory); + public DefaultReadinessChecker( + BrokerTopicAvailabilityChecker brokerTopicAvailabilityChecker, + AdminReadinessService adminReadinessService, + boolean enabled, + boolean topicsCheckEnabled, + Duration interval) { + this.enabled = enabled; + this.topicsCheckEnabled = topicsCheckEnabled; + this.interval = interval; + this.brokerTopicAvailabilityChecker = brokerTopicAvailabilityChecker; + this.adminReadinessService = adminReadinessService; + ThreadFactory threadFactory = + new ThreadFactoryBuilder().setNameFormat("ReadinessChecker-%d").build(); + this.scheduler = Executors.newSingleThreadScheduledExecutor(threadFactory); - logger.info("Default readiness checker enabled: {}, readiness check based on topic metadata enabled: {}", enabled, topicsCheckEnabled); - } + logger.info( + "Default readiness checker enabled: {}, readiness check based on topic metadata enabled: {}", + enabled, + topicsCheckEnabled); + } - @Override - public boolean isReady() { - if (!enabled) { - return true; - } - return ready; + @Override + public boolean isReady() { + if (!enabled) { + return true; } + return ready; + } - @Override - public void start() { - if (enabled) { - ReadinessCheckerJob job = new ReadinessCheckerJob(); - job.run(); - scheduler.scheduleAtFixedRate(job, interval.toSeconds(), interval.toSeconds(), TimeUnit.SECONDS); - } + @Override + public void start() { + if (enabled) { + ReadinessCheckerJob job = new ReadinessCheckerJob(); + job.run(); + scheduler.scheduleAtFixedRate( + job, interval.toSeconds(), interval.toSeconds(), TimeUnit.SECONDS); } + } - @Override - public void stop() throws InterruptedException { - scheduler.shutdown(); - scheduler.awaitTermination(1, TimeUnit.MINUTES); - } + @Override + public void stop() throws InterruptedException { + scheduler.shutdown(); + scheduler.awaitTermination(1, TimeUnit.MINUTES); + } - private class ReadinessCheckerJob implements Runnable { - private volatile boolean allTopicsAvailable = false; + private class ReadinessCheckerJob implements Runnable { + private volatile boolean allTopicsAvailable = false; - @Override - public void run() { - if (!adminReadinessService.isLocalDatacenterReady()) { - ready = false; - } else if (allTopicsAvailable) { - ready = true; - } else if (topicsCheckEnabled) { - allTopicsAvailable = brokerTopicAvailabilityChecker.areAllTopicsAvailable(); - logger.info("All topics available: {}, setting ready to: {}", allTopicsAvailable, ready); - ready = allTopicsAvailable; - } else { - allTopicsAvailable = true; - ready = true; - } - } + @Override + public void run() { + if (!adminReadinessService.isLocalDatacenterReady()) { + ready = false; + } else if (allTopicsAvailable) { + ready = true; + } else if (topicsCheckEnabled) { + allTopicsAvailable = brokerTopicAvailabilityChecker.areAllTopicsAvailable(); + logger.info("All topics available: {}, setting ready to: {}", allTopicsAvailable, ready); + ready = allTopicsAvailable; + } else { + allTopicsAvailable = true; + ready = true; + } } + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/readiness/HealthCheckService.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/readiness/HealthCheckService.java index f63cf9733a..5f569a88a3 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/readiness/HealthCheckService.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/readiness/HealthCheckService.java @@ -2,18 +2,17 @@ public class HealthCheckService { - private volatile boolean shutdown = true; + private volatile boolean shutdown = true; - public boolean isShutdown() { - return shutdown; - } + public boolean isShutdown() { + return shutdown; + } - public void shutdown() { - this.shutdown = true; - } - - public void startup() { - this.shutdown = false; - } + public void shutdown() { + this.shutdown = true; + } + public void startup() { + this.shutdown = false; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/readiness/ReadinessChecker.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/readiness/ReadinessChecker.java index 17c9c36c33..0a3b7221f4 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/readiness/ReadinessChecker.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/readiness/ReadinessChecker.java @@ -2,9 +2,9 @@ public interface ReadinessChecker { - boolean isReady(); + boolean isReady(); - void start(); + void start(); - void stop() throws InterruptedException; + void stop() throws InterruptedException; } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/HealthCheckHandler.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/HealthCheckHandler.java index c55db74fe2..b665673d54 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/HealthCheckHandler.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/HealthCheckHandler.java @@ -1,40 +1,39 @@ package pl.allegro.tech.hermes.frontend.server; +import static io.undertow.util.StatusCodes.OK; +import static io.undertow.util.StatusCodes.SERVICE_UNAVAILABLE; + import io.undertow.server.HttpHandler; import io.undertow.server.HttpServerExchange; import pl.allegro.tech.hermes.frontend.readiness.HealthCheckService; -import static io.undertow.util.StatusCodes.OK; -import static io.undertow.util.StatusCodes.SERVICE_UNAVAILABLE; - public class HealthCheckHandler implements HttpHandler { - private final HealthCheckService healthCheckService; + private final HealthCheckService healthCheckService; - public HealthCheckHandler(HealthCheckService healthCheckService) { - this.healthCheckService = healthCheckService; - } + public HealthCheckHandler(HealthCheckService healthCheckService) { + this.healthCheckService = healthCheckService; + } - @Override - public void handleRequest(HttpServerExchange exchange) { - if (healthCheckService.isShutdown()) { - unavailable(exchange); - } else { - success(exchange); - } + @Override + public void handleRequest(HttpServerExchange exchange) { + if (healthCheckService.isShutdown()) { + unavailable(exchange); + } else { + success(exchange); } + } - private void success(HttpServerExchange exchange) { - response(exchange, OK, "UP"); - } + private void success(HttpServerExchange exchange) { + response(exchange, OK, "UP"); + } - private void unavailable(HttpServerExchange exchange) { - response(exchange, SERVICE_UNAVAILABLE, "SHUTDOWN"); - } - - private void response(HttpServerExchange exchange, int status, String data) { - exchange.setStatusCode(status); - exchange.getResponseSender().send(data); - } + private void unavailable(HttpServerExchange exchange) { + response(exchange, SERVICE_UNAVAILABLE, "SHUTDOWN"); + } + private void response(HttpServerExchange exchange, int status, String data) { + exchange.setStatusCode(status); + exchange.getResponseSender().send(data); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/HermesServer.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/HermesServer.java index aa68eab291..796c613eb4 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/HermesServer.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/HermesServer.java @@ -1,10 +1,22 @@ package pl.allegro.tech.hermes.frontend.server; +import static io.undertow.UndertowOptions.ALWAYS_SET_KEEP_ALIVE; +import static io.undertow.UndertowOptions.ENABLE_HTTP2; +import static io.undertow.UndertowOptions.MAX_COOKIES; +import static io.undertow.UndertowOptions.MAX_HEADERS; +import static io.undertow.UndertowOptions.MAX_PARAMETERS; +import static io.undertow.UndertowOptions.REQUEST_PARSE_TIMEOUT; +import static org.xnio.Options.BACKLOG; +import static org.xnio.Options.KEEP_ALIVE; +import static org.xnio.Options.READ_TIMEOUT; +import static org.xnio.Options.SSL_CLIENT_AUTH_MODE; + import io.micrometer.prometheus.PrometheusMeterRegistry; import io.undertow.Undertow; import io.undertow.server.HttpHandler; import io.undertow.server.RoutingHandler; import io.undertow.server.handlers.RequestDumpingHandler; +import java.net.InetSocketAddress; import org.xnio.SslClientAuthMode; import pl.allegro.tech.hermes.common.metric.MetricsFacade; import pl.allegro.tech.hermes.frontend.publishing.handlers.ThroughputLimiter; @@ -12,151 +24,151 @@ import pl.allegro.tech.hermes.frontend.readiness.HealthCheckService; import pl.allegro.tech.hermes.frontend.readiness.ReadinessChecker; -import java.net.InetSocketAddress; - -import static io.undertow.UndertowOptions.ALWAYS_SET_KEEP_ALIVE; -import static io.undertow.UndertowOptions.ENABLE_HTTP2; -import static io.undertow.UndertowOptions.MAX_COOKIES; -import static io.undertow.UndertowOptions.MAX_HEADERS; -import static io.undertow.UndertowOptions.MAX_PARAMETERS; -import static io.undertow.UndertowOptions.REQUEST_PARSE_TIMEOUT; -import static org.xnio.Options.BACKLOG; -import static org.xnio.Options.KEEP_ALIVE; -import static org.xnio.Options.READ_TIMEOUT; -import static org.xnio.Options.SSL_CLIENT_AUTH_MODE; - public class HermesServer { - private final MetricsFacade metricsFacade; - private final HermesServerParameters hermesServerParameters; - private final SslParameters sslParameters; - private final HttpHandler publishingHandler; - private final HealthCheckService healthCheckService; - private final ReadinessChecker readinessChecker; - private final MessagePreviewPersister messagePreviewPersister; - private final ThroughputLimiter throughputLimiter; - private final SslContextFactoryProvider sslContextFactoryProvider; - private final PrometheusMeterRegistry prometheusMeterRegistry; - private Undertow undertow; - private HermesShutdownHandler gracefulShutdown; - - public HermesServer( - SslParameters sslParameters, - HermesServerParameters hermesServerParameters, - MetricsFacade metricsFacade, - HttpHandler publishingHandler, - HealthCheckService healthCheckService, - ReadinessChecker readinessChecker, - MessagePreviewPersister messagePreviewPersister, - ThroughputLimiter throughputLimiter, - SslContextFactoryProvider sslContextFactoryProvider, - PrometheusMeterRegistry prometheusMeterRegistry) { - - this.sslParameters = sslParameters; - this.hermesServerParameters = hermesServerParameters; - this.metricsFacade = metricsFacade; - this.publishingHandler = publishingHandler; - this.prometheusMeterRegistry = prometheusMeterRegistry; - this.healthCheckService = healthCheckService; - this.readinessChecker = readinessChecker; - this.messagePreviewPersister = messagePreviewPersister; - this.sslContextFactoryProvider = sslContextFactoryProvider; - this.throughputLimiter = throughputLimiter; - } - - public void start() { - configureServer().start(); - messagePreviewPersister.start(); - throughputLimiter.start(); - healthCheckService.startup(); - readinessChecker.start(); - } - - public void stop() throws InterruptedException { - if (hermesServerParameters.isGracefulShutdownEnabled()) { - prepareForGracefulShutdown(); - } - shutdown(); + private final MetricsFacade metricsFacade; + private final HermesServerParameters hermesServerParameters; + private final SslParameters sslParameters; + private final HttpHandler publishingHandler; + private final HealthCheckService healthCheckService; + private final ReadinessChecker readinessChecker; + private final MessagePreviewPersister messagePreviewPersister; + private final ThroughputLimiter throughputLimiter; + private final SslContextFactoryProvider sslContextFactoryProvider; + private final PrometheusMeterRegistry prometheusMeterRegistry; + private Undertow undertow; + private HermesShutdownHandler gracefulShutdown; + + public HermesServer( + SslParameters sslParameters, + HermesServerParameters hermesServerParameters, + MetricsFacade metricsFacade, + HttpHandler publishingHandler, + HealthCheckService healthCheckService, + ReadinessChecker readinessChecker, + MessagePreviewPersister messagePreviewPersister, + ThroughputLimiter throughputLimiter, + SslContextFactoryProvider sslContextFactoryProvider, + PrometheusMeterRegistry prometheusMeterRegistry) { + + this.sslParameters = sslParameters; + this.hermesServerParameters = hermesServerParameters; + this.metricsFacade = metricsFacade; + this.publishingHandler = publishingHandler; + this.prometheusMeterRegistry = prometheusMeterRegistry; + this.healthCheckService = healthCheckService; + this.readinessChecker = readinessChecker; + this.messagePreviewPersister = messagePreviewPersister; + this.sslContextFactoryProvider = sslContextFactoryProvider; + this.throughputLimiter = throughputLimiter; + } + + public void start() { + configureServer().start(); + messagePreviewPersister.start(); + throughputLimiter.start(); + healthCheckService.startup(); + readinessChecker.start(); + } + + public void stop() throws InterruptedException { + if (hermesServerParameters.isGracefulShutdownEnabled()) { + prepareForGracefulShutdown(); } - - public void prepareForGracefulShutdown() throws InterruptedException { - healthCheckService.shutdown(); - - Thread.sleep(hermesServerParameters.getGracefulShutdownInitialWait().toMillis()); - - gracefulShutdown.handleShutdown(); + shutdown(); + } + + public void prepareForGracefulShutdown() throws InterruptedException { + healthCheckService.shutdown(); + + Thread.sleep(hermesServerParameters.getGracefulShutdownInitialWait().toMillis()); + + gracefulShutdown.handleShutdown(); + } + + public void shutdown() throws InterruptedException { + undertow.stop(); + messagePreviewPersister.shutdown(); + throughputLimiter.stop(); + readinessChecker.stop(); + } + + private Undertow configureServer() { + gracefulShutdown = new HermesShutdownHandler(handlers(), metricsFacade); + Undertow.Builder builder = + Undertow.builder() + .addHttpListener(hermesServerParameters.getPort(), hermesServerParameters.getHost()) + .setServerOption( + REQUEST_PARSE_TIMEOUT, + (int) hermesServerParameters.getRequestParseTimeout().toMillis()) + .setServerOption(MAX_HEADERS, hermesServerParameters.getMaxHeaders()) + .setServerOption(MAX_PARAMETERS, hermesServerParameters.getMaxParameters()) + .setServerOption(MAX_COOKIES, hermesServerParameters.getMaxCookies()) + .setServerOption(ALWAYS_SET_KEEP_ALIVE, hermesServerParameters.isAlwaysKeepAlive()) + .setServerOption(KEEP_ALIVE, hermesServerParameters.isKeepAlive()) + .setSocketOption(BACKLOG, hermesServerParameters.getBacklogSize()) + .setSocketOption(READ_TIMEOUT, (int) hermesServerParameters.getReadTimeout().toMillis()) + .setIoThreads(hermesServerParameters.getIoThreadsCount()) + .setWorkerThreads(hermesServerParameters.getWorkerThreadCount()) + .setBufferSize(hermesServerParameters.getBufferSize()) + .setHandler(gracefulShutdown); + + if (sslParameters.isEnabled()) { + builder + .addHttpsListener( + sslParameters.getPort(), + hermesServerParameters.getHost(), + sslContextFactoryProvider.getSslContextFactory().create().getSslContext()) + .setSocketOption( + SSL_CLIENT_AUTH_MODE, + SslClientAuthMode.valueOf(sslParameters.getClientAuthMode().toUpperCase())) + .setServerOption(ENABLE_HTTP2, hermesServerParameters.isHttp2Enabled()); } - - public void shutdown() throws InterruptedException { - undertow.stop(); - messagePreviewPersister.shutdown(); - throughputLimiter.stop(); - readinessChecker.stop(); - } - - private Undertow configureServer() { - gracefulShutdown = new HermesShutdownHandler(handlers(), metricsFacade); - Undertow.Builder builder = Undertow.builder() - .addHttpListener(hermesServerParameters.getPort(), hermesServerParameters.getHost()) - .setServerOption(REQUEST_PARSE_TIMEOUT, (int) hermesServerParameters.getRequestParseTimeout().toMillis()) - .setServerOption(MAX_HEADERS, hermesServerParameters.getMaxHeaders()) - .setServerOption(MAX_PARAMETERS, hermesServerParameters.getMaxParameters()) - .setServerOption(MAX_COOKIES, hermesServerParameters.getMaxCookies()) - .setServerOption(ALWAYS_SET_KEEP_ALIVE, hermesServerParameters.isAlwaysKeepAlive()) - .setServerOption(KEEP_ALIVE, hermesServerParameters.isKeepAlive()) - .setSocketOption(BACKLOG, hermesServerParameters.getBacklogSize()) - .setSocketOption(READ_TIMEOUT, (int) hermesServerParameters.getReadTimeout().toMillis()) - .setIoThreads(hermesServerParameters.getIoThreadsCount()) - .setWorkerThreads(hermesServerParameters.getWorkerThreadCount()) - .setBufferSize(hermesServerParameters.getBufferSize()) - .setHandler(gracefulShutdown); - - if (sslParameters.isEnabled()) { - builder.addHttpsListener(sslParameters.getPort(), hermesServerParameters.getHost(), - sslContextFactoryProvider.getSslContextFactory().create().getSslContext()) - .setSocketOption(SSL_CLIENT_AUTH_MODE, - SslClientAuthMode.valueOf(sslParameters.getClientAuthMode().toUpperCase())) - .setServerOption(ENABLE_HTTP2, hermesServerParameters.isHttp2Enabled()); - } - this.undertow = builder.build(); - return undertow; - } - - private HttpHandler handlers() { - HttpHandler healthCheckHandler = new HealthCheckHandler(healthCheckService); - HttpHandler readinessHandler = new ReadinessCheckHandler(readinessChecker, healthCheckService); - HttpHandler prometheusHandler = new PrometheusMetricsHandler(prometheusMeterRegistry); - - RoutingHandler routingHandler = new RoutingHandler() - .post("/topics/{qualifiedTopicName}", publishingHandler) - .get("/status/ping", healthCheckHandler) - .get("/status/health", healthCheckHandler) - .get("/status/ready", readinessHandler) - .get("/status/prometheus", prometheusHandler) - .get("/", healthCheckHandler); - - return isFrontendRequestDumperEnabled() ? new RequestDumpingHandler(routingHandler) : routingHandler; - } - - private boolean isFrontendRequestDumperEnabled() { - return hermesServerParameters.isRequestDumperEnabled(); - } - - public int getPort() { - InetSocketAddress socketAddress = (InetSocketAddress) undertow.getListenerInfo().stream() + this.undertow = builder.build(); + return undertow; + } + + private HttpHandler handlers() { + HttpHandler healthCheckHandler = new HealthCheckHandler(healthCheckService); + HttpHandler readinessHandler = new ReadinessCheckHandler(readinessChecker, healthCheckService); + HttpHandler prometheusHandler = new PrometheusMetricsHandler(prometheusMeterRegistry); + + RoutingHandler routingHandler = + new RoutingHandler() + .post("/topics/{qualifiedTopicName}", publishingHandler) + .get("/status/ping", healthCheckHandler) + .get("/status/health", healthCheckHandler) + .get("/status/ready", readinessHandler) + .get("/status/prometheus", prometheusHandler) + .get("/", healthCheckHandler); + + return isFrontendRequestDumperEnabled() + ? new RequestDumpingHandler(routingHandler) + : routingHandler; + } + + private boolean isFrontendRequestDumperEnabled() { + return hermesServerParameters.isRequestDumperEnabled(); + } + + public int getPort() { + InetSocketAddress socketAddress = + (InetSocketAddress) + undertow.getListenerInfo().stream() .findFirst() .orElseThrow(() -> new IllegalStateException("No port available yet.")) .getAddress(); - return socketAddress.getPort(); - } + return socketAddress.getPort(); + } - public int getSSLPort() { - InetSocketAddress socketAddress = (InetSocketAddress) undertow.getListenerInfo().stream() + public int getSSLPort() { + InetSocketAddress socketAddress = + (InetSocketAddress) + undertow.getListenerInfo().stream() .filter(listener -> listener.getSslContext() != null) .findFirst() .orElseThrow(() -> new IllegalStateException("No SSL port available yet.")) .getAddress(); - return socketAddress.getPort(); - } - + return socketAddress.getPort(); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/HermesServerParameters.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/HermesServerParameters.java index 5c2f7bf549..6a400e3edb 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/HermesServerParameters.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/HermesServerParameters.java @@ -4,37 +4,37 @@ public interface HermesServerParameters { - int getPort(); + int getPort(); - String getHost(); + String getHost(); - Duration getReadTimeout(); + Duration getReadTimeout(); - Duration getRequestParseTimeout(); + Duration getRequestParseTimeout(); - int getMaxHeaders(); + int getMaxHeaders(); - int getMaxParameters(); + int getMaxParameters(); - int getMaxCookies(); + int getMaxCookies(); - int getBacklogSize(); + int getBacklogSize(); - int getIoThreadsCount(); + int getIoThreadsCount(); - int getWorkerThreadCount(); + int getWorkerThreadCount(); - boolean isAlwaysKeepAlive(); + boolean isAlwaysKeepAlive(); - boolean isKeepAlive(); + boolean isKeepAlive(); - boolean isRequestDumperEnabled(); + boolean isRequestDumperEnabled(); - int getBufferSize(); + int getBufferSize(); - boolean isGracefulShutdownEnabled(); + boolean isGracefulShutdownEnabled(); - Duration getGracefulShutdownInitialWait(); + Duration getGracefulShutdownInitialWait(); - boolean isHttp2Enabled(); + boolean isHttp2Enabled(); } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/HermesShutdownHandler.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/HermesShutdownHandler.java index 05925cecdd..7a48a08314 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/HermesShutdownHandler.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/HermesShutdownHandler.java @@ -4,82 +4,84 @@ import io.undertow.server.HttpHandler; import io.undertow.server.HttpServerExchange; import io.undertow.util.StatusCodes; +import java.util.concurrent.atomic.AtomicInteger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.common.metric.MetricsFacade; -import java.util.concurrent.atomic.AtomicInteger; - public class HermesShutdownHandler implements HttpHandler { - private static final Logger logger = LoggerFactory.getLogger(HermesShutdownHandler.class); + private static final Logger logger = LoggerFactory.getLogger(HermesShutdownHandler.class); - private static final int MILLIS = 1000; - private static final int MAX_INFLIGHT_RETRIES = 20; - private static final int TOLERANCE_BYTES = 5; + private static final int MILLIS = 1000; + private static final int MAX_INFLIGHT_RETRIES = 20; + private static final int TOLERANCE_BYTES = 5; - private final HttpHandler next; - private final MetricsFacade metrics; - private final ExchangeCompletionListener completionListener = new GracefulExchangeCompletionListener(); - private final AtomicInteger inflightRequests = new AtomicInteger(); - private volatile boolean shutdown = false; + private final HttpHandler next; + private final MetricsFacade metrics; + private final ExchangeCompletionListener completionListener = + new GracefulExchangeCompletionListener(); + private final AtomicInteger inflightRequests = new AtomicInteger(); + private volatile boolean shutdown = false; + public HermesShutdownHandler(HttpHandler next, MetricsFacade metrics) { + this.next = next; + this.metrics = metrics; + metrics.producer().registerProducerInflightRequestGauge(inflightRequests, AtomicInteger::get); + } - public HermesShutdownHandler(HttpHandler next, MetricsFacade metrics) { - this.next = next; - this.metrics = metrics; - metrics.producer().registerProducerInflightRequestGauge(inflightRequests, AtomicInteger::get); + @Override + public void handleRequest(HttpServerExchange exchange) throws Exception { + if (shutdown) { + exchange.setStatusCode(StatusCodes.SERVICE_UNAVAILABLE); + exchange.endExchange(); + return; } + exchange.addExchangeCompleteListener(completionListener); + inflightRequests.incrementAndGet(); + next.handleRequest(exchange); + } - @Override - public void handleRequest(HttpServerExchange exchange) throws Exception { - if (shutdown) { - exchange.setStatusCode(StatusCodes.SERVICE_UNAVAILABLE); - exchange.endExchange(); - return; - } - exchange.addExchangeCompleteListener(completionListener); - inflightRequests.incrementAndGet(); - next.handleRequest(exchange); - } + public void handleShutdown() throws InterruptedException { + shutdown = true; + logger.info("Waiting for inflight requests to complete"); + awaitRequestsComplete(); + logger.info("Awaiting buffer flush"); + awaitBufferFlush(); + logger.info("Shutdown complete"); + } - public void handleShutdown() throws InterruptedException { - shutdown = true; - logger.info("Waiting for inflight requests to complete"); - awaitRequestsComplete(); - logger.info("Awaiting buffer flush"); - awaitBufferFlush(); - logger.info("Shutdown complete"); + private void awaitRequestsComplete() throws InterruptedException { + int retries = MAX_INFLIGHT_RETRIES; + while (inflightRequests.get() > 0 && retries > 0) { + logger.info( + "Inflight requests: {}, timing out in {} ms", inflightRequests.get(), retries * MILLIS); + retries--; + Thread.sleep(MILLIS); } + } - private void awaitRequestsComplete() throws InterruptedException { - int retries = MAX_INFLIGHT_RETRIES; - while (inflightRequests.get() > 0 && retries > 0) { - logger.info("Inflight requests: {}, timing out in {} ms", inflightRequests.get(), retries * MILLIS); - retries--; - Thread.sleep(MILLIS); - } + private void awaitBufferFlush() throws InterruptedException { + while (!isBufferEmpty()) { + Thread.sleep(MILLIS); } + } - private void awaitBufferFlush() throws InterruptedException { - while (!isBufferEmpty()) { - Thread.sleep(MILLIS); - } - } - - private boolean isBufferEmpty() { - long bufferUsedBytes = (long) (metrics.producer().getBufferTotalBytes() + private boolean isBufferEmpty() { + long bufferUsedBytes = + (long) + (metrics.producer().getBufferTotalBytes() - metrics.producer().getBufferAvailableBytes()); - logger.info("Buffer flush: {} bytes still in use", bufferUsedBytes); - return bufferUsedBytes < TOLERANCE_BYTES; - } + logger.info("Buffer flush: {} bytes still in use", bufferUsedBytes); + return bufferUsedBytes < TOLERANCE_BYTES; + } - private final class GracefulExchangeCompletionListener implements ExchangeCompletionListener { + private final class GracefulExchangeCompletionListener implements ExchangeCompletionListener { - @Override - public void exchangeEvent(HttpServerExchange exchange, NextListener nextListener) { - inflightRequests.decrementAndGet(); - nextListener.proceed(); - } + @Override + public void exchangeEvent(HttpServerExchange exchange, NextListener nextListener) { + inflightRequests.decrementAndGet(); + nextListener.proceed(); } + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/PrometheusMetricsHandler.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/PrometheusMetricsHandler.java index dcfd45af38..c941a6d587 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/PrometheusMetricsHandler.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/PrometheusMetricsHandler.java @@ -1,27 +1,26 @@ package pl.allegro.tech.hermes.frontend.server; +import static io.undertow.util.StatusCodes.OK; + import io.micrometer.prometheus.PrometheusMeterRegistry; import io.undertow.server.HttpHandler; import io.undertow.server.HttpServerExchange; -import static io.undertow.util.StatusCodes.OK; - public class PrometheusMetricsHandler implements HttpHandler { - private final PrometheusMeterRegistry meterRegistry; - - public PrometheusMetricsHandler(PrometheusMeterRegistry meterRegistry) { - this.meterRegistry = meterRegistry; - } + private final PrometheusMeterRegistry meterRegistry; - @Override - public void handleRequest(HttpServerExchange exchange) { - response(exchange, OK, meterRegistry.scrape()); - } + public PrometheusMetricsHandler(PrometheusMeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } - private void response(HttpServerExchange exchange, int status, String data) { - exchange.setStatusCode(status); - exchange.getResponseSender().send(data); - } + @Override + public void handleRequest(HttpServerExchange exchange) { + response(exchange, OK, meterRegistry.scrape()); + } + private void response(HttpServerExchange exchange, int status, String data) { + exchange.setStatusCode(status); + exchange.getResponseSender().send(data); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/ReadinessCheckHandler.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/ReadinessCheckHandler.java index cb93af2333..d745595a2b 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/ReadinessCheckHandler.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/ReadinessCheckHandler.java @@ -1,43 +1,43 @@ package pl.allegro.tech.hermes.frontend.server; +import static io.undertow.util.StatusCodes.OK; +import static io.undertow.util.StatusCodes.SERVICE_UNAVAILABLE; + import io.undertow.server.HttpHandler; import io.undertow.server.HttpServerExchange; import pl.allegro.tech.hermes.frontend.readiness.HealthCheckService; import pl.allegro.tech.hermes.frontend.readiness.ReadinessChecker; -import static io.undertow.util.StatusCodes.OK; -import static io.undertow.util.StatusCodes.SERVICE_UNAVAILABLE; - public class ReadinessCheckHandler implements HttpHandler { - private final ReadinessChecker readinessChecker; - private final HealthCheckService healthCheckService; - - public ReadinessCheckHandler(ReadinessChecker readinessChecker, HealthCheckService healthCheckService) { - this.readinessChecker = readinessChecker; - this.healthCheckService = healthCheckService; + private final ReadinessChecker readinessChecker; + private final HealthCheckService healthCheckService; + + public ReadinessCheckHandler( + ReadinessChecker readinessChecker, HealthCheckService healthCheckService) { + this.readinessChecker = readinessChecker; + this.healthCheckService = healthCheckService; + } + + @Override + public void handleRequest(HttpServerExchange exchange) { + if (!healthCheckService.isShutdown() && readinessChecker.isReady()) { + success(exchange); + } else { + unavailable(exchange); } + } - @Override - public void handleRequest(HttpServerExchange exchange) { - if (!healthCheckService.isShutdown() && readinessChecker.isReady()) { - success(exchange); - } else { - unavailable(exchange); - } - } - - private void success(HttpServerExchange exchange) { - response(exchange, OK, "READY"); - } + private void success(HttpServerExchange exchange) { + response(exchange, OK, "READY"); + } - private void unavailable(HttpServerExchange exchange) { - response(exchange, SERVICE_UNAVAILABLE, "NOT_READY"); - } - - private void response(HttpServerExchange exchange, int status, String data) { - exchange.setStatusCode(status); - exchange.getResponseSender().send(data); - } + private void unavailable(HttpServerExchange exchange) { + response(exchange, SERVICE_UNAVAILABLE, "NOT_READY"); + } + private void response(HttpServerExchange exchange, int status, String data) { + exchange.setStatusCode(status); + exchange.getResponseSender().send(data); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/SchemaLoadingResult.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/SchemaLoadingResult.java index 052f5aa71d..2b4081a139 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/SchemaLoadingResult.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/SchemaLoadingResult.java @@ -4,40 +4,42 @@ final class SchemaLoadingResult { - enum Type { - SUCCESS, FAILURE, MISSING - } + enum Type { + SUCCESS, + FAILURE, + MISSING + } - private final Type type; + private final Type type; - private final Topic topic; + private final Topic topic; - private SchemaLoadingResult(Type type, Topic topic) { - this.type = type; - this.topic = topic; - } + private SchemaLoadingResult(Type type, Topic topic) { + this.type = type; + this.topic = topic; + } - static SchemaLoadingResult success(Topic topic) { - return new SchemaLoadingResult(Type.SUCCESS, topic); - } + static SchemaLoadingResult success(Topic topic) { + return new SchemaLoadingResult(Type.SUCCESS, topic); + } - static SchemaLoadingResult failure(Topic topic) { - return new SchemaLoadingResult(Type.FAILURE, topic); - } + static SchemaLoadingResult failure(Topic topic) { + return new SchemaLoadingResult(Type.FAILURE, topic); + } - static SchemaLoadingResult missing(Topic topic) { - return new SchemaLoadingResult(Type.MISSING, topic); - } + static SchemaLoadingResult missing(Topic topic) { + return new SchemaLoadingResult(Type.MISSING, topic); + } - Type getType() { - return type; - } + Type getType() { + return type; + } - Topic getTopic() { - return topic; - } + Topic getTopic() { + return topic; + } - boolean isFailure() { - return Type.FAILURE == type; - } + boolean isFailure() { + return Type.FAILURE == type; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/SslContextFactoryProvider.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/SslContextFactoryProvider.java index 256065084e..9e9d9aded6 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/SslContextFactoryProvider.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/SslContextFactoryProvider.java @@ -1,5 +1,9 @@ package pl.allegro.tech.hermes.frontend.server; +import static pl.allegro.tech.hermes.common.ssl.KeystoreSource.JRE; +import static pl.allegro.tech.hermes.common.ssl.KeystoreSource.PROVIDED; + +import java.util.Optional; import pl.allegro.tech.hermes.common.ssl.DefaultSslContextFactory; import pl.allegro.tech.hermes.common.ssl.KeyManagersProvider; import pl.allegro.tech.hermes.common.ssl.KeystoreConfigurationException; @@ -12,61 +16,57 @@ import pl.allegro.tech.hermes.common.ssl.provided.ProvidedKeyManagersProvider; import pl.allegro.tech.hermes.common.ssl.provided.ProvidedTrustManagersProvider; -import java.util.Optional; +public class SslContextFactoryProvider { -import static pl.allegro.tech.hermes.common.ssl.KeystoreSource.JRE; -import static pl.allegro.tech.hermes.common.ssl.KeystoreSource.PROVIDED; + private final SslContextFactory sslContextFactory; + private final SslParameters sslParameters; -public class SslContextFactoryProvider { + public SslContextFactoryProvider( + SslContextFactory sslContextFactory, SslParameters sslParameters) { + this.sslContextFactory = sslContextFactory; + this.sslParameters = sslParameters; + } - private final SslContextFactory sslContextFactory; - private final SslParameters sslParameters; + public SslContextFactory getSslContextFactory() { + return Optional.ofNullable(sslContextFactory).orElse(getDefault()); + } - public SslContextFactoryProvider(SslContextFactory sslContextFactory, SslParameters sslParameters) { - this.sslContextFactory = sslContextFactory; - this.sslParameters = sslParameters; - } + private SslContextFactory getDefault() { + String protocol = sslParameters.getProtocol(); + KeyManagersProvider keyManagersProvider = createKeyManagersProvider(); + TrustManagersProvider trustManagersProvider = createTrustManagersProvider(); + return new DefaultSslContextFactory(protocol, keyManagersProvider, trustManagersProvider); + } - public SslContextFactory getSslContextFactory() { - return Optional.ofNullable(sslContextFactory).orElse(getDefault()); + private KeyManagersProvider createKeyManagersProvider() { + String keystoreSource = sslParameters.getKeystoreSource(); + if (PROVIDED.getValue().equals(keystoreSource)) { + KeystoreProperties properties = + new KeystoreProperties( + sslParameters.getKeystoreLocation(), + sslParameters.getKeystoreFormat(), + sslParameters.getKeystorePassword()); + return new ProvidedKeyManagersProvider(properties); } - - private SslContextFactory getDefault() { - String protocol = sslParameters.getProtocol(); - KeyManagersProvider keyManagersProvider = createKeyManagersProvider(); - TrustManagersProvider trustManagersProvider = createTrustManagersProvider(); - return new DefaultSslContextFactory(protocol, keyManagersProvider, trustManagersProvider); + if (JRE.getValue().equals(keystoreSource)) { + return new JvmKeyManagersProvider(); } + throw new KeystoreConfigurationException(keystoreSource); + } - private KeyManagersProvider createKeyManagersProvider() { - String keystoreSource = sslParameters.getKeystoreSource(); - if (PROVIDED.getValue().equals(keystoreSource)) { - KeystoreProperties properties = new KeystoreProperties( - sslParameters.getKeystoreLocation(), - sslParameters.getKeystoreFormat(), - sslParameters.getKeystorePassword() - ); - return new ProvidedKeyManagersProvider(properties); - } - if (JRE.getValue().equals(keystoreSource)) { - return new JvmKeyManagersProvider(); - } - throw new KeystoreConfigurationException(keystoreSource); + public TrustManagersProvider createTrustManagersProvider() { + String truststoreSource = sslParameters.getTruststoreSource(); + if (PROVIDED.getValue().equals(truststoreSource)) { + KeystoreProperties properties = + new KeystoreProperties( + sslParameters.getTruststoreLocation(), + sslParameters.getTruststoreFormat(), + sslParameters.getTruststorePassword()); + return new ProvidedTrustManagersProvider(properties); } - - public TrustManagersProvider createTrustManagersProvider() { - String truststoreSource = sslParameters.getTruststoreSource(); - if (PROVIDED.getValue().equals(truststoreSource)) { - KeystoreProperties properties = new KeystoreProperties( - sslParameters.getTruststoreLocation(), - sslParameters.getTruststoreFormat(), - sslParameters.getTruststorePassword() - ); - return new ProvidedTrustManagersProvider(properties); - } - if (JRE.getValue().equals(truststoreSource)) { - return new JvmTrustManagerProvider(); - } - throw new TruststoreConfigurationException(truststoreSource); + if (JRE.getValue().equals(truststoreSource)) { + return new JvmTrustManagerProvider(); } + throw new TruststoreConfigurationException(truststoreSource); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/SslParameters.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/SslParameters.java index cf2ff2ff43..971f1b9d68 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/SslParameters.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/SslParameters.java @@ -2,27 +2,27 @@ public interface SslParameters { - boolean isEnabled(); + boolean isEnabled(); - int getPort(); + int getPort(); - String getClientAuthMode(); + String getClientAuthMode(); - String getProtocol(); + String getProtocol(); - String getKeystoreSource(); + String getKeystoreSource(); - String getKeystoreLocation(); + String getKeystoreLocation(); - String getKeystorePassword(); + String getKeystorePassword(); - String getKeystoreFormat(); + String getKeystoreFormat(); - String getTruststoreSource(); + String getTruststoreSource(); - String getTruststoreLocation(); + String getTruststoreLocation(); - String getTruststorePassword(); + String getTruststorePassword(); - String getTruststoreFormat(); + String getTruststoreFormat(); } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/TopicSchemaLoader.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/TopicSchemaLoader.java index 375cea9cc5..2b091c9600 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/TopicSchemaLoader.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/TopicSchemaLoader.java @@ -1,6 +1,13 @@ package pl.allegro.tech.hermes.frontend.server; +import static java.util.concurrent.CompletableFuture.completedFuture; + import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; import net.jodah.failsafe.ExecutionContext; import net.jodah.failsafe.Failsafe; import net.jodah.failsafe.RetryPolicy; @@ -11,57 +18,60 @@ import pl.allegro.tech.hermes.schema.SchemaNotFoundException; import pl.allegro.tech.hermes.schema.SchemaRepository; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; - -import static java.util.concurrent.CompletableFuture.completedFuture; - class TopicSchemaLoader implements AutoCloseable { - private static final Logger logger = LoggerFactory.getLogger(TopicSchemaLoader.class); + private static final Logger logger = LoggerFactory.getLogger(TopicSchemaLoader.class); - private final SchemaRepository schemaRepository; + private final SchemaRepository schemaRepository; - private final ScheduledExecutorService scheduler; + private final ScheduledExecutorService scheduler; - private final RetryPolicy retryPolicy; + private final RetryPolicy retryPolicy; - TopicSchemaLoader(SchemaRepository schemaRepository, int retryCount, int threadPoolSize) { - this.schemaRepository = schemaRepository; + TopicSchemaLoader(SchemaRepository schemaRepository, int retryCount, int threadPoolSize) { + this.schemaRepository = schemaRepository; - ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat("topic-schema-loader-%d").build(); - this.scheduler = Executors.newScheduledThreadPool(threadPoolSize, threadFactory); - this.retryPolicy = new RetryPolicy() - .withMaxRetries(retryCount) - .handleIf((resp, cause) -> resp.isFailure()); - } + ThreadFactory threadFactory = + new ThreadFactoryBuilder().setNameFormat("topic-schema-loader-%d").build(); + this.scheduler = Executors.newScheduledThreadPool(threadPoolSize, threadFactory); + this.retryPolicy = + new RetryPolicy() + .withMaxRetries(retryCount) + .handleIf((resp, cause) -> resp.isFailure()); + } - CompletableFuture loadTopicSchema(Topic topic) { - return Failsafe.with(retryPolicy).with(scheduler) - .getStageAsync((context) -> completedFuture(loadLatestSchema(topic, context))); - } + CompletableFuture loadTopicSchema(Topic topic) { + return Failsafe.with(retryPolicy) + .with(scheduler) + .getStageAsync((context) -> completedFuture(loadLatestSchema(topic, context))); + } - private SchemaLoadingResult loadLatestSchema(Topic topic, ExecutionContext context) { - int attempt = context.getAttemptCount(); - try { - schemaRepository.getLatestAvroSchema(topic); - logger.info("Successfully loaded schema for topic {}, attempt #{}", topic.getQualifiedName(), attempt); - return SchemaLoadingResult.success(topic); - } catch (SchemaNotFoundException e) { - logger.warn("Failed to load schema for topic {}, attempt #{}. {}", topic.getQualifiedName(), attempt, e.getMessage()); - return SchemaLoadingResult.missing(topic); - } catch (CouldNotLoadSchemaException e) { - logger.error("Failed to load schema for topic {}, attempt #{}", topic.getQualifiedName(), attempt, e); - } - return SchemaLoadingResult.failure(topic); + private SchemaLoadingResult loadLatestSchema(Topic topic, ExecutionContext context) { + int attempt = context.getAttemptCount(); + try { + schemaRepository.getLatestAvroSchema(topic); + logger.info( + "Successfully loaded schema for topic {}, attempt #{}", + topic.getQualifiedName(), + attempt); + return SchemaLoadingResult.success(topic); + } catch (SchemaNotFoundException e) { + logger.warn( + "Failed to load schema for topic {}, attempt #{}. {}", + topic.getQualifiedName(), + attempt, + e.getMessage()); + return SchemaLoadingResult.missing(topic); + } catch (CouldNotLoadSchemaException e) { + logger.error( + "Failed to load schema for topic {}, attempt #{}", topic.getQualifiedName(), attempt, e); } + return SchemaLoadingResult.failure(topic); + } - @Override - public void close() throws Exception { - scheduler.shutdown(); - scheduler.awaitTermination(1, TimeUnit.SECONDS); - } + @Override + public void close() throws Exception { + scheduler.shutdown(); + scheduler.awaitTermination(1, TimeUnit.SECONDS); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/TopicSchemaLoadingStartupHook.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/TopicSchemaLoadingStartupHook.java index 33fad952ad..f668c780a1 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/TopicSchemaLoadingStartupHook.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/TopicSchemaLoadingStartupHook.java @@ -1,5 +1,17 @@ package pl.allegro.tech.hermes.frontend.server; +import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toList; +import static pl.allegro.tech.hermes.frontend.server.SchemaLoadingResult.Type.FAILURE; +import static pl.allegro.tech.hermes.frontend.server.SchemaLoadingResult.Type.MISSING; +import static pl.allegro.tech.hermes.frontend.server.SchemaLoadingResult.Type.SUCCESS; +import static pl.allegro.tech.hermes.frontend.utils.CompletableFuturesHelper.allComplete; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.ContentType; @@ -9,94 +21,94 @@ import pl.allegro.tech.hermes.frontend.server.SchemaLoadingResult.Type; import pl.allegro.tech.hermes.schema.SchemaRepository; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; - -import static java.util.stream.Collectors.joining; -import static java.util.stream.Collectors.toList; -import static pl.allegro.tech.hermes.frontend.server.SchemaLoadingResult.Type.FAILURE; -import static pl.allegro.tech.hermes.frontend.server.SchemaLoadingResult.Type.MISSING; -import static pl.allegro.tech.hermes.frontend.server.SchemaLoadingResult.Type.SUCCESS; -import static pl.allegro.tech.hermes.frontend.utils.CompletableFuturesHelper.allComplete; - public class TopicSchemaLoadingStartupHook { - private static final Logger logger = LoggerFactory.getLogger(TopicSchemaLoadingStartupHook.class); + private static final Logger logger = LoggerFactory.getLogger(TopicSchemaLoadingStartupHook.class); - private final TopicsCache topicsCache; + private final TopicsCache topicsCache; - private final SchemaRepository schemaRepository; + private final SchemaRepository schemaRepository; - private final int retryCount; + private final int retryCount; - private final int threadPoolSize; + private final int threadPoolSize; - private final boolean isTopicSchemaLoadingStartupHookEnabled; + private final boolean isTopicSchemaLoadingStartupHookEnabled; - public TopicSchemaLoadingStartupHook(TopicsCache topicsCache, - SchemaRepository schemaRepository, - int retryCount, - int threadPoolSize, - boolean isTopicSchemaLoadingStartupHookEnabled) { - this.topicsCache = topicsCache; - this.schemaRepository = schemaRepository; - this.retryCount = retryCount; - this.threadPoolSize = threadPoolSize; - this.isTopicSchemaLoadingStartupHookEnabled = isTopicSchemaLoadingStartupHookEnabled; - } + public TopicSchemaLoadingStartupHook( + TopicsCache topicsCache, + SchemaRepository schemaRepository, + int retryCount, + int threadPoolSize, + boolean isTopicSchemaLoadingStartupHookEnabled) { + this.topicsCache = topicsCache; + this.schemaRepository = schemaRepository; + this.retryCount = retryCount; + this.threadPoolSize = threadPoolSize; + this.isTopicSchemaLoadingStartupHookEnabled = isTopicSchemaLoadingStartupHookEnabled; + } - public void run() { - if (isTopicSchemaLoadingStartupHookEnabled) { - long start = System.currentTimeMillis(); - logger.info("Loading topic schemas"); - List topics = getAvroTopics(); - List allResults = loadSchemasForTopics(topics); - logResultInfo(allResults, System.currentTimeMillis() - start); - } else { - logger.info("Loading topic schemas is disabled"); - } + public void run() { + if (isTopicSchemaLoadingStartupHookEnabled) { + long start = System.currentTimeMillis(); + logger.info("Loading topic schemas"); + List topics = getAvroTopics(); + List allResults = loadSchemasForTopics(topics); + logResultInfo(allResults, System.currentTimeMillis() - start); + } else { + logger.info("Loading topic schemas is disabled"); } - - private List getAvroTopics() { - return topicsCache.getTopics().stream() - .map(CachedTopic::getTopic) - .filter(topic -> ContentType.AVRO == topic.getContentType()) - .collect(toList()); - } - - private List loadSchemasForTopics(List topics) { - try (TopicSchemaLoader loader = new TopicSchemaLoader(schemaRepository, retryCount, threadPoolSize)) { - return allComplete(topics.stream().map(loader::loadTopicSchema).collect(toList())).join(); - } catch (Exception e) { - logger.error("An error occurred while loading schema topics", e); - return Collections.emptyList(); - } - } - - private void logResultInfo(List allResults, long elapsed) { - Map> groupedResults = getGroupedResults(allResults); - Optional> successes = Optional.ofNullable(groupedResults.get(SUCCESS)); - Optional> missing = Optional.ofNullable(groupedResults.get(MISSING)); - Optional> failures = Optional.ofNullable(groupedResults.get(FAILURE)); - - logger.info("Finished loading schemas for {} topics in {}ms [successful: {}, missing: {}, failed: {}]. {}{}", - allResults.size(), elapsed, successes.map(List::size).orElse(0), - missing.map(List::size).orElse(0), failures.map(List::size).orElse(0), - missing.map(results -> String.format("Missing schema for: [%s]", topicsOfResults(results))).orElse(""), - failures.map(results -> String.format("Failed for: [%s]. ", topicsOfResults(results))).orElse("")); - } - - private String topicsOfResults(List results) { - return results.stream() - .map(SchemaLoadingResult::getTopic) - .map(Topic::getQualifiedName) - .collect(joining(", ")); - } - - private Map> getGroupedResults(List allResults) { - return allResults.stream().collect(Collectors.groupingBy(SchemaLoadingResult::getType, Collectors.toList())); + } + + private List getAvroTopics() { + return topicsCache.getTopics().stream() + .map(CachedTopic::getTopic) + .filter(topic -> ContentType.AVRO == topic.getContentType()) + .collect(toList()); + } + + private List loadSchemasForTopics(List topics) { + try (TopicSchemaLoader loader = + new TopicSchemaLoader(schemaRepository, retryCount, threadPoolSize)) { + return allComplete(topics.stream().map(loader::loadTopicSchema).collect(toList())).join(); + } catch (Exception e) { + logger.error("An error occurred while loading schema topics", e); + return Collections.emptyList(); } + } + + private void logResultInfo(List allResults, long elapsed) { + Map> groupedResults = getGroupedResults(allResults); + Optional> successes = + Optional.ofNullable(groupedResults.get(SUCCESS)); + Optional> missing = Optional.ofNullable(groupedResults.get(MISSING)); + Optional> failures = Optional.ofNullable(groupedResults.get(FAILURE)); + + logger.info( + "Finished loading schemas for {} topics in {}ms [successful: {}, missing: {}, failed: {}]. {}{}", + allResults.size(), + elapsed, + successes.map(List::size).orElse(0), + missing.map(List::size).orElse(0), + failures.map(List::size).orElse(0), + missing + .map(results -> String.format("Missing schema for: [%s]", topicsOfResults(results))) + .orElse(""), + failures + .map(results -> String.format("Failed for: [%s]. ", topicsOfResults(results))) + .orElse("")); + } + + private String topicsOfResults(List results) { + return results.stream() + .map(SchemaLoadingResult::getTopic) + .map(Topic::getQualifiedName) + .collect(joining(", ")); + } + + private Map> getGroupedResults( + List allResults) { + return allResults.stream() + .collect(Collectors.groupingBy(SchemaLoadingResult::getType, Collectors.toList())); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/auth/AuthenticationConfiguration.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/auth/AuthenticationConfiguration.java index a5b9cc63c5..8ae24c101c 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/auth/AuthenticationConfiguration.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/auth/AuthenticationConfiguration.java @@ -4,37 +4,39 @@ import io.undertow.security.api.AuthenticationMechanism; import io.undertow.security.idm.IdentityManager; import io.undertow.server.HttpServerExchange; - import java.util.List; import java.util.function.Predicate; public class AuthenticationConfiguration { - private final Predicate isAuthenticationRequiredPredicate; - private final List authMechanisms; - private final IdentityManager identityManager; - - public AuthenticationConfiguration(Predicate isAuthenticationRequiredPredicate, - List authMechanisms, - IdentityManager identityManager) { - Preconditions.checkNotNull(isAuthenticationRequiredPredicate, "IsAuthenticationRequired predicate has to be provided"); - Preconditions.checkArgument(!authMechanisms.isEmpty(), "At least one AuthenticationMechanism has to be provided."); - Preconditions.checkNotNull(identityManager, "IdentityManager has to be provided"); - - this.isAuthenticationRequiredPredicate = isAuthenticationRequiredPredicate; - this.authMechanisms = authMechanisms; - this.identityManager = identityManager; - } - - public Predicate getIsAuthenticationRequiredPredicate() { - return isAuthenticationRequiredPredicate; - } - - public List getAuthMechanisms() { - return authMechanisms; - } - - public IdentityManager getIdentityManager() { - return identityManager; - } + private final Predicate isAuthenticationRequiredPredicate; + private final List authMechanisms; + private final IdentityManager identityManager; + + public AuthenticationConfiguration( + Predicate isAuthenticationRequiredPredicate, + List authMechanisms, + IdentityManager identityManager) { + Preconditions.checkNotNull( + isAuthenticationRequiredPredicate, "IsAuthenticationRequired predicate has to be provided"); + Preconditions.checkArgument( + !authMechanisms.isEmpty(), "At least one AuthenticationMechanism has to be provided."); + Preconditions.checkNotNull(identityManager, "IdentityManager has to be provided"); + + this.isAuthenticationRequiredPredicate = isAuthenticationRequiredPredicate; + this.authMechanisms = authMechanisms; + this.identityManager = identityManager; + } + + public Predicate getIsAuthenticationRequiredPredicate() { + return isAuthenticationRequiredPredicate; + } + + public List getAuthMechanisms() { + return authMechanisms; + } + + public IdentityManager getIdentityManager() { + return identityManager; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/auth/AuthenticationPredicateAwareConstraintHandler.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/auth/AuthenticationPredicateAwareConstraintHandler.java index f6c60d0899..9598772c9e 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/auth/AuthenticationPredicateAwareConstraintHandler.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/auth/AuthenticationPredicateAwareConstraintHandler.java @@ -3,20 +3,20 @@ import io.undertow.security.handlers.AuthenticationConstraintHandler; import io.undertow.server.HttpHandler; import io.undertow.server.HttpServerExchange; - import java.util.function.Predicate; public class AuthenticationPredicateAwareConstraintHandler extends AuthenticationConstraintHandler { - private final Predicate predicate; + private final Predicate predicate; - public AuthenticationPredicateAwareConstraintHandler(HttpHandler next, Predicate predicate) { - super(next); - this.predicate = predicate; - } + public AuthenticationPredicateAwareConstraintHandler( + HttpHandler next, Predicate predicate) { + super(next); + this.predicate = predicate; + } - @Override - protected boolean isAuthenticationRequired(HttpServerExchange exchange) { - return predicate.test(exchange); - } + @Override + protected boolean isAuthenticationRequired(HttpServerExchange exchange) { + return predicate.test(exchange); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/auth/Roles.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/auth/Roles.java index a996ddaf4a..0fc702aeec 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/auth/Roles.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/server/auth/Roles.java @@ -1,5 +1,5 @@ package pl.allegro.tech.hermes.frontend.server.auth; public interface Roles { - String PUBLISHER = "PUBLISHER"; + String PUBLISHER = "PUBLISHER"; } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/utils/CompletableFuturesHelper.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/utils/CompletableFuturesHelper.java index b3db4b9760..a5952dd397 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/utils/CompletableFuturesHelper.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/utils/CompletableFuturesHelper.java @@ -1,14 +1,14 @@ package pl.allegro.tech.hermes.frontend.utils; +import static java.util.stream.Collectors.toList; + import java.util.List; import java.util.concurrent.CompletableFuture; -import static java.util.stream.Collectors.toList; - public class CompletableFuturesHelper { - public static CompletableFuture> allComplete(List> futures) { - return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) - .thenApply(v -> futures.stream().map(CompletableFuture::join).collect(toList())); - } + public static CompletableFuture> allComplete(List> futures) { + return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) + .thenApply(v -> futures.stream().map(CompletableFuture::join).collect(toList())); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/validator/InvalidMessageException.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/validator/InvalidMessageException.java index 49d2c74616..1d545b31ec 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/validator/InvalidMessageException.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/validator/InvalidMessageException.java @@ -1,19 +1,18 @@ package pl.allegro.tech.hermes.frontend.validator; import com.google.common.base.Joiner; +import java.util.List; import pl.allegro.tech.hermes.api.ErrorCode; import pl.allegro.tech.hermes.common.exception.HermesException; -import java.util.List; - public class InvalidMessageException extends HermesException { - public InvalidMessageException(String msg, List reasons) { - super(String.format("%s Errors: %s", msg, Joiner.on(";").join(reasons))); - } + public InvalidMessageException(String msg, List reasons) { + super(String.format("%s Errors: %s", msg, Joiner.on(";").join(reasons))); + } - @Override - public ErrorCode getCode() { - return ErrorCode.FORMAT_ERROR; - } + @Override + public ErrorCode getCode() { + return ErrorCode.FORMAT_ERROR; + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/validator/MessageValidators.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/validator/MessageValidators.java index 3ae4de3a3b..67301dd412 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/validator/MessageValidators.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/validator/MessageValidators.java @@ -1,20 +1,18 @@ package pl.allegro.tech.hermes.frontend.validator; +import java.util.List; import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.frontend.publishing.message.Message; -import java.util.List; - public class MessageValidators { - private final List messageValidators; - - public MessageValidators(List messageValidators) { - this.messageValidators = messageValidators; - } + private final List messageValidators; - public void check(Topic topic, Message message) { - messageValidators.forEach(v -> v.check(message, topic)); - } + public MessageValidators(List messageValidators) { + this.messageValidators = messageValidators; + } + public void check(Topic topic, Message message) { + messageValidators.forEach(v -> v.check(message, topic)); + } } diff --git a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/validator/TopicMessageValidator.java b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/validator/TopicMessageValidator.java index f5f37610c3..5cc002fb73 100644 --- a/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/validator/TopicMessageValidator.java +++ b/hermes-frontend/src/main/java/pl/allegro/tech/hermes/frontend/validator/TopicMessageValidator.java @@ -4,5 +4,5 @@ import pl.allegro.tech.hermes.frontend.publishing.message.Message; public interface TopicMessageValidator { - void check(Message message, Topic topic); + void check(Message message, Topic topic); } diff --git a/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/buffer/BackupFilesManagerTest.java b/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/buffer/BackupFilesManagerTest.java index 3e83fd364c..febb4c09bc 100644 --- a/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/buffer/BackupFilesManagerTest.java +++ b/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/buffer/BackupFilesManagerTest.java @@ -1,11 +1,8 @@ package pl.allegro.tech.hermes.frontend.buffer; -import com.google.common.io.Files; -import org.apache.commons.io.FileUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import static org.assertj.core.api.Assertions.assertThat; +import com.google.common.io.Files; import java.io.File; import java.io.IOException; import java.time.Clock; @@ -13,110 +10,118 @@ import java.time.ZoneId; import java.util.List; import java.util.Optional; - -import static org.assertj.core.api.Assertions.assertThat; +import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; public class BackupFilesManagerTest { - private final Clock clock = Clock.fixed(Instant.ofEpochMilli(12345), ZoneId.systemDefault()); - - private File tempDir; - - @Before - public void setup() { - tempDir = Files.createTempDir(); - } - - @After - public void cleanup() throws IOException { - FileUtils.deleteDirectory(tempDir); - } - - @Test - public void shouldRolloverExistingBackupFile() throws IOException { - // given - new File(tempDir, "hermes-buffer-v3.dat").createNewFile(); - BackupFilesManager backupFilesManager = new BackupFilesManager(tempDir.getAbsolutePath(), clock); - - // when - Optional backupFile = backupFilesManager.rolloverBackupFileIfExists(); - - // then - assertThat(backupFile.get().getName()).isEqualTo("hermes-buffer-v3-12345.dat"); - } - - @Test - public void shouldReadBackupFilesList() throws IOException { - // given - final BackupFilesManager backupFilesManager = new BackupFilesManager(tempDir.getAbsolutePath(), clock); - File timestampedBackup1 = new File(tempDir, "hermes-buffer-v3-001.dat"); - File timestampedBackup2 = new File(tempDir, "hermes-buffer-v3-002.dat"); - File customBackup = new File(tempDir, "hermes-buffer-v3-old.dat"); - - // and - timestampedBackup1.createNewFile(); - timestampedBackup2.createNewFile(); - customBackup.createNewFile(); - - // when - List backups = backupFilesManager.getRolledBackupFiles(); - - // then - assertThat(backups).containsOnly(timestampedBackup1, timestampedBackup2); - } - - @Test - public void shouldReadEmptyBackupFileList() { - // given - BackupFilesManager backupFilesManager = new BackupFilesManager(tempDir.getAbsolutePath(), clock); - - // when - List backups = backupFilesManager.getRolledBackupFiles(); - - // then - assertThat(backups).isEmpty(); - } - - @Test - public void shouldNotRolloverNotExistsBackupFile() { - // given - BackupFilesManager backupFilesManager = new BackupFilesManager(tempDir.getAbsolutePath(), clock); - // when - Optional backupFile = backupFilesManager.rolloverBackupFileIfExists(); - - // when & then - assertThat(backupFile.isPresent()).isFalse(); - } - - @Test - public void shouldUseNewNamingSchemeWhenNoFilesAvailable() { - // given - BackupFilesManager backupFilesManager = new BackupFilesManager(tempDir.getAbsolutePath(), clock); - - // when - File file = backupFilesManager.getCurrentBackupFile(); - - // then - assertThat(file.getName()).isEqualTo("hermes-buffer-v3.dat"); - } - - @Test - public void shouldLoadAllTemporaryBackupV2Files() throws IOException { - // given - final BackupFilesManager backupFilesManager = new BackupFilesManager(tempDir.getAbsolutePath(), clock); - File temporaryBackup1 = new File(tempDir, "hermes-buffer-v2-old.tmp"); - File temporaryBackup2 = new File(tempDir, "hermes-buffer-002-v2-old.tmp"); - File customBackup = new File(tempDir, "hermes-buffer.dat"); - - // and - temporaryBackup1.createNewFile(); - temporaryBackup2.createNewFile(); - customBackup.createNewFile(); - - // when - List files = backupFilesManager.getTemporaryBackupV2Files(tempDir.getAbsolutePath()); - - // then - assertThat(files.size()).isEqualTo(2); - } -} \ No newline at end of file + private final Clock clock = Clock.fixed(Instant.ofEpochMilli(12345), ZoneId.systemDefault()); + + private File tempDir; + + @Before + public void setup() { + tempDir = Files.createTempDir(); + } + + @After + public void cleanup() throws IOException { + FileUtils.deleteDirectory(tempDir); + } + + @Test + public void shouldRolloverExistingBackupFile() throws IOException { + // given + new File(tempDir, "hermes-buffer-v3.dat").createNewFile(); + BackupFilesManager backupFilesManager = + new BackupFilesManager(tempDir.getAbsolutePath(), clock); + + // when + Optional backupFile = backupFilesManager.rolloverBackupFileIfExists(); + + // then + assertThat(backupFile.get().getName()).isEqualTo("hermes-buffer-v3-12345.dat"); + } + + @Test + public void shouldReadBackupFilesList() throws IOException { + // given + final BackupFilesManager backupFilesManager = + new BackupFilesManager(tempDir.getAbsolutePath(), clock); + File timestampedBackup1 = new File(tempDir, "hermes-buffer-v3-001.dat"); + File timestampedBackup2 = new File(tempDir, "hermes-buffer-v3-002.dat"); + File customBackup = new File(tempDir, "hermes-buffer-v3-old.dat"); + + // and + timestampedBackup1.createNewFile(); + timestampedBackup2.createNewFile(); + customBackup.createNewFile(); + + // when + List backups = backupFilesManager.getRolledBackupFiles(); + + // then + assertThat(backups).containsOnly(timestampedBackup1, timestampedBackup2); + } + + @Test + public void shouldReadEmptyBackupFileList() { + // given + BackupFilesManager backupFilesManager = + new BackupFilesManager(tempDir.getAbsolutePath(), clock); + + // when + List backups = backupFilesManager.getRolledBackupFiles(); + + // then + assertThat(backups).isEmpty(); + } + + @Test + public void shouldNotRolloverNotExistsBackupFile() { + // given + BackupFilesManager backupFilesManager = + new BackupFilesManager(tempDir.getAbsolutePath(), clock); + // when + Optional backupFile = backupFilesManager.rolloverBackupFileIfExists(); + + // when & then + assertThat(backupFile.isPresent()).isFalse(); + } + + @Test + public void shouldUseNewNamingSchemeWhenNoFilesAvailable() { + // given + BackupFilesManager backupFilesManager = + new BackupFilesManager(tempDir.getAbsolutePath(), clock); + + // when + File file = backupFilesManager.getCurrentBackupFile(); + + // then + assertThat(file.getName()).isEqualTo("hermes-buffer-v3.dat"); + } + + @Test + public void shouldLoadAllTemporaryBackupV2Files() throws IOException { + // given + final BackupFilesManager backupFilesManager = + new BackupFilesManager(tempDir.getAbsolutePath(), clock); + File temporaryBackup1 = new File(tempDir, "hermes-buffer-v2-old.tmp"); + File temporaryBackup2 = new File(tempDir, "hermes-buffer-002-v2-old.tmp"); + File customBackup = new File(tempDir, "hermes-buffer.dat"); + + // and + temporaryBackup1.createNewFile(); + temporaryBackup2.createNewFile(); + customBackup.createNewFile(); + + // when + List files = backupFilesManager.getTemporaryBackupV2Files(tempDir.getAbsolutePath()); + + // then + assertThat(files.size()).isEqualTo(2); + } +} diff --git a/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/buffer/BackupMessagesLoaderTest.java b/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/buffer/BackupMessagesLoaderTest.java index 528b0a0e38..ec40e054e9 100644 --- a/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/buffer/BackupMessagesLoaderTest.java +++ b/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/buffer/BackupMessagesLoaderTest.java @@ -1,8 +1,24 @@ package pl.allegro.tech.hermes.frontend.buffer; +import static java.time.LocalDateTime.now; +import static java.time.ZoneOffset.UTC; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + import com.google.common.io.Files; import io.micrometer.core.instrument.Timer; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import java.io.File; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; import org.apache.commons.io.FileUtils; import org.junit.After; import org.junit.Before; @@ -26,225 +42,213 @@ import pl.allegro.tech.hermes.tracker.frontend.NoOperationPublishingTracker; import pl.allegro.tech.hermes.tracker.frontend.Trackers; -import java.io.File; -import java.time.Duration; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import static java.time.LocalDateTime.now; -import static java.time.ZoneOffset.UTC; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - public class BackupMessagesLoaderTest { - private static final int ENTRIES = 100; - private static final int AVERAGE_MESSAGE_SIZE = 600; - - private final BrokerMessageProducer producer = mock(BrokerMessageProducer.class); - - private final BrokerListeners listeners = mock(BrokerListeners.class); - - private final TopicsCache topicsCache = mock(TopicsCache.class); - - private final Trackers trackers = mock(Trackers.class); - - private final CachedTopic cachedTopic = mock(CachedTopic.class); - - private final SchemaRepository schemaRepository = mock(SchemaRepository.class); - - private final SchemaExistenceEnsurer schemaExistenceEnsurer = mock(SchemaExistenceEnsurer.class); - - private File tempDir; - - private final Topic topic = TopicBuilder.topic("pl.allegro.tech.hermes.test").build(); - - @Before - public void setUp() { - tempDir = Files.createTempDir(); - - Timer micrometerTimer = new SimpleMeterRegistry().timer("broker-latency"); - when(cachedTopic.getTopic()).thenReturn(topic); - when(cachedTopic.startBrokerLatencyTimer()).thenReturn(HermesTimerContext.from(micrometerTimer)); - when(topicsCache.getTopic(topic.getQualifiedName())).thenReturn(Optional.of(cachedTopic)); - when(producer.isTopicAvailable(cachedTopic)).thenReturn(true); - } - - @After - public void tearDown() throws Exception { - FileUtils.deleteDirectory(tempDir); - } - - @Test - public void shouldNotSendOldMessages() { - //given - LocalMessageStorageProperties localMessageStorageProperties = new LocalMessageStorageProperties(); - localMessageStorageProperties.setMaxAge(Duration.ofHours(8)); - localMessageStorageProperties.setMaxResendRetries(2); - - MessageRepository messageRepository = new ChronicleMapMessageRepository( - new File(tempDir.getAbsoluteFile(), "messages.dat"), - ENTRIES, - AVERAGE_MESSAGE_SIZE - ); - - final BackupMessagesLoader backupMessagesLoader = - new BackupMessagesLoader( - producer, - producer, - listeners, - topicsCache, - schemaRepository, - schemaExistenceEnsurer, - trackers, - localMessageStorageProperties - ); - - messageRepository.save(messageOfAge(1), topic); - messageRepository.save(messageOfAge(10), topic); - messageRepository.save(messageOfAge(10), topic); - - //when - backupMessagesLoader.loadMessages(messageRepository.findAll()); - - //then - verify(producer, times(1)).send(any(JsonMessage.class), eq(cachedTopic), any(PublishingCallback.class)); - } - - @Test - public void shouldSendAndResendMessages() { - //given - int noOfSentCalls = 2; - LocalMessageStorageProperties localMessageStorageProperties = new LocalMessageStorageProperties(); - localMessageStorageProperties.setMaxAge(Duration.ofHours(8)); - localMessageStorageProperties.setMaxResendRetries(noOfSentCalls - 1); - when(trackers.get(eq(topic))).thenReturn(new NoOperationPublishingTracker()); - - doAnswer(invocation -> { - Object[] args = invocation.getArguments(); - ((PublishingCallback) args[2]).onUnpublished((Message) args[0], ((CachedTopic) args[1]).getTopic(), new Exception("test")); - return ""; - }).when(producer).send(any(JsonMessage.class), eq(cachedTopic), any(PublishingCallback.class)); - - - MessageRepository messageRepository = new ChronicleMapMessageRepository( - new File(tempDir.getAbsoluteFile(), "messages.dat"), - ENTRIES, - AVERAGE_MESSAGE_SIZE - ); - BackupMessagesLoader backupMessagesLoader = - new BackupMessagesLoader( - producer, - producer, - listeners, - topicsCache, - schemaRepository, - schemaExistenceEnsurer, - trackers, - localMessageStorageProperties - ); - - messageRepository.save(messageOfAge(1), topic); - - //when - backupMessagesLoader.loadMessages(messageRepository.findAll()); - - //then - verify(producer, times(noOfSentCalls)).send(any(JsonMessage.class), eq(cachedTopic), any(PublishingCallback.class)); - verify(listeners, times(noOfSentCalls)).onError(any(JsonMessage.class), eq(topic), any(Exception.class)); - } - - @Test - public void shouldSendOnlyWhenBrokerTopicIsAvailable() { - // given - LocalMessageStorageProperties localMessageStorageProperties = new LocalMessageStorageProperties(); - localMessageStorageProperties.setMaxAge(Duration.ofHours(8)); - - when(producer.isTopicAvailable(cachedTopic)).thenReturn(false).thenReturn(false).thenReturn(true); - - BackupMessagesLoader backupMessagesLoader = - new BackupMessagesLoader( - producer, - producer, - listeners, - topicsCache, - schemaRepository, - schemaExistenceEnsurer, - trackers, - localMessageStorageProperties - ); - MessageRepository messageRepository = new ChronicleMapMessageRepository( - new File(tempDir.getAbsoluteFile(), "messages.dat"), - ENTRIES, - AVERAGE_MESSAGE_SIZE - ); - messageRepository.save(messageOfAge(1), topic); - - // when - backupMessagesLoader.loadMessages(messageRepository.findAll()); - - // then - verify(producer, times(3)).isTopicAvailable(cachedTopic); - verify(producer, times(1)).send(any(JsonMessage.class), eq(cachedTopic), any(PublishingCallback.class)); - } - - @Test - public void shouldSendMessageWithAllArgumentsFromBackupMessage() { - //given - LocalMessageStorageProperties localMessageStorageProperties = new LocalMessageStorageProperties(); - localMessageStorageProperties.setMaxAge(Duration.ofHours(8)); - localMessageStorageProperties.setMaxResendRetries(2); - - MessageRepository messageRepository = new ChronicleMapMessageRepository( - new File(tempDir.getAbsoluteFile(), "messages.dat"), - ENTRIES, - AVERAGE_MESSAGE_SIZE - ); - - BackupMessagesLoader backupMessagesLoader = - new BackupMessagesLoader( - producer, - producer, - listeners, - topicsCache, - schemaRepository, - schemaExistenceEnsurer, - trackers, - localMessageStorageProperties - ); - - messageRepository.save(messageOfAge(1), topic); - - //when - List backupMessages = messageRepository.findAll(); - backupMessagesLoader.loadMessages(backupMessages); - - //then - ArgumentCaptor messageArgument = ArgumentCaptor.forClass(Message.class); - - verify(producer, times(1)).send(messageArgument.capture(), eq(cachedTopic), any(PublishingCallback.class)); - Message sendMessage = messageArgument.getValue(); - assertThat(sendMessage.getPartitionKey()).isEqualTo(backupMessages.get(0).getPartitionKey()); - assertThat(sendMessage.getData()).isEqualTo(backupMessages.get(0).getData()); - assertThat(sendMessage.getId()).isEqualTo(backupMessages.get(0).getMessageId()); - assertThat(sendMessage.getTimestamp()).isEqualTo(backupMessages.get(0).getTimestamp()); - assertThat(sendMessage.getHTTPHeaders().get("propagated-http-header")).isEqualTo("example-value"); - } - - private Message messageOfAge(int ageHours) { - return new JsonMessage( - MessageIdGenerator.generate(), - "{'a':'b'}".getBytes(), - now().minusHours(ageHours).toInstant(UTC).toEpochMilli(), - "partition-key", - Map.of("propagated-http-header", "example-value") - ); - } + private static final int ENTRIES = 100; + private static final int AVERAGE_MESSAGE_SIZE = 600; + + private final BrokerMessageProducer producer = mock(BrokerMessageProducer.class); + + private final BrokerListeners listeners = mock(BrokerListeners.class); + + private final TopicsCache topicsCache = mock(TopicsCache.class); + + private final Trackers trackers = mock(Trackers.class); + + private final CachedTopic cachedTopic = mock(CachedTopic.class); + + private final SchemaRepository schemaRepository = mock(SchemaRepository.class); + + private final SchemaExistenceEnsurer schemaExistenceEnsurer = mock(SchemaExistenceEnsurer.class); + + private File tempDir; + + private final Topic topic = TopicBuilder.topic("pl.allegro.tech.hermes.test").build(); + + @Before + public void setUp() { + tempDir = Files.createTempDir(); + + Timer micrometerTimer = new SimpleMeterRegistry().timer("broker-latency"); + when(cachedTopic.getTopic()).thenReturn(topic); + when(cachedTopic.startBrokerLatencyTimer()) + .thenReturn(HermesTimerContext.from(micrometerTimer)); + when(topicsCache.getTopic(topic.getQualifiedName())).thenReturn(Optional.of(cachedTopic)); + when(producer.isTopicAvailable(cachedTopic)).thenReturn(true); + } + + @After + public void tearDown() throws Exception { + FileUtils.deleteDirectory(tempDir); + } + + @Test + public void shouldNotSendOldMessages() { + // given + LocalMessageStorageProperties localMessageStorageProperties = + new LocalMessageStorageProperties(); + localMessageStorageProperties.setMaxAge(Duration.ofHours(8)); + localMessageStorageProperties.setMaxResendRetries(2); + + MessageRepository messageRepository = + new ChronicleMapMessageRepository( + new File(tempDir.getAbsoluteFile(), "messages.dat"), ENTRIES, AVERAGE_MESSAGE_SIZE); + + final BackupMessagesLoader backupMessagesLoader = + new BackupMessagesLoader( + producer, + producer, + listeners, + topicsCache, + schemaRepository, + schemaExistenceEnsurer, + trackers, + localMessageStorageProperties); + + messageRepository.save(messageOfAge(1), topic); + messageRepository.save(messageOfAge(10), topic); + messageRepository.save(messageOfAge(10), topic); + + // when + backupMessagesLoader.loadMessages(messageRepository.findAll()); + + // then + verify(producer, times(1)) + .send(any(JsonMessage.class), eq(cachedTopic), any(PublishingCallback.class)); + } + + @Test + public void shouldSendAndResendMessages() { + // given + int noOfSentCalls = 2; + LocalMessageStorageProperties localMessageStorageProperties = + new LocalMessageStorageProperties(); + localMessageStorageProperties.setMaxAge(Duration.ofHours(8)); + localMessageStorageProperties.setMaxResendRetries(noOfSentCalls - 1); + when(trackers.get(eq(topic))).thenReturn(new NoOperationPublishingTracker()); + + doAnswer( + invocation -> { + Object[] args = invocation.getArguments(); + ((PublishingCallback) args[2]) + .onUnpublished( + (Message) args[0], ((CachedTopic) args[1]).getTopic(), new Exception("test")); + return ""; + }) + .when(producer) + .send(any(JsonMessage.class), eq(cachedTopic), any(PublishingCallback.class)); + + MessageRepository messageRepository = + new ChronicleMapMessageRepository( + new File(tempDir.getAbsoluteFile(), "messages.dat"), ENTRIES, AVERAGE_MESSAGE_SIZE); + BackupMessagesLoader backupMessagesLoader = + new BackupMessagesLoader( + producer, + producer, + listeners, + topicsCache, + schemaRepository, + schemaExistenceEnsurer, + trackers, + localMessageStorageProperties); + + messageRepository.save(messageOfAge(1), topic); + + // when + backupMessagesLoader.loadMessages(messageRepository.findAll()); + + // then + verify(producer, times(noOfSentCalls)) + .send(any(JsonMessage.class), eq(cachedTopic), any(PublishingCallback.class)); + verify(listeners, times(noOfSentCalls)) + .onError(any(JsonMessage.class), eq(topic), any(Exception.class)); + } + + @Test + public void shouldSendOnlyWhenBrokerTopicIsAvailable() { + // given + LocalMessageStorageProperties localMessageStorageProperties = + new LocalMessageStorageProperties(); + localMessageStorageProperties.setMaxAge(Duration.ofHours(8)); + + when(producer.isTopicAvailable(cachedTopic)) + .thenReturn(false) + .thenReturn(false) + .thenReturn(true); + + BackupMessagesLoader backupMessagesLoader = + new BackupMessagesLoader( + producer, + producer, + listeners, + topicsCache, + schemaRepository, + schemaExistenceEnsurer, + trackers, + localMessageStorageProperties); + MessageRepository messageRepository = + new ChronicleMapMessageRepository( + new File(tempDir.getAbsoluteFile(), "messages.dat"), ENTRIES, AVERAGE_MESSAGE_SIZE); + messageRepository.save(messageOfAge(1), topic); + + // when + backupMessagesLoader.loadMessages(messageRepository.findAll()); + + // then + verify(producer, times(3)).isTopicAvailable(cachedTopic); + verify(producer, times(1)) + .send(any(JsonMessage.class), eq(cachedTopic), any(PublishingCallback.class)); + } + + @Test + public void shouldSendMessageWithAllArgumentsFromBackupMessage() { + // given + LocalMessageStorageProperties localMessageStorageProperties = + new LocalMessageStorageProperties(); + localMessageStorageProperties.setMaxAge(Duration.ofHours(8)); + localMessageStorageProperties.setMaxResendRetries(2); + + MessageRepository messageRepository = + new ChronicleMapMessageRepository( + new File(tempDir.getAbsoluteFile(), "messages.dat"), ENTRIES, AVERAGE_MESSAGE_SIZE); + + BackupMessagesLoader backupMessagesLoader = + new BackupMessagesLoader( + producer, + producer, + listeners, + topicsCache, + schemaRepository, + schemaExistenceEnsurer, + trackers, + localMessageStorageProperties); + + messageRepository.save(messageOfAge(1), topic); + + // when + List backupMessages = messageRepository.findAll(); + backupMessagesLoader.loadMessages(backupMessages); + + // then + ArgumentCaptor messageArgument = ArgumentCaptor.forClass(Message.class); + + verify(producer, times(1)) + .send(messageArgument.capture(), eq(cachedTopic), any(PublishingCallback.class)); + Message sendMessage = messageArgument.getValue(); + assertThat(sendMessage.getPartitionKey()).isEqualTo(backupMessages.get(0).getPartitionKey()); + assertThat(sendMessage.getData()).isEqualTo(backupMessages.get(0).getData()); + assertThat(sendMessage.getId()).isEqualTo(backupMessages.get(0).getMessageId()); + assertThat(sendMessage.getTimestamp()).isEqualTo(backupMessages.get(0).getTimestamp()); + assertThat(sendMessage.getHTTPHeaders().get("propagated-http-header")) + .isEqualTo("example-value"); + } + + private Message messageOfAge(int ageHours) { + return new JsonMessage( + MessageIdGenerator.generate(), + "{'a':'b'}".getBytes(), + now().minusHours(ageHours).toInstant(UTC).toEpochMilli(), + "partition-key", + Map.of("propagated-http-header", "example-value")); + } } diff --git a/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/buffer/BufferSerializationCompatibilityTest.java b/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/buffer/BufferSerializationCompatibilityTest.java index 204a3a5282..e738284700 100644 --- a/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/buffer/BufferSerializationCompatibilityTest.java +++ b/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/buffer/BufferSerializationCompatibilityTest.java @@ -1,23 +1,22 @@ package pl.allegro.tech.hermes.frontend.buffer; +import static org.assertj.core.api.Assertions.assertThat; + import org.junit.Test; import pl.allegro.tech.hermes.frontend.buffer.chronicle.ChronicleMapEntryValue; -import static org.assertj.core.api.Assertions.assertThat; - /** - * This test protects against unintended changes to ChronicleMapEntryValue class. As ChronicleMap uses Java serialization - * when writing the class to offheap (and to the file), any change in canonical class name breaks the compatibility - * of buffers. Incompatible buffer will not be read from disk, which might lead to losing some messages between the - * updates. + * This test protects against unintended changes to ChronicleMapEntryValue class. As ChronicleMap + * uses Java serialization when writing the class to offheap (and to the file), any change in + * canonical class name breaks the compatibility of buffers. Incompatible buffer will not be read + * from disk, which might lead to losing some messages between the updates. */ public class BufferSerializationCompatibilityTest { - @Test - public void shouldKeepChronicleMapEntryValueClassCanonicalNameConsistent() { - // then - assertThat(ChronicleMapEntryValue.class.getCanonicalName()) - .isEqualTo("pl.allegro.tech.hermes.frontend.buffer.chronicle.ChronicleMapEntryValue"); - } - + @Test + public void shouldKeepChronicleMapEntryValueClassCanonicalNameConsistent() { + // then + assertThat(ChronicleMapEntryValue.class.getCanonicalName()) + .isEqualTo("pl.allegro.tech.hermes.frontend.buffer.chronicle.ChronicleMapEntryValue"); + } } diff --git a/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/buffer/ChronicleMapMessageRepositoryTest.java b/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/buffer/ChronicleMapMessageRepositoryTest.java index cfb6fe9cf3..d8689373c6 100644 --- a/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/buffer/ChronicleMapMessageRepositoryTest.java +++ b/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/buffer/ChronicleMapMessageRepositoryTest.java @@ -1,6 +1,19 @@ package pl.allegro.tech.hermes.frontend.buffer; +import static java.nio.charset.Charset.defaultCharset; +import static java.util.Collections.emptyMap; +import static org.assertj.core.api.Assertions.assertThat; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topic; + import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.UUID; import org.junit.Before; import org.junit.Test; import pl.allegro.tech.hermes.api.Topic; @@ -15,200 +28,203 @@ import pl.allegro.tech.hermes.test.helper.avro.AvroUser; import pl.allegro.tech.hermes.test.helper.avro.AvroUserSchemaLoader; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -import static java.nio.charset.Charset.defaultCharset; -import static java.util.Collections.emptyMap; -import static org.assertj.core.api.Assertions.assertThat; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topic; - public class ChronicleMapMessageRepositoryTest { - private static final int ENTRIES = 100; - private static final int AVERAGE_MESSAGE_SIZE = 600; - - private File file; - private MessageRepository messageRepository; + private static final int ENTRIES = 100; + private static final int AVERAGE_MESSAGE_SIZE = 600; - @Before - public void setUp() throws Throwable { - file = File.createTempFile("local_backup", ".dat"); - messageRepository = new ChronicleMapMessageRepository(file, ENTRIES, AVERAGE_MESSAGE_SIZE); - } + private File file; + private MessageRepository messageRepository; - @Test - public void shouldSaveFindAndDeleteMessage() { - //given - String qualifiedName = "groupName.topic"; + @Before + public void setUp() throws Throwable { + file = File.createTempFile("local_backup", ".dat"); + messageRepository = new ChronicleMapMessageRepository(file, ENTRIES, AVERAGE_MESSAGE_SIZE); + } - Message message = generateJsonMessage(); - String id = message.getId(); + @Test + public void shouldSaveFindAndDeleteMessage() { + // given + String qualifiedName = "groupName.topic"; - Topic topic = topic(qualifiedName).build(); + Message message = generateJsonMessage(); + String id = message.getId(); - //when - messageRepository.save(message, topic); + Topic topic = topic(qualifiedName).build(); - //then - assertThat(messageRepository.findAll()).contains(backupMessage(message, qualifiedName)); + // when + messageRepository.save(message, topic); - //when - messageRepository.delete(id); + // then + assertThat(messageRepository.findAll()).contains(backupMessage(message, qualifiedName)); - //then - assertThat(messageRepository.findAll()).isEmpty(); - } + // when + messageRepository.delete(id); - @Test - public void shouldSaveMultipleTimesFindAndDeleteMessage() { - //given - String messageContent = "hello world"; - Message message1 = generateJsonMessage(messageContent); - Message message2 = generateJsonMessage(messageContent); - String qualifiedName = "groupName.topic"; + // then + assertThat(messageRepository.findAll()).isEmpty(); + } - Topic topic = topic(qualifiedName).build(); + @Test + public void shouldSaveMultipleTimesFindAndDeleteMessage() { + // given + String messageContent = "hello world"; + Message message1 = generateJsonMessage(messageContent); + Message message2 = generateJsonMessage(messageContent); + String qualifiedName = "groupName.topic"; - //when - messageRepository.save(message1, topic); - messageRepository.save(message2, topic); - messageRepository.save(message1, topic); - messageRepository.save(message2, topic); - - //then - assertThat(messageRepository.findAll()).contains(backupMessage(message1, qualifiedName)); + Topic topic = topic(qualifiedName).build(); - //when - messageRepository.delete(message1.getId()); - - //then - assertThat(messageRepository.findAll()).hasSize(1); - assertThat(messageRepository.findAll()).contains(backupMessage(message2, qualifiedName)); - } - - @Test - public void shouldCreateRepositoryFromFile() throws IOException { - //given - File file = Files.createTempFile("backup", "messages.dat").toFile(); - - //when - new ChronicleMapMessageRepository(file, ENTRIES, AVERAGE_MESSAGE_SIZE); - - //then - assertThat(file).exists(); - } - - @Test - public void shouldCreateRepositoryThenCloseAndRestore() throws IOException { - //given - Message message = generateJsonMessage(); - String qualifiedName = "groupName.topic"; - Topic topic = topic(qualifiedName).build(); - - File file = Files.createTempFile("backup", "messages.dat").toFile(); - - messageRepository = new ChronicleMapMessageRepository(file, ENTRIES, AVERAGE_MESSAGE_SIZE); - - //when - messageRepository.save(message, topic); - - //then - messageRepository.close(); - - //when - messageRepository = new ChronicleMapMessageRepository(file, ENTRIES, AVERAGE_MESSAGE_SIZE); - - //then - assertThat(messageRepository.findAll()).contains(backupMessage(message, qualifiedName)); - } - - @Test - public void shouldSaveFindAndDeleteMessageAvroMessage() { - //given - String qualifiedName = "groupName.topic"; - - AvroUser avroUser = new AvroUser("Bob", 18, "blue"); - String id = MessageIdGenerator.generate(); - Message message = new AvroMessage(id, avroUser.asBytes(), System.currentTimeMillis(), - CompiledSchema.of(AvroUserSchemaLoader.load(), 1, 1), "partition-key", emptyMap()); - - Topic topic = topic(qualifiedName).build(); - - //when - messageRepository.save(message, topic); - - //then - assertThat(messageRepository.findAll()).contains(backupMessage(message, qualifiedName)); - - //when - messageRepository.delete(id); - - //then - assertThat(messageRepository.findAll()).isEmpty(); - } - - @Test - public void shouldLoadMessagesSavedByVersion2_5_0() throws IOException { - // given - // The hermes-buffer-v3_2-5-0.dat file has been generated by running - // pl.allegro.tech.hermes.integration.MessageBufferLoadingTest#backupFileWithOneMessage - // against version 2.5.0 (https://github.com/allegro/hermes/tree/hermes-2.5.0). - InputStream fileSavedBy2_5_0 = getClass().getResourceAsStream("/hermes-buffer-v3_2-5-0.dat"); - String tempDirPath = Files.createTempDirectory("backup").toAbsolutePath().toString(); - Path backupFile = Path.of(tempDirPath, "/hermes-buffer-v3.dat"); - Files.copy(fileSavedBy2_5_0, backupFile); - - ChronicleMapMessageRepository messageRepository = new ChronicleMapMessageRepository( - backupFile.toFile(), - ENTRIES, - AVERAGE_MESSAGE_SIZE - ); - - // when - List loadedMessages = messageRepository.findAll(); - - // then - assertThat(loadedMessages).hasSize(1); - BackupMessage message = loadedMessages.get(0); - assertThat(message.getMessageId()).isEqualTo("573e570c-890d-49c6-8916-434a1ecb6c66"); - assertThat(message.getTimestamp()).isEqualTo(1704280749690L); - assertThat(message.getQualifiedTopicName()).isEqualTo("backupGroup.topic-2e760871-954e-4823-935c-dfdadbb1be09"); - assertThat(message.getPartitionKey()).isNull(); - assertThat(message.getPropagatedHTTPHeaders()).isEmpty(); - assertThat(message.getSchemaVersion()).isNull(); - assertThat(message.getSchemaId()).isNull(); - - JsonMessageContentWrapper contentWrapper = new JsonMessageContentWrapper("message", "metadata", new ObjectMapper()); - CompositeMessageContentWrapper wrapper = new CompositeMessageContentWrapper(contentWrapper, null, null, null, null, null); - byte[] content = wrapper.wrapJson("message".getBytes(defaultCharset()), message.getMessageId(), message.getTimestamp(), emptyMap()); - assertThat(message.getData()).isEqualTo(content); - } - - private Message generateJsonMessage() { - return generateJsonMessage(UUID.randomUUID().toString()); - } - - private Message generateJsonMessage(String content) { - return generateJsonMessage(content, System.currentTimeMillis()); - } - - private Message generateJsonMessage(String content, long timestamp) { - byte[] messageContent = content.getBytes(); - String id = MessageIdGenerator.generate(); - return new JsonMessage(id, messageContent, timestamp, "partition-key", Map.of("propagated-http-header", "value")); - } - - private BackupMessage backupMessage(Message m, String qualifiedTopicName) { - return new BackupMessage(m.getId(), m.getData(), m.getTimestamp(), qualifiedTopicName, m.getPartitionKey(), - m.getCompiledSchema().map(cs -> cs.getVersion().value()).orElse(null), - m.getCompiledSchema().map(cs -> cs.getId().value()).orElse(null), - m.getHTTPHeaders()); - } + // when + messageRepository.save(message1, topic); + messageRepository.save(message2, topic); + messageRepository.save(message1, topic); + messageRepository.save(message2, topic); + + // then + assertThat(messageRepository.findAll()).contains(backupMessage(message1, qualifiedName)); + + // when + messageRepository.delete(message1.getId()); + + // then + assertThat(messageRepository.findAll()).hasSize(1); + assertThat(messageRepository.findAll()).contains(backupMessage(message2, qualifiedName)); + } + + @Test + public void shouldCreateRepositoryFromFile() throws IOException { + // given + File file = Files.createTempFile("backup", "messages.dat").toFile(); + + // when + new ChronicleMapMessageRepository(file, ENTRIES, AVERAGE_MESSAGE_SIZE); + + // then + assertThat(file).exists(); + } + + @Test + public void shouldCreateRepositoryThenCloseAndRestore() throws IOException { + // given + Message message = generateJsonMessage(); + String qualifiedName = "groupName.topic"; + Topic topic = topic(qualifiedName).build(); + + File file = Files.createTempFile("backup", "messages.dat").toFile(); + + messageRepository = new ChronicleMapMessageRepository(file, ENTRIES, AVERAGE_MESSAGE_SIZE); + + // when + messageRepository.save(message, topic); + + // then + messageRepository.close(); + + // when + messageRepository = new ChronicleMapMessageRepository(file, ENTRIES, AVERAGE_MESSAGE_SIZE); + + // then + assertThat(messageRepository.findAll()).contains(backupMessage(message, qualifiedName)); + } + + @Test + public void shouldSaveFindAndDeleteMessageAvroMessage() { + // given + String qualifiedName = "groupName.topic"; + + AvroUser avroUser = new AvroUser("Bob", 18, "blue"); + String id = MessageIdGenerator.generate(); + Message message = + new AvroMessage( + id, + avroUser.asBytes(), + System.currentTimeMillis(), + CompiledSchema.of(AvroUserSchemaLoader.load(), 1, 1), + "partition-key", + emptyMap()); + + Topic topic = topic(qualifiedName).build(); + + // when + messageRepository.save(message, topic); + + // then + assertThat(messageRepository.findAll()).contains(backupMessage(message, qualifiedName)); + + // when + messageRepository.delete(id); + + // then + assertThat(messageRepository.findAll()).isEmpty(); + } + + @Test + public void shouldLoadMessagesSavedByVersion2_5_0() throws IOException { + // given + // The hermes-buffer-v3_2-5-0.dat file has been generated by running + // pl.allegro.tech.hermes.integration.MessageBufferLoadingTest#backupFileWithOneMessage + // against version 2.5.0 (https://github.com/allegro/hermes/tree/hermes-2.5.0). + InputStream fileSavedBy2_5_0 = getClass().getResourceAsStream("/hermes-buffer-v3_2-5-0.dat"); + String tempDirPath = Files.createTempDirectory("backup").toAbsolutePath().toString(); + Path backupFile = Path.of(tempDirPath, "/hermes-buffer-v3.dat"); + Files.copy(fileSavedBy2_5_0, backupFile); + + ChronicleMapMessageRepository messageRepository = + new ChronicleMapMessageRepository(backupFile.toFile(), ENTRIES, AVERAGE_MESSAGE_SIZE); + + // when + List loadedMessages = messageRepository.findAll(); + + // then + assertThat(loadedMessages).hasSize(1); + BackupMessage message = loadedMessages.get(0); + assertThat(message.getMessageId()).isEqualTo("573e570c-890d-49c6-8916-434a1ecb6c66"); + assertThat(message.getTimestamp()).isEqualTo(1704280749690L); + assertThat(message.getQualifiedTopicName()) + .isEqualTo("backupGroup.topic-2e760871-954e-4823-935c-dfdadbb1be09"); + assertThat(message.getPartitionKey()).isNull(); + assertThat(message.getPropagatedHTTPHeaders()).isEmpty(); + assertThat(message.getSchemaVersion()).isNull(); + assertThat(message.getSchemaId()).isNull(); + + JsonMessageContentWrapper contentWrapper = + new JsonMessageContentWrapper("message", "metadata", new ObjectMapper()); + CompositeMessageContentWrapper wrapper = + new CompositeMessageContentWrapper(contentWrapper, null, null, null, null, null); + byte[] content = + wrapper.wrapJson( + "message".getBytes(defaultCharset()), + message.getMessageId(), + message.getTimestamp(), + emptyMap()); + assertThat(message.getData()).isEqualTo(content); + } + + private Message generateJsonMessage() { + return generateJsonMessage(UUID.randomUUID().toString()); + } + + private Message generateJsonMessage(String content) { + return generateJsonMessage(content, System.currentTimeMillis()); + } + + private Message generateJsonMessage(String content, long timestamp) { + byte[] messageContent = content.getBytes(); + String id = MessageIdGenerator.generate(); + return new JsonMessage( + id, messageContent, timestamp, "partition-key", Map.of("propagated-http-header", "value")); + } + + private BackupMessage backupMessage(Message m, String qualifiedTopicName) { + return new BackupMessage( + m.getId(), + m.getData(), + m.getTimestamp(), + qualifiedTopicName, + m.getPartitionKey(), + m.getCompiledSchema().map(cs -> cs.getVersion().value()).orElse(null), + m.getCompiledSchema().map(cs -> cs.getId().value()).orElse(null), + m.getHTTPHeaders()); + } } diff --git a/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/producer/kafka/LocalDatacenterMessageProducerTest.java b/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/producer/kafka/LocalDatacenterMessageProducerTest.java index 3866830892..81d9eafa10 100644 --- a/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/producer/kafka/LocalDatacenterMessageProducerTest.java +++ b/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/producer/kafka/LocalDatacenterMessageProducerTest.java @@ -1,6 +1,18 @@ package pl.allegro.tech.hermes.frontend.producer.kafka; +import static com.google.common.base.Charsets.UTF_8; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topic; + import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import java.time.Duration; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.producer.MockProducer; import org.apache.kafka.clients.producer.ProducerRecord; @@ -27,164 +39,172 @@ import pl.allegro.tech.hermes.frontend.publishing.message.JsonMessage; import pl.allegro.tech.hermes.frontend.publishing.message.Message; -import java.time.Duration; -import java.util.List; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.atomic.AtomicBoolean; - -import static com.google.common.base.Charsets.UTF_8; -import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topic; - @RunWith(MockitoJUnitRunner.class) public class LocalDatacenterMessageProducerTest { - private static final Long TIMESTAMP = 1L; - private static final String PARTITION_KEY = "partition-key"; - private static final String MESSAGE_ID = "id"; - private static final String datacenter = "dc"; - private static final Topic TOPIC = topic("group.topic").build(); - private static final byte[] CONTENT = "{\"data\":\"json\"}".getBytes(UTF_8); - private static final Message MESSAGE = new JsonMessage(MESSAGE_ID, CONTENT, TIMESTAMP, PARTITION_KEY, emptyMap()); - - private final ByteArraySerializer serializer = new ByteArraySerializer(); - private final MetricsFacade metricsFacade = new MetricsFacade(new SimpleMeterRegistry()); - - private final BrokerLatencyReporter brokerLatencyReporter = new BrokerLatencyReporter(false, metricsFacade, Duration.ZERO, Executors.newSingleThreadExecutor()); - - private final ScheduledExecutorService chaosScheduler = Executors.newSingleThreadScheduledExecutor(); - - private final MockProducer leaderConfirmsProducer = new MockProducer<>(true, serializer, serializer); - private final MockProducer everyoneConfirmProducer = new MockProducer<>(true, serializer, serializer); - private final KafkaMessageSender leaderConfirmsProduceWrapper = new KafkaMessageSender<>(leaderConfirmsProducer, brokerLatencyReporter, metricsFacade, datacenter, chaosScheduler); - private final KafkaMessageSender everyoneConfirmsProduceWrapper = new KafkaMessageSender<>(everyoneConfirmProducer, brokerLatencyReporter, metricsFacade, datacenter, chaosScheduler); - - private final KafkaHeaderNameProperties kafkaHeaderNameProperties = new KafkaHeaderNameProperties(); - private final HTTPHeadersPropagationAsKafkaHeadersProperties httpHeadersPropagationAsKafkaHeadersProperties = - new HTTPHeadersProperties.PropagationAsKafkaHeadersProperties(); - @Mock - private TopicsCache topicsCache; - private final TopicMetadataLoadingExecutor topicMetadataLoadingExecutor = new TopicMetadataLoadingExecutor( - topicsCache, 2, Duration.ofSeconds(10), 2 - ); - @Mock - private AdminClient adminClient; - private final MinInSyncReplicasLoader localMinInSyncReplicasLoader = new MinInSyncReplicasLoader(adminClient, Duration.ofMinutes(1)); - private final KafkaMessageSenders kafkaMessageSenders = new KafkaMessageSenders( - topicMetadataLoadingExecutor, - localMinInSyncReplicasLoader, - new KafkaMessageSenders.Tuple(leaderConfirmsProduceWrapper, everyoneConfirmsProduceWrapper), - emptyList() - ); - - private LocalDatacenterMessageProducer producer; - private final KafkaNamesMapper kafkaNamesMapper = new NamespaceKafkaNamesMapper("ns", "_"); - private final KafkaHeaderFactory kafkaHeaderFactory = new KafkaHeaderFactory(kafkaHeaderNameProperties, - httpHeadersPropagationAsKafkaHeadersProperties); - - private CachedTopic cachedTopic; - - private final SchemaProperties schemaProperties = new SchemaProperties(); - - @Mock - private ThroughputRegistry throughputRegistry; - - @Before - public void before() { - cachedTopic = new CachedTopic(TOPIC, metricsFacade, throughputRegistry, kafkaNamesMapper.toKafkaTopics(TOPIC)); - MessageToKafkaProducerRecordConverter messageConverter = - new MessageToKafkaProducerRecordConverter(kafkaHeaderFactory, schemaProperties.isIdHeaderEnabled()); - producer = new LocalDatacenterMessageProducer(kafkaMessageSenders, messageConverter); - } - - @After - public void after() { - leaderConfirmsProducer.clear(); - everyoneConfirmProducer.clear(); - } - - @Test - public void shouldPublishOnTopicUsingKafkaTopicName() { - //when - producer.send(MESSAGE, cachedTopic, new DoNothing()); - - //then - List> records = leaderConfirmsProducer.history(); - assertThat(records.size()).isEqualTo(1); - assertThat(records.get(0).topic()).isEqualTo("ns_group.topic"); - } - - @Test - public void shouldPublishOnTheSamePartition() { - //when - producer.send(MESSAGE, cachedTopic, new DoNothing()); - producer.send(MESSAGE, cachedTopic, new DoNothing()); - producer.send(MESSAGE, cachedTopic, new DoNothing()); - - //then - List> records = leaderConfirmsProducer.history(); - assertThat(records.size()).isEqualTo(3); - assertThat(records.stream().filter(record -> PARTITION_KEY.equals(new String(record.key()))).count()).isEqualTo(3); - } - - @Test - public void shouldCallCallbackOnSend() { - //given - final AtomicBoolean callbackCalled = new AtomicBoolean(false); - - //when - producer.send(MESSAGE, cachedTopic, new PublishingCallback() { - @Override - public void onUnpublished(Message message, Topic topic, Exception exception) { - callbackCalled.set(true); - } - - @Override - public void onPublished(Message message, Topic topic) { - callbackCalled.set(true); - } - - @Override - public void onEachPublished(Message message, Topic topic, String datacenter) { - callbackCalled.set(true); - } + private static final Long TIMESTAMP = 1L; + private static final String PARTITION_KEY = "partition-key"; + private static final String MESSAGE_ID = "id"; + private static final String datacenter = "dc"; + private static final Topic TOPIC = topic("group.topic").build(); + private static final byte[] CONTENT = "{\"data\":\"json\"}".getBytes(UTF_8); + private static final Message MESSAGE = + new JsonMessage(MESSAGE_ID, CONTENT, TIMESTAMP, PARTITION_KEY, emptyMap()); + + private final ByteArraySerializer serializer = new ByteArraySerializer(); + private final MetricsFacade metricsFacade = new MetricsFacade(new SimpleMeterRegistry()); + + private final BrokerLatencyReporter brokerLatencyReporter = + new BrokerLatencyReporter( + false, metricsFacade, Duration.ZERO, Executors.newSingleThreadExecutor()); + + private final ScheduledExecutorService chaosScheduler = + Executors.newSingleThreadScheduledExecutor(); + + private final MockProducer leaderConfirmsProducer = + new MockProducer<>(true, serializer, serializer); + private final MockProducer everyoneConfirmProducer = + new MockProducer<>(true, serializer, serializer); + private final KafkaMessageSender leaderConfirmsProduceWrapper = + new KafkaMessageSender<>( + leaderConfirmsProducer, brokerLatencyReporter, metricsFacade, datacenter, chaosScheduler); + private final KafkaMessageSender everyoneConfirmsProduceWrapper = + new KafkaMessageSender<>( + everyoneConfirmProducer, + brokerLatencyReporter, + metricsFacade, + datacenter, + chaosScheduler); + + private final KafkaHeaderNameProperties kafkaHeaderNameProperties = + new KafkaHeaderNameProperties(); + private final HTTPHeadersPropagationAsKafkaHeadersProperties + httpHeadersPropagationAsKafkaHeadersProperties = + new HTTPHeadersProperties.PropagationAsKafkaHeadersProperties(); + @Mock private TopicsCache topicsCache; + private final TopicMetadataLoadingExecutor topicMetadataLoadingExecutor = + new TopicMetadataLoadingExecutor(topicsCache, 2, Duration.ofSeconds(10), 2); + @Mock private AdminClient adminClient; + private final MinInSyncReplicasLoader localMinInSyncReplicasLoader = + new MinInSyncReplicasLoader(adminClient, Duration.ofMinutes(1)); + private final KafkaMessageSenders kafkaMessageSenders = + new KafkaMessageSenders( + topicMetadataLoadingExecutor, + localMinInSyncReplicasLoader, + new KafkaMessageSenders.Tuple( + leaderConfirmsProduceWrapper, everyoneConfirmsProduceWrapper), + emptyList()); + + private LocalDatacenterMessageProducer producer; + private final KafkaNamesMapper kafkaNamesMapper = new NamespaceKafkaNamesMapper("ns", "_"); + private final KafkaHeaderFactory kafkaHeaderFactory = + new KafkaHeaderFactory( + kafkaHeaderNameProperties, httpHeadersPropagationAsKafkaHeadersProperties); + + private CachedTopic cachedTopic; + + private final SchemaProperties schemaProperties = new SchemaProperties(); + + @Mock private ThroughputRegistry throughputRegistry; + + @Before + public void before() { + cachedTopic = + new CachedTopic( + TOPIC, metricsFacade, throughputRegistry, kafkaNamesMapper.toKafkaTopics(TOPIC)); + MessageToKafkaProducerRecordConverter messageConverter = + new MessageToKafkaProducerRecordConverter( + kafkaHeaderFactory, schemaProperties.isIdHeaderEnabled()); + producer = new LocalDatacenterMessageProducer(kafkaMessageSenders, messageConverter); + } + + @After + public void after() { + leaderConfirmsProducer.clear(); + everyoneConfirmProducer.clear(); + } + + @Test + public void shouldPublishOnTopicUsingKafkaTopicName() { + // when + producer.send(MESSAGE, cachedTopic, new DoNothing()); + + // then + List> records = leaderConfirmsProducer.history(); + assertThat(records.size()).isEqualTo(1); + assertThat(records.get(0).topic()).isEqualTo("ns_group.topic"); + } + + @Test + public void shouldPublishOnTheSamePartition() { + // when + producer.send(MESSAGE, cachedTopic, new DoNothing()); + producer.send(MESSAGE, cachedTopic, new DoNothing()); + producer.send(MESSAGE, cachedTopic, new DoNothing()); + + // then + List> records = leaderConfirmsProducer.history(); + assertThat(records.size()).isEqualTo(3); + assertThat( + records.stream() + .filter(record -> PARTITION_KEY.equals(new String(record.key()))) + .count()) + .isEqualTo(3); + } + + @Test + public void shouldCallCallbackOnSend() { + // given + final AtomicBoolean callbackCalled = new AtomicBoolean(false); + + // when + producer.send( + MESSAGE, + cachedTopic, + new PublishingCallback() { + @Override + public void onUnpublished(Message message, Topic topic, Exception exception) { + callbackCalled.set(true); + } + + @Override + public void onPublished(Message message, Topic topic) { + callbackCalled.set(true); + } + + @Override + public void onEachPublished(Message message, Topic topic, String datacenter) { + callbackCalled.set(true); + } }); - //then - await().until(callbackCalled::get); - } - - @Test - public void shouldUseEveryoneConfirmProducerForTopicWithAckAll() { - //given - Topic topic = topic("group.all").withAck(Topic.Ack.ALL).build(); - CachedTopic cachedTopic = new CachedTopic(topic, metricsFacade, throughputRegistry, - kafkaNamesMapper.toKafkaTopics(topic)); - - //when - producer.send(MESSAGE, cachedTopic, new DoNothing()); + // then + await().until(callbackCalled::get); + } - //then - List> records = everyoneConfirmProducer.history(); - assertThat(records.size()).isEqualTo(1); - assertThat(records.get(0).topic()).isEqualTo("ns_group.all"); - } + @Test + public void shouldUseEveryoneConfirmProducerForTopicWithAckAll() { + // given + Topic topic = topic("group.all").withAck(Topic.Ack.ALL).build(); + CachedTopic cachedTopic = + new CachedTopic( + topic, metricsFacade, throughputRegistry, kafkaNamesMapper.toKafkaTopics(topic)); - private static class DoNothing implements PublishingCallback { - public void onUnpublished(Message message, Topic topic, Exception exception) { - } + // when + producer.send(MESSAGE, cachedTopic, new DoNothing()); - public void onPublished(Message message, Topic topic) { - } + // then + List> records = everyoneConfirmProducer.history(); + assertThat(records.size()).isEqualTo(1); + assertThat(records.get(0).topic()).isEqualTo("ns_group.all"); + } - @Override - public void onEachPublished(Message message, Topic topic, String datacenter) { + private static class DoNothing implements PublishingCallback { + public void onUnpublished(Message message, Topic topic, Exception exception) {} - } - } + public void onPublished(Message message, Topic topic) {} + @Override + public void onEachPublished(Message message, Topic topic, String datacenter) {} + } } diff --git a/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageContentTypeEnforcerTest.java b/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageContentTypeEnforcerTest.java index c76b7d81d1..bab826c62d 100644 --- a/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageContentTypeEnforcerTest.java +++ b/hermes-frontend/src/test/java/pl/allegro/tech/hermes/frontend/publishing/message/MessageContentTypeEnforcerTest.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.frontend.publishing.message; +import static org.assertj.core.api.Assertions.assertThat; + import jakarta.ws.rs.core.MediaType; import org.apache.avro.Schema; import org.junit.Test; @@ -11,76 +13,87 @@ import pl.allegro.tech.hermes.test.helper.avro.AvroUserSchemaLoader; import pl.allegro.tech.hermes.test.helper.builder.TopicBuilder; -import static org.assertj.core.api.Assertions.assertThat; - public class MessageContentTypeEnforcerTest { - private final MessageContentTypeEnforcer enforcer = new MessageContentTypeEnforcer(); - - private final Topic topic = TopicBuilder.topic("test.Topic").withContentType(ContentType.AVRO).build(); - private final AvroUser avroMessage = new AvroUser("Bob", 30, "black"); - private final CompiledSchema schema = CompiledSchema.of(avroMessage.getSchema(), 1, 0); - private final CompiledSchema testSchema = CompiledSchema.of(AvroUserSchemaLoader.load(), 1, 0); - - @Test - public void shouldConvertToAvroWhenReceivedJSONOnAvroTopic() { - // when - byte[] enforcedMessage = enforcer.enforceAvro("application/json", avroMessage.asJson().getBytes(), schema.getSchema(), topic); - - // then - assertThat(enforcedMessage).isEqualTo(avroMessage.asBytes()); - } - - @Test - public void sh1ouldConvertToAvroWhenReceivedJSONOnAvroTopic() { - // when - byte[] enforcedMessage = enforcer.enforceAvro("application/json", avroMessage.asJson().getBytes(), testSchema.getSchema(), topic); - - // then - assertThat(enforcedMessage).isEqualTo(avroMessage.asBytes()); - } - - - @Test - public void shouldConvertToAvroWhenReceivedAvroJSONOnAvroTopic() { - // when - byte[] enforcedMessage = enforcer.enforceAvro("avro/json", avroMessage.asAvroEncodedJson().getBytes(), schema.getSchema(), topic); - - // then - assertThat(enforcedMessage).isEqualTo(avroMessage.asBytes()); - } - - @Test - public void shouldStringContentTypeOfAdditionalOptionsWhenInterpretingIt() { - // when - byte[] enforcedMessage = - enforcer.enforceAvro("application/json;encoding=utf-8", avroMessage.asJson().getBytes(), schema.getSchema(), topic); - - // then - assertThat(enforcedMessage).isEqualTo(avroMessage.asBytes()); - } - - @Test - public void shouldNotConvertWhenReceivingAvroOnAvroTopic() { - // when - byte[] enforcedMessage = enforcer.enforceAvro("avro/binary", avroMessage.asBytes(), schema.getSchema(), topic); - - // then - assertThat(enforcedMessage).isEqualTo(avroMessage.asBytes()); - } - - @Test - public void shouldBeCaseInsensitiveForPayloadContentType() { - // when - byte[] enforcedMessage = enforcer.enforceAvro("AVRO/Binary", avroMessage.asBytes(), schema.getSchema(), topic); - - // then - assertThat(enforcedMessage).isEqualTo(avroMessage.asBytes()); - } - - @Test(expected = UnsupportedContentTypeException.class) - public void shouldThrowUnsupportedContentTypeExceptionWhenReceivedWrongContentType() { - // when - enforcer.enforceAvro(MediaType.TEXT_PLAIN, avroMessage.asBytes(), schema.getSchema(), topic); - } + private final MessageContentTypeEnforcer enforcer = new MessageContentTypeEnforcer(); + + private final Topic topic = + TopicBuilder.topic("test.Topic").withContentType(ContentType.AVRO).build(); + private final AvroUser avroMessage = new AvroUser("Bob", 30, "black"); + private final CompiledSchema schema = CompiledSchema.of(avroMessage.getSchema(), 1, 0); + private final CompiledSchema testSchema = + CompiledSchema.of(AvroUserSchemaLoader.load(), 1, 0); + + @Test + public void shouldConvertToAvroWhenReceivedJSONOnAvroTopic() { + // when + byte[] enforcedMessage = + enforcer.enforceAvro( + "application/json", avroMessage.asJson().getBytes(), schema.getSchema(), topic); + + // then + assertThat(enforcedMessage).isEqualTo(avroMessage.asBytes()); + } + + @Test + public void sh1ouldConvertToAvroWhenReceivedJSONOnAvroTopic() { + // when + byte[] enforcedMessage = + enforcer.enforceAvro( + "application/json", avroMessage.asJson().getBytes(), testSchema.getSchema(), topic); + + // then + assertThat(enforcedMessage).isEqualTo(avroMessage.asBytes()); + } + + @Test + public void shouldConvertToAvroWhenReceivedAvroJSONOnAvroTopic() { + // when + byte[] enforcedMessage = + enforcer.enforceAvro( + "avro/json", avroMessage.asAvroEncodedJson().getBytes(), schema.getSchema(), topic); + + // then + assertThat(enforcedMessage).isEqualTo(avroMessage.asBytes()); + } + + @Test + public void shouldStringContentTypeOfAdditionalOptionsWhenInterpretingIt() { + // when + byte[] enforcedMessage = + enforcer.enforceAvro( + "application/json;encoding=utf-8", + avroMessage.asJson().getBytes(), + schema.getSchema(), + topic); + + // then + assertThat(enforcedMessage).isEqualTo(avroMessage.asBytes()); + } + + @Test + public void shouldNotConvertWhenReceivingAvroOnAvroTopic() { + // when + byte[] enforcedMessage = + enforcer.enforceAvro("avro/binary", avroMessage.asBytes(), schema.getSchema(), topic); + + // then + assertThat(enforcedMessage).isEqualTo(avroMessage.asBytes()); + } + + @Test + public void shouldBeCaseInsensitiveForPayloadContentType() { + // when + byte[] enforcedMessage = + enforcer.enforceAvro("AVRO/Binary", avroMessage.asBytes(), schema.getSchema(), topic); + + // then + assertThat(enforcedMessage).isEqualTo(avroMessage.asBytes()); + } + + @Test(expected = UnsupportedContentTypeException.class) + public void shouldThrowUnsupportedContentTypeExceptionWhenReceivedWrongContentType() { + // when + enforcer.enforceAvro(MediaType.TEXT_PLAIN, avroMessage.asBytes(), schema.getSchema(), topic); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/HermesManagement.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/HermesManagement.java index 284681c868..3372816d20 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/HermesManagement.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/HermesManagement.java @@ -6,8 +6,7 @@ @SpringBootApplication public class HermesManagement { - public static void main(String[] args) { - SpringApplication.run(HermesManagement.class, args); - } - + public static void main(String[] args) { + SpringApplication.run(HermesManagement.class, args); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/AllTopicClientsEndpoint.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/AllTopicClientsEndpoint.java index b6038c2740..aa0060f119 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/AllTopicClientsEndpoint.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/AllTopicClientsEndpoint.java @@ -1,30 +1,29 @@ package pl.allegro.tech.hermes.management.api; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; +import static pl.allegro.tech.hermes.api.TopicName.fromQualifiedName; + import jakarta.ws.rs.GET; import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import jakarta.ws.rs.Produces; +import java.util.List; import org.springframework.stereotype.Component; import pl.allegro.tech.hermes.management.domain.clients.AllTopicClientsService; -import java.util.List; - -import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; -import static pl.allegro.tech.hermes.api.TopicName.fromQualifiedName; - @Component @Path("topics/{topic}/clients") public class AllTopicClientsEndpoint { - private final AllTopicClientsService allTopicClientsService; + private final AllTopicClientsService allTopicClientsService; - public AllTopicClientsEndpoint(AllTopicClientsService allTopicClientsService) { - this.allTopicClientsService = allTopicClientsService; - } + public AllTopicClientsEndpoint(AllTopicClientsService allTopicClientsService) { + this.allTopicClientsService = allTopicClientsService; + } - @GET - @Produces(APPLICATION_JSON) - public List getTopicClients(@PathParam("topic") String topic) { - return allTopicClientsService.getAllClientsByTopic(fromQualifiedName(topic)); - } + @GET + @Produces(APPLICATION_JSON) + public List getTopicClients(@PathParam("topic") String topic) { + return allTopicClientsService.getAllClientsByTopic(fromQualifiedName(topic)); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/BlacklistEndpoint.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/BlacklistEndpoint.java index bd796708ad..60d2600382 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/BlacklistEndpoint.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/BlacklistEndpoint.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.management.api; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; +import static jakarta.ws.rs.core.Response.status; +import static pl.allegro.tech.hermes.api.BlacklistStatus.BLACKLISTED; +import static pl.allegro.tech.hermes.api.BlacklistStatus.NOT_BLACKLISTED; + import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import jakarta.annotation.security.RolesAllowed; @@ -14,6 +19,7 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.Response; +import java.util.List; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import pl.allegro.tech.hermes.api.BlacklistStatus; @@ -22,65 +28,58 @@ import pl.allegro.tech.hermes.management.domain.auth.RequestUser; import pl.allegro.tech.hermes.management.domain.blacklist.TopicBlacklistService; -import java.util.List; - -import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; -import static jakarta.ws.rs.core.Response.status; -import static pl.allegro.tech.hermes.api.BlacklistStatus.BLACKLISTED; -import static pl.allegro.tech.hermes.api.BlacklistStatus.NOT_BLACKLISTED; - @Component @Path("/blacklist") @Api(value = "/blacklist", description = "Operations on topics") public class BlacklistEndpoint { - private final TopicBlacklistService topicBlacklistService; + private final TopicBlacklistService topicBlacklistService; - @Autowired - public BlacklistEndpoint(TopicBlacklistService topicBlacklistService) { - this.topicBlacklistService = topicBlacklistService; - } + @Autowired + public BlacklistEndpoint(TopicBlacklistService topicBlacklistService) { + this.topicBlacklistService = topicBlacklistService; + } - @GET - @Produces(APPLICATION_JSON) - @Path("/topics/{topicName}") - @ApiOperation(value = "Is topic blacklisted", httpMethod = HttpMethod.GET) - public BlacklistStatus isTopicBlacklisted(@PathParam("topicName") String qualifiedTopicName) { - return topicBlacklistService.isBlacklisted(qualifiedTopicName) ? BLACKLISTED : NOT_BLACKLISTED; - } + @GET + @Produces(APPLICATION_JSON) + @Path("/topics/{topicName}") + @ApiOperation(value = "Is topic blacklisted", httpMethod = HttpMethod.GET) + public BlacklistStatus isTopicBlacklisted(@PathParam("topicName") String qualifiedTopicName) { + return topicBlacklistService.isBlacklisted(qualifiedTopicName) ? BLACKLISTED : NOT_BLACKLISTED; + } - @GET - @Produces(APPLICATION_JSON) - @Path("/topics") - @RolesAllowed(Roles.ADMIN) - public List topicsBlacklist() { - return topicBlacklistService.list(); - } + @GET + @Produces(APPLICATION_JSON) + @Path("/topics") + @RolesAllowed(Roles.ADMIN) + public List topicsBlacklist() { + return topicBlacklistService.list(); + } - @POST - @Produces(APPLICATION_JSON) - @Consumes(APPLICATION_JSON) - @Path("/topics") - @RolesAllowed(Roles.ADMIN) - @ApiOperation(value = "Blacklist topics", httpMethod = HttpMethod.POST) - public Response blacklistTopics( - List qualifiedTopicNames, - @Context ContainerRequestContext requestContext) { - RequestUser blacklistRequester = new HermesSecurityAwareRequestUser(requestContext); - qualifiedTopicNames.forEach(topicName -> topicBlacklistService.blacklist(topicName, blacklistRequester)); - return status(Response.Status.OK).build(); - } + @POST + @Produces(APPLICATION_JSON) + @Consumes(APPLICATION_JSON) + @Path("/topics") + @RolesAllowed(Roles.ADMIN) + @ApiOperation(value = "Blacklist topics", httpMethod = HttpMethod.POST) + public Response blacklistTopics( + List qualifiedTopicNames, @Context ContainerRequestContext requestContext) { + RequestUser blacklistRequester = new HermesSecurityAwareRequestUser(requestContext); + qualifiedTopicNames.forEach( + topicName -> topicBlacklistService.blacklist(topicName, blacklistRequester)); + return status(Response.Status.OK).build(); + } - @DELETE - @Produces(APPLICATION_JSON) - @Path("/topics/{topicName}") - @RolesAllowed(Roles.ADMIN) - @ApiOperation(value = "Unblacklist topic", httpMethod = HttpMethod.DELETE) - public Response unblacklistTopic( - @PathParam("topicName") String qualifiedTopicName, - @Context ContainerRequestContext requestContext) { - RequestUser unblacklistRequester = new HermesSecurityAwareRequestUser(requestContext); - topicBlacklistService.unblacklist(qualifiedTopicName, unblacklistRequester); - return status(Response.Status.OK).build(); - } + @DELETE + @Produces(APPLICATION_JSON) + @Path("/topics/{topicName}") + @RolesAllowed(Roles.ADMIN) + @ApiOperation(value = "Unblacklist topic", httpMethod = HttpMethod.DELETE) + public Response unblacklistTopic( + @PathParam("topicName") String qualifiedTopicName, + @Context ContainerRequestContext requestContext) { + RequestUser unblacklistRequester = new HermesSecurityAwareRequestUser(requestContext); + topicBlacklistService.unblacklist(qualifiedTopicName, unblacklistRequester); + return status(Response.Status.OK).build(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/ConsistencyEndpoint.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/ConsistencyEndpoint.java index 5da9a9593a..2bd3cbf987 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/ConsistencyEndpoint.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/ConsistencyEndpoint.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.management.api; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; + import jakarta.annotation.security.RolesAllowed; import jakarta.ws.rs.DELETE; import jakarta.ws.rs.GET; @@ -12,6 +14,9 @@ import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.GenericEntity; import jakarta.ws.rs.core.Response; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import org.springframework.stereotype.Component; import pl.allegro.tech.hermes.api.InconsistentGroup; import pl.allegro.tech.hermes.api.SubscriptionName; @@ -21,94 +26,89 @@ import pl.allegro.tech.hermes.management.domain.consistency.DcConsistencyService; import pl.allegro.tech.hermes.management.domain.consistency.KafkaHermesConsistencyService; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; - @Component @RolesAllowed(Roles.ADMIN) @Path("consistency") public class ConsistencyEndpoint { - private final DcConsistencyService dcConsistencyService; - private final KafkaHermesConsistencyService kafkaHermesConsistencyService; + private final DcConsistencyService dcConsistencyService; + private final KafkaHermesConsistencyService kafkaHermesConsistencyService; - public ConsistencyEndpoint(DcConsistencyService dcConsistencyService, - KafkaHermesConsistencyService kafkaHermesConsistencyService) { - this.dcConsistencyService = dcConsistencyService; - this.kafkaHermesConsistencyService = kafkaHermesConsistencyService; - } + public ConsistencyEndpoint( + DcConsistencyService dcConsistencyService, + KafkaHermesConsistencyService kafkaHermesConsistencyService) { + this.dcConsistencyService = dcConsistencyService; + this.kafkaHermesConsistencyService = kafkaHermesConsistencyService; + } - @GET - @Produces({APPLICATION_JSON}) - @Path("/inconsistencies/groups") - public Response listInconsistentGroups(@QueryParam("groupNames") List groupNames) { - List inconsistentGroups = dcConsistencyService.listInconsistentGroups(new HashSet<>(groupNames)); - return Response.ok() - .entity(new GenericEntity>(inconsistentGroups) { - }) - .build(); - } + @GET + @Produces({APPLICATION_JSON}) + @Path("/inconsistencies/groups") + public Response listInconsistentGroups(@QueryParam("groupNames") List groupNames) { + List inconsistentGroups = + dcConsistencyService.listInconsistentGroups(new HashSet<>(groupNames)); + return Response.ok() + .entity(new GenericEntity>(inconsistentGroups) {}) + .build(); + } - @POST - @Produces({APPLICATION_JSON}) - @Path("/sync/groups/{groupName}") - public Response syncGroup(@PathParam("groupName") String groupName, - @QueryParam("primaryDatacenter") String primaryDatacenter - ) { - dcConsistencyService.syncGroup(groupName, primaryDatacenter); - return Response.ok().build(); - } + @POST + @Produces({APPLICATION_JSON}) + @Path("/sync/groups/{groupName}") + public Response syncGroup( + @PathParam("groupName") String groupName, + @QueryParam("primaryDatacenter") String primaryDatacenter) { + dcConsistencyService.syncGroup(groupName, primaryDatacenter); + return Response.ok().build(); + } - @POST - @Produces({APPLICATION_JSON}) - @Path("/sync/topics/{topicName}") - public Response syncTopic(@PathParam("topicName") String topicName, - @QueryParam("primaryDatacenter") String primaryDatacenter - ) { - dcConsistencyService.syncTopic(TopicName.fromQualifiedName(topicName), primaryDatacenter); - return Response.ok().build(); - } + @POST + @Produces({APPLICATION_JSON}) + @Path("/sync/topics/{topicName}") + public Response syncTopic( + @PathParam("topicName") String topicName, + @QueryParam("primaryDatacenter") String primaryDatacenter) { + dcConsistencyService.syncTopic(TopicName.fromQualifiedName(topicName), primaryDatacenter); + return Response.ok().build(); + } - @POST - @Produces({APPLICATION_JSON}) - @Path("/sync/topics/{topicName}/subscriptions/{subscriptionName}") - public Response syncSubscription(@PathParam("topicName") String topicName, - @PathParam("subscriptionName") String subscriptionName, - @QueryParam("primaryDatacenter") String primaryDatacenter - ) { - SubscriptionName name = new SubscriptionName(subscriptionName, TopicName.fromQualifiedName(topicName)); - dcConsistencyService.syncSubscription(name, primaryDatacenter); - return Response.ok().build(); - } + @POST + @Produces({APPLICATION_JSON}) + @Path("/sync/topics/{topicName}/subscriptions/{subscriptionName}") + public Response syncSubscription( + @PathParam("topicName") String topicName, + @PathParam("subscriptionName") String subscriptionName, + @QueryParam("primaryDatacenter") String primaryDatacenter) { + SubscriptionName name = + new SubscriptionName(subscriptionName, TopicName.fromQualifiedName(topicName)); + dcConsistencyService.syncSubscription(name, primaryDatacenter); + return Response.ok().build(); + } - @GET - @Produces({APPLICATION_JSON}) - @Path("/inconsistencies/topics") - public Response listInconsistentTopics() { - return Response - .ok(new GenericEntity>(kafkaHermesConsistencyService.listInconsistentTopics()) { - }) - .build(); - } + @GET + @Produces({APPLICATION_JSON}) + @Path("/inconsistencies/topics") + public Response listInconsistentTopics() { + return Response.ok( + new GenericEntity>( + kafkaHermesConsistencyService.listInconsistentTopics()) {}) + .build(); + } - @DELETE - @Produces({APPLICATION_JSON}) - @Path("/inconsistencies/topics") - public Response removeTopicByName(@QueryParam("topicName") String topicName, @Context ContainerRequestContext requestContext) { - kafkaHermesConsistencyService.removeTopic(topicName, new HermesSecurityAwareRequestUser(requestContext)); - return Response.ok().build(); - } + @DELETE + @Produces({APPLICATION_JSON}) + @Path("/inconsistencies/topics") + public Response removeTopicByName( + @QueryParam("topicName") String topicName, @Context ContainerRequestContext requestContext) { + kafkaHermesConsistencyService.removeTopic( + topicName, new HermesSecurityAwareRequestUser(requestContext)); + return Response.ok().build(); + } - @GET - @Produces({APPLICATION_JSON}) - @Path("/groups") - public Response listAllGroups() { - Set groupNames = dcConsistencyService.listAllGroupNames(); - return Response.ok() - .entity(new GenericEntity>(groupNames) { - }) - .build(); - } + @GET + @Produces({APPLICATION_JSON}) + @Path("/groups") + public Response listAllGroups() { + Set groupNames = dcConsistencyService.listAllGroupNames(); + return Response.ok().entity(new GenericEntity>(groupNames) {}).build(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/ConsoleEndpoint.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/ConsoleEndpoint.java index e2b4fa5288..b31f3e6e68 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/ConsoleEndpoint.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/ConsoleEndpoint.java @@ -14,26 +14,26 @@ @Api(value = "/", description = "Hermes console") public class ConsoleEndpoint { - private ConsoleService service; + private ConsoleService service; - public ConsoleEndpoint(ConsoleService service) { - this.service = service; - } + public ConsoleEndpoint(ConsoleService service) { + this.service = service; + } - @GET - @Path("/console") - @Produces("application/javascript") - @ApiOperation(value = "Hermes console configuration", httpMethod = HttpMethod.GET) - @Deprecated - public String getConfiguration() { - return service.getConfiguration(); - } + @GET + @Path("/console") + @Produces("application/javascript") + @ApiOperation(value = "Hermes console configuration", httpMethod = HttpMethod.GET) + @Deprecated + public String getConfiguration() { + return service.getConfiguration(); + } - @GET - @Path("/console") - @Produces("application/json") - @ApiOperation(value = "Hermes console configuration", httpMethod = HttpMethod.GET) - public String getConfigurationJson() { - return service.getConfigurationJson(); - } + @GET + @Path("/console") + @Produces("application/json") + @ApiOperation(value = "Hermes console configuration", httpMethod = HttpMethod.GET) + public String getConfigurationJson() { + return service.getConfigurationJson(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/CorsFilter.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/CorsFilter.java index 0cc80867fe..7247f47692 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/CorsFilter.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/CorsFilter.java @@ -11,27 +11,27 @@ @Provider public class CorsFilter implements ContainerResponseFilter { - private final CorsProperties corsProperties; + private final CorsProperties corsProperties; - @Autowired - public CorsFilter(CorsProperties corsProperties) { - this.corsProperties = corsProperties; - } + @Autowired + public CorsFilter(CorsProperties corsProperties) { + this.corsProperties = corsProperties; + } - @Override - public void filter(ContainerRequestContext requestContext, ContainerResponseContext responseContext) { - MultivaluedMap headers = responseContext.getHeaders(); - headers.add("Access-Control-Allow-Origin", corsProperties.getAllowedOrigin()); - headers.add("Access-Control-Allow-Methods", "POST,PUT,GET,HEAD,DELETE"); - headers.add("Access-Control-Max-Age", "1209600"); - headers.addAll( - "Access-Control-Allow-Headers", - "X-Requested-With", - "Content-Type", - "Accept", - "Origin", - "Authorization", - "Hermes-Admin-Password" - ); - } + @Override + public void filter( + ContainerRequestContext requestContext, ContainerResponseContext responseContext) { + MultivaluedMap headers = responseContext.getHeaders(); + headers.add("Access-Control-Allow-Origin", corsProperties.getAllowedOrigin()); + headers.add("Access-Control-Allow-Methods", "POST,PUT,GET,HEAD,DELETE"); + headers.add("Access-Control-Max-Age", "1209600"); + headers.addAll( + "Access-Control-Allow-Headers", + "X-Requested-With", + "Content-Type", + "Accept", + "Origin", + "Authorization", + "Hermes-Admin-Password"); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/FilterEndpoint.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/FilterEndpoint.java index 104eaa6553..446d4ab725 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/FilterEndpoint.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/FilterEndpoint.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.management.api; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; + import jakarta.ws.rs.Consumes; import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; @@ -11,23 +13,21 @@ import pl.allegro.tech.hermes.api.TopicName; import pl.allegro.tech.hermes.management.domain.filtering.FilteringService; -import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; - @Component @Path("filters") public class FilterEndpoint { - private final FilteringService filteringService; + private final FilteringService filteringService; - public FilterEndpoint(FilteringService filteringService) { - this.filteringService = filteringService; - } + public FilterEndpoint(FilteringService filteringService) { + this.filteringService = filteringService; + } - @POST - @Consumes(APPLICATION_JSON) - @Produces(APPLICATION_JSON) - @Path("/{topicName}") - public MessageFiltersVerificationResult verify(@PathParam("topicName") String qualifiedTopicName, - MessageFiltersVerificationInput input) { - return filteringService.verify(input, TopicName.fromQualifiedName(qualifiedTopicName)); - } + @POST + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON) + @Path("/{topicName}") + public MessageFiltersVerificationResult verify( + @PathParam("topicName") String qualifiedTopicName, MessageFiltersVerificationInput input) { + return filteringService.verify(input, TopicName.fromQualifiedName(qualifiedTopicName)); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/GroupsEndpoint.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/GroupsEndpoint.java index 58a0663ae7..b0a9ae82a5 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/GroupsEndpoint.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/GroupsEndpoint.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.management.api; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; + import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import jakarta.annotation.security.RolesAllowed; @@ -15,6 +17,7 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.Response; +import java.util.List; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import pl.allegro.tech.hermes.api.Group; @@ -25,83 +28,81 @@ import pl.allegro.tech.hermes.management.api.validator.ApiPreconditions; import pl.allegro.tech.hermes.management.domain.group.GroupService; -import java.util.List; - -import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; - @Component @Path("/groups") @Api(value = "/groups", description = "Operations on groups") public class GroupsEndpoint { - private final GroupService groupService; + private final GroupService groupService; - private final ApiPreconditions preconditions; + private final ApiPreconditions preconditions; - private final ManagementRights managementRights; + private final ManagementRights managementRights; - @Autowired - public GroupsEndpoint(GroupService groupService, - ApiPreconditions preconditions, - ManagementRights managementRights) { - this.groupService = groupService; - this.preconditions = preconditions; - this.managementRights = managementRights; - } + @Autowired + public GroupsEndpoint( + GroupService groupService, + ApiPreconditions preconditions, + ManagementRights managementRights) { + this.groupService = groupService; + this.preconditions = preconditions; + this.managementRights = managementRights; + } - @GET - @Produces(APPLICATION_JSON) - @ApiOperation(value = "List groups", response = List.class, httpMethod = HttpMethod.GET) - public List list() { - return groupService.listGroupNames(); - } + @GET + @Produces(APPLICATION_JSON) + @ApiOperation(value = "List groups", response = List.class, httpMethod = HttpMethod.GET) + public List list() { + return groupService.listGroupNames(); + } - @GET - @Produces(APPLICATION_JSON) - @Path("/{groupName}") - @ApiOperation(value = "Get group details", response = Group.class, httpMethod = HttpMethod.GET) - public Group get(@PathParam("groupName") String groupName) { - return groupService.getGroupDetails(groupName); - } + @GET + @Produces(APPLICATION_JSON) + @Path("/{groupName}") + @ApiOperation(value = "Get group details", response = Group.class, httpMethod = HttpMethod.GET) + public Group get(@PathParam("groupName") String groupName) { + return groupService.getGroupDetails(groupName); + } - @POST - @Consumes(APPLICATION_JSON) - @Produces(APPLICATION_JSON) - @ApiOperation(value = "Create group", response = String.class, httpMethod = HttpMethod.POST) - @RolesAllowed(Roles.ANY) - public Response create(Group group, @Context ContainerRequestContext requestContext) { - preconditions.checkConstraints(group, false); - groupService.createGroup( - group, - new HermesSecurityAwareRequestUser(requestContext), - managementRights.getGroupCreatorRights(requestContext) - ); - return Response.status(Response.Status.CREATED).build(); - } + @POST + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON) + @ApiOperation(value = "Create group", response = String.class, httpMethod = HttpMethod.POST) + @RolesAllowed(Roles.ANY) + public Response create(Group group, @Context ContainerRequestContext requestContext) { + preconditions.checkConstraints(group, false); + groupService.createGroup( + group, + new HermesSecurityAwareRequestUser(requestContext), + managementRights.getGroupCreatorRights(requestContext)); + return Response.status(Response.Status.CREATED).build(); + } - @PUT - @Consumes(APPLICATION_JSON) - @Produces(APPLICATION_JSON) - @Path("/{groupName}") - @ApiOperation(value = "Update group", response = String.class, httpMethod = HttpMethod.PUT) - @RolesAllowed(Roles.ADMIN) - public Response update(@PathParam("groupName") String groupName, - PatchData patch, - @Context ContainerRequestContext requestContext) { - groupService.updateGroup(groupName, patch, new HermesSecurityAwareRequestUser(requestContext)); - return responseStatus(Response.Status.NO_CONTENT); - } + @PUT + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON) + @Path("/{groupName}") + @ApiOperation(value = "Update group", response = String.class, httpMethod = HttpMethod.PUT) + @RolesAllowed(Roles.ADMIN) + public Response update( + @PathParam("groupName") String groupName, + PatchData patch, + @Context ContainerRequestContext requestContext) { + groupService.updateGroup(groupName, patch, new HermesSecurityAwareRequestUser(requestContext)); + return responseStatus(Response.Status.NO_CONTENT); + } - @DELETE - @Path("/{groupName}") - @ApiOperation(value = "Remove group", response = String.class, httpMethod = HttpMethod.DELETE) - @RolesAllowed(Roles.ANY) - public Response delete(@PathParam("groupName") String groupName, @Context ContainerRequestContext requestContext) { - groupService.removeGroup(groupName, new HermesSecurityAwareRequestUser(requestContext)); - return responseStatus(Response.Status.OK); - } + @DELETE + @Path("/{groupName}") + @ApiOperation(value = "Remove group", response = String.class, httpMethod = HttpMethod.DELETE) + @RolesAllowed(Roles.ANY) + public Response delete( + @PathParam("groupName") String groupName, @Context ContainerRequestContext requestContext) { + groupService.removeGroup(groupName, new HermesSecurityAwareRequestUser(requestContext)); + return responseStatus(Response.Status.OK); + } - private Response responseStatus(Response.Status responseStatus) { - return Response.status(responseStatus).build(); - } + private Response responseStatus(Response.Status responseStatus) { + return Response.status(responseStatus).build(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/MetricsDashboardUrlEndpoint.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/MetricsDashboardUrlEndpoint.java index 9491e193d8..6d71dbf1b2 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/MetricsDashboardUrlEndpoint.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/MetricsDashboardUrlEndpoint.java @@ -1,10 +1,13 @@ package pl.allegro.tech.hermes.management.api; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; + import jakarta.ws.rs.GET; import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import jakarta.ws.rs.Produces; import jakarta.ws.rs.core.Response; +import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -13,52 +16,53 @@ import pl.allegro.tech.hermes.management.domain.MetricsDashboardUrl; import pl.allegro.tech.hermes.management.domain.MetricsDashboardUrlService; -import java.util.Optional; - -import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; - @Component @Path("/dashboards") public class MetricsDashboardUrlEndpoint { - private static final Logger logger = LoggerFactory.getLogger(MetricsDashboardUrlEndpoint.class); + private static final Logger logger = LoggerFactory.getLogger(MetricsDashboardUrlEndpoint.class); - private final Optional metricsDashboardUrlService; + private final Optional metricsDashboardUrlService; - MetricsDashboardUrlEndpoint(Optional metricsDashboardUrlService) { - if (metricsDashboardUrlService.isEmpty()) { - logger.info("Dashboards url bean is absent"); - } - this.metricsDashboardUrlService = metricsDashboardUrlService; + MetricsDashboardUrlEndpoint(Optional metricsDashboardUrlService) { + if (metricsDashboardUrlService.isEmpty()) { + logger.info("Dashboards url bean is absent"); } + this.metricsDashboardUrlService = metricsDashboardUrlService; + } - @GET - @Path("/topics/{topic}") - @Produces(APPLICATION_JSON) - public Response fetchUrlForTopic(@PathParam("topic") String topic) { - return metricsDashboardUrlService - .map(service -> Response.ok(new MetricsDashboardUrl(service.getUrlForTopic(topic))).build()) - .orElseThrow(MetricsDashboardUrlEndpoint.MetricsDashboardUrlAbsentException::new); - } + @GET + @Path("/topics/{topic}") + @Produces(APPLICATION_JSON) + public Response fetchUrlForTopic(@PathParam("topic") String topic) { + return metricsDashboardUrlService + .map(service -> Response.ok(new MetricsDashboardUrl(service.getUrlForTopic(topic))).build()) + .orElseThrow(MetricsDashboardUrlEndpoint.MetricsDashboardUrlAbsentException::new); + } - @GET - @Path("/topics/{topic}/subscriptions/{subscription}") - @Produces(APPLICATION_JSON) - public Response fetchUrlForSubscription(@PathParam("topic") String topic, @PathParam("subscription") String subscription) { - return metricsDashboardUrlService - .map(service -> Response.ok(new MetricsDashboardUrl(service.getUrlForSubscription(topic, subscription))).build()) - .orElseThrow(MetricsDashboardUrlEndpoint.MetricsDashboardUrlAbsentException::new); - } + @GET + @Path("/topics/{topic}/subscriptions/{subscription}") + @Produces(APPLICATION_JSON) + public Response fetchUrlForSubscription( + @PathParam("topic") String topic, @PathParam("subscription") String subscription) { + return metricsDashboardUrlService + .map( + service -> + Response.ok( + new MetricsDashboardUrl(service.getUrlForSubscription(topic, subscription))) + .build()) + .orElseThrow(MetricsDashboardUrlEndpoint.MetricsDashboardUrlAbsentException::new); + } - private static class MetricsDashboardUrlAbsentException extends ManagementException { + private static class MetricsDashboardUrlAbsentException extends ManagementException { - MetricsDashboardUrlAbsentException() { - super("Dashboard url implementation is absent"); - } + MetricsDashboardUrlAbsentException() { + super("Dashboard url implementation is absent"); + } - @Override - public ErrorCode getCode() { - return ErrorCode.IMPLEMENTATION_ABSENT; - } + @Override + public ErrorCode getCode() { + return ErrorCode.IMPLEMENTATION_ABSENT; } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/ModeEndpoint.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/ModeEndpoint.java index b8abc1c208..e3fdb8d5b0 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/ModeEndpoint.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/ModeEndpoint.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.management.api; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; +import static jakarta.ws.rs.core.MediaType.TEXT_PLAIN; +import static pl.allegro.tech.hermes.management.domain.mode.ModeService.ManagementMode.READ_ONLY_ADMIN; +import static pl.allegro.tech.hermes.management.domain.mode.ModeService.ManagementMode.READ_WRITE; + import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import jakarta.annotation.security.RolesAllowed; @@ -14,48 +19,46 @@ import pl.allegro.tech.hermes.management.api.auth.Roles; import pl.allegro.tech.hermes.management.domain.mode.ModeService; -import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; -import static jakarta.ws.rs.core.MediaType.TEXT_PLAIN; -import static pl.allegro.tech.hermes.management.domain.mode.ModeService.ManagementMode.READ_ONLY_ADMIN; -import static pl.allegro.tech.hermes.management.domain.mode.ModeService.ManagementMode.READ_WRITE; - @Component @Path("/mode") @Api(value = "/mode", description = "Operations on management mode") public class ModeEndpoint { - private final ModeService modeService; + private final ModeService modeService; - public ModeEndpoint(ModeService modeService) { - this.modeService = modeService; - } + public ModeEndpoint(ModeService modeService) { + this.modeService = modeService; + } - @GET - @Produces(TEXT_PLAIN) - @ApiOperation(value = "Get management mode", response = String.class, httpMethod = HttpMethod.GET) - public String getMode() { - return modeService.getMode().toString(); - } + @GET + @Produces(TEXT_PLAIN) + @ApiOperation(value = "Get management mode", response = String.class, httpMethod = HttpMethod.GET) + public String getMode() { + return modeService.getMode().toString(); + } - @POST - @Produces(APPLICATION_JSON) - @ApiOperation(value = "Set management mode", response = String.class, httpMethod = HttpMethod.POST) - @RolesAllowed(Roles.ADMIN) - public Response setMode(@QueryParam("mode") String mode) { - if (mode == null) { - return Response.status(Response.Status.BAD_REQUEST).build(); - } - switch (mode) { - case ModeService.READ_WRITE: - modeService.setModeByAdmin(READ_WRITE); - break; - case ModeService.READ_ONLY: - case ModeService.READ_ONLY_ADMIN: - modeService.setModeByAdmin(READ_ONLY_ADMIN); - break; - default: - return Response.status(Response.Status.BAD_REQUEST).build(); - } - return Response.status(Response.Status.OK).build(); + @POST + @Produces(APPLICATION_JSON) + @ApiOperation( + value = "Set management mode", + response = String.class, + httpMethod = HttpMethod.POST) + @RolesAllowed(Roles.ADMIN) + public Response setMode(@QueryParam("mode") String mode) { + if (mode == null) { + return Response.status(Response.Status.BAD_REQUEST).build(); + } + switch (mode) { + case ModeService.READ_WRITE: + modeService.setModeByAdmin(READ_WRITE); + break; + case ModeService.READ_ONLY: + case ModeService.READ_ONLY_ADMIN: + modeService.setModeByAdmin(READ_ONLY_ADMIN); + break; + default: + return Response.status(Response.Status.BAD_REQUEST).build(); } + return Response.status(Response.Status.OK).build(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/OAuthProvidersEndpoint.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/OAuthProvidersEndpoint.java index c4daf90171..06316bd363 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/OAuthProvidersEndpoint.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/OAuthProvidersEndpoint.java @@ -1,5 +1,8 @@ package pl.allegro.tech.hermes.management.api; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; +import static jakarta.ws.rs.core.Response.status; + import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import jakarta.annotation.security.RolesAllowed; @@ -15,6 +18,7 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.Response; +import java.util.List; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import pl.allegro.tech.hermes.api.OAuthProvider; @@ -23,69 +27,69 @@ import pl.allegro.tech.hermes.management.api.auth.Roles; import pl.allegro.tech.hermes.management.domain.oauth.OAuthProviderService; -import java.util.List; - -import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; -import static jakarta.ws.rs.core.Response.status; - @Component @Path("/oauth/providers") @Api(value = "/oauth/providers", description = "Operations on OAuth providers") public class OAuthProvidersEndpoint { - private final OAuthProviderService service; + private final OAuthProviderService service; - @Autowired - public OAuthProvidersEndpoint(OAuthProviderService service) { - this.service = service; - } + @Autowired + public OAuthProvidersEndpoint(OAuthProviderService service) { + this.service = service; + } - @GET - @Produces(APPLICATION_JSON) - @ApiOperation(value = "List OAuth providers", httpMethod = HttpMethod.GET) - public List list() { - return service.listOAuthProviderNames(); - } + @GET + @Produces(APPLICATION_JSON) + @ApiOperation(value = "List OAuth providers", httpMethod = HttpMethod.GET) + public List list() { + return service.listOAuthProviderNames(); + } - @GET - @Produces(APPLICATION_JSON) - @Path("/{oAuthProviderName}") - @ApiOperation(value = "OAuth provider details", httpMethod = HttpMethod.GET) - public OAuthProvider get(@PathParam("oAuthProviderName") String oAuthProviderName) { - return service.getOAuthProviderDetails(oAuthProviderName); - } + @GET + @Produces(APPLICATION_JSON) + @Path("/{oAuthProviderName}") + @ApiOperation(value = "OAuth provider details", httpMethod = HttpMethod.GET) + public OAuthProvider get(@PathParam("oAuthProviderName") String oAuthProviderName) { + return service.getOAuthProviderDetails(oAuthProviderName); + } - @POST - @Consumes(APPLICATION_JSON) - @Produces(APPLICATION_JSON) - @RolesAllowed(Roles.ADMIN) - @ApiOperation(value = "Create OAuth provider", httpMethod = HttpMethod.POST) - public Response create(OAuthProvider oAuthProvider, - @Context ContainerRequestContext requestContext) { - service.createOAuthProvider(oAuthProvider, new HermesSecurityAwareRequestUser(requestContext)); - return status(Response.Status.CREATED).build(); - } + @POST + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON) + @RolesAllowed(Roles.ADMIN) + @ApiOperation(value = "Create OAuth provider", httpMethod = HttpMethod.POST) + public Response create( + OAuthProvider oAuthProvider, @Context ContainerRequestContext requestContext) { + service.createOAuthProvider(oAuthProvider, new HermesSecurityAwareRequestUser(requestContext)); + return status(Response.Status.CREATED).build(); + } - @PUT - @Consumes(APPLICATION_JSON) - @Produces(APPLICATION_JSON) - @RolesAllowed(Roles.ADMIN) - @Path("/{oAuthProviderName}") - @ApiOperation(value = "Update OAuth provider", httpMethod = HttpMethod.PUT) - public Response update(@PathParam("oAuthProviderName") String oAuthProviderName, PatchData patch, - @Context ContainerRequestContext requestContext) { - service.updateOAuthProvider(oAuthProviderName, patch, new HermesSecurityAwareRequestUser(requestContext)); - return status(Response.Status.OK).build(); - } + @PUT + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON) + @RolesAllowed(Roles.ADMIN) + @Path("/{oAuthProviderName}") + @ApiOperation(value = "Update OAuth provider", httpMethod = HttpMethod.PUT) + public Response update( + @PathParam("oAuthProviderName") String oAuthProviderName, + PatchData patch, + @Context ContainerRequestContext requestContext) { + service.updateOAuthProvider( + oAuthProviderName, patch, new HermesSecurityAwareRequestUser(requestContext)); + return status(Response.Status.OK).build(); + } - @DELETE - @Produces(APPLICATION_JSON) - @RolesAllowed(Roles.ADMIN) - @Path("/{oAuthProviderName}") - @ApiOperation(value = "Remove OAuth provider", httpMethod = HttpMethod.DELETE) - public Response remove(@PathParam("oAuthProviderName") String oAuthProviderName, - @Context ContainerRequestContext requestContext) { - service.removeOAuthProvider(oAuthProviderName, new HermesSecurityAwareRequestUser(requestContext)); - return status(Response.Status.OK).build(); - } + @DELETE + @Produces(APPLICATION_JSON) + @RolesAllowed(Roles.ADMIN) + @Path("/{oAuthProviderName}") + @ApiOperation(value = "Remove OAuth provider", httpMethod = HttpMethod.DELETE) + public Response remove( + @PathParam("oAuthProviderName") String oAuthProviderName, + @Context ContainerRequestContext requestContext) { + service.removeOAuthProvider( + oAuthProviderName, new HermesSecurityAwareRequestUser(requestContext)); + return status(Response.Status.OK).build(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/OfflineClientsEndpoint.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/OfflineClientsEndpoint.java index 25bc4d382a..aeb03c99cc 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/OfflineClientsEndpoint.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/OfflineClientsEndpoint.java @@ -1,10 +1,13 @@ package pl.allegro.tech.hermes.management.api; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; + import jakarta.ws.rs.GET; import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import jakarta.ws.rs.Produces; import jakarta.ws.rs.core.Response; +import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -14,44 +17,44 @@ import pl.allegro.tech.hermes.management.domain.clients.IframeSource; import pl.allegro.tech.hermes.management.domain.clients.OfflineClientsService; -import java.util.Optional; - -import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; - @Component @Path("/topics") public class OfflineClientsEndpoint { - private static final Logger logger = LoggerFactory.getLogger(OfflineClientsEndpoint.class); + private static final Logger logger = LoggerFactory.getLogger(OfflineClientsEndpoint.class); - private final Optional offlineClientsService; + private final Optional offlineClientsService; - OfflineClientsEndpoint(Optional offlineClientsService) { - if (!offlineClientsService.isPresent()) { - logger.info("Offline clients bean is absent"); - } - this.offlineClientsService = offlineClientsService; + OfflineClientsEndpoint(Optional offlineClientsService) { + if (!offlineClientsService.isPresent()) { + logger.info("Offline clients bean is absent"); } - - @GET - @Path("/{topic}/offline-clients-source") - @Produces(APPLICATION_JSON) - public Response find(@PathParam("topic") String topic) { - return offlineClientsService - .map(service -> Response.ok(new IframeSource(service.getIframeSource(TopicName.fromQualifiedName(topic)))).build()) - .orElseThrow(OfflineClientsServiceAbsentException::new); + this.offlineClientsService = offlineClientsService; + } + + @GET + @Path("/{topic}/offline-clients-source") + @Produces(APPLICATION_JSON) + public Response find(@PathParam("topic") String topic) { + return offlineClientsService + .map( + service -> + Response.ok( + new IframeSource( + service.getIframeSource(TopicName.fromQualifiedName(topic)))) + .build()) + .orElseThrow(OfflineClientsServiceAbsentException::new); + } + + private static class OfflineClientsServiceAbsentException extends ManagementException { + + OfflineClientsServiceAbsentException() { + super("Offline clients implementation is absent"); } - private static class OfflineClientsServiceAbsentException extends ManagementException { - - OfflineClientsServiceAbsentException() { - super("Offline clients implementation is absent"); - } - - @Override - public ErrorCode getCode() { - return ErrorCode.IMPLEMENTATION_ABSENT; - } + @Override + public ErrorCode getCode() { + return ErrorCode.IMPLEMENTATION_ABSENT; } + } } - diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/OfflineRetransmissionEndpoint.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/OfflineRetransmissionEndpoint.java index 2cbd5ea6ed..1b3fe564a8 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/OfflineRetransmissionEndpoint.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/OfflineRetransmissionEndpoint.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.management.api; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; + import io.swagger.annotations.Api; import jakarta.validation.Valid; import jakarta.ws.rs.Consumes; @@ -12,6 +14,8 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.Response; +import java.util.List; +import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -23,90 +27,105 @@ import pl.allegro.tech.hermes.management.domain.PermissionDeniedException; import pl.allegro.tech.hermes.management.domain.retransmit.OfflineRetransmissionService; -import java.util.List; -import java.util.Optional; - -import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; - @Component @Path("offline-retransmission/tasks") @Api(value = "offline-retransmission/tasks", description = "Offline retransmission operations") public class OfflineRetransmissionEndpoint { - private final OfflineRetransmissionService retransmissionService; - private final RetransmissionPermissions permissions; - private final OfflineRetransmissionAuditor auditor; - private final Logger logger = LoggerFactory.getLogger(OfflineRetransmissionEndpoint.class); - - public OfflineRetransmissionEndpoint(OfflineRetransmissionService retransmissionService, - TopicRepository topicRepository, ManagementRights managementRights) { - this.retransmissionService = retransmissionService; - this.permissions = new RetransmissionPermissions(topicRepository, managementRights); - this.auditor = new OfflineRetransmissionAuditor(); + private final OfflineRetransmissionService retransmissionService; + private final RetransmissionPermissions permissions; + private final OfflineRetransmissionAuditor auditor; + private final Logger logger = LoggerFactory.getLogger(OfflineRetransmissionEndpoint.class); + + public OfflineRetransmissionEndpoint( + OfflineRetransmissionService retransmissionService, + TopicRepository topicRepository, + ManagementRights managementRights) { + this.retransmissionService = retransmissionService; + this.permissions = new RetransmissionPermissions(topicRepository, managementRights); + this.auditor = new OfflineRetransmissionAuditor(); + } + + @POST + @Consumes(APPLICATION_JSON) + public Response createRetransmissionTask( + @Valid OfflineRetransmissionRequest request, + @Context ContainerRequestContext requestContext) { + logger.info("Offline retransmission request: {}", request); + retransmissionService.validateRequest(request); + permissions.ensurePermissions(request, requestContext); + var task = retransmissionService.createTask(request); + auditor.auditRetransmissionCreation(request, requestContext, task); + return Response.status(Response.Status.CREATED).build(); + } + + @GET + @Produces(APPLICATION_JSON) + public List getAllRetransmissionTasks() { + return retransmissionService.getAllTasks(); + } + + @DELETE + @Path("/{taskId}") + public Response deleteRetransmissionTask(@PathParam("taskId") String taskId) { + retransmissionService.deleteTask(taskId); + return Response.status(Response.Status.OK).build(); + } + + private static class RetransmissionPermissions { + private final Logger logger = LoggerFactory.getLogger(RetransmissionPermissions.class); + private final TopicRepository topicRepository; + private final ManagementRights managementRights; + + private RetransmissionPermissions( + TopicRepository topicRepository, ManagementRights managementRights) { + this.topicRepository = topicRepository; + this.managementRights = managementRights; } - @POST - @Consumes(APPLICATION_JSON) - public Response createRetransmissionTask(@Valid OfflineRetransmissionRequest request, @Context ContainerRequestContext requestContext) { - logger.info("Offline retransmission request: {}", request); - retransmissionService.validateRequest(request); - permissions.ensurePermissions(request, requestContext); - var task = retransmissionService.createTask(request); - auditor.auditRetransmissionCreation(request, requestContext, task); - return Response.status(Response.Status.CREATED).build(); + private void ensurePermissions( + OfflineRetransmissionRequest request, ContainerRequestContext requestContext) { + var targetTopic = + topicRepository.getTopicDetails(TopicName.fromQualifiedName(request.getTargetTopic())); + var hasPermissions = + validateSourceTopic(request.getSourceTopic(), requestContext) + && managementRights.isUserAllowedToManageTopic(targetTopic, requestContext); + if (!hasPermissions) { + logger.info( + "User {} has no permissions to make offline retransmission {}", + requestContext.getSecurityContext().getUserPrincipal(), + request); + throw new PermissionDeniedException("User needs permissions to source and target topics."); + } } - @GET - @Produces(APPLICATION_JSON) - public List getAllRetransmissionTasks() { - return retransmissionService.getAllTasks(); + private boolean validateSourceTopic( + Optional sourceTopic, ContainerRequestContext requestContext) { + return sourceTopic.isEmpty() + || managementRights.isUserAllowedToManageTopic( + topicRepository.getTopicDetails(TopicName.fromQualifiedName(sourceTopic.get())), + requestContext); } - - @DELETE - @Path("/{taskId}") - public Response deleteRetransmissionTask(@PathParam("taskId") String taskId) { - retransmissionService.deleteTask(taskId); - return Response.status(Response.Status.OK).build(); + } + + private static class OfflineRetransmissionAuditor { + private static final Logger logger = + LoggerFactory.getLogger(OfflineRetransmissionAuditor.class); + + public void auditRetransmissionCreation( + OfflineRetransmissionRequest request, + ContainerRequestContext requestContext, + OfflineRetransmissionTask task) { + String username = extractUsername(requestContext); + logger.info( + "User {} created offline retransmission task: {}, taskId: {}", + username, + request, + task.getTaskId()); } - private static class RetransmissionPermissions { - private final Logger logger = LoggerFactory.getLogger(RetransmissionPermissions.class); - private final TopicRepository topicRepository; - private final ManagementRights managementRights; - - private RetransmissionPermissions(TopicRepository topicRepository, ManagementRights managementRights) { - this.topicRepository = topicRepository; - this.managementRights = managementRights; - } - - private void ensurePermissions(OfflineRetransmissionRequest request, ContainerRequestContext requestContext) { - var targetTopic = topicRepository.getTopicDetails(TopicName.fromQualifiedName(request.getTargetTopic())); - var hasPermissions = validateSourceTopic(request.getSourceTopic(), requestContext) && managementRights.isUserAllowedToManageTopic(targetTopic, requestContext); - if (!hasPermissions) { - logger.info("User {} has no permissions to make offline retransmission {}", requestContext.getSecurityContext().getUserPrincipal(), request); - throw new PermissionDeniedException("User needs permissions to source and target topics."); - } - } - - private boolean validateSourceTopic(Optional sourceTopic, ContainerRequestContext requestContext) { - return sourceTopic.isEmpty() || managementRights.isUserAllowedToManageTopic( - topicRepository.getTopicDetails(TopicName.fromQualifiedName(sourceTopic.get())), - requestContext - ); - } - } - - private static class OfflineRetransmissionAuditor { - private static final Logger logger = LoggerFactory.getLogger(OfflineRetransmissionAuditor.class); - - public void auditRetransmissionCreation(OfflineRetransmissionRequest request, ContainerRequestContext requestContext, OfflineRetransmissionTask task) { - String username = extractUsername(requestContext); - logger.info("User {} created offline retransmission task: {}, taskId: {}", username, request, task.getTaskId()); - } - - private String extractUsername(ContainerRequestContext requestContext) { - return requestContext.getSecurityContext().getUserPrincipal().getName(); - } + private String extractUsername(ContainerRequestContext requestContext) { + return requestContext.getSecurityContext().getUserPrincipal().getName(); } + } } - diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/OwnersEndpoint.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/OwnersEndpoint.java index e6b9cb7ccc..225d1e6ab6 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/OwnersEndpoint.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/OwnersEndpoint.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.management.api; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; + import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; @@ -9,6 +11,9 @@ import jakarta.ws.rs.PathParam; import jakarta.ws.rs.Produces; import jakarta.ws.rs.QueryParam; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import pl.allegro.tech.hermes.api.Owner; @@ -16,76 +21,73 @@ import pl.allegro.tech.hermes.management.domain.owner.OwnerSourceNotFound; import pl.allegro.tech.hermes.management.domain.owner.OwnerSources; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; - -import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; - @Component @Path("/owners") @Api(value = "/owners", description = "Provides owners information") public class OwnersEndpoint { - private final OwnerSources ownerSources; - - @Autowired - public OwnersEndpoint(OwnerSources ownerSources) { - this.ownerSources = ownerSources; + private final OwnerSources ownerSources; + + @Autowired + public OwnersEndpoint(OwnerSources ownerSources) { + this.ownerSources = ownerSources; + } + + @GET + @Path("/sources/{source}") + @Produces(APPLICATION_JSON) + @ApiOperation( + value = "Lists owners from the given source matching the search string", + response = List.class, + httpMethod = HttpMethod.GET) + public List search( + @PathParam("source") String source, @QueryParam("search") String searchString) { + return ownerSources.getAutocompletionFor(source).ownersMatching(searchString); + } + + @GET + @Path("/sources/{source}/{id}") + @Produces(APPLICATION_JSON) + @ApiOperation( + value = "Returns owner from the given source of the given id", + response = List.class, + httpMethod = HttpMethod.GET) + public Owner get(@PathParam("source") String source, @PathParam("id") String id) { + return ownerSources + .getByName(source) + .map(s -> s.get(id)) + .orElseThrow(() -> new OwnerSourceNotFound(source)); + } + + @GET + @Path("/sources") + @Produces(APPLICATION_JSON) + @ApiOperation(value = "Lists owner sources", response = List.class, httpMethod = HttpMethod.GET) + public List listSources() { + return StreamSupport.stream(ownerSources.spliterator(), false) + .map(SourceDescriptor::of) + .collect(Collectors.toList()); + } + + private static class SourceDescriptor { + + @JsonProperty("name") + private final String name; + + @JsonProperty("autocomplete") + private final boolean autocomplete; + + @JsonProperty("deprecated") + private final boolean deprecated; + + private SourceDescriptor(OwnerSource source) { + this.name = source.name(); + this.autocomplete = source.autocompletion().isPresent(); + this.deprecated = source.isDeprecated(); } - @GET - @Path("/sources/{source}") - @Produces(APPLICATION_JSON) - @ApiOperation(value = "Lists owners from the given source matching the search string", - response = List.class, httpMethod = HttpMethod.GET) - public List search(@PathParam("source") String source, - @QueryParam("search") String searchString) { - return ownerSources.getAutocompletionFor(source).ownersMatching(searchString); + static SourceDescriptor of(OwnerSource source) { + return new SourceDescriptor(source); } - - @GET - @Path("/sources/{source}/{id}") - @Produces(APPLICATION_JSON) - @ApiOperation(value = "Returns owner from the given source of the given id", response = List.class, httpMethod = HttpMethod.GET) - public Owner get(@PathParam("source") String source, - @PathParam("id") String id) { - return ownerSources.getByName(source) - .map(s -> s.get(id)) - .orElseThrow(() -> new OwnerSourceNotFound(source)); - } - - @GET - @Path("/sources") - @Produces(APPLICATION_JSON) - @ApiOperation(value = "Lists owner sources", response = List.class, httpMethod = HttpMethod.GET) - public List listSources() { - return StreamSupport.stream(ownerSources.spliterator(), false) - .map(SourceDescriptor::of) - .collect(Collectors.toList()); - } - - private static class SourceDescriptor { - - @JsonProperty("name") - private final String name; - - @JsonProperty("autocomplete") - private final boolean autocomplete; - - @JsonProperty("deprecated") - private final boolean deprecated; - - private SourceDescriptor(OwnerSource source) { - this.name = source.name(); - this.autocomplete = source.autocompletion().isPresent(); - this.deprecated = source.isDeprecated(); - } - - static SourceDescriptor of(OwnerSource source) { - return new SourceDescriptor(source); - } - - } - + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/QueryEndpoint.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/QueryEndpoint.java index c7c8bf183d..fc895347db 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/QueryEndpoint.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/QueryEndpoint.java @@ -1,9 +1,12 @@ package pl.allegro.tech.hermes.management.api; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; + import jakarta.ws.rs.Consumes; import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; import jakarta.ws.rs.Produces; +import java.util.List; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import pl.allegro.tech.hermes.api.Group; @@ -16,63 +19,62 @@ import pl.allegro.tech.hermes.management.domain.subscription.SubscriptionService; import pl.allegro.tech.hermes.management.domain.topic.TopicService; -import java.util.List; - -import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; - @Path("query") @Component public class QueryEndpoint { - private final SubscriptionService subscriptionService; - private final TopicService topicService; - private final GroupService groupService; - - @Autowired - public QueryEndpoint(SubscriptionService subscriptionService, TopicService topicService, GroupService groupService) { - this.subscriptionService = subscriptionService; - this.topicService = topicService; - this.groupService = groupService; - } + private final SubscriptionService subscriptionService; + private final TopicService topicService; + private final GroupService groupService; - @POST - @Consumes(APPLICATION_JSON) - @Produces(APPLICATION_JSON) - @Path("/groups") - public List queryGroups(Query query) { - return groupService.queryGroup(query); - } + @Autowired + public QueryEndpoint( + SubscriptionService subscriptionService, + TopicService topicService, + GroupService groupService) { + this.subscriptionService = subscriptionService; + this.topicService = topicService; + this.groupService = groupService; + } - @POST - @Consumes(APPLICATION_JSON) - @Produces(APPLICATION_JSON) - @Path("/topics") - public List queryTopics(Query query) { - return topicService.queryTopic(query); - } + @POST + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON) + @Path("/groups") + public List queryGroups(Query query) { + return groupService.queryGroup(query); + } - @POST - @Consumes(APPLICATION_JSON) - @Produces(APPLICATION_JSON) - @Path("/subscriptions") - public List querySubscriptions(Query query) { - return subscriptionService.querySubscription(query); - } + @POST + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON) + @Path("/topics") + public List queryTopics(Query query) { + return topicService.queryTopic(query); + } - @POST - @Consumes(APPLICATION_JSON) - @Produces(APPLICATION_JSON) - @Path("/topics/metrics") - public List queryTopicsMetrics(Query query) { - return topicService.queryTopicsMetrics(query); - } + @POST + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON) + @Path("/subscriptions") + public List querySubscriptions(Query query) { + return subscriptionService.querySubscription(query); + } - @POST - @Consumes(APPLICATION_JSON) - @Produces(APPLICATION_JSON) - @Path("subscriptions/metrics") - public List querySubscriptionsMetrics(Query query) { - return subscriptionService.querySubscriptionsMetrics(query); - } + @POST + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON) + @Path("/topics/metrics") + public List queryTopicsMetrics(Query query) { + return topicService.queryTopicsMetrics(query); + } + @POST + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON) + @Path("subscriptions/metrics") + public List querySubscriptionsMetrics( + Query query) { + return subscriptionService.querySubscriptionsMetrics(query); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/ReadOnlyFilter.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/ReadOnlyFilter.java index 1a552e6003..369c3a6d37 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/ReadOnlyFilter.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/ReadOnlyFilter.java @@ -5,54 +5,53 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.container.ContainerRequestFilter; import jakarta.ws.rs.ext.Provider; +import java.io.IOException; import org.glassfish.jersey.server.ContainerRequest; import org.springframework.beans.factory.annotation.Autowired; import pl.allegro.tech.hermes.management.api.auth.AuthorizationFilter; import pl.allegro.tech.hermes.management.api.auth.Roles; import pl.allegro.tech.hermes.management.domain.mode.ModeService; -import java.io.IOException; - @Provider @Priority(AuthorizationFilter.AUTHORIZATION_FILTER_PRIORITY + 2) public class ReadOnlyFilter implements ContainerRequestFilter { - private static final String READ_ONLY_ERROR_MESSAGE = "Action forbidden due to read-only mode"; + private static final String READ_ONLY_ERROR_MESSAGE = "Action forbidden due to read-only mode"; + + private final ModeService modeService; + + @Autowired + public ReadOnlyFilter(ModeService modeService) { + this.modeService = modeService; + } + + @Override + public void filter(ContainerRequestContext requestContext) throws IOException { + if (modeService.isReadOnlyEnabled()) { + ContainerRequest req = (ContainerRequest) requestContext.getRequest(); + if (!isWhitelisted(req) && !isAdmin(requestContext)) { + throw new ServiceUnavailableException(READ_ONLY_ERROR_MESSAGE); + } + } + } - private final ModeService modeService; + private boolean isAdmin(ContainerRequestContext requestContext) { + return requestContext.getSecurityContext().isUserInRole(Roles.ADMIN); + } - @Autowired - public ReadOnlyFilter(ModeService modeService) { - this.modeService = modeService; + private boolean isWhitelisted(ContainerRequest req) { + if (req.getMethod().equals("GET")) { + return true; } - - @Override - public void filter(ContainerRequestContext requestContext) throws IOException { - if (modeService.isReadOnlyEnabled()) { - ContainerRequest req = (ContainerRequest) requestContext.getRequest(); - if (!isWhitelisted(req) && !isAdmin(requestContext)) { - throw new ServiceUnavailableException(READ_ONLY_ERROR_MESSAGE); - } - } + String requestURI = req.getUriInfo().getPath(); + if (requestURI.startsWith("/query")) { + return true; } - - private boolean isAdmin(ContainerRequestContext requestContext) { - return requestContext.getSecurityContext().isUserInRole(Roles.ADMIN); + if (requestURI.startsWith("/mode")) { + return true; } - - private boolean isWhitelisted(ContainerRequest req) { - if (req.getMethod().equals("GET")) { - return true; - } - String requestURI = req.getUriInfo().getPath(); - if (requestURI.startsWith("/query")) { - return true; - } - if (requestURI.startsWith("/mode")) { - return true; - } - if (requestURI.startsWith("/topics") && requestURI.endsWith("query")) { - return true; - } - return false; + if (requestURI.startsWith("/topics") && requestURI.endsWith("query")) { + return true; } + return false; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/ReadinessEndpoint.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/ReadinessEndpoint.java index be8a50732c..fa181ccd92 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/ReadinessEndpoint.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/ReadinessEndpoint.java @@ -1,5 +1,9 @@ package pl.allegro.tech.hermes.management.api; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; +import static pl.allegro.tech.hermes.api.DatacenterReadiness.ReadinessStatus.NOT_READY; +import static pl.allegro.tech.hermes.api.DatacenterReadiness.ReadinessStatus.READY; + import jakarta.annotation.security.RolesAllowed; import jakarta.ws.rs.Consumes; import jakarta.ws.rs.GET; @@ -8,43 +12,39 @@ import jakarta.ws.rs.PathParam; import jakarta.ws.rs.Produces; import jakarta.ws.rs.core.Response; +import java.util.List; import org.springframework.stereotype.Component; import pl.allegro.tech.hermes.api.DatacenterReadiness; import pl.allegro.tech.hermes.api.Readiness; import pl.allegro.tech.hermes.management.api.auth.Roles; import pl.allegro.tech.hermes.management.domain.readiness.ReadinessService; -import java.util.List; - -import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; -import static pl.allegro.tech.hermes.api.DatacenterReadiness.ReadinessStatus.NOT_READY; -import static pl.allegro.tech.hermes.api.DatacenterReadiness.ReadinessStatus.READY; - @Path("readiness/datacenters") @Component public class ReadinessEndpoint { - private final ReadinessService readinessService; - - public ReadinessEndpoint(ReadinessService readinessService) { - this.readinessService = readinessService; - } - - @POST - @Consumes(APPLICATION_JSON) - @Produces(APPLICATION_JSON) - @RolesAllowed(Roles.ADMIN) - @Path("/{datacenter}") - public Response setReadiness(@PathParam("datacenter") String datacenter, Readiness readiness) { - readinessService.setReady(new DatacenterReadiness(datacenter, readiness.isReady() ? READY : NOT_READY)); - return Response.status(Response.Status.ACCEPTED).build(); - } - - @GET - @Consumes(APPLICATION_JSON) - @Produces(APPLICATION_JSON) - @RolesAllowed(Roles.ADMIN) - public List getReadiness() { - return readinessService.getDatacentersReadiness(); - } + private final ReadinessService readinessService; + + public ReadinessEndpoint(ReadinessService readinessService) { + this.readinessService = readinessService; + } + + @POST + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON) + @RolesAllowed(Roles.ADMIN) + @Path("/{datacenter}") + public Response setReadiness(@PathParam("datacenter") String datacenter, Readiness readiness) { + readinessService.setReady( + new DatacenterReadiness(datacenter, readiness.isReady() ? READY : NOT_READY)); + return Response.status(Response.Status.ACCEPTED).build(); + } + + @GET + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON) + @RolesAllowed(Roles.ADMIN) + public List getReadiness() { + return readinessService.getDatacentersReadiness(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/RolesEndpoint.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/RolesEndpoint.java index 3ef288a9ed..80e7ea5231 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/RolesEndpoint.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/RolesEndpoint.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.management.api; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; + import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import jakarta.ws.rs.GET; @@ -8,63 +10,61 @@ import jakarta.ws.rs.Produces; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.SecurityContext; -import org.springframework.stereotype.Component; -import pl.allegro.tech.hermes.management.api.auth.Roles; - import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.function.Consumer; - -import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; +import org.springframework.stereotype.Component; +import pl.allegro.tech.hermes.management.api.auth.Roles; @Component @Path("/roles") @Api(value = "/roles", description = "Get user roles for given resource") public class RolesEndpoint { - @GET - @Produces(APPLICATION_JSON) - @ApiOperation(value = "Get general user roles", httpMethod = HttpMethod.GET) - public Collection getRoles(ContainerRequestContext requestContext) { - return getRoles(requestContext, Collections.emptyList()); - } - - private Collection getRoles(ContainerRequestContext requestContext, Collection additionalRoles) { - SecurityContext securityContext = requestContext.getSecurityContext(); - Collection roles = new ArrayList<>(); + @GET + @Produces(APPLICATION_JSON) + @ApiOperation(value = "Get general user roles", httpMethod = HttpMethod.GET) + public Collection getRoles(ContainerRequestContext requestContext) { + return getRoles(requestContext, Collections.emptyList()); + } - ifUserInRoleDo(securityContext, Roles.ADMIN, roles::add); - ifUserInRoleDo(securityContext, Roles.ANY, roles::add); + private Collection getRoles( + ContainerRequestContext requestContext, Collection additionalRoles) { + SecurityContext securityContext = requestContext.getSecurityContext(); + Collection roles = new ArrayList<>(); - for (String role : additionalRoles) { - ifUserInRoleDo(securityContext, role, roles::add); - } + ifUserInRoleDo(securityContext, Roles.ADMIN, roles::add); + ifUserInRoleDo(securityContext, Roles.ANY, roles::add); - return roles; + for (String role : additionalRoles) { + ifUserInRoleDo(securityContext, role, roles::add); } - @GET - @Produces(APPLICATION_JSON) - @Path("/topics/{topicName}") - @ApiOperation(value = "Get topic user roles", httpMethod = HttpMethod.GET) - public Collection getTopicRoles(ContainerRequestContext requestContext) { - return getRoles(requestContext, Collections.singletonList(Roles.TOPIC_OWNER)); - } + return roles; + } - @GET - @Produces(APPLICATION_JSON) - @Path("/topics/{topicName}/subscriptions/{subscriptionName}") - @ApiOperation(value = "Get subscription user roles", httpMethod = HttpMethod.GET) - public Collection getSubscriptionRoles(ContainerRequestContext requestContext) { - return getRoles(requestContext, Arrays.asList(Roles.TOPIC_OWNER, Roles.SUBSCRIPTION_OWNER)); - } + @GET + @Produces(APPLICATION_JSON) + @Path("/topics/{topicName}") + @ApiOperation(value = "Get topic user roles", httpMethod = HttpMethod.GET) + public Collection getTopicRoles(ContainerRequestContext requestContext) { + return getRoles(requestContext, Collections.singletonList(Roles.TOPIC_OWNER)); + } - private void ifUserInRoleDo(SecurityContext securityContext, String role, Consumer consumer) { - if (securityContext.isUserInRole(role)) { - consumer.accept(role); - } - } + @GET + @Produces(APPLICATION_JSON) + @Path("/topics/{topicName}/subscriptions/{subscriptionName}") + @ApiOperation(value = "Get subscription user roles", httpMethod = HttpMethod.GET) + public Collection getSubscriptionRoles(ContainerRequestContext requestContext) { + return getRoles(requestContext, Arrays.asList(Roles.TOPIC_OWNER, Roles.SUBSCRIPTION_OWNER)); + } + private void ifUserInRoleDo( + SecurityContext securityContext, String role, Consumer consumer) { + if (securityContext.isUserInRole(role)) { + consumer.accept(role); + } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/SchemaEndpoint.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/SchemaEndpoint.java index 6403dd031e..d3d2a4505e 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/SchemaEndpoint.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/SchemaEndpoint.java @@ -1,5 +1,8 @@ package pl.allegro.tech.hermes.management.api; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; +import static pl.allegro.tech.hermes.api.TopicName.fromQualifiedName; + import io.swagger.annotations.ApiOperation; import jakarta.annotation.security.RolesAllowed; import jakarta.ws.rs.Consumes; @@ -15,6 +18,7 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.Response; +import java.util.Optional; import org.springframework.beans.factory.annotation.Autowired; import pl.allegro.tech.hermes.api.RawSchema; import pl.allegro.tech.hermes.api.Topic; @@ -26,79 +30,82 @@ import pl.allegro.tech.hermes.schema.SchemaId; import pl.allegro.tech.hermes.schema.SchemaVersion; -import java.util.Optional; - -import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; -import static pl.allegro.tech.hermes.api.TopicName.fromQualifiedName; - @Path("topics/{topicName}/schema") public class SchemaEndpoint { - private final SchemaService schemaService; - private final TopicService topicService; + private final SchemaService schemaService; + private final TopicService topicService; - @Autowired - public SchemaEndpoint(SchemaService schemaService, TopicService topicService) { - this.schemaService = schemaService; - this.topicService = topicService; - } + @Autowired + public SchemaEndpoint(SchemaService schemaService, TopicService topicService) { + this.schemaService = schemaService; + this.topicService = topicService; + } - @GET - @Produces(APPLICATION_JSON) - @ApiOperation(value = "Get schema", httpMethod = HttpMethod.GET) - public Response get(@PathParam("topicName") String qualifiedTopicName) { - Optional rawSchema = schemaService.getSchema(qualifiedTopicName); - return rawSchema.map(RawSchema::value) - .map(v -> Response.ok(v).build()) - .orElse(Response.noContent().build()); - } + @GET + @Produces(APPLICATION_JSON) + @ApiOperation(value = "Get schema", httpMethod = HttpMethod.GET) + public Response get(@PathParam("topicName") String qualifiedTopicName) { + Optional rawSchema = schemaService.getSchema(qualifiedTopicName); + return rawSchema + .map(RawSchema::value) + .map(v -> Response.ok(v).build()) + .orElse(Response.noContent().build()); + } - @GET - @Path("versions/{version}") - @Produces(APPLICATION_JSON) - @ApiOperation(value = "Get schema", httpMethod = HttpMethod.GET) - public Response getByVersion(@PathParam("topicName") String qualifiedTopicName, @PathParam("version") int version) { - Optional rawSchema = schemaService.getSchema(qualifiedTopicName, SchemaVersion.valueOf(version)); - return rawSchema.map(RawSchema::value) - .map(v -> Response.ok(v).build()) - .orElse(Response.noContent().build()); - } + @GET + @Path("versions/{version}") + @Produces(APPLICATION_JSON) + @ApiOperation(value = "Get schema", httpMethod = HttpMethod.GET) + public Response getByVersion( + @PathParam("topicName") String qualifiedTopicName, @PathParam("version") int version) { + Optional rawSchema = + schemaService.getSchema(qualifiedTopicName, SchemaVersion.valueOf(version)); + return rawSchema + .map(RawSchema::value) + .map(v -> Response.ok(v).build()) + .orElse(Response.noContent().build()); + } - @GET - @Path("ids/{id}") - @Produces(APPLICATION_JSON) - @ApiOperation(value = "Get schema", httpMethod = HttpMethod.GET) - public Response getById(@PathParam("topicName") String qualifiedTopicName, @PathParam("id") int id) { - Optional rawSchema = schemaService.getSchema(qualifiedTopicName, SchemaId.valueOf(id)); - return rawSchema.map(RawSchema::value) - .map(v -> Response.ok(v).build()) - .orElse(Response.noContent().build()); - } + @GET + @Path("ids/{id}") + @Produces(APPLICATION_JSON) + @ApiOperation(value = "Get schema", httpMethod = HttpMethod.GET) + public Response getById( + @PathParam("topicName") String qualifiedTopicName, @PathParam("id") int id) { + Optional rawSchema = + schemaService.getSchema(qualifiedTopicName, SchemaId.valueOf(id)); + return rawSchema + .map(RawSchema::value) + .map(v -> Response.ok(v).build()) + .orElse(Response.noContent().build()); + } - @POST - @Consumes(APPLICATION_JSON) - @RolesAllowed({Roles.TOPIC_OWNER, Roles.ADMIN}) - @ApiOperation(value = "Save schema", httpMethod = HttpMethod.POST) - public Response save(@PathParam("topicName") String qualifiedTopicName, - @DefaultValue("true") @QueryParam(value = "validate") boolean validate, - @Context ContainerRequestContext requestContext, - String schema) { - Topic topic = topicService.getTopicDetails(fromQualifiedName(qualifiedTopicName)); - RequestUser user = new HermesSecurityAwareRequestUser(requestContext); - schemaService.registerSchema(topic, schema, validate); - notifyFrontendSchemaChanged(qualifiedTopicName, user); - return Response.status(Response.Status.CREATED).build(); - } + @POST + @Consumes(APPLICATION_JSON) + @RolesAllowed({Roles.TOPIC_OWNER, Roles.ADMIN}) + @ApiOperation(value = "Save schema", httpMethod = HttpMethod.POST) + public Response save( + @PathParam("topicName") String qualifiedTopicName, + @DefaultValue("true") @QueryParam(value = "validate") boolean validate, + @Context ContainerRequestContext requestContext, + String schema) { + Topic topic = topicService.getTopicDetails(fromQualifiedName(qualifiedTopicName)); + RequestUser user = new HermesSecurityAwareRequestUser(requestContext); + schemaService.registerSchema(topic, schema, validate); + notifyFrontendSchemaChanged(qualifiedTopicName, user); + return Response.status(Response.Status.CREATED).build(); + } - private void notifyFrontendSchemaChanged(String qualifiedTopicName, RequestUser changedBy) { - topicService.scheduleTouchTopic(fromQualifiedName(qualifiedTopicName), changedBy); - } + private void notifyFrontendSchemaChanged(String qualifiedTopicName, RequestUser changedBy) { + topicService.scheduleTouchTopic(fromQualifiedName(qualifiedTopicName), changedBy); + } - @DELETE - @RolesAllowed({Roles.TOPIC_OWNER, Roles.ADMIN}) - @ApiOperation(value = "Delete schema", httpMethod = HttpMethod.DELETE) - public Response delete(@PathParam("topicName") String qualifiedTopicName) { - schemaService.deleteAllSchemaVersions(qualifiedTopicName); - return Response.status(Response.Status.OK).build(); - } + @DELETE + @RolesAllowed({Roles.TOPIC_OWNER, Roles.ADMIN}) + @ApiOperation(value = "Delete schema", httpMethod = HttpMethod.DELETE) + public Response delete(@PathParam("topicName") String qualifiedTopicName) { + schemaService.deleteAllSchemaVersions(qualifiedTopicName); + return Response.status(Response.Status.OK).build(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/StatsEndpoint.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/StatsEndpoint.java index ef998c6ae6..99ef1fbc3f 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/StatsEndpoint.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/StatsEndpoint.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.management.api; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; + import io.swagger.annotations.ApiOperation; import jakarta.ws.rs.GET; import jakarta.ws.rs.HttpMethod; @@ -12,29 +14,26 @@ import pl.allegro.tech.hermes.management.domain.subscription.SubscriptionService; import pl.allegro.tech.hermes.management.domain.topic.TopicService; -import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; - @Path("stats") public class StatsEndpoint { - private final SubscriptionService subscriptionService; - private final TopicService topicService; - - @Autowired - public StatsEndpoint(SubscriptionService subscriptionService, TopicService topicService) { - this.subscriptionService = subscriptionService; - this.topicService = topicService; - } + private final SubscriptionService subscriptionService; + private final TopicService topicService; - @GET - @Produces(APPLICATION_JSON) - @ApiOperation(value = "Get topic and subscription stats", response = Stats.class, httpMethod = HttpMethod.GET) - public Stats getStats() { - TopicStats topicStats = topicService.getStats(); - SubscriptionStats subscriptionStats = subscriptionService.getStats(); - return new Stats( - topicStats, - subscriptionStats - ); - } + @Autowired + public StatsEndpoint(SubscriptionService subscriptionService, TopicService topicService) { + this.subscriptionService = subscriptionService; + this.topicService = topicService; + } + @GET + @Produces(APPLICATION_JSON) + @ApiOperation( + value = "Get topic and subscription stats", + response = Stats.class, + httpMethod = HttpMethod.GET) + public Stats getStats() { + TopicStats topicStats = topicService.getStats(); + SubscriptionStats subscriptionStats = subscriptionService.getStats(); + return new Stats(topicStats, subscriptionStats); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/SubscriptionsEndpoint.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/SubscriptionsEndpoint.java index 9eaf69334e..4c92836529 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/SubscriptionsEndpoint.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/SubscriptionsEndpoint.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.management.api; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; +import static jakarta.ws.rs.core.Response.Status.NOT_FOUND; +import static jakarta.ws.rs.core.Response.Status.OK; +import static pl.allegro.tech.hermes.api.TopicName.fromQualifiedName; + import io.swagger.annotations.ApiOperation; import jakarta.annotation.security.RolesAllowed; import jakarta.validation.Valid; @@ -17,6 +22,8 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.Response; +import java.util.List; +import java.util.Optional; import org.springframework.beans.factory.annotation.Autowired; import pl.allegro.tech.hermes.api.ConsumerGroup; import pl.allegro.tech.hermes.api.MessageTrace; @@ -38,252 +45,283 @@ import pl.allegro.tech.hermes.management.infrastructure.kafka.MultiDCAwareService; import pl.allegro.tech.hermes.management.infrastructure.kafka.MultiDCOffsetChangeSummary; -import java.util.List; -import java.util.Optional; - -import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; -import static jakarta.ws.rs.core.Response.Status.NOT_FOUND; -import static jakarta.ws.rs.core.Response.Status.OK; -import static pl.allegro.tech.hermes.api.TopicName.fromQualifiedName; - @Path("topics/{topicName}/subscriptions") public class SubscriptionsEndpoint { - private final SubscriptionService subscriptionService; - private final TopicService topicService; - private final MultiDCAwareService multiDCAwareService; - - @Autowired - public SubscriptionsEndpoint(SubscriptionService subscriptionService, - TopicService topicService, - MultiDCAwareService multiDCAwareService) { - this.subscriptionService = subscriptionService; - this.topicService = topicService; - this.multiDCAwareService = multiDCAwareService; - } + private final SubscriptionService subscriptionService; + private final TopicService topicService; + private final MultiDCAwareService multiDCAwareService; - @GET - @Produces(APPLICATION_JSON) - @ApiOperation(value = "Lists subscriptions", response = List.class, httpMethod = HttpMethod.GET) - public List list( - @PathParam("topicName") String qualifiedTopicName, - @DefaultValue("false") @QueryParam("tracked") boolean tracked) { + @Autowired + public SubscriptionsEndpoint( + SubscriptionService subscriptionService, + TopicService topicService, + MultiDCAwareService multiDCAwareService) { + this.subscriptionService = subscriptionService; + this.topicService = topicService; + this.multiDCAwareService = multiDCAwareService; + } - return tracked - ? subscriptionService.listTrackedSubscriptionNames(fromQualifiedName(qualifiedTopicName)) - : subscriptionService.listSubscriptionNames(fromQualifiedName(qualifiedTopicName)); - } + @GET + @Produces(APPLICATION_JSON) + @ApiOperation(value = "Lists subscriptions", response = List.class, httpMethod = HttpMethod.GET) + public List list( + @PathParam("topicName") String qualifiedTopicName, + @DefaultValue("false") @QueryParam("tracked") boolean tracked) { - @POST - @Consumes(APPLICATION_JSON) - @Produces(APPLICATION_JSON) - @Path("/query") - @ApiOperation(value = "Queries subscriptions", response = List.class, httpMethod = HttpMethod.POST) - public List queryList( - @PathParam("topicName") String qualifiedTopicName, - Query query) { - return subscriptionService.listFilteredSubscriptionNames(fromQualifiedName(qualifiedTopicName), query); - } + return tracked + ? subscriptionService.listTrackedSubscriptionNames(fromQualifiedName(qualifiedTopicName)) + : subscriptionService.listSubscriptionNames(fromQualifiedName(qualifiedTopicName)); + } - @POST - @Consumes(APPLICATION_JSON) - @RolesAllowed({Roles.ANY}) - @ApiOperation(value = "Create subscription", httpMethod = HttpMethod.POST) - public Response create(@PathParam("topicName") String qualifiedTopicName, - Subscription subscription, - @Context ContainerRequestContext requestContext) { - subscriptionService.createSubscription( - subscription, - new HermesSecurityAwareRequestUser(requestContext), - qualifiedTopicName - ); - return responseStatus(Response.Status.CREATED); - } + @POST + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON) + @Path("/query") + @ApiOperation( + value = "Queries subscriptions", + response = List.class, + httpMethod = HttpMethod.POST) + public List queryList( + @PathParam("topicName") String qualifiedTopicName, Query query) { + return subscriptionService.listFilteredSubscriptionNames( + fromQualifiedName(qualifiedTopicName), query); + } - @GET - @Produces(APPLICATION_JSON) - @Path("/{subscriptionName}") - @ApiOperation(value = "Get subscription details", response = Subscription.class, httpMethod = HttpMethod.GET) - public Subscription get(@PathParam("topicName") String qualifiedTopicName, - @PathParam("subscriptionName") String subscriptionName) { - return subscriptionService.getSubscriptionDetails(fromQualifiedName(qualifiedTopicName), subscriptionName); - } + @POST + @Consumes(APPLICATION_JSON) + @RolesAllowed({Roles.ANY}) + @ApiOperation(value = "Create subscription", httpMethod = HttpMethod.POST) + public Response create( + @PathParam("topicName") String qualifiedTopicName, + Subscription subscription, + @Context ContainerRequestContext requestContext) { + subscriptionService.createSubscription( + subscription, new HermesSecurityAwareRequestUser(requestContext), qualifiedTopicName); + return responseStatus(Response.Status.CREATED); + } - @GET - @Produces(APPLICATION_JSON) - @Path("/{subscriptionName}/state") - @ApiOperation(value = "Get subscription state", response = Subscription.State.class, httpMethod = HttpMethod.GET) - public Subscription.State getState(@PathParam("topicName") String qualifiedTopicName, - @PathParam("subscriptionName") String subscriptionName) { - return subscriptionService.getSubscriptionState(fromQualifiedName(qualifiedTopicName), subscriptionName); - } + @GET + @Produces(APPLICATION_JSON) + @Path("/{subscriptionName}") + @ApiOperation( + value = "Get subscription details", + response = Subscription.class, + httpMethod = HttpMethod.GET) + public Subscription get( + @PathParam("topicName") String qualifiedTopicName, + @PathParam("subscriptionName") String subscriptionName) { + return subscriptionService.getSubscriptionDetails( + fromQualifiedName(qualifiedTopicName), subscriptionName); + } - @GET - @Produces(APPLICATION_JSON) - @RolesAllowed({Roles.ADMIN, Roles.TOPIC_OWNER, Roles.SUBSCRIPTION_OWNER}) - @Path("/{subscriptionName}/undelivered/last") - @ApiOperation(value = "Get latest undelivered message", response = SentMessageTrace.class, httpMethod = HttpMethod.GET) - public Response getLatestUndeliveredMessage(@PathParam("topicName") String qualifiedTopicName, - @PathParam("subscriptionName") String subscriptionName) { - Optional result = subscriptionService.getLatestUndeliveredMessage( - fromQualifiedName(qualifiedTopicName), subscriptionName - ); + @GET + @Produces(APPLICATION_JSON) + @Path("/{subscriptionName}/state") + @ApiOperation( + value = "Get subscription state", + response = Subscription.State.class, + httpMethod = HttpMethod.GET) + public Subscription.State getState( + @PathParam("topicName") String qualifiedTopicName, + @PathParam("subscriptionName") String subscriptionName) { + return subscriptionService.getSubscriptionState( + fromQualifiedName(qualifiedTopicName), subscriptionName); + } - return result.isPresent() ? Response.status(OK).entity(result.get()).build() : responseStatus(NOT_FOUND); - } + @GET + @Produces(APPLICATION_JSON) + @RolesAllowed({Roles.ADMIN, Roles.TOPIC_OWNER, Roles.SUBSCRIPTION_OWNER}) + @Path("/{subscriptionName}/undelivered/last") + @ApiOperation( + value = "Get latest undelivered message", + response = SentMessageTrace.class, + httpMethod = HttpMethod.GET) + public Response getLatestUndeliveredMessage( + @PathParam("topicName") String qualifiedTopicName, + @PathParam("subscriptionName") String subscriptionName) { + Optional result = + subscriptionService.getLatestUndeliveredMessage( + fromQualifiedName(qualifiedTopicName), subscriptionName); - @GET - @Produces(APPLICATION_JSON) - @RolesAllowed({Roles.ADMIN, Roles.TOPIC_OWNER, Roles.SUBSCRIPTION_OWNER}) - @Path("/{subscriptionName}/undelivered") - @ApiOperation(value = "Get latest undelivered messages", response = List.class, httpMethod = HttpMethod.GET) - public Response getLatestUndeliveredMessages(@PathParam("topicName") String qualifiedTopicName, - @PathParam("subscriptionName") String subscriptionName) { - List result = subscriptionService.getLatestUndeliveredMessagesTrackerLogs( - fromQualifiedName(qualifiedTopicName), subscriptionName - ); - return Response.status(OK).entity(result).build(); - } + return result.isPresent() + ? Response.status(OK).entity(result.get()).build() + : responseStatus(NOT_FOUND); + } - @GET - @Produces(APPLICATION_JSON) - @Path("/{subscriptionName}/metrics") - @ApiOperation(value = "Get subscription metrics", response = SubscriptionMetrics.class, httpMethod = HttpMethod.GET) - public SubscriptionMetrics getMetrics(@PathParam("topicName") String qualifiedTopicName, - @PathParam("subscriptionName") String subscriptionName) { - return subscriptionService.getSubscriptionMetrics(fromQualifiedName(qualifiedTopicName), subscriptionName); - } + @GET + @Produces(APPLICATION_JSON) + @RolesAllowed({Roles.ADMIN, Roles.TOPIC_OWNER, Roles.SUBSCRIPTION_OWNER}) + @Path("/{subscriptionName}/undelivered") + @ApiOperation( + value = "Get latest undelivered messages", + response = List.class, + httpMethod = HttpMethod.GET) + public Response getLatestUndeliveredMessages( + @PathParam("topicName") String qualifiedTopicName, + @PathParam("subscriptionName") String subscriptionName) { + List result = + subscriptionService.getLatestUndeliveredMessagesTrackerLogs( + fromQualifiedName(qualifiedTopicName), subscriptionName); + return Response.status(OK).entity(result).build(); + } - @GET - @Produces(APPLICATION_JSON) - @Path("/{subscriptionName}/metrics/persistent") - @ApiOperation( - value = "Get persistent subscription metrics", response = PersistentSubscriptionMetrics.class, httpMethod = HttpMethod.GET) - public PersistentSubscriptionMetrics getPersistentMetrics(@PathParam("topicName") String qualifiedTopicName, - @PathParam("subscriptionName") String subscriptionName) { - return subscriptionService.getPersistentSubscriptionMetrics(fromQualifiedName(qualifiedTopicName), subscriptionName); - } + @GET + @Produces(APPLICATION_JSON) + @Path("/{subscriptionName}/metrics") + @ApiOperation( + value = "Get subscription metrics", + response = SubscriptionMetrics.class, + httpMethod = HttpMethod.GET) + public SubscriptionMetrics getMetrics( + @PathParam("topicName") String qualifiedTopicName, + @PathParam("subscriptionName") String subscriptionName) { + return subscriptionService.getSubscriptionMetrics( + fromQualifiedName(qualifiedTopicName), subscriptionName); + } - @GET - @Produces(APPLICATION_JSON) - @Path("/{subscriptionName}/health") - @ApiOperation(value = "Get subscription health", response = SubscriptionHealth.class, httpMethod = HttpMethod.GET) - public SubscriptionHealth getHealth(@PathParam("topicName") String qualifiedTopicName, - @PathParam("subscriptionName") String subscriptionName) { - return subscriptionService.getSubscriptionHealth(fromQualifiedName(qualifiedTopicName), subscriptionName); - } + @GET + @Produces(APPLICATION_JSON) + @Path("/{subscriptionName}/metrics/persistent") + @ApiOperation( + value = "Get persistent subscription metrics", + response = PersistentSubscriptionMetrics.class, + httpMethod = HttpMethod.GET) + public PersistentSubscriptionMetrics getPersistentMetrics( + @PathParam("topicName") String qualifiedTopicName, + @PathParam("subscriptionName") String subscriptionName) { + return subscriptionService.getPersistentSubscriptionMetrics( + fromQualifiedName(qualifiedTopicName), subscriptionName); + } - @PUT - @Consumes(APPLICATION_JSON) - @Path("/{subscriptionName}/state") - @RolesAllowed({Roles.ADMIN, Roles.TOPIC_OWNER, Roles.SUBSCRIPTION_OWNER}) - @ApiOperation(value = "Update subscription state", httpMethod = HttpMethod.PUT) - public Response updateState(@PathParam("topicName") String qualifiedTopicName, - @PathParam("subscriptionName") String subscriptionName, - Subscription.State state, - @Context ContainerRequestContext requestContext) { - subscriptionService.updateSubscriptionState( - fromQualifiedName(qualifiedTopicName), - subscriptionName, - state, - new HermesSecurityAwareRequestUser(requestContext) - ); - return responseStatus(OK); - } + @GET + @Produces(APPLICATION_JSON) + @Path("/{subscriptionName}/health") + @ApiOperation( + value = "Get subscription health", + response = SubscriptionHealth.class, + httpMethod = HttpMethod.GET) + public SubscriptionHealth getHealth( + @PathParam("topicName") String qualifiedTopicName, + @PathParam("subscriptionName") String subscriptionName) { + return subscriptionService.getSubscriptionHealth( + fromQualifiedName(qualifiedTopicName), subscriptionName); + } - @DELETE - @Path("/{subscriptionName}") - @RolesAllowed({Roles.ADMIN, Roles.TOPIC_OWNER, Roles.SUBSCRIPTION_OWNER}) - @ApiOperation(value = "Remove subscription", httpMethod = HttpMethod.DELETE) - public Response remove(@PathParam("topicName") String qualifiedTopicName, - @PathParam("subscriptionName") String subscriptionId, - @Context ContainerRequestContext requestContext) { - subscriptionService.removeSubscription( - fromQualifiedName(qualifiedTopicName), - subscriptionId, - new HermesSecurityAwareRequestUser(requestContext) - ); - return responseStatus(OK); - } + @PUT + @Consumes(APPLICATION_JSON) + @Path("/{subscriptionName}/state") + @RolesAllowed({Roles.ADMIN, Roles.TOPIC_OWNER, Roles.SUBSCRIPTION_OWNER}) + @ApiOperation(value = "Update subscription state", httpMethod = HttpMethod.PUT) + public Response updateState( + @PathParam("topicName") String qualifiedTopicName, + @PathParam("subscriptionName") String subscriptionName, + Subscription.State state, + @Context ContainerRequestContext requestContext) { + subscriptionService.updateSubscriptionState( + fromQualifiedName(qualifiedTopicName), + subscriptionName, + state, + new HermesSecurityAwareRequestUser(requestContext)); + return responseStatus(OK); + } - @PUT - @Consumes(APPLICATION_JSON) - @Path("/{subscriptionName}") - @RolesAllowed({Roles.ADMIN, Roles.TOPIC_OWNER, Roles.SUBSCRIPTION_OWNER}) - @ApiOperation(value = "Update subscription", httpMethod = HttpMethod.PUT) - public Response update(@PathParam("topicName") String qualifiedTopicName, - @PathParam("subscriptionName") String subscriptionName, - PatchData patch, - @Context ContainerRequestContext requestContext) { - subscriptionService.updateSubscription( - TopicName.fromQualifiedName(qualifiedTopicName), - subscriptionName, - patch, - new HermesSecurityAwareRequestUser(requestContext) - ); - return responseStatus(OK); - } + @DELETE + @Path("/{subscriptionName}") + @RolesAllowed({Roles.ADMIN, Roles.TOPIC_OWNER, Roles.SUBSCRIPTION_OWNER}) + @ApiOperation(value = "Remove subscription", httpMethod = HttpMethod.DELETE) + public Response remove( + @PathParam("topicName") String qualifiedTopicName, + @PathParam("subscriptionName") String subscriptionId, + @Context ContainerRequestContext requestContext) { + subscriptionService.removeSubscription( + fromQualifiedName(qualifiedTopicName), + subscriptionId, + new HermesSecurityAwareRequestUser(requestContext)); + return responseStatus(OK); + } - @PUT - @Consumes(APPLICATION_JSON) - @Produces(APPLICATION_JSON) - @Path("/{subscriptionName}/retransmission") - @RolesAllowed({Roles.ADMIN, Roles.TOPIC_OWNER, Roles.SUBSCRIPTION_OWNER}) - @ApiOperation(value = "Update subscription offset", httpMethod = HttpMethod.PUT) - public Response retransmit(@PathParam("topicName") String qualifiedTopicName, - @PathParam("subscriptionName") String subscriptionName, - @DefaultValue("false") @QueryParam("dryRun") boolean dryRun, - @Valid OffsetRetransmissionDate offsetRetransmissionDate, - @Context ContainerRequestContext requestContext) { + @PUT + @Consumes(APPLICATION_JSON) + @Path("/{subscriptionName}") + @RolesAllowed({Roles.ADMIN, Roles.TOPIC_OWNER, Roles.SUBSCRIPTION_OWNER}) + @ApiOperation(value = "Update subscription", httpMethod = HttpMethod.PUT) + public Response update( + @PathParam("topicName") String qualifiedTopicName, + @PathParam("subscriptionName") String subscriptionName, + PatchData patch, + @Context ContainerRequestContext requestContext) { + subscriptionService.updateSubscription( + TopicName.fromQualifiedName(qualifiedTopicName), + subscriptionName, + patch, + new HermesSecurityAwareRequestUser(requestContext)); + return responseStatus(OK); + } - MultiDCOffsetChangeSummary summary = multiDCAwareService.retransmit( - topicService.getTopicDetails(TopicName.fromQualifiedName(qualifiedTopicName)), - subscriptionName, - offsetRetransmissionDate.getRetransmissionDate().toInstant().toEpochMilli(), - dryRun, - new HermesSecurityAwareRequestUser(requestContext) - ); + @PUT + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON) + @Path("/{subscriptionName}/retransmission") + @RolesAllowed({Roles.ADMIN, Roles.TOPIC_OWNER, Roles.SUBSCRIPTION_OWNER}) + @ApiOperation(value = "Update subscription offset", httpMethod = HttpMethod.PUT) + public Response retransmit( + @PathParam("topicName") String qualifiedTopicName, + @PathParam("subscriptionName") String subscriptionName, + @DefaultValue("false") @QueryParam("dryRun") boolean dryRun, + @Valid OffsetRetransmissionDate offsetRetransmissionDate, + @Context ContainerRequestContext requestContext) { - return Response.status(OK).entity(summary).build(); - } + MultiDCOffsetChangeSummary summary = + multiDCAwareService.retransmit( + topicService.getTopicDetails(TopicName.fromQualifiedName(qualifiedTopicName)), + subscriptionName, + offsetRetransmissionDate.getRetransmissionDate().toInstant().toEpochMilli(), + dryRun, + new HermesSecurityAwareRequestUser(requestContext)); - @POST - @Consumes(APPLICATION_JSON) - @Produces(APPLICATION_JSON) - @RolesAllowed({Roles.ADMIN}) - @Path("/{subscriptionName}/moveOffsetsToTheEnd") - public Response moveOffsetsToTheEnd(@PathParam("topicName") String qualifiedTopicName, - @PathParam("subscriptionName") String subscriptionName) { - TopicName topicName = fromQualifiedName(qualifiedTopicName); - multiDCAwareService.moveOffsetsToTheEnd( - topicService.getTopicDetails(topicName), - new SubscriptionName(subscriptionName, topicName)); - return responseStatus(OK); - } + return Response.status(OK).entity(summary).build(); + } - @GET - @Produces(APPLICATION_JSON) - @Path("/{subscriptionName}/events/{messageId}/trace") - public Response getMessageTrace(@PathParam("topicName") String qualifiedTopicName, - @PathParam("subscriptionName") String subscriptionName, - @PathParam("messageId") String messageId) { + @POST + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON) + @RolesAllowed({Roles.ADMIN}) + @Path("/{subscriptionName}/moveOffsetsToTheEnd") + public Response moveOffsetsToTheEnd( + @PathParam("topicName") String qualifiedTopicName, + @PathParam("subscriptionName") String subscriptionName) { + TopicName topicName = fromQualifiedName(qualifiedTopicName); + multiDCAwareService.moveOffsetsToTheEnd( + topicService.getTopicDetails(topicName), new SubscriptionName(subscriptionName, topicName)); + return responseStatus(OK); + } - List status = subscriptionService.getMessageStatus(qualifiedTopicName, subscriptionName, messageId); + @GET + @Produces(APPLICATION_JSON) + @Path("/{subscriptionName}/events/{messageId}/trace") + public Response getMessageTrace( + @PathParam("topicName") String qualifiedTopicName, + @PathParam("subscriptionName") String subscriptionName, + @PathParam("messageId") String messageId) { - return Response.status(OK).entity(status).build(); - } + List status = + subscriptionService.getMessageStatus(qualifiedTopicName, subscriptionName, messageId); - @GET - @Produces(APPLICATION_JSON) - @Path("/{subscriptionName}/consumer-groups") - public List describeConsumerGroups(@PathParam("topicName") String qualifiedTopicName, - @PathParam("subscriptionName") String subscriptionName) { - Topic topic = topicService.getTopicDetails(fromQualifiedName(qualifiedTopicName)); - return multiDCAwareService.describeConsumerGroups(topic, subscriptionName); - } + return Response.status(OK).entity(status).build(); + } - private Response responseStatus(Response.Status responseStatus) { - return Response.status(responseStatus).build(); - } + @GET + @Produces(APPLICATION_JSON) + @Path("/{subscriptionName}/consumer-groups") + public List describeConsumerGroups( + @PathParam("topicName") String qualifiedTopicName, + @PathParam("subscriptionName") String subscriptionName) { + Topic topic = topicService.getTopicDetails(fromQualifiedName(qualifiedTopicName)); + return multiDCAwareService.describeConsumerGroups(topic, subscriptionName); + } + private Response responseStatus(Response.Status responseStatus) { + return Response.status(responseStatus).build(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/SubscriptionsOwnershipEndpoint.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/SubscriptionsOwnershipEndpoint.java index c6ad63e80c..8406550335 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/SubscriptionsOwnershipEndpoint.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/SubscriptionsOwnershipEndpoint.java @@ -1,9 +1,12 @@ package pl.allegro.tech.hermes.management.api; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; + import jakarta.ws.rs.GET; import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import jakarta.ws.rs.Produces; +import java.util.List; import org.springframework.beans.factory.annotation.Autowired; import pl.allegro.tech.hermes.api.OwnerId; import pl.allegro.tech.hermes.api.Subscription; @@ -12,37 +15,36 @@ import pl.allegro.tech.hermes.management.domain.owner.OwnerSources; import pl.allegro.tech.hermes.management.domain.subscription.SubscriptionService; -import java.util.List; - -import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; - @Path("subscriptions/owner") public class SubscriptionsOwnershipEndpoint { - private final OwnerSources ownerSources; - private final SubscriptionService subscriptionService; - - @Autowired - public SubscriptionsOwnershipEndpoint(OwnerSources ownerSources, - SubscriptionService subscriptionService) { - this.ownerSources = ownerSources; - this.subscriptionService = subscriptionService; - } - - @GET - @Produces(APPLICATION_JSON) - @Path("/{ownerSourceName}/{ownerId}") - public List listForOwner(@PathParam("ownerSourceName") String ownerSourceName, @PathParam("ownerId") String id) { - OwnerId ownerId = resolveOwnerId(ownerSourceName, id); - return subscriptionService.getForOwnerId(ownerId); - } - - private OwnerId resolveOwnerId(String ownerSourceName, String id) { - OwnerSource ownerSource = ownerSources.getByName(ownerSourceName) - .orElseThrow(() -> new OwnerSourceNotFound(ownerSourceName)); - if (!ownerSource.exists(id)) { - throw new OwnerSource.OwnerNotFound(ownerSourceName, id); - } - return new OwnerId(ownerSource.name(), id); + private final OwnerSources ownerSources; + private final SubscriptionService subscriptionService; + + @Autowired + public SubscriptionsOwnershipEndpoint( + OwnerSources ownerSources, SubscriptionService subscriptionService) { + this.ownerSources = ownerSources; + this.subscriptionService = subscriptionService; + } + + @GET + @Produces(APPLICATION_JSON) + @Path("/{ownerSourceName}/{ownerId}") + public List listForOwner( + @PathParam("ownerSourceName") String ownerSourceName, @PathParam("ownerId") String id) { + OwnerId ownerId = resolveOwnerId(ownerSourceName, id); + return subscriptionService.getForOwnerId(ownerId); + } + + private OwnerId resolveOwnerId(String ownerSourceName, String id) { + OwnerSource ownerSource = + ownerSources + .getByName(ownerSourceName) + .orElseThrow(() -> new OwnerSourceNotFound(ownerSourceName)); + if (!ownerSource.exists(id)) { + throw new OwnerSource.OwnerNotFound(ownerSourceName, id); } + return new OwnerId(ownerSource.name(), id); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/TopicsEndpoint.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/TopicsEndpoint.java index 7f87849f30..8f2f53284e 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/TopicsEndpoint.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/TopicsEndpoint.java @@ -1,5 +1,11 @@ package pl.allegro.tech.hermes.management.api; +import static com.google.common.base.Strings.isNullOrEmpty; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; +import static jakarta.ws.rs.core.MediaType.APPLICATION_OCTET_STREAM; +import static jakarta.ws.rs.core.Response.status; +import static java.lang.String.format; + import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import jakarta.annotation.security.RolesAllowed; @@ -18,6 +24,8 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.Response; +import java.util.List; +import java.util.Optional; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import pl.allegro.tech.hermes.api.MessageTextPreview; @@ -40,172 +48,184 @@ import pl.allegro.tech.hermes.management.domain.topic.SingleMessageReaderException; import pl.allegro.tech.hermes.management.domain.topic.TopicService; -import java.util.List; -import java.util.Optional; - -import static com.google.common.base.Strings.isNullOrEmpty; -import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; -import static jakarta.ws.rs.core.MediaType.APPLICATION_OCTET_STREAM; -import static jakarta.ws.rs.core.Response.status; -import static java.lang.String.format; - @Component @Path("/topics") @Api(value = "/topics", description = "Operations on topics") public class TopicsEndpoint { - private final TopicService topicService; - private final ManagementRights managementRights; - private final OwnerSources ownerSources; - - @Autowired - public TopicsEndpoint(TopicService topicService, - ManagementRights managementRights, - OwnerSources ownerSources) { - this.topicService = topicService; - this.managementRights = managementRights; - this.ownerSources = ownerSources; - } - - @GET - @Produces(APPLICATION_JSON) - @ApiOperation(value = "List topics from group", response = List.class, httpMethod = HttpMethod.GET) - public List list( - @DefaultValue("") @QueryParam("groupName") String groupName, - @DefaultValue("false") @QueryParam("tracked") boolean tracked) { - - return tracked ? listTracked(groupName) : listNames(groupName); - } - - @POST - @Consumes(APPLICATION_JSON) - @Produces(APPLICATION_JSON) - @Path("/query") - @ApiOperation(value = "Queries topics from group", response = List.class, httpMethod = HttpMethod.POST) - public List queryList( - @DefaultValue("") @QueryParam("groupName") String groupName, - Query query) { - - return isNullOrEmpty(groupName) - ? topicService.listFilteredTopicNames(query) - : topicService.listFilteredTopicNames(groupName, query); - } - - @POST - @Consumes(APPLICATION_JSON) - @Produces(APPLICATION_JSON) - @RolesAllowed(Roles.ANY) - @ApiOperation(value = "Create topic", httpMethod = HttpMethod.POST) - public Response create(TopicWithSchema topicWithSchema, @Context ContainerRequestContext requestContext) { - RequestUser requestUser = new HermesSecurityAwareRequestUser(requestContext); - CreatorRights isAllowedToManage = checkedTopic -> managementRights.isUserAllowedToManageTopic(checkedTopic, requestContext); - topicService.createTopicWithSchema(topicWithSchema, requestUser, isAllowedToManage); - return status(Response.Status.CREATED).build(); + private final TopicService topicService; + private final ManagementRights managementRights; + private final OwnerSources ownerSources; + + @Autowired + public TopicsEndpoint( + TopicService topicService, ManagementRights managementRights, OwnerSources ownerSources) { + this.topicService = topicService; + this.managementRights = managementRights; + this.ownerSources = ownerSources; + } + + @GET + @Produces(APPLICATION_JSON) + @ApiOperation( + value = "List topics from group", + response = List.class, + httpMethod = HttpMethod.GET) + public List list( + @DefaultValue("") @QueryParam("groupName") String groupName, + @DefaultValue("false") @QueryParam("tracked") boolean tracked) { + + return tracked ? listTracked(groupName) : listNames(groupName); + } + + @POST + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON) + @Path("/query") + @ApiOperation( + value = "Queries topics from group", + response = List.class, + httpMethod = HttpMethod.POST) + public List queryList( + @DefaultValue("") @QueryParam("groupName") String groupName, Query query) { + + return isNullOrEmpty(groupName) + ? topicService.listFilteredTopicNames(query) + : topicService.listFilteredTopicNames(groupName, query); + } + + @POST + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON) + @RolesAllowed(Roles.ANY) + @ApiOperation(value = "Create topic", httpMethod = HttpMethod.POST) + public Response create( + TopicWithSchema topicWithSchema, @Context ContainerRequestContext requestContext) { + RequestUser requestUser = new HermesSecurityAwareRequestUser(requestContext); + CreatorRights isAllowedToManage = + checkedTopic -> managementRights.isUserAllowedToManageTopic(checkedTopic, requestContext); + topicService.createTopicWithSchema(topicWithSchema, requestUser, isAllowedToManage); + return status(Response.Status.CREATED).build(); + } + + @DELETE + @Produces(APPLICATION_JSON) + @Path("/{topicName}") + @RolesAllowed({Roles.ADMIN, Roles.TOPIC_OWNER}) + @ApiOperation(value = "Remove topic", httpMethod = HttpMethod.DELETE) + public Response remove( + @PathParam("topicName") String qualifiedTopicName, + @Context ContainerRequestContext requestContext) { + RequestUser requestUser = new HermesSecurityAwareRequestUser(requestContext); + topicService.removeTopicWithSchema( + topicService.getTopicDetails(TopicName.fromQualifiedName(qualifiedTopicName)), requestUser); + return status(Response.Status.OK).build(); + } + + @PUT + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON) + @Path("/{topicName}") + @RolesAllowed({Roles.ADMIN, Roles.TOPIC_OWNER}) + @ApiOperation(value = "Update topic", httpMethod = HttpMethod.PUT) + public Response update( + @PathParam("topicName") String qualifiedTopicName, + PatchData patch, + @Context ContainerRequestContext requestContext) { + RequestUser requestUser = new HermesSecurityAwareRequestUser(requestContext); + topicService.updateTopicWithSchema( + TopicName.fromQualifiedName(qualifiedTopicName), patch, requestUser); + return status(Response.Status.OK).build(); + } + + @GET + @Produces(APPLICATION_JSON) + @Path("/{topicName}") + @ApiOperation(value = "Topic details", httpMethod = HttpMethod.GET) + public TopicWithSchema get(@PathParam("topicName") String qualifiedTopicName) { + return topicService.getTopicWithSchema(TopicName.fromQualifiedName(qualifiedTopicName)); + } + + @GET + @Produces(APPLICATION_JSON) + @Path("/{topicName}/metrics") + @ApiOperation(value = "Topic metrics", httpMethod = HttpMethod.GET) + public TopicMetrics getMetrics(@PathParam("topicName") String qualifiedTopicName) { + return topicService.getTopicMetrics(TopicName.fromQualifiedName(qualifiedTopicName)); + } + + @GET + @Produces(APPLICATION_JSON) + @Path("/{topicName}/preview") + @ApiOperation(value = "Topic publisher preview", httpMethod = HttpMethod.GET) + public List getPreview(@PathParam("topicName") String qualifiedTopicName) { + return topicService.previewText(TopicName.fromQualifiedName(qualifiedTopicName)); + } + + @GET + @Produces(APPLICATION_OCTET_STREAM) + @Path("/{topicName}/preview/{idx}") + @ApiOperation(value = "Topic publisher preview", httpMethod = HttpMethod.GET) + public byte[] getPreviewRaw( + @PathParam("topicName") String qualifiedTopicName, @PathParam("idx") Integer idx) { + TopicName topicName = TopicName.fromQualifiedName(qualifiedTopicName); + Optional preview = topicService.preview(topicName, idx); + if (preview.isPresent()) { + return preview.get(); + } else { + throw new NotFoundException( + format("Message preview not found for topic %s and offset %d", topicName, idx)); } - - @DELETE - @Produces(APPLICATION_JSON) - @Path("/{topicName}") - @RolesAllowed({Roles.ADMIN, Roles.TOPIC_OWNER}) - @ApiOperation(value = "Remove topic", httpMethod = HttpMethod.DELETE) - public Response remove(@PathParam("topicName") String qualifiedTopicName, @Context ContainerRequestContext requestContext) { - RequestUser requestUser = new HermesSecurityAwareRequestUser(requestContext); - topicService.removeTopicWithSchema(topicService.getTopicDetails(TopicName.fromQualifiedName(qualifiedTopicName)), - requestUser); - return status(Response.Status.OK).build(); - } - - @PUT - @Consumes(APPLICATION_JSON) - @Produces(APPLICATION_JSON) - @Path("/{topicName}") - @RolesAllowed({Roles.ADMIN, Roles.TOPIC_OWNER}) - @ApiOperation(value = "Update topic", httpMethod = HttpMethod.PUT) - public Response update(@PathParam("topicName") String qualifiedTopicName, PatchData patch, - @Context ContainerRequestContext requestContext) { - RequestUser requestUser = new HermesSecurityAwareRequestUser(requestContext); - topicService.updateTopicWithSchema(TopicName.fromQualifiedName(qualifiedTopicName), patch, requestUser); - return status(Response.Status.OK).build(); + } + + @GET + @Produces(APPLICATION_JSON) + @Path("/{topicName}/preview/cluster/{brokersClusterName}/partition/{partition}/offset/{offset}") + @RolesAllowed({Roles.ADMIN}) + @ApiOperation( + value = "Fetch single message from specified brokers cluster", + httpMethod = HttpMethod.GET) + public String preview( + @PathParam("topicName") String qualifiedTopicName, + @PathParam("brokersClusterName") String brokersClusterName, + @PathParam("partition") Integer partition, + @PathParam("offset") Long offset) { + try { + return topicService.fetchSingleMessageFromPrimary( + brokersClusterName, TopicName.fromQualifiedName(qualifiedTopicName), partition, offset); + } catch (BrokerNotFoundForPartitionException | SingleMessageReaderException exception) { + throw new NotFoundException( + format( + "Message not found for brokers cluster %s, topic %s, partition %d and offset %d", + brokersClusterName, qualifiedTopicName, partition, offset)); } - - @GET - @Produces(APPLICATION_JSON) - @Path("/{topicName}") - @ApiOperation(value = "Topic details", httpMethod = HttpMethod.GET) - public TopicWithSchema get(@PathParam("topicName") String qualifiedTopicName) { - return topicService.getTopicWithSchema(TopicName.fromQualifiedName(qualifiedTopicName)); - } - - @GET - @Produces(APPLICATION_JSON) - @Path("/{topicName}/metrics") - @ApiOperation(value = "Topic metrics", httpMethod = HttpMethod.GET) - public TopicMetrics getMetrics(@PathParam("topicName") String qualifiedTopicName) { - return topicService.getTopicMetrics(TopicName.fromQualifiedName(qualifiedTopicName)); - } - - @GET - @Produces(APPLICATION_JSON) - @Path("/{topicName}/preview") - @ApiOperation(value = "Topic publisher preview", httpMethod = HttpMethod.GET) - public List getPreview(@PathParam("topicName") String qualifiedTopicName) { - return topicService.previewText(TopicName.fromQualifiedName(qualifiedTopicName)); - } - - @GET - @Produces(APPLICATION_OCTET_STREAM) - @Path("/{topicName}/preview/{idx}") - @ApiOperation(value = "Topic publisher preview", httpMethod = HttpMethod.GET) - public byte[] getPreviewRaw(@PathParam("topicName") String qualifiedTopicName, @PathParam("idx") Integer idx) { - TopicName topicName = TopicName.fromQualifiedName(qualifiedTopicName); - Optional preview = topicService.preview(topicName, idx); - if (preview.isPresent()) { - return preview.get(); - } else { - throw new NotFoundException(format( - "Message preview not found for topic %s and offset %d", - topicName, idx - )); - } - } - - @GET - @Produces(APPLICATION_JSON) - @Path("/{topicName}/preview/cluster/{brokersClusterName}/partition/{partition}/offset/{offset}") - @RolesAllowed({Roles.ADMIN}) - @ApiOperation(value = "Fetch single message from specified brokers cluster", httpMethod = HttpMethod.GET) - public String preview(@PathParam("topicName") String qualifiedTopicName, - @PathParam("brokersClusterName") String brokersClusterName, - @PathParam("partition") Integer partition, - @PathParam("offset") Long offset) { - try { - return topicService.fetchSingleMessageFromPrimary(brokersClusterName, TopicName.fromQualifiedName(qualifiedTopicName), - partition, offset); - } catch (BrokerNotFoundForPartitionException | SingleMessageReaderException exception) { - throw new NotFoundException( - format("Message not found for brokers cluster %s, topic %s, partition %d and offset %d", brokersClusterName, - qualifiedTopicName, partition, offset)); - } - } - - @GET - @Produces(APPLICATION_JSON) - @Path("/owner/{ownerSourceName}/{ownerId}") - public List listForOwner(@PathParam("ownerSourceName") String ownerSourceName, @PathParam("ownerId") String id) { - OwnerSource ownerSource = ownerSources.getByName(ownerSourceName) - .orElseThrow(() -> new OwnerSourceNotFound(ownerSourceName)); - if (!ownerSource.exists(id)) { - throw new OwnerSource.OwnerNotFound(ownerSourceName, id); - } - OwnerId ownerId = new OwnerId(ownerSource.name(), id); - return topicService.listForOwnerId(ownerId); - } - - private List listTracked(String groupName) { - return isNullOrEmpty(groupName) ? topicService.listTrackedTopicNames() : topicService.listTrackedTopicNames(groupName); - } - - private List listNames(String groupName) { - return isNullOrEmpty(groupName) ? topicService.listQualifiedTopicNames() : topicService.listQualifiedTopicNames(groupName); + } + + @GET + @Produces(APPLICATION_JSON) + @Path("/owner/{ownerSourceName}/{ownerId}") + public List listForOwner( + @PathParam("ownerSourceName") String ownerSourceName, @PathParam("ownerId") String id) { + OwnerSource ownerSource = + ownerSources + .getByName(ownerSourceName) + .orElseThrow(() -> new OwnerSourceNotFound(ownerSourceName)); + if (!ownerSource.exists(id)) { + throw new OwnerSource.OwnerNotFound(ownerSourceName, id); } + OwnerId ownerId = new OwnerId(ownerSource.name(), id); + return topicService.listForOwnerId(ownerId); + } + + private List listTracked(String groupName) { + return isNullOrEmpty(groupName) + ? topicService.listTrackedTopicNames() + : topicService.listTrackedTopicNames(groupName); + } + + private List listNames(String groupName) { + return isNullOrEmpty(groupName) + ? topicService.listQualifiedTopicNames() + : topicService.listQualifiedTopicNames(groupName); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/UiResource.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/UiResource.java index 6876248077..fc845778d7 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/UiResource.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/UiResource.java @@ -1,17 +1,17 @@ package pl.allegro.tech.hermes.management.api; +import static jakarta.ws.rs.core.MediaType.TEXT_HTML; + import jakarta.ws.rs.GET; import jakarta.ws.rs.Path; import jakarta.ws.rs.Produces; import org.glassfish.jersey.server.mvc.Viewable; -import static jakarta.ws.rs.core.MediaType.TEXT_HTML; - @Path("/") public class UiResource { - @GET - @Produces(TEXT_HTML) - public Viewable getIndex() { - return new Viewable("/index.html"); - } + @GET + @Produces(TEXT_HTML) + public Viewable getIndex() { + return new Viewable("/index.html"); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/UnhealthyEndpoint.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/UnhealthyEndpoint.java index b28ac97321..beae414ee4 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/UnhealthyEndpoint.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/UnhealthyEndpoint.java @@ -1,5 +1,8 @@ package pl.allegro.tech.hermes.management.api; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; +import static jakarta.ws.rs.core.MediaType.TEXT_PLAIN; + import jakarta.ws.rs.DefaultValue; import jakarta.ws.rs.GET; import jakarta.ws.rs.Path; @@ -7,6 +10,8 @@ import jakarta.ws.rs.QueryParam; import jakarta.ws.rs.core.GenericEntity; import jakarta.ws.rs.core.Response; +import java.util.List; +import java.util.Optional; import org.springframework.beans.factory.annotation.Autowired; import pl.allegro.tech.hermes.api.OwnerId; import pl.allegro.tech.hermes.api.UnhealthySubscription; @@ -14,53 +19,52 @@ import pl.allegro.tech.hermes.management.domain.owner.OwnerSources; import pl.allegro.tech.hermes.management.domain.subscription.SubscriptionService; -import java.util.List; -import java.util.Optional; - -import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; -import static jakarta.ws.rs.core.MediaType.TEXT_PLAIN; - @Path("unhealthy") public class UnhealthyEndpoint { - private final OwnerSources ownerSources; - private final SubscriptionService subscriptionService; + private final OwnerSources ownerSources; + private final SubscriptionService subscriptionService; - @Autowired - public UnhealthyEndpoint(OwnerSources ownerSources, - SubscriptionService subscriptionService) { - this.ownerSources = ownerSources; - this.subscriptionService = subscriptionService; - } + @Autowired + public UnhealthyEndpoint(OwnerSources ownerSources, SubscriptionService subscriptionService) { + this.ownerSources = ownerSources; + this.subscriptionService = subscriptionService; + } - @GET - @Produces({APPLICATION_JSON, TEXT_PLAIN}) - @Path("/") - public Response listUnhealthy( - @QueryParam("ownerSourceName") String ownerSourceName, - @QueryParam("ownerId") String id, - @DefaultValue("true") @QueryParam("respectMonitoringSeverity") boolean respectMonitoringSeverity, - @QueryParam("subscriptionNames") List subscriptionNames, - @QueryParam("qualifiedTopicNames") List qualifiedTopicNames) { + @GET + @Produces({APPLICATION_JSON, TEXT_PLAIN}) + @Path("/") + public Response listUnhealthy( + @QueryParam("ownerSourceName") String ownerSourceName, + @QueryParam("ownerId") String id, + @DefaultValue("true") @QueryParam("respectMonitoringSeverity") + boolean respectMonitoringSeverity, + @QueryParam("subscriptionNames") List subscriptionNames, + @QueryParam("qualifiedTopicNames") List qualifiedTopicNames) { - List unhealthySubscriptions = areEmpty(ownerSourceName, id) - ? subscriptionService.getAllUnhealthy(respectMonitoringSeverity, subscriptionNames, qualifiedTopicNames) - : resolveOwnerId(ownerSourceName, id) - .map(ownerId -> subscriptionService.getUnhealthyForOwner( - ownerId, respectMonitoringSeverity, subscriptionNames, qualifiedTopicNames - )) + List unhealthySubscriptions = + areEmpty(ownerSourceName, id) + ? subscriptionService.getAllUnhealthy( + respectMonitoringSeverity, subscriptionNames, qualifiedTopicNames) + : resolveOwnerId(ownerSourceName, id) + .map( + ownerId -> + subscriptionService.getUnhealthyForOwner( + ownerId, + respectMonitoringSeverity, + subscriptionNames, + qualifiedTopicNames)) .orElseThrow(() -> new OwnerSource.OwnerNotFound(ownerSourceName, id)); - return Response.ok() - .entity(new GenericEntity<>(unhealthySubscriptions) { - }) - .build(); - } + return Response.ok().entity(new GenericEntity<>(unhealthySubscriptions) {}).build(); + } - private boolean areEmpty(String ownerSourceName, String id) { - return ownerSourceName == null && id == null; - } + private boolean areEmpty(String ownerSourceName, String id) { + return ownerSourceName == null && id == null; + } - private Optional resolveOwnerId(String ownerSourceName, String id) { - return ownerSources.getByName(ownerSourceName).map(ownerSource -> new OwnerId(ownerSource.name(), id)); - } + private Optional resolveOwnerId(String ownerSourceName, String id) { + return ownerSources + .getByName(ownerSourceName) + .map(ownerSource -> new OwnerId(ownerSource.name(), id)); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/WorkloadConstraintsEndpoint.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/WorkloadConstraintsEndpoint.java index 6a43d32208..9b29c8e283 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/WorkloadConstraintsEndpoint.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/WorkloadConstraintsEndpoint.java @@ -1,5 +1,9 @@ package pl.allegro.tech.hermes.management.api; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; +import static jakarta.ws.rs.core.Response.Status.CREATED; +import static jakarta.ws.rs.core.Response.Status.OK; + import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import jakarta.annotation.security.RolesAllowed; @@ -15,6 +19,7 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.Response; +import java.util.List; import org.springframework.stereotype.Component; import pl.allegro.tech.hermes.api.SubscriptionConstraints; import pl.allegro.tech.hermes.api.SubscriptionName; @@ -26,91 +31,107 @@ import pl.allegro.tech.hermes.management.domain.auth.RequestUser; import pl.allegro.tech.hermes.management.domain.workload.constraints.WorkloadConstraintsService; -import java.util.List; - -import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; -import static jakarta.ws.rs.core.Response.Status.CREATED; -import static jakarta.ws.rs.core.Response.Status.OK; - @Component @Path("/workload-constraints") @Api(value = "/workload-constraints", description = "Operations on workload constraints") public class WorkloadConstraintsEndpoint { - private final WorkloadConstraintsService service; + private final WorkloadConstraintsService service; - public WorkloadConstraintsEndpoint(WorkloadConstraintsService service) { - this.service = service; - } + public WorkloadConstraintsEndpoint(WorkloadConstraintsService service) { + this.service = service; + } - @GET - @Produces(APPLICATION_JSON) - @RolesAllowed(Roles.ANY) - @ApiOperation(value = "All workload constraints", response = List.class, httpMethod = HttpMethod.GET) - public ConsumersWorkloadConstraints getConsumersWorkloadConstraints() { - return service.getConsumersWorkloadConstraints(); - } + @GET + @Produces(APPLICATION_JSON) + @RolesAllowed(Roles.ANY) + @ApiOperation( + value = "All workload constraints", + response = List.class, + httpMethod = HttpMethod.GET) + public ConsumersWorkloadConstraints getConsumersWorkloadConstraints() { + return service.getConsumersWorkloadConstraints(); + } - @PUT - @Path("/topic") - @Consumes(APPLICATION_JSON) - @Produces(APPLICATION_JSON) - @RolesAllowed(Roles.ADMIN) - @ApiOperation(value = "Create or update topic constraints", response = String.class, httpMethod = HttpMethod.PUT) - public Response createOrUpdateTopicConstraints( - @Valid TopicConstraints topicConstraints, - @Context ContainerRequestContext requestContext) { - RequestUser requestUser = new HermesSecurityAwareRequestUser(requestContext); - if (service.constraintsExist(topicConstraints.getTopicName())) { - service.updateConstraints(topicConstraints.getTopicName(), topicConstraints.getConstraints(), requestUser); - return Response.status(OK).build(); - } else { - service.createConstraints(topicConstraints.getTopicName(), topicConstraints.getConstraints(), requestUser); - return Response.status(CREATED).build(); - } + @PUT + @Path("/topic") + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON) + @RolesAllowed(Roles.ADMIN) + @ApiOperation( + value = "Create or update topic constraints", + response = String.class, + httpMethod = HttpMethod.PUT) + public Response createOrUpdateTopicConstraints( + @Valid TopicConstraints topicConstraints, @Context ContainerRequestContext requestContext) { + RequestUser requestUser = new HermesSecurityAwareRequestUser(requestContext); + if (service.constraintsExist(topicConstraints.getTopicName())) { + service.updateConstraints( + topicConstraints.getTopicName(), topicConstraints.getConstraints(), requestUser); + return Response.status(OK).build(); + } else { + service.createConstraints( + topicConstraints.getTopicName(), topicConstraints.getConstraints(), requestUser); + return Response.status(CREATED).build(); } + } - @DELETE - @Path("/topic/{topicName}") - @RolesAllowed(Roles.ADMIN) - @ApiOperation(value = "Remove topic constraints", response = String.class, httpMethod = HttpMethod.DELETE) - public Response deleteTopicConstraints( - @PathParam("topicName") String topicName, - @Context ContainerRequestContext requestContext) { - service.deleteConstraints(TopicName.fromQualifiedName(topicName), new HermesSecurityAwareRequestUser(requestContext)); - return Response.status(OK).build(); - } + @DELETE + @Path("/topic/{topicName}") + @RolesAllowed(Roles.ADMIN) + @ApiOperation( + value = "Remove topic constraints", + response = String.class, + httpMethod = HttpMethod.DELETE) + public Response deleteTopicConstraints( + @PathParam("topicName") String topicName, @Context ContainerRequestContext requestContext) { + service.deleteConstraints( + TopicName.fromQualifiedName(topicName), new HermesSecurityAwareRequestUser(requestContext)); + return Response.status(OK).build(); + } - @PUT - @Path("/subscription") - @Consumes(APPLICATION_JSON) - @Produces(APPLICATION_JSON) - @RolesAllowed(Roles.ADMIN) - @ApiOperation(value = "Create or update subscription constraints", response = String.class, httpMethod = HttpMethod.PUT) - public Response createOrUpdateSubscriptionConstraints( - @Valid SubscriptionConstraints subscriptionConstraints, - @Context ContainerRequestContext requestContext) { - RequestUser requestUser = new HermesSecurityAwareRequestUser(requestContext); - if (service.constraintsExist(subscriptionConstraints.getSubscriptionName())) { - service.updateConstraints(subscriptionConstraints.getSubscriptionName(), subscriptionConstraints.getConstraints(), requestUser); - return Response.status(OK).build(); - } else { - service.createConstraints(subscriptionConstraints.getSubscriptionName(), subscriptionConstraints.getConstraints(), requestUser); - return Response.status(CREATED).build(); - } + @PUT + @Path("/subscription") + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON) + @RolesAllowed(Roles.ADMIN) + @ApiOperation( + value = "Create or update subscription constraints", + response = String.class, + httpMethod = HttpMethod.PUT) + public Response createOrUpdateSubscriptionConstraints( + @Valid SubscriptionConstraints subscriptionConstraints, + @Context ContainerRequestContext requestContext) { + RequestUser requestUser = new HermesSecurityAwareRequestUser(requestContext); + if (service.constraintsExist(subscriptionConstraints.getSubscriptionName())) { + service.updateConstraints( + subscriptionConstraints.getSubscriptionName(), + subscriptionConstraints.getConstraints(), + requestUser); + return Response.status(OK).build(); + } else { + service.createConstraints( + subscriptionConstraints.getSubscriptionName(), + subscriptionConstraints.getConstraints(), + requestUser); + return Response.status(CREATED).build(); } + } - @DELETE - @Path("/subscription/{topicName}/{subscriptionName}") - @RolesAllowed(Roles.ADMIN) - @ApiOperation(value = "Remove subscription constraints", response = String.class, httpMethod = HttpMethod.DELETE) - public Response deleteSubscriptionConstraints(@PathParam("topicName") String topicName, - @PathParam("subscriptionName") String subscriptionName, - @Context ContainerRequestContext requestContext) { - service.deleteConstraints( - new SubscriptionName(subscriptionName, TopicName.fromQualifiedName(topicName)), - new HermesSecurityAwareRequestUser(requestContext) - ); - return Response.status(OK).build(); - } + @DELETE + @Path("/subscription/{topicName}/{subscriptionName}") + @RolesAllowed(Roles.ADMIN) + @ApiOperation( + value = "Remove subscription constraints", + response = String.class, + httpMethod = HttpMethod.DELETE) + public Response deleteSubscriptionConstraints( + @PathParam("topicName") String topicName, + @PathParam("subscriptionName") String subscriptionName, + @Context ContainerRequestContext requestContext) { + service.deleteConstraints( + new SubscriptionName(subscriptionName, TopicName.fromQualifiedName(topicName)), + new HermesSecurityAwareRequestUser(requestContext)); + return Response.status(OK).build(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/AllowAllSecurityProvider.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/AllowAllSecurityProvider.java index 395429516c..37cfe540d9 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/AllowAllSecurityProvider.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/AllowAllSecurityProvider.java @@ -2,46 +2,45 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.SecurityContext; -import org.apache.commons.lang3.NotImplementedException; - import java.security.Principal; +import org.apache.commons.lang3.NotImplementedException; public class AllowAllSecurityProvider implements SecurityProvider { - @Override - public HermesSecurity security(ContainerRequestContext requestContext) { - return new HermesSecurity(securityContext(requestContext), ownerId -> true); - } + @Override + public HermesSecurity security(ContainerRequestContext requestContext) { + return new HermesSecurity(securityContext(requestContext), ownerId -> true); + } + + private SecurityContext securityContext(ContainerRequestContext requestContext) { + return new SecurityContext() { + @Override + public Principal getUserPrincipal() { + return new AnonymousUserPrincipal(); + } + + @Override + public boolean isUserInRole(String role) { + return true; + } + + @Override + public boolean isSecure() { + throw new NotImplementedException(); + } + + @Override + public String getAuthenticationScheme() { + throw new NotImplementedException(); + } + }; + } + + private static class AnonymousUserPrincipal implements Principal { - private SecurityContext securityContext(ContainerRequestContext requestContext) { - return new SecurityContext() { - @Override - public Principal getUserPrincipal() { - return new AnonymousUserPrincipal(); - } - - @Override - public boolean isUserInRole(String role) { - return true; - } - - @Override - public boolean isSecure() { - throw new NotImplementedException(); - } - - @Override - public String getAuthenticationScheme() { - throw new NotImplementedException(); - } - }; - } - - private static class AnonymousUserPrincipal implements Principal { - - @Override - public String getName() { - return "[anonymous user]"; - } + @Override + public String getName() { + return "[anonymous user]"; } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/AuthException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/AuthException.java index 3c5906e062..3ba296bb34 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/AuthException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/AuthException.java @@ -5,16 +5,16 @@ public class AuthException extends ManagementException { - public AuthException(String message) { - super(message); - } - - public AuthException(RuntimeException ex) { - super("Exception while authorization: " + ex.getMessage(), ex); - } + public AuthException(String message) { + super(message); + } - @Override - public ErrorCode getCode() { - return ErrorCode.AUTH_ERROR; - } + public AuthException(RuntimeException ex) { + super("Exception while authorization: " + ex.getMessage(), ex); + } + + @Override + public ErrorCode getCode() { + return ErrorCode.AUTH_ERROR; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/AuthorizationFilter.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/AuthorizationFilter.java index 9e8ba437fd..197a39e634 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/AuthorizationFilter.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/AuthorizationFilter.java @@ -4,30 +4,30 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.container.ContainerRequestFilter; import jakarta.ws.rs.ext.Provider; -import org.springframework.beans.factory.annotation.Autowired; - import java.io.IOException; +import org.springframework.beans.factory.annotation.Autowired; @Provider @Priority(AuthorizationFilter.AUTHORIZATION_FILTER_PRIORITY) public class AuthorizationFilter implements ContainerRequestFilter { - public static final String OWNERSHIP_RESOLVER = "ownership-resolver"; + public static final String OWNERSHIP_RESOLVER = "ownership-resolver"; - // fixing equal values reordering issue of Jersey's 2.23.2 RankedComparator (Priorities.AUTHORIZATION=2000) - public static final int AUTHORIZATION_FILTER_PRIORITY = 1999; + // fixing equal values reordering issue of Jersey's 2.23.2 RankedComparator + // (Priorities.AUTHORIZATION=2000) + public static final int AUTHORIZATION_FILTER_PRIORITY = 1999; - private final SecurityProvider securityProvider; + private final SecurityProvider securityProvider; - @Autowired - public AuthorizationFilter(SecurityProvider securityProvider) { - this.securityProvider = securityProvider; - } + @Autowired + public AuthorizationFilter(SecurityProvider securityProvider) { + this.securityProvider = securityProvider; + } - @Override - public void filter(ContainerRequestContext requestContext) throws IOException { - SecurityProvider.HermesSecurity security = securityProvider.security(requestContext); - requestContext.setSecurityContext(security.getSecurityContext()); - requestContext.setProperty(OWNERSHIP_RESOLVER, security.getOwnershipResolver()); - } + @Override + public void filter(ContainerRequestContext requestContext) throws IOException { + SecurityProvider.HermesSecurity security = securityProvider.security(requestContext); + requestContext.setSecurityContext(security.getSecurityContext()); + requestContext.setProperty(OWNERSHIP_RESOLVER, security.getOwnershipResolver()); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/CreatorRights.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/CreatorRights.java index fd2daad8bb..2e95c3f3b0 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/CreatorRights.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/CreatorRights.java @@ -2,7 +2,7 @@ public interface CreatorRights { - boolean allowedToManage(T entity); + boolean allowedToManage(T entity); - boolean allowedToCreate(T entity); + boolean allowedToCreate(T entity); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/HermesSecurityAwareRequestUser.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/HermesSecurityAwareRequestUser.java index d137b15701..316217f367 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/HermesSecurityAwareRequestUser.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/HermesSecurityAwareRequestUser.java @@ -7,29 +7,30 @@ import pl.allegro.tech.hermes.management.domain.auth.RequestUser; public class HermesSecurityAwareRequestUser implements RequestUser { - private final String username; - private final boolean isAdmin; - private final OwnershipResolver ownershipResolver; + private final String username; + private final boolean isAdmin; + private final OwnershipResolver ownershipResolver; - public HermesSecurityAwareRequestUser(ContainerRequestContext requestContext) { - SecurityContext securityContext = requestContext.getSecurityContext(); - username = securityContext.getUserPrincipal().getName(); - isAdmin = securityContext.isUserInRole(Roles.ADMIN); - ownershipResolver = (OwnershipResolver) requestContext.getProperty(AuthorizationFilter.OWNERSHIP_RESOLVER); - } + public HermesSecurityAwareRequestUser(ContainerRequestContext requestContext) { + SecurityContext securityContext = requestContext.getSecurityContext(); + username = securityContext.getUserPrincipal().getName(); + isAdmin = securityContext.isUserInRole(Roles.ADMIN); + ownershipResolver = + (OwnershipResolver) requestContext.getProperty(AuthorizationFilter.OWNERSHIP_RESOLVER); + } - @Override - public String getUsername() { - return username; - } + @Override + public String getUsername() { + return username; + } - @Override - public boolean isAdmin() { - return isAdmin; - } + @Override + public boolean isAdmin() { + return isAdmin; + } - @Override - public boolean isOwner(OwnerId ownerId) { - return ownershipResolver.isUserAnOwner(ownerId); - } + @Override + public boolean isOwner(OwnerId ownerId) { + return ownershipResolver.isUserAnOwner(ownerId); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/ManagementRights.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/ManagementRights.java index 65c8c3b83e..5584e886e4 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/ManagementRights.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/ManagementRights.java @@ -8,58 +8,61 @@ import pl.allegro.tech.hermes.management.config.GroupProperties; /** - * Make sure these implementations conform to what is configured via RolesAllowed annotations in endpoints. + * Make sure these implementations conform to what is configured via RolesAllowed annotations in + * endpoints. */ @Component public class ManagementRights { - private final GroupProperties groupProperties; + private final GroupProperties groupProperties; - @Autowired - public ManagementRights(GroupProperties groupProperties) { - this.groupProperties = groupProperties; - } + @Autowired + public ManagementRights(GroupProperties groupProperties) { + this.groupProperties = groupProperties; + } - public boolean isUserAllowedToManageTopic(Topic topic, ContainerRequestContext requestContext) { - return isAdmin(requestContext) - || getOwnershipResolver(requestContext).isUserAnOwner(topic.getOwner()); - } + public boolean isUserAllowedToManageTopic(Topic topic, ContainerRequestContext requestContext) { + return isAdmin(requestContext) + || getOwnershipResolver(requestContext).isUserAnOwner(topic.getOwner()); + } - public boolean isUserAllowedToCreateGroup(ContainerRequestContext requestContext) { - return isAdmin(requestContext) || groupProperties.isNonAdminCreationEnabled(); - } + public boolean isUserAllowedToCreateGroup(ContainerRequestContext requestContext) { + return isAdmin(requestContext) || groupProperties.isNonAdminCreationEnabled(); + } - private boolean isUserAllowedToManageGroup(ContainerRequestContext requestContext) { - return isAdmin(requestContext); - } + private boolean isUserAllowedToManageGroup(ContainerRequestContext requestContext) { + return isAdmin(requestContext); + } - private boolean isAdmin(ContainerRequestContext requestContext) { - return requestContext.getSecurityContext().isUserInRole(Roles.ADMIN); - } + private boolean isAdmin(ContainerRequestContext requestContext) { + return requestContext.getSecurityContext().isUserInRole(Roles.ADMIN); + } - private SecurityProvider.OwnershipResolver getOwnershipResolver(ContainerRequestContext requestContext) { - return (SecurityProvider.OwnershipResolver) requestContext.getProperty(AuthorizationFilter.OWNERSHIP_RESOLVER); - } + private SecurityProvider.OwnershipResolver getOwnershipResolver( + ContainerRequestContext requestContext) { + return (SecurityProvider.OwnershipResolver) + requestContext.getProperty(AuthorizationFilter.OWNERSHIP_RESOLVER); + } - public CreatorRights getGroupCreatorRights(ContainerRequestContext requestContext) { - return new GroupCreatorRights(requestContext); - } + public CreatorRights getGroupCreatorRights(ContainerRequestContext requestContext) { + return new GroupCreatorRights(requestContext); + } - class GroupCreatorRights implements CreatorRights { - private ContainerRequestContext requestContext; + class GroupCreatorRights implements CreatorRights { + private ContainerRequestContext requestContext; - GroupCreatorRights(ContainerRequestContext requestContext) { - this.requestContext = requestContext; - } + GroupCreatorRights(ContainerRequestContext requestContext) { + this.requestContext = requestContext; + } - @Override - public boolean allowedToManage(Group group) { - return ManagementRights.this.isUserAllowedToManageGroup(requestContext); - } + @Override + public boolean allowedToManage(Group group) { + return ManagementRights.this.isUserAllowedToManageGroup(requestContext); + } - @Override - public boolean allowedToCreate(Group group) { - return ManagementRights.this.isUserAllowedToCreateGroup(requestContext); - } + @Override + public boolean allowedToCreate(Group group) { + return ManagementRights.this.isUserAllowedToCreateGroup(requestContext); } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/Roles.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/Roles.java index 2a8f7a84d9..5bff68ddb2 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/Roles.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/Roles.java @@ -1,8 +1,8 @@ package pl.allegro.tech.hermes.management.api.auth; public final class Roles { - public static final String ANY = "any"; - public static final String ADMIN = "admin"; - public static final String TOPIC_OWNER = "topicOwner"; - public static final String SUBSCRIPTION_OWNER = "subscriptionOwner"; + public static final String ANY = "any"; + public static final String ADMIN = "admin"; + public static final String TOPIC_OWNER = "topicOwner"; + public static final String SUBSCRIPTION_OWNER = "subscriptionOwner"; } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/SecurityProvider.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/SecurityProvider.java index 7806a61420..ccfd7ae87b 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/SecurityProvider.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/auth/SecurityProvider.java @@ -6,28 +6,27 @@ public interface SecurityProvider { - HermesSecurity security(ContainerRequestContext requestContext); + HermesSecurity security(ContainerRequestContext requestContext); - class HermesSecurity { - private final SecurityContext securityContext; - private final OwnershipResolver ownershipResolver; + class HermesSecurity { + private final SecurityContext securityContext; + private final OwnershipResolver ownershipResolver; - public HermesSecurity(SecurityContext securityContext, OwnershipResolver ownershipResolver) { - this.securityContext = securityContext; - this.ownershipResolver = ownershipResolver; - } - - public SecurityContext getSecurityContext() { - return securityContext; - } + public HermesSecurity(SecurityContext securityContext, OwnershipResolver ownershipResolver) { + this.securityContext = securityContext; + this.ownershipResolver = ownershipResolver; + } - public OwnershipResolver getOwnershipResolver() { - return ownershipResolver; - } + public SecurityContext getSecurityContext() { + return securityContext; } - interface OwnershipResolver { - boolean isUserAnOwner(OwnerId owner); + public OwnershipResolver getOwnershipResolver() { + return ownershipResolver; } + } + interface OwnershipResolver { + boolean isUserAnOwner(OwnerId owner); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/AbstractExceptionMapper.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/AbstractExceptionMapper.java index 512ba3b3e6..46aa32d93b 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/AbstractExceptionMapper.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/AbstractExceptionMapper.java @@ -8,20 +8,19 @@ abstract class AbstractExceptionMapper implements ExceptionMapper { - @Override - public Response toResponse(T exception) { - return Response - .status(httpStatus()) - .type(MediaType.APPLICATION_JSON_TYPE) - .entity(new ErrorDescription(errorMessage(exception), errorCode())) - .build(); - } + @Override + public Response toResponse(T exception) { + return Response.status(httpStatus()) + .type(MediaType.APPLICATION_JSON_TYPE) + .entity(new ErrorDescription(errorMessage(exception), errorCode())) + .build(); + } - String errorMessage(T exception) { - return exception.getMessage(); - } + String errorMessage(T exception) { + return exception.getMessage(); + } - abstract Response.Status httpStatus(); + abstract Response.Status httpStatus(); - abstract ErrorCode errorCode(); + abstract ErrorCode errorCode(); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/AuthExceptionMapper.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/AuthExceptionMapper.java index c10e42caab..2ff79c0b05 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/AuthExceptionMapper.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/AuthExceptionMapper.java @@ -8,13 +8,13 @@ @Provider public class AuthExceptionMapper extends AbstractExceptionMapper { - @Override - Response.Status httpStatus() { - return Response.Status.FORBIDDEN; - } + @Override + Response.Status httpStatus() { + return Response.Status.FORBIDDEN; + } - @Override - ErrorCode errorCode() { - return ErrorCode.AUTH_ERROR; - } -} \ No newline at end of file + @Override + ErrorCode errorCode() { + return ErrorCode.AUTH_ERROR; + } +} diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/ConstraintViolationMapper.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/ConstraintViolationMapper.java index 69a6054a31..a16acfa69a 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/ConstraintViolationMapper.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/ConstraintViolationMapper.java @@ -6,43 +6,44 @@ import jakarta.validation.ConstraintViolationException; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.ext.Provider; +import java.util.List; import org.glassfish.jersey.server.validation.ValidationErrorData; import org.glassfish.jersey.server.validation.internal.ValidationHelper; import pl.allegro.tech.hermes.api.ErrorCode; -import java.util.List; - @Provider -public class ConstraintViolationMapper extends AbstractExceptionMapper { - +public class ConstraintViolationMapper + extends AbstractExceptionMapper { + + @Override + Response.Status httpStatus() { + return Response.Status.BAD_REQUEST; + } + + @Override + ErrorCode errorCode() { + return ErrorCode.VALIDATION_ERROR; + } + + @Override + public String errorMessage(ConstraintViolationException exception) { + return prepareMessage(exception); + } + + private String prepareMessage(ConstraintViolationException ex) { + List errors = + Lists.transform( + ValidationHelper.constraintViolationToValidationErrors(ex), + new ValidationErrorDataConverter()); + + return Joiner.on("; ").join(errors); + } + + private static final class ValidationErrorDataConverter + implements Function { @Override - Response.Status httpStatus() { - return Response.Status.BAD_REQUEST; - } - - @Override - ErrorCode errorCode() { - return ErrorCode.VALIDATION_ERROR; - } - - @Override - public String errorMessage(ConstraintViolationException exception) { - return prepareMessage(exception); - } - - private String prepareMessage(ConstraintViolationException ex) { - List errors = Lists.transform( - ValidationHelper.constraintViolationToValidationErrors(ex), - new ValidationErrorDataConverter() - ); - - return Joiner.on("; ").join(errors); - } - - private static final class ValidationErrorDataConverter implements Function { - @Override - public String apply(ValidationErrorData input) { - return input.getPath() + " " + input.getMessage(); - } + public String apply(ValidationErrorData input) { + return input.getPath() + " " + input.getMessage(); } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/HermesExceptionMapper.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/HermesExceptionMapper.java index 0939d066f6..0b96a6ed8c 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/HermesExceptionMapper.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/HermesExceptionMapper.java @@ -10,12 +10,11 @@ @Provider public class HermesExceptionMapper implements ExceptionMapper { - @Override - public Response toResponse(HermesException exception) { - return Response - .status(exception.getCode().getHttpCode()) - .type(MediaType.APPLICATION_JSON_TYPE) - .entity(new ErrorDescription(exception.getMessage(), exception.getCode())) - .build(); - } + @Override + public Response toResponse(HermesException exception) { + return Response.status(exception.getCode().getHttpCode()) + .type(MediaType.APPLICATION_JSON_TYPE) + .entity(new ErrorDescription(exception.getMessage(), exception.getCode())) + .build(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/IOExceptionMapper.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/IOExceptionMapper.java index 7c2e24a17a..c0dc20b126 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/IOExceptionMapper.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/IOExceptionMapper.java @@ -2,20 +2,19 @@ import jakarta.ws.rs.core.Response; import jakarta.ws.rs.ext.Provider; -import pl.allegro.tech.hermes.api.ErrorCode; - import java.io.IOException; +import pl.allegro.tech.hermes.api.ErrorCode; @Provider public class IOExceptionMapper extends AbstractExceptionMapper { - @Override - Response.Status httpStatus() { - return Response.Status.BAD_REQUEST; - } + @Override + Response.Status httpStatus() { + return Response.Status.BAD_REQUEST; + } - @Override - ErrorCode errorCode() { - return ErrorCode.OTHER; - } + @Override + ErrorCode errorCode() { + return ErrorCode.OTHER; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/IllegalArgumentExceptionMapper.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/IllegalArgumentExceptionMapper.java index 2e0fa74bf1..5353a52c33 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/IllegalArgumentExceptionMapper.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/IllegalArgumentExceptionMapper.java @@ -5,15 +5,16 @@ import pl.allegro.tech.hermes.api.ErrorCode; @Provider -public class IllegalArgumentExceptionMapper extends AbstractExceptionMapper { +public class IllegalArgumentExceptionMapper + extends AbstractExceptionMapper { - @Override - Response.Status httpStatus() { - return Response.Status.BAD_REQUEST; - } + @Override + Response.Status httpStatus() { + return Response.Status.BAD_REQUEST; + } - @Override - ErrorCode errorCode() { - return ErrorCode.OTHER; - } + @Override + ErrorCode errorCode() { + return ErrorCode.OTHER; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/JsonMappingExceptionMapper.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/JsonMappingExceptionMapper.java index 145ac5d5dd..6d6587b8b2 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/JsonMappingExceptionMapper.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/JsonMappingExceptionMapper.java @@ -8,13 +8,13 @@ @Provider public class JsonMappingExceptionMapper extends AbstractExceptionMapper { - @Override - Response.Status httpStatus() { - return Response.Status.BAD_REQUEST; - } + @Override + Response.Status httpStatus() { + return Response.Status.BAD_REQUEST; + } - @Override - ErrorCode errorCode() { - return ErrorCode.FORMAT_ERROR; - } + @Override + ErrorCode errorCode() { + return ErrorCode.FORMAT_ERROR; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/JsonParseExceptionMapper.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/JsonParseExceptionMapper.java index df1c7b5e50..4d7a85579f 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/JsonParseExceptionMapper.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/JsonParseExceptionMapper.java @@ -8,13 +8,13 @@ @Provider public class JsonParseExceptionMapper extends AbstractExceptionMapper { - @Override - Response.Status httpStatus() { - return Response.Status.BAD_REQUEST; - } + @Override + Response.Status httpStatus() { + return Response.Status.BAD_REQUEST; + } - @Override - ErrorCode errorCode() { - return ErrorCode.FORMAT_ERROR; - } + @Override + ErrorCode errorCode() { + return ErrorCode.FORMAT_ERROR; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/ManagementExceptionMapper.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/ManagementExceptionMapper.java index 8b713d23cd..d28950cfb3 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/ManagementExceptionMapper.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/ManagementExceptionMapper.java @@ -10,12 +10,11 @@ @Provider public class ManagementExceptionMapper implements ExceptionMapper { - @Override - public Response toResponse(ManagementException exception) { - return Response - .status(exception.getCode().getHttpCode()) - .type(MediaType.APPLICATION_JSON_TYPE) - .entity(new ErrorDescription(exception.getMessage(), exception.getCode())) - .build(); - } + @Override + public Response toResponse(ManagementException exception) { + return Response.status(exception.getCode().getHttpCode()) + .type(MediaType.APPLICATION_JSON_TYPE) + .entity(new ErrorDescription(exception.getMessage(), exception.getCode())) + .build(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/NotSupportedExceptionMapper.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/NotSupportedExceptionMapper.java index 199880661e..dea6828f4f 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/NotSupportedExceptionMapper.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/NotSupportedExceptionMapper.java @@ -8,13 +8,13 @@ @Provider public class NotSupportedExceptionMapper extends AbstractExceptionMapper { - @Override - Response.Status httpStatus() { - return Response.Status.UNSUPPORTED_MEDIA_TYPE; - } + @Override + Response.Status httpStatus() { + return Response.Status.UNSUPPORTED_MEDIA_TYPE; + } - @Override - ErrorCode errorCode() { - return ErrorCode.VALIDATION_ERROR; - } + @Override + ErrorCode errorCode() { + return ErrorCode.VALIDATION_ERROR; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/ParseExceptionMapper.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/ParseExceptionMapper.java index 1818c91371..d57ee86b6f 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/ParseExceptionMapper.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/ParseExceptionMapper.java @@ -8,13 +8,13 @@ @Provider public class ParseExceptionMapper extends AbstractExceptionMapper { - @Override - Response.Status httpStatus() { - return Response.Status.BAD_REQUEST; - } + @Override + Response.Status httpStatus() { + return Response.Status.BAD_REQUEST; + } - @Override - ErrorCode errorCode() { - return ErrorCode.FORMAT_ERROR; - } + @Override + ErrorCode errorCode() { + return ErrorCode.FORMAT_ERROR; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/RuntimeExceptionMapper.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/RuntimeExceptionMapper.java index 18884b3171..5960a476cb 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/RuntimeExceptionMapper.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/RuntimeExceptionMapper.java @@ -12,15 +12,14 @@ @Provider public class RuntimeExceptionMapper implements ExceptionMapper { - private final Logger logger = LoggerFactory.getLogger(RuntimeExceptionMapper.class); + private final Logger logger = LoggerFactory.getLogger(RuntimeExceptionMapper.class); - @Override - public Response toResponse(RuntimeException exception) { - logger.warn("Caught unmapped exception: {}", exception.getClass().getSimpleName(), exception); - return Response - .status(Response.Status.INTERNAL_SERVER_ERROR) - .type(MediaType.APPLICATION_JSON_TYPE) - .entity(new ErrorDescription(exception.getMessage(), ErrorCode.OTHER)) - .build(); - } + @Override + public Response toResponse(RuntimeException exception) { + logger.warn("Caught unmapped exception: {}", exception.getClass().getSimpleName(), exception); + return Response.status(Response.Status.INTERNAL_SERVER_ERROR) + .type(MediaType.APPLICATION_JSON_TYPE) + .entity(new ErrorDescription(exception.getMessage(), ErrorCode.OTHER)) + .build(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/SchemaExceptionMapper.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/SchemaExceptionMapper.java index 547661b082..f1ac42de30 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/SchemaExceptionMapper.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/SchemaExceptionMapper.java @@ -10,12 +10,11 @@ @Provider public class SchemaExceptionMapper implements ExceptionMapper { - @Override - public Response toResponse(SchemaException exception) { - return Response - .status(exception.getCode().getHttpCode()) - .type(MediaType.APPLICATION_JSON_TYPE) - .entity(new ErrorDescription(exception.getMessage(), exception.getCode())) - .build(); - } + @Override + public Response toResponse(SchemaException exception) { + return Response.status(exception.getCode().getHttpCode()) + .type(MediaType.APPLICATION_JSON_TYPE) + .entity(new ErrorDescription(exception.getMessage(), exception.getCode())) + .build(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/WebApplicationExceptionMapper.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/WebApplicationExceptionMapper.java index e3680aced5..53b5e6c20d 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/WebApplicationExceptionMapper.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/mappers/WebApplicationExceptionMapper.java @@ -11,12 +11,11 @@ @Provider public class WebApplicationExceptionMapper implements ExceptionMapper { - @Override - public Response toResponse(WebApplicationException exception) { - return Response - .status(exception.getResponse().getStatus()) - .type(MediaType.APPLICATION_JSON_TYPE) - .entity(new ErrorDescription(exception.getMessage(), ErrorCode.OTHER)) - .build(); - } + @Override + public Response toResponse(WebApplicationException exception) { + return Response.status(exception.getResponse().getStatus()) + .type(MediaType.APPLICATION_JSON_TYPE) + .entity(new ErrorDescription(exception.getMessage(), ErrorCode.OTHER)) + .build(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/reader/QueryBodyReader.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/reader/QueryBodyReader.java index 33cdedf61d..2d06bb9a9e 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/reader/QueryBodyReader.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/reader/QueryBodyReader.java @@ -6,45 +6,47 @@ import jakarta.ws.rs.core.MultivaluedMap; import jakarta.ws.rs.ext.MessageBodyReader; import jakarta.ws.rs.ext.Provider; -import org.springframework.beans.factory.annotation.Autowired; -import pl.allegro.tech.hermes.api.Query; -import pl.allegro.tech.hermes.management.infrastructure.query.parser.QueryParser; -import pl.allegro.tech.hermes.management.infrastructure.query.parser.json.JsonQueryParser; - import java.io.IOException; import java.io.InputStream; import java.lang.annotation.Annotation; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; +import org.springframework.beans.factory.annotation.Autowired; +import pl.allegro.tech.hermes.api.Query; +import pl.allegro.tech.hermes.management.infrastructure.query.parser.QueryParser; +import pl.allegro.tech.hermes.management.infrastructure.query.parser.json.JsonQueryParser; @Provider public class QueryBodyReader implements MessageBodyReader { - private final ObjectMapper objectMapper; - - @Autowired - public QueryBodyReader(ObjectMapper objectMapper) { - this.objectMapper = objectMapper; - } - - @Override - public boolean isReadable(Class type, Type genericType, Annotation[] annotations, MediaType mediaType) { - return type == Query.class; - } - - @Override - public Query readFrom(Class type, - Type genericType, - Annotation[] annotations, - MediaType mediaType, - MultivaluedMap httpHeaders, - InputStream entityStream) throws IOException, WebApplicationException { - - Class queryType = Object.class; - if (genericType instanceof ParameterizedType) { - queryType = (Class) ((ParameterizedType) genericType).getRawType(); - } - QueryParser parser = new JsonQueryParser(objectMapper); - return parser.parse(entityStream, queryType); + private final ObjectMapper objectMapper; + + @Autowired + public QueryBodyReader(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + @Override + public boolean isReadable( + Class type, Type genericType, Annotation[] annotations, MediaType mediaType) { + return type == Query.class; + } + + @Override + public Query readFrom( + Class type, + Type genericType, + Annotation[] annotations, + MediaType mediaType, + MultivaluedMap httpHeaders, + InputStream entityStream) + throws IOException, WebApplicationException { + + Class queryType = Object.class; + if (genericType instanceof ParameterizedType) { + queryType = (Class) ((ParameterizedType) genericType).getRawType(); } + QueryParser parser = new JsonQueryParser(objectMapper); + return parser.parse(entityStream, queryType); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/validator/ApiPreconditions.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/validator/ApiPreconditions.java index 4c4b8cadd1..ff3fd65501 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/validator/ApiPreconditions.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/validator/ApiPreconditions.java @@ -5,29 +5,28 @@ import jakarta.validation.Validation; import jakarta.validation.Validator; import jakarta.validation.groups.Default; +import java.util.Set; import org.springframework.stereotype.Component; import pl.allegro.tech.hermes.api.constraints.AdminPermitted; -import java.util.Set; - @Component public class ApiPreconditions { - private final Validator validator = Validation.buildDefaultValidatorFactory().getValidator(); + private final Validator validator = Validation.buildDefaultValidatorFactory().getValidator(); - public void checkConstraints(T object, boolean skipAdminPermitted) { - Class[] groups = groupsToValidate(skipAdminPermitted); - Set> violations = validator.validate(object, groups); - if (!violations.isEmpty()) { - throw new ConstraintViolationException(violations); - } + public void checkConstraints(T object, boolean skipAdminPermitted) { + Class[] groups = groupsToValidate(skipAdminPermitted); + Set> violations = validator.validate(object, groups); + if (!violations.isEmpty()) { + throw new ConstraintViolationException(violations); } + } - private Class[] groupsToValidate(boolean skipAdminPermitted) { - if (skipAdminPermitted) { - return new Class[] {Default.class}; - } else { - return new Class[] {Default.class, AdminPermitted.class}; - } + private Class[] groupsToValidate(boolean skipAdminPermitted) { + if (skipAdminPermitted) { + return new Class[] {Default.class}; + } else { + return new Class[] {Default.class, AdminPermitted.class}; } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/writer/UnhealthySubscriptionListPlainTextBodyWriter.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/writer/UnhealthySubscriptionListPlainTextBodyWriter.java index 0226d34d77..9517209b89 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/writer/UnhealthySubscriptionListPlainTextBodyWriter.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/api/writer/UnhealthySubscriptionListPlainTextBodyWriter.java @@ -1,65 +1,71 @@ package pl.allegro.tech.hermes.management.api.writer; +import static jakarta.ws.rs.core.MediaType.TEXT_PLAIN; +import static java.util.stream.Collectors.joining; + import jakarta.ws.rs.Produces; import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.MultivaluedMap; import jakarta.ws.rs.ext.MessageBodyWriter; import jakarta.ws.rs.ext.Provider; -import pl.allegro.tech.hermes.api.SubscriptionHealthProblem; -import pl.allegro.tech.hermes.api.UnhealthySubscription; - import java.io.IOException; import java.io.OutputStream; import java.lang.annotation.Annotation; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; import java.util.List; - -import static jakarta.ws.rs.core.MediaType.TEXT_PLAIN; -import static java.util.stream.Collectors.joining; +import pl.allegro.tech.hermes.api.SubscriptionHealthProblem; +import pl.allegro.tech.hermes.api.UnhealthySubscription; @Provider @Produces(TEXT_PLAIN) -public class UnhealthySubscriptionListPlainTextBodyWriter implements MessageBodyWriter> { +public class UnhealthySubscriptionListPlainTextBodyWriter + implements MessageBodyWriter> { - private static String toPlainText(UnhealthySubscription unhealthySubscription) { - String problemDescriptions = unhealthySubscription.getProblems().stream() - .map(SubscriptionHealthProblem::getDescription) - .collect(joining("; ")); - return unhealthySubscription.getName() + " - " + problemDescriptions; - } + private static String toPlainText(UnhealthySubscription unhealthySubscription) { + String problemDescriptions = + unhealthySubscription.getProblems().stream() + .map(SubscriptionHealthProblem::getDescription) + .collect(joining("; ")); + return unhealthySubscription.getName() + " - " + problemDescriptions; + } - @Override - public boolean isWriteable(Class type, Type genericType, Annotation[] annotations, MediaType mediaType) { - if (List.class.isAssignableFrom(type) && genericType instanceof ParameterizedType) { - ParameterizedType parameterizedType = (ParameterizedType) genericType; - Type[] actualTypeArgs = parameterizedType.getActualTypeArguments(); - return actualTypeArgs.length == 1 && actualTypeArgs[0].equals(UnhealthySubscription.class); - } - return false; + @Override + public boolean isWriteable( + Class type, Type genericType, Annotation[] annotations, MediaType mediaType) { + if (List.class.isAssignableFrom(type) && genericType instanceof ParameterizedType) { + ParameterizedType parameterizedType = (ParameterizedType) genericType; + Type[] actualTypeArgs = parameterizedType.getActualTypeArguments(); + return actualTypeArgs.length == 1 && actualTypeArgs[0].equals(UnhealthySubscription.class); } + return false; + } - @Override - public long getSize(List unhealthySubscriptionList, - Class type, - Type genericType, - Annotation[] annotations, - MediaType mediaType) { - return -1; // According to JAX-RS 2.0 spec this method is deprecated and should return -1 - } + @Override + public long getSize( + List unhealthySubscriptionList, + Class type, + Type genericType, + Annotation[] annotations, + MediaType mediaType) { + return -1; // According to JAX-RS 2.0 spec this method is deprecated and should return -1 + } - @Override - public void writeTo(List unhealthySubscriptionList, - Class type, - Type genericType, - Annotation[] annotations, - MediaType mediaType, - MultivaluedMap httpHeaders, - OutputStream entityStream) throws IOException, WebApplicationException { - String body = unhealthySubscriptionList.stream() - .map(UnhealthySubscriptionListPlainTextBodyWriter::toPlainText) - .collect(joining("\r\n")); - entityStream.write(body.getBytes()); - } + @Override + public void writeTo( + List unhealthySubscriptionList, + Class type, + Type genericType, + Annotation[] annotations, + MediaType mediaType, + MultivaluedMap httpHeaders, + OutputStream entityStream) + throws IOException, WebApplicationException { + String body = + unhealthySubscriptionList.stream() + .map(UnhealthySubscriptionListPlainTextBodyWriter::toPlainText) + .collect(joining("\r\n")); + entityStream.write(body.getBytes()); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/AllTopicClientsConfiguration.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/AllTopicClientsConfiguration.java index d2286cdcfc..4e40b57c72 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/AllTopicClientsConfiguration.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/AllTopicClientsConfiguration.java @@ -12,12 +12,13 @@ @Configuration public class AllTopicClientsConfiguration { - private static final Logger logger = LoggerFactory.getLogger(AllTopicClientsConfiguration.class); + private static final Logger logger = LoggerFactory.getLogger(AllTopicClientsConfiguration.class); - @Bean - @ConditionalOnMissingBean(AllTopicClientsService.class) - public AllTopicClientsService allTopicClientsService(SubscriptionRepository subscriptionRepository) { - logger.info("Creating allTopicClientsService bean"); - return new DefaultAllTopicClientsService(subscriptionRepository); - } + @Bean + @ConditionalOnMissingBean(AllTopicClientsService.class) + public AllTopicClientsService allTopicClientsService( + SubscriptionRepository subscriptionRepository) { + logger.info("Creating allTopicClientsService bean"); + return new DefaultAllTopicClientsService(subscriptionRepository); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/AuditConfiguration.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/AuditConfiguration.java index 9b6c4b3f4f..b2d4c44eef 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/AuditConfiguration.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/AuditConfiguration.java @@ -1,6 +1,7 @@ package pl.allegro.tech.hermes.management.config; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Collection; import org.javers.core.Javers; import org.javers.core.JaversBuilder; import org.javers.core.metamodel.clazz.EntityDefinitionBuilder; @@ -24,58 +25,60 @@ import pl.allegro.tech.hermes.management.infrastructure.audit.EventAuditor; import pl.allegro.tech.hermes.management.infrastructure.audit.LoggingAuditor; -import java.util.Collection; - @Configuration @EnableConfigurationProperties({AuditProperties.class}) public class AuditConfiguration { - private static final Logger logger = LoggerFactory.getLogger(AuditConfiguration.class); + private static final Logger logger = LoggerFactory.getLogger(AuditConfiguration.class); - @Bean(name = "eventAuditorRestTemplate") - @ConditionalOnMissingBean(name = "eventAuditorRestTemplate") - public RestTemplate eventAuditorRestTemplate() { - logger.info("Creating eventAuditorRestTemplate bean"); - return new RestTemplateBuilder().build(); - } + @Bean(name = "eventAuditorRestTemplate") + @ConditionalOnMissingBean(name = "eventAuditorRestTemplate") + public RestTemplate eventAuditorRestTemplate() { + logger.info("Creating eventAuditorRestTemplate bean"); + return new RestTemplateBuilder().build(); + } - @Bean - @ConditionalOnProperty(prefix = "audit", value = "isLoggingAuditEnabled", havingValue = "true") - public LoggingAuditor loggingAuditor(ObjectMapper objectMapper) { - return new LoggingAuditor(javers(), objectMapper); - } + @Bean + @ConditionalOnProperty(prefix = "audit", value = "isLoggingAuditEnabled", havingValue = "true") + public LoggingAuditor loggingAuditor(ObjectMapper objectMapper) { + return new LoggingAuditor(javers(), objectMapper); + } - @Bean - @ConditionalOnProperty(prefix = "audit", value = "isEventAuditEnabled", havingValue = "true") - public EventAuditor eventAuditor( - AuditProperties auditProperties, - @Qualifier("eventAuditorRestTemplate") RestTemplate eventAuditorRestTemplate, - ObjectMapper objectMapper - ) { - return new EventAuditor(javers(), eventAuditorRestTemplate, auditProperties.getEventUrl(), objectMapper); - } + @Bean + @ConditionalOnProperty(prefix = "audit", value = "isEventAuditEnabled", havingValue = "true") + public EventAuditor eventAuditor( + AuditProperties auditProperties, + @Qualifier("eventAuditorRestTemplate") RestTemplate eventAuditorRestTemplate, + ObjectMapper objectMapper) { + return new EventAuditor( + javers(), eventAuditorRestTemplate, auditProperties.getEventUrl(), objectMapper); + } - @Bean - @Primary - public CompositeAuditor compositeAuditor(Collection auditors) { - return new CompositeAuditor(auditors); - } + @Bean + @Primary + public CompositeAuditor compositeAuditor(Collection auditors) { + return new CompositeAuditor(auditors); + } - private Javers javers() { - return JaversBuilder.javers() - .withPrettyPrint(false) - .registerEntity(EntityDefinitionBuilder.entityDefinition(Group.class) - .withIdPropertyName("groupName") - .build()) - .registerEntity(EntityDefinitionBuilder.entityDefinition(Topic.class) - .withIdPropertyName("name") - .build()) - .registerEntity(EntityDefinitionBuilder.entityDefinition(Subscription.class) - .withIdPropertyName("name") - .build()) - .registerEntity(EntityDefinitionBuilder.entityDefinition(OAuthProvider.class) - .withIdPropertyName("name") - .build()) - .build(); - } + private Javers javers() { + return JaversBuilder.javers() + .withPrettyPrint(false) + .registerEntity( + EntityDefinitionBuilder.entityDefinition(Group.class) + .withIdPropertyName("groupName") + .build()) + .registerEntity( + EntityDefinitionBuilder.entityDefinition(Topic.class) + .withIdPropertyName("name") + .build()) + .registerEntity( + EntityDefinitionBuilder.entityDefinition(Subscription.class) + .withIdPropertyName("name") + .build()) + .registerEntity( + EntityDefinitionBuilder.entityDefinition(OAuthProvider.class) + .withIdPropertyName("name") + .build()) + .build(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/AuditProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/AuditProperties.java index f4d63f41d6..b1c2797e8b 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/AuditProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/AuditProperties.java @@ -5,13 +5,13 @@ @ConfigurationProperties(prefix = "audit") class AuditProperties { - private String eventUrl = null; + private String eventUrl = null; - public String getEventUrl() { - return eventUrl; - } + public String getEventUrl() { + return eventUrl; + } - public void setEventUrl(String eventUrl) { - this.eventUrl = eventUrl; - } + public void setEventUrl(String eventUrl) { + this.eventUrl = eventUrl; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/AvroConfiguration.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/AvroConfiguration.java index a1f87c8376..4dd4feeaa5 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/AvroConfiguration.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/AvroConfiguration.java @@ -7,8 +7,8 @@ @Configuration public class AvroConfiguration { - @Bean - JsonAvroConverter jsonAvroConverter() { - return new JsonAvroConverter(); - } + @Bean + JsonAvroConverter jsonAvroConverter() { + return new JsonAvroConverter(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ConsistencyCheckerProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ConsistencyCheckerProperties.java index fba1977eb3..87388fa4ad 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ConsistencyCheckerProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ConsistencyCheckerProperties.java @@ -1,48 +1,45 @@ package pl.allegro.tech.hermes.management.config; -import org.springframework.boot.context.properties.ConfigurationProperties; - import java.time.Duration; +import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties(prefix = "consistency-checker") public class ConsistencyCheckerProperties { - private int threadPoolSize = 2; - private boolean periodicCheckEnabled = false; - private Duration refreshInterval = Duration.ofMinutes(15); - private Duration initialRefreshDelay = Duration.ofMinutes(2); - - public int getThreadPoolSize() { - return threadPoolSize; - } - - public void setThreadPoolSize(int threadPoolSize) { - this.threadPoolSize = threadPoolSize; - } + private int threadPoolSize = 2; + private boolean periodicCheckEnabled = false; + private Duration refreshInterval = Duration.ofMinutes(15); + private Duration initialRefreshDelay = Duration.ofMinutes(2); + public int getThreadPoolSize() { + return threadPoolSize; + } - public boolean isPeriodicCheckEnabled() { - return periodicCheckEnabled; - } + public void setThreadPoolSize(int threadPoolSize) { + this.threadPoolSize = threadPoolSize; + } - public void setPeriodicCheckEnabled(boolean periodicCheckEnabled) { - this.periodicCheckEnabled = periodicCheckEnabled; - } + public boolean isPeriodicCheckEnabled() { + return periodicCheckEnabled; + } + public void setPeriodicCheckEnabled(boolean periodicCheckEnabled) { + this.periodicCheckEnabled = periodicCheckEnabled; + } - public Duration getRefreshInterval() { - return refreshInterval; - } + public Duration getRefreshInterval() { + return refreshInterval; + } - public void setRefreshInterval(Duration refreshInterval) { - this.refreshInterval = refreshInterval; - } + public void setRefreshInterval(Duration refreshInterval) { + this.refreshInterval = refreshInterval; + } - public Duration getInitialRefreshDelay() { - return initialRefreshDelay; - } + public Duration getInitialRefreshDelay() { + return initialRefreshDelay; + } - public void setInitialRefreshDelay(Duration initialRefreshDelay) { - this.initialRefreshDelay = initialRefreshDelay; - } + public void setInitialRefreshDelay(Duration initialRefreshDelay) { + this.initialRefreshDelay = initialRefreshDelay; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/CorsConfiguration.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/CorsConfiguration.java index 093f9d1ff4..aef7d39534 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/CorsConfiguration.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/CorsConfiguration.java @@ -5,6 +5,4 @@ @Configuration @EnableConfigurationProperties(CorsProperties.class) -public class CorsConfiguration { - -} +public class CorsConfiguration {} diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/CorsProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/CorsProperties.java index 249b842b38..3558e46e79 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/CorsProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/CorsProperties.java @@ -5,14 +5,13 @@ @ConfigurationProperties("cors") public class CorsProperties { - private String allowedOrigin = "*"; + private String allowedOrigin = "*"; - public String getAllowedOrigin() { - return allowedOrigin; - } - - public void setAllowedOrigin(String allowedOrigin) { - this.allowedOrigin = allowedOrigin; - } + public String getAllowedOrigin() { + return allowedOrigin; + } + public void setAllowedOrigin(String allowedOrigin) { + this.allowedOrigin = allowedOrigin; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/EndpointConfiguration.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/EndpointConfiguration.java index a349cc666d..f087a3f9b6 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/EndpointConfiguration.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/EndpointConfiguration.java @@ -12,17 +12,16 @@ @EnableConfigurationProperties(JerseyProperties.class) public class EndpointConfiguration { - @Autowired - private JerseyProperties jerseyProperties; + @Autowired private JerseyProperties jerseyProperties; - @Bean - JerseyResourceConfig resourceConfig() { - return new JerseyResourceConfig(jerseyProperties); - } + @Bean + JerseyResourceConfig resourceConfig() { + return new JerseyResourceConfig(jerseyProperties); + } - @Bean - @ConditionalOnMissingBean(SecurityProvider.class) - SecurityProvider authorization() { - return new AllowAllSecurityProvider(); - } + @Bean + @ConditionalOnMissingBean(SecurityProvider.class) + SecurityProvider authorization() { + return new AllowAllSecurityProvider(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ExternalMonitoringClientProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ExternalMonitoringClientProperties.java index 4b50c67e32..43bd2fc5fa 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ExternalMonitoringClientProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ExternalMonitoringClientProperties.java @@ -2,93 +2,92 @@ public class ExternalMonitoringClientProperties { - private int connectionTimeoutMillis = 1000; + private int connectionTimeoutMillis = 1000; - private int socketTimeoutMillis = 2000; + private int socketTimeoutMillis = 2000; - private int maxConnections = 100; + private int maxConnections = 100; - private int maxConnectionsPerRoute = 100; + private int maxConnectionsPerRoute = 100; - private int cacheTtlSeconds = 55; + private int cacheTtlSeconds = 55; - private int cacheSize = 100_000; + private int cacheSize = 100_000; - private int fetchingTimeoutMillis = 5000; - private int fetchingThreads = 30; + private int fetchingTimeoutMillis = 5000; + private int fetchingThreads = 30; - private String externalMonitoringUrl = "http://localhost:18090"; + private String externalMonitoringUrl = "http://localhost:18090"; + public int getConnectionTimeoutMillis() { + return connectionTimeoutMillis; + } - public int getConnectionTimeoutMillis() { - return connectionTimeoutMillis; - } + public void setConnectionTimeoutMillis(int connectionTimeoutMillis) { + this.connectionTimeoutMillis = connectionTimeoutMillis; + } - public void setConnectionTimeoutMillis(int connectionTimeoutMillis) { - this.connectionTimeoutMillis = connectionTimeoutMillis; - } + public int getSocketTimeoutMillis() { + return socketTimeoutMillis; + } - public int getSocketTimeoutMillis() { - return socketTimeoutMillis; - } + public void setSocketTimeoutMillis(int socketTimeoutMillis) { + this.socketTimeoutMillis = socketTimeoutMillis; + } - public void setSocketTimeoutMillis(int socketTimeoutMillis) { - this.socketTimeoutMillis = socketTimeoutMillis; - } + public int getCacheTtlSeconds() { + return cacheTtlSeconds; + } - public int getCacheTtlSeconds() { - return cacheTtlSeconds; - } + public void setCacheTtlSeconds(int cacheTtlSeconds) { + this.cacheTtlSeconds = cacheTtlSeconds; + } - public void setCacheTtlSeconds(int cacheTtlSeconds) { - this.cacheTtlSeconds = cacheTtlSeconds; - } + public int getCacheSize() { + return cacheSize; + } - public int getCacheSize() { - return cacheSize; - } + public void setCacheSize(int cacheSize) { + this.cacheSize = cacheSize; + } - public void setCacheSize(int cacheSize) { - this.cacheSize = cacheSize; - } + public int getMaxConnections() { + return maxConnections; + } - public int getMaxConnections() { - return maxConnections; - } + public void setMaxConnections(int maxConnections) { + this.maxConnections = maxConnections; + } - public void setMaxConnections(int maxConnections) { - this.maxConnections = maxConnections; - } + public int getMaxConnectionsPerRoute() { + return maxConnectionsPerRoute; + } - public int getMaxConnectionsPerRoute() { - return maxConnectionsPerRoute; - } + public void setMaxConnectionsPerRoute(int maxConnectionsPerRoute) { + this.maxConnectionsPerRoute = maxConnectionsPerRoute; + } - public void setMaxConnectionsPerRoute(int maxConnectionsPerRoute) { - this.maxConnectionsPerRoute = maxConnectionsPerRoute; - } + public String getExternalMonitoringUrl() { + return externalMonitoringUrl; + } - public String getExternalMonitoringUrl() { - return externalMonitoringUrl; - } + public void setExternalMonitoringUrl(String externalMonitoringUrl) { + this.externalMonitoringUrl = externalMonitoringUrl; + } - public void setExternalMonitoringUrl(String externalMonitoringUrl) { - this.externalMonitoringUrl = externalMonitoringUrl; - } + public int getFetchingThreads() { + return fetchingThreads; + } - public int getFetchingThreads() { - return fetchingThreads; - } + public void setFetchingThreads(int fetchingThreads) { + this.fetchingThreads = fetchingThreads; + } - public void setFetchingThreads(int fetchingThreads) { - this.fetchingThreads = fetchingThreads; - } + public int getFetchingTimeoutMillis() { + return fetchingTimeoutMillis; + } - public int getFetchingTimeoutMillis() { - return fetchingTimeoutMillis; - } - - public void setFetchingTimeoutMillis(int fetchingTimeoutMillis) { - this.fetchingTimeoutMillis = fetchingTimeoutMillis; - } + public void setFetchingTimeoutMillis(int fetchingTimeoutMillis) { + this.fetchingTimeoutMillis = fetchingTimeoutMillis; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ExternalMonitoringConfiguration.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ExternalMonitoringConfiguration.java index 96149ec69f..714de3a5fb 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ExternalMonitoringConfiguration.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ExternalMonitoringConfiguration.java @@ -1,7 +1,13 @@ package pl.allegro.tech.hermes.management.config; +import static com.google.common.base.Ticker.systemTicker; + import com.google.common.util.concurrent.ThreadFactoryBuilder; import io.micrometer.core.instrument.MeterRegistry; +import java.net.URI; +import java.time.Duration; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import org.apache.hc.client5.http.classic.HttpClient; import org.apache.hc.client5.http.config.RequestConfig; import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; @@ -21,72 +27,72 @@ import pl.allegro.tech.hermes.management.infrastructure.prometheus.PrometheusMetricsProvider; import pl.allegro.tech.hermes.management.infrastructure.prometheus.RestTemplatePrometheusClient; -import java.net.URI; -import java.time.Duration; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import static com.google.common.base.Ticker.systemTicker; - @Configuration @ConditionalOnProperty(value = "prometheus.client.enabled", havingValue = "true") public class ExternalMonitoringConfiguration { - @Bean - public PrometheusMetricsProvider prometheusMetricsProvider(PrometheusClient prometheusClient, - PrometheusMonitoringClientProperties properties) { - return new PrometheusMetricsProvider(prometheusClient, - properties.getConsumersMetricsPrefix(), properties.getFrontendMetricsPrefix(), - properties.getAdditionalFilters()); - } + @Bean + public PrometheusMetricsProvider prometheusMetricsProvider( + PrometheusClient prometheusClient, PrometheusMonitoringClientProperties properties) { + return new PrometheusMetricsProvider( + prometheusClient, + properties.getConsumersMetricsPrefix(), + properties.getFrontendMetricsPrefix(), + properties.getAdditionalFilters()); + } - @Bean - public PrometheusClient prometheusClient(@Qualifier("monitoringRestTemplate") RestTemplate monitoringRestTemplate, - PrometheusMonitoringClientProperties clientProperties, - @Qualifier("prometheusFetcherExecutorService") ExecutorService executorService, - MeterRegistry meterRegistry) { - RestTemplatePrometheusClient underlyingPrometheusClient = - new RestTemplatePrometheusClient( - monitoringRestTemplate, - URI.create(clientProperties.getExternalMonitoringUrl()), - executorService, - Duration.ofMillis(clientProperties.getFetchingTimeoutMillis()), - meterRegistry); - return new CachingPrometheusClient( - underlyingPrometheusClient, - systemTicker(), - clientProperties.getCacheTtlSeconds(), - clientProperties.getCacheSize() - ); - } + @Bean + public PrometheusClient prometheusClient( + @Qualifier("monitoringRestTemplate") RestTemplate monitoringRestTemplate, + PrometheusMonitoringClientProperties clientProperties, + @Qualifier("prometheusFetcherExecutorService") ExecutorService executorService, + MeterRegistry meterRegistry) { + RestTemplatePrometheusClient underlyingPrometheusClient = + new RestTemplatePrometheusClient( + monitoringRestTemplate, + URI.create(clientProperties.getExternalMonitoringUrl()), + executorService, + Duration.ofMillis(clientProperties.getFetchingTimeoutMillis()), + meterRegistry); + return new CachingPrometheusClient( + underlyingPrometheusClient, + systemTicker(), + clientProperties.getCacheTtlSeconds(), + clientProperties.getCacheSize()); + } - @Bean("monitoringRestTemplate") - @ConditionalOnMissingBean(name = "monitoringRestTemplate") - public RestTemplate restTemplate(ExternalMonitoringClientProperties clientProperties) { - PoolingHttpClientConnectionManager connectionManager = PoolingHttpClientConnectionManagerBuilder.create() - .setMaxConnTotal(clientProperties.getMaxConnections()) - .setMaxConnPerRoute(clientProperties.getMaxConnectionsPerRoute()) - .build(); + @Bean("monitoringRestTemplate") + @ConditionalOnMissingBean(name = "monitoringRestTemplate") + public RestTemplate restTemplate(ExternalMonitoringClientProperties clientProperties) { + PoolingHttpClientConnectionManager connectionManager = + PoolingHttpClientConnectionManagerBuilder.create() + .setMaxConnTotal(clientProperties.getMaxConnections()) + .setMaxConnPerRoute(clientProperties.getMaxConnectionsPerRoute()) + .build(); - RequestConfig requestConfig = RequestConfig.custom() - .setConnectTimeout(Timeout.ofMilliseconds(clientProperties.getConnectionTimeoutMillis())) - .setResponseTimeout(Timeout.ofMilliseconds(clientProperties.getSocketTimeoutMillis())) - .build(); + RequestConfig requestConfig = + RequestConfig.custom() + .setConnectTimeout( + Timeout.ofMilliseconds(clientProperties.getConnectionTimeoutMillis())) + .setResponseTimeout(Timeout.ofMilliseconds(clientProperties.getSocketTimeoutMillis())) + .build(); - HttpClient client = HttpClientBuilder.create() - .setDefaultRequestConfig(requestConfig) - .setConnectionManager(connectionManager) - .build(); + HttpClient client = + HttpClientBuilder.create() + .setDefaultRequestConfig(requestConfig) + .setConnectionManager(connectionManager) + .build(); - ClientHttpRequestFactory clientHttpRequestFactory = new HttpComponentsClientHttpRequestFactory(client); - return new RestTemplate(clientHttpRequestFactory); - } + ClientHttpRequestFactory clientHttpRequestFactory = + new HttpComponentsClientHttpRequestFactory(client); + return new RestTemplate(clientHttpRequestFactory); + } - @Bean("prometheusFetcherExecutorService") - @ConditionalOnMissingBean(name = "prometheusFetcherExecutorService") - public ExecutorService executorService(ExternalMonitoringClientProperties clientProperties) { - return Executors.newFixedThreadPool(clientProperties.getFetchingThreads(), - new ThreadFactoryBuilder().setNameFormat("prometheus-metrics-fetcher-%d").build() - ); - } + @Bean("prometheusFetcherExecutorService") + @ConditionalOnMissingBean(name = "prometheusFetcherExecutorService") + public ExecutorService executorService(ExternalMonitoringClientProperties clientProperties) { + return Executors.newFixedThreadPool( + clientProperties.getFetchingThreads(), + new ThreadFactoryBuilder().setNameFormat("prometheus-metrics-fetcher-%d").build()); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/FilteringConfiguration.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/FilteringConfiguration.java index c10411f5a3..cb2ecedd30 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/FilteringConfiguration.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/FilteringConfiguration.java @@ -1,5 +1,9 @@ package pl.allegro.tech.hermes.management.config; +import static java.util.Collections.emptyList; + +import java.util.Arrays; +import java.util.List; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import pl.allegro.tech.hermes.domain.filtering.MessageFilters; @@ -9,22 +13,17 @@ import pl.allegro.tech.hermes.domain.filtering.header.HeaderSubscriptionMessageFilterCompiler; import pl.allegro.tech.hermes.domain.filtering.json.JsonPathSubscriptionMessageFilterCompiler; -import java.util.Arrays; -import java.util.List; - -import static java.util.Collections.emptyList; - @Configuration public class FilteringConfiguration { - @Bean - FilterChainFactory filterChainFactory() { - List subscriptionFilterCompilers = Arrays.asList( - new AvroPathSubscriptionMessageFilterCompiler(), - new JsonPathSubscriptionMessageFilterCompiler(), - new HeaderSubscriptionMessageFilterCompiler() - ); - MessageFilters messageFilters = new MessageFilters(emptyList(), subscriptionFilterCompilers); - return new FilterChainFactory(messageFilters); - } + @Bean + FilterChainFactory filterChainFactory() { + List subscriptionFilterCompilers = + Arrays.asList( + new AvroPathSubscriptionMessageFilterCompiler(), + new JsonPathSubscriptionMessageFilterCompiler(), + new HeaderSubscriptionMessageFilterCompiler()); + MessageFilters messageFilters = new MessageFilters(emptyList(), subscriptionFilterCompilers); + return new FilterChainFactory(messageFilters); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/GroupProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/GroupProperties.java index 3c5dfe0688..ce26f8ecfe 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/GroupProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/GroupProperties.java @@ -7,23 +7,23 @@ @ConfigurationProperties(prefix = "group") public class GroupProperties { - private boolean nonAdminCreationEnabled = false; + private boolean nonAdminCreationEnabled = false; - private String allowedGroupNameRegex = "^[a-zA-Z0-9.]+$"; + private String allowedGroupNameRegex = "^[a-zA-Z0-9.]+$"; - public boolean isNonAdminCreationEnabled() { - return nonAdminCreationEnabled; - } + public boolean isNonAdminCreationEnabled() { + return nonAdminCreationEnabled; + } - public void setNonAdminCreationEnabled(boolean nonAdminCreationEnabled) { - this.nonAdminCreationEnabled = nonAdminCreationEnabled; - } + public void setNonAdminCreationEnabled(boolean nonAdminCreationEnabled) { + this.nonAdminCreationEnabled = nonAdminCreationEnabled; + } - public String getAllowedGroupNameRegex() { - return allowedGroupNameRegex; - } + public String getAllowedGroupNameRegex() { + return allowedGroupNameRegex; + } - public void setAllowedGroupNameRegex(String allowedGroupNameRegex) { - this.allowedGroupNameRegex = allowedGroupNameRegex; - } + public void setAllowedGroupNameRegex(String allowedGroupNameRegex) { + this.allowedGroupNameRegex = allowedGroupNameRegex; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/HttpClientProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/HttpClientProperties.java index 654e749e44..a3432003c9 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/HttpClientProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/HttpClientProperties.java @@ -5,22 +5,22 @@ @ConfigurationProperties(prefix = "http-client") public class HttpClientProperties { - private int readTimeout = 3000; - private int connectTimeout = 3000; + private int readTimeout = 3000; + private int connectTimeout = 3000; - public int getReadTimeout() { - return readTimeout; - } + public int getReadTimeout() { + return readTimeout; + } - public void setReadTimeout(int readTimeout) { - this.readTimeout = readTimeout; - } + public void setReadTimeout(int readTimeout) { + this.readTimeout = readTimeout; + } - public int getConnectTimeout() { - return connectTimeout; - } + public int getConnectTimeout() { + return connectTimeout; + } - public void setConnectTimeout(int connectTimeout) { - this.connectTimeout = connectTimeout; - } + public void setConnectTimeout(int connectTimeout) { + this.connectTimeout = connectTimeout; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/JerseyProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/JerseyProperties.java index 16d285a1e3..7fc4ded509 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/JerseyProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/JerseyProperties.java @@ -1,29 +1,28 @@ package pl.allegro.tech.hermes.management.config; -import org.springframework.boot.context.properties.ConfigurationProperties; - import java.util.ArrayList; import java.util.List; +import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties(prefix = "jersey") public class JerseyProperties { - private List packagesToScan = new ArrayList<>(); - private String filterStaticContentRegexp = "(/status/|/assets/|/favicon.ico).*"; + private List packagesToScan = new ArrayList<>(); + private String filterStaticContentRegexp = "(/status/|/assets/|/favicon.ico).*"; - public List getPackagesToScan() { - return packagesToScan; - } + public List getPackagesToScan() { + return packagesToScan; + } - public void setPackagesToScan(List packagesToScan) { - this.packagesToScan = packagesToScan; - } + public void setPackagesToScan(List packagesToScan) { + this.packagesToScan = packagesToScan; + } - public String getFilterStaticContentRegexp() { - return filterStaticContentRegexp; - } + public String getFilterStaticContentRegexp() { + return filterStaticContentRegexp; + } - public void setFilterStaticContentRegexp(String filterStaticContentRegexp) { - this.filterStaticContentRegexp = filterStaticContentRegexp; - } + public void setFilterStaticContentRegexp(String filterStaticContentRegexp) { + this.filterStaticContentRegexp = filterStaticContentRegexp; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/JerseyResourceConfig.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/JerseyResourceConfig.java index a183f9ed0b..4eff4e2ece 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/JerseyResourceConfig.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/JerseyResourceConfig.java @@ -12,19 +12,21 @@ @ApplicationPath("/") public class JerseyResourceConfig extends ResourceConfig { - private static final Logger logger = LoggerFactory.getLogger(JerseyResourceConfig.class); + private static final Logger logger = LoggerFactory.getLogger(JerseyResourceConfig.class); - public JerseyResourceConfig(JerseyProperties jerseyProperties) { - packages(true, "pl.allegro.tech.hermes.management.api"); + public JerseyResourceConfig(JerseyProperties jerseyProperties) { + packages(true, "pl.allegro.tech.hermes.management.api"); - for (String packageToScan : jerseyProperties.getPackagesToScan()) { - packages(true, packageToScan); - logger.info("Scanning Jersey resources in: {}", packageToScan); - } - property(ServerProperties.BV_SEND_ERROR_IN_RESPONSE, true); - property(ServletProperties.FILTER_STATIC_CONTENT_REGEX, jerseyProperties.getFilterStaticContentRegexp()); - register(RolesAllowedDynamicFeature.class); - property(FreemarkerMvcFeature.TEMPLATE_BASE_PATH, "static"); - register(FreemarkerMvcFeature.class); + for (String packageToScan : jerseyProperties.getPackagesToScan()) { + packages(true, packageToScan); + logger.info("Scanning Jersey resources in: {}", packageToScan); } + property(ServerProperties.BV_SEND_ERROR_IN_RESPONSE, true); + property( + ServletProperties.FILTER_STATIC_CONTENT_REGEX, + jerseyProperties.getFilterStaticContentRegexp()); + register(RolesAllowedDynamicFeature.class); + property(FreemarkerMvcFeature.TEMPLATE_BASE_PATH, "static"); + register(FreemarkerMvcFeature.class); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/LogRepositoryConfiguration.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/LogRepositoryConfiguration.java index 609f919df8..da2b9c195d 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/LogRepositoryConfiguration.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/LogRepositoryConfiguration.java @@ -9,10 +9,9 @@ @Configuration public class LogRepositoryConfiguration { - @Bean - @ConditionalOnMissingBean(LogRepository.class) - public LogRepository logRepository() { - return new NoOperationLogRepository(); - } - + @Bean + @ConditionalOnMissingBean(LogRepository.class) + public LogRepository logRepository() { + return new NoOperationLogRepository(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ManagementConfiguration.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ManagementConfiguration.java index ffaaf0ba99..93139d4803 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ManagementConfiguration.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ManagementConfiguration.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import io.micrometer.core.instrument.MeterRegistry; +import java.time.Clock; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.context.properties.EnableConfigurationProperties; @@ -22,64 +23,66 @@ import pl.allegro.tech.hermes.management.infrastructure.metrics.NoOpSubscriptionLagSource; import pl.allegro.tech.hermes.metrics.PathsCompiler; -import java.time.Clock; - @Configuration @EnableConfigurationProperties({ - TopicProperties.class, - HttpClientProperties.class, - ConsistencyCheckerProperties.class, - PrometheusProperties.class, - MicrometerRegistryProperties.class + TopicProperties.class, + HttpClientProperties.class, + ConsistencyCheckerProperties.class, + PrometheusProperties.class, + MicrometerRegistryProperties.class }) public class ManagementConfiguration { - @Autowired - TopicProperties topicProperties; + @Autowired TopicProperties topicProperties; - @Bean - public ObjectMapper objectMapper() { - ObjectMapper mapper = new ObjectMapper(); - mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); - mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); - mapper.disable(SerializationFeature.WRITE_NULL_MAP_VALUES); - mapper.registerModules(new JavaTimeModule(), new Jdk8Module()); // Jdk8Module is required for Jackson to serialize & deserialize Optional type + @Bean + public ObjectMapper objectMapper() { + ObjectMapper mapper = new ObjectMapper(); + mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + mapper.disable(SerializationFeature.WRITE_NULL_MAP_VALUES); + mapper.registerModules( + new JavaTimeModule(), + new Jdk8Module()); // Jdk8Module is required for Jackson to serialize & deserialize Optional + // type - final InjectableValues defaultSchemaIdAwareSerializationEnabled = new InjectableValues.Std().addValue( + final InjectableValues defaultSchemaIdAwareSerializationEnabled = + new InjectableValues.Std() + .addValue( Topic.DEFAULT_SCHEMA_ID_SERIALIZATION_ENABLED_KEY, topicProperties.isDefaultSchemaIdAwareSerializationEnabled()) - .addValue(Topic.DEFAULT_FALLBACK_TO_REMOTE_DATACENTER_KEY, topicProperties.isDefaultFallbackToRemoteDatacenterEnabled()); - - mapper.setInjectableValues(defaultSchemaIdAwareSerializationEnabled); - - return mapper; - } + .addValue( + Topic.DEFAULT_FALLBACK_TO_REMOTE_DATACENTER_KEY, + topicProperties.isDefaultFallbackToRemoteDatacenterEnabled()); - @Bean - public InstanceIdResolver instanceIdResolver() { - return new InetAddressInstanceIdResolver(); - } + mapper.setInjectableValues(defaultSchemaIdAwareSerializationEnabled); - @Bean - public PathsCompiler pathsCompiler(InstanceIdResolver instanceIdResolver) { - return new PathsCompiler(instanceIdResolver.resolve()); - } + return mapper; + } - @Bean - public MetricsFacade micrometerHermesMetrics(MeterRegistry meterRegistry) { - return new MetricsFacade(meterRegistry); - } + @Bean + public InstanceIdResolver instanceIdResolver() { + return new InetAddressInstanceIdResolver(); + } - @Bean - @ConditionalOnMissingBean - public SubscriptionLagSource consumerLagSource() { - return new NoOpSubscriptionLagSource(); - } + @Bean + public PathsCompiler pathsCompiler(InstanceIdResolver instanceIdResolver) { + return new PathsCompiler(instanceIdResolver.resolve()); + } - @Bean - public Clock clock() { - return new ClockFactory().provide(); - } + @Bean + public MetricsFacade micrometerHermesMetrics(MeterRegistry meterRegistry) { + return new MetricsFacade(meterRegistry); + } + @Bean + @ConditionalOnMissingBean + public SubscriptionLagSource consumerLagSource() { + return new NoOpSubscriptionLagSource(); + } + @Bean + public Clock clock() { + return new ClockFactory().provide(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/MessageConfiguration.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/MessageConfiguration.java index cb4ae5dfa3..1f71bc9b8e 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/MessageConfiguration.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/MessageConfiguration.java @@ -1,6 +1,7 @@ package pl.allegro.tech.hermes.management.config; import com.fasterxml.jackson.databind.ObjectMapper; +import java.time.Clock; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -14,47 +15,56 @@ import pl.allegro.tech.hermes.common.metric.MetricsFacade; import pl.allegro.tech.hermes.schema.SchemaRepository; -import java.time.Clock; - @Configuration @EnableConfigurationProperties(MessageProperties.class) public class MessageConfiguration { - @Bean - CompositeMessageContentWrapper messageContentWrapper( - MessageProperties messageProperties, - Clock clock, - ObjectMapper objectMapper, - SchemaRepository schemaRepository, - MetricsFacade metricsFacade) { - AvroMessageContentWrapper avroWrapper = new AvroMessageContentWrapper(clock); - JsonMessageContentWrapper jsonWrapper = jsonMessageContentWrapper(messageProperties, objectMapper); - - AvroMessageHeaderSchemaVersionContentWrapper headerSchemaVersionWrapper = - new AvroMessageHeaderSchemaVersionContentWrapper(schemaRepository, avroWrapper, metricsFacade); - - AvroMessageHeaderSchemaIdContentWrapper headerSchemaIdWrapper = - new AvroMessageHeaderSchemaIdContentWrapper(schemaRepository, avroWrapper, metricsFacade, - messageProperties.isSchemaIdHeaderEnabled()); - - AvroMessageSchemaIdAwareContentWrapper schemaAwareWrapper = - new AvroMessageSchemaIdAwareContentWrapper(schemaRepository, avroWrapper, metricsFacade); - - AvroMessageSchemaVersionTruncationContentWrapper schemaVersionTruncationContentWrapper = - new AvroMessageSchemaVersionTruncationContentWrapper(schemaRepository, avroWrapper, metricsFacade, - messageProperties.isSchemaVersionTruncationEnabled()); - - return new CompositeMessageContentWrapper( - jsonWrapper, - avroWrapper, - schemaAwareWrapper, - headerSchemaVersionWrapper, - headerSchemaIdWrapper, - schemaVersionTruncationContentWrapper); - } - - private JsonMessageContentWrapper jsonMessageContentWrapper(MessageProperties messageProperties, ObjectMapper objectMapper) { - return new JsonMessageContentWrapper(messageProperties.getContentRoot(), messageProperties.getMetadataContentRoot(), objectMapper); - } + @Bean + CompositeMessageContentWrapper messageContentWrapper( + MessageProperties messageProperties, + Clock clock, + ObjectMapper objectMapper, + SchemaRepository schemaRepository, + MetricsFacade metricsFacade) { + AvroMessageContentWrapper avroWrapper = new AvroMessageContentWrapper(clock); + JsonMessageContentWrapper jsonWrapper = + jsonMessageContentWrapper(messageProperties, objectMapper); + + AvroMessageHeaderSchemaVersionContentWrapper headerSchemaVersionWrapper = + new AvroMessageHeaderSchemaVersionContentWrapper( + schemaRepository, avroWrapper, metricsFacade); + + AvroMessageHeaderSchemaIdContentWrapper headerSchemaIdWrapper = + new AvroMessageHeaderSchemaIdContentWrapper( + schemaRepository, + avroWrapper, + metricsFacade, + messageProperties.isSchemaIdHeaderEnabled()); + + AvroMessageSchemaIdAwareContentWrapper schemaAwareWrapper = + new AvroMessageSchemaIdAwareContentWrapper(schemaRepository, avroWrapper, metricsFacade); + + AvroMessageSchemaVersionTruncationContentWrapper schemaVersionTruncationContentWrapper = + new AvroMessageSchemaVersionTruncationContentWrapper( + schemaRepository, + avroWrapper, + metricsFacade, + messageProperties.isSchemaVersionTruncationEnabled()); + + return new CompositeMessageContentWrapper( + jsonWrapper, + avroWrapper, + schemaAwareWrapper, + headerSchemaVersionWrapper, + headerSchemaIdWrapper, + schemaVersionTruncationContentWrapper); + } + private JsonMessageContentWrapper jsonMessageContentWrapper( + MessageProperties messageProperties, ObjectMapper objectMapper) { + return new JsonMessageContentWrapper( + messageProperties.getContentRoot(), + messageProperties.getMetadataContentRoot(), + objectMapper); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/MessageProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/MessageProperties.java index cedb11437c..a7c127c199 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/MessageProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/MessageProperties.java @@ -5,43 +5,43 @@ @ConfigurationProperties(prefix = "message") public class MessageProperties { - private String contentRoot = "message"; + private String contentRoot = "message"; - private String metadataContentRoot = "metadata"; + private String metadataContentRoot = "metadata"; - private boolean schemaIdHeaderEnabled = false; + private boolean schemaIdHeaderEnabled = false; - private boolean schemaVersionTruncationEnabled = false; + private boolean schemaVersionTruncationEnabled = false; - public String getContentRoot() { - return contentRoot; - } + public String getContentRoot() { + return contentRoot; + } - public void setContentRoot(String contentRoot) { - this.contentRoot = contentRoot; - } + public void setContentRoot(String contentRoot) { + this.contentRoot = contentRoot; + } - public String getMetadataContentRoot() { - return metadataContentRoot; - } + public String getMetadataContentRoot() { + return metadataContentRoot; + } - public void setMetadataContentRoot(String metadataContentRoot) { - this.metadataContentRoot = metadataContentRoot; - } + public void setMetadataContentRoot(String metadataContentRoot) { + this.metadataContentRoot = metadataContentRoot; + } - public boolean isSchemaIdHeaderEnabled() { - return schemaIdHeaderEnabled; - } + public boolean isSchemaIdHeaderEnabled() { + return schemaIdHeaderEnabled; + } - public void setSchemaIdHeaderEnabled(boolean schemaIdHeaderEnabled) { - this.schemaIdHeaderEnabled = schemaIdHeaderEnabled; - } + public void setSchemaIdHeaderEnabled(boolean schemaIdHeaderEnabled) { + this.schemaIdHeaderEnabled = schemaIdHeaderEnabled; + } - public boolean isSchemaVersionTruncationEnabled() { - return schemaVersionTruncationEnabled; - } + public boolean isSchemaVersionTruncationEnabled() { + return schemaVersionTruncationEnabled; + } - public void setSchemaVersionTruncationEnabled(boolean schemaVersionTruncationEnabled) { - this.schemaVersionTruncationEnabled = schemaVersionTruncationEnabled; - } + public void setSchemaVersionTruncationEnabled(boolean schemaVersionTruncationEnabled) { + this.schemaVersionTruncationEnabled = schemaVersionTruncationEnabled; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/MicrometerRegistryProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/MicrometerRegistryProperties.java index 9f131b66c8..65b012a02a 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/MicrometerRegistryProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/MicrometerRegistryProperties.java @@ -1,19 +1,18 @@ package pl.allegro.tech.hermes.management.config; -import org.springframework.boot.context.properties.ConfigurationProperties; - import java.util.List; +import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties(prefix = "metrics.micrometer") public class MicrometerRegistryProperties { - private List percentiles = List.of(0.5, 0.99, 0.999); + private List percentiles = List.of(0.5, 0.99, 0.999); - public List getPercentiles() { - return percentiles; - } + public List getPercentiles() { + return percentiles; + } - public void setPercentiles(List percentiles) { - this.percentiles = percentiles; - } + public void setPercentiles(List percentiles) { + this.percentiles = percentiles; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/MonitoringClientPropertiesConfiguration.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/MonitoringClientPropertiesConfiguration.java index ed4b733b0d..8f2018ca86 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/MonitoringClientPropertiesConfiguration.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/MonitoringClientPropertiesConfiguration.java @@ -12,10 +12,10 @@ @Configuration public class MonitoringClientPropertiesConfiguration { - @Bean - @ConfigurationProperties("prometheus.client") - @ConditionalOnProperty(value = "prometheus.client.enabled", havingValue = "true") - public PrometheusMonitoringClientProperties prometheusMonitoringClientProperties() { - return new PrometheusMonitoringClientProperties(); - } + @Bean + @ConfigurationProperties("prometheus.client") + @ConditionalOnProperty(value = "prometheus.client.enabled", havingValue = "true") + public PrometheusMonitoringClientProperties prometheusMonitoringClientProperties() { + return new PrometheusMonitoringClientProperties(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/OfflineRetransmissionConfiguration.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/OfflineRetransmissionConfiguration.java index faaf98e25a..f7c16ac3f7 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/OfflineRetransmissionConfiguration.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/OfflineRetransmissionConfiguration.java @@ -11,18 +11,21 @@ @Configuration public class OfflineRetransmissionConfiguration { - @Bean - @Qualifier("dcAwareOfflineRetransmissionRepository") - OfflineRetransmissionRepository dcAwareOfflineRetransmissionRepository( - MultiDatacenterRepositoryCommandExecutor commandExecutor, - @Qualifier("zookeeperOfflineRetransmissionRepository") OfflineRetransmissionRepository offlineRetransmissionRepository) { - return new DcAwareOfflineRetransmissionRepository(commandExecutor, offlineRetransmissionRepository); - } + @Bean + @Qualifier("dcAwareOfflineRetransmissionRepository") + OfflineRetransmissionRepository dcAwareOfflineRetransmissionRepository( + MultiDatacenterRepositoryCommandExecutor commandExecutor, + @Qualifier("zookeeperOfflineRetransmissionRepository") + OfflineRetransmissionRepository offlineRetransmissionRepository) { + return new DcAwareOfflineRetransmissionRepository( + commandExecutor, offlineRetransmissionRepository); + } - @Bean - OfflineRetransmissionService offlineRetransmissionService( - @Qualifier("dcAwareOfflineRetransmissionRepository") OfflineRetransmissionRepository taskRepository, - TopicRepository topicRepository) { - return new OfflineRetransmissionService(taskRepository, topicRepository); - } + @Bean + OfflineRetransmissionService offlineRetransmissionService( + @Qualifier("dcAwareOfflineRetransmissionRepository") + OfflineRetransmissionRepository taskRepository, + TopicRepository topicRepository) { + return new OfflineRetransmissionService(taskRepository, topicRepository); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/OwnerConfiguration.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/OwnerConfiguration.java index 96f67f78cd..b7bde4ffad 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/OwnerConfiguration.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/OwnerConfiguration.java @@ -1,20 +1,18 @@ package pl.allegro.tech.hermes.management.config; +import java.util.List; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import pl.allegro.tech.hermes.management.domain.owner.OwnerSource; import pl.allegro.tech.hermes.management.domain.owner.OwnerSources; -import java.util.List; - @Configuration public class OwnerConfiguration { - @Bean - @ConditionalOnMissingBean - public OwnerSources ownerSources(List ownerSources) { - return new OwnerSources(ownerSources); - } - + @Bean + @ConditionalOnMissingBean + public OwnerSources ownerSources(List ownerSources) { + return new OwnerSources(ownerSources); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/PrometheusConfigAdapter.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/PrometheusConfigAdapter.java index b7bc154b40..4d4eba794d 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/PrometheusConfigAdapter.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/PrometheusConfigAdapter.java @@ -1,32 +1,29 @@ package pl.allegro.tech.hermes.management.config; import io.micrometer.prometheus.PrometheusConfig; - import java.time.Duration; public class PrometheusConfigAdapter implements PrometheusConfig { - private final PrometheusProperties prometheusReporterProperties; + private final PrometheusProperties prometheusReporterProperties; - public PrometheusConfigAdapter(PrometheusProperties prometheusReporterProperties) { - this.prometheusReporterProperties = prometheusReporterProperties; - } + public PrometheusConfigAdapter(PrometheusProperties prometheusReporterProperties) { + this.prometheusReporterProperties = prometheusReporterProperties; + } - @Override - public boolean descriptions() { - return prometheusReporterProperties.getDescriptions(); - } + @Override + public boolean descriptions() { + return prometheusReporterProperties.getDescriptions(); + } - @Override - public Duration step() { - return prometheusReporterProperties.getStep(); - } + @Override + public Duration step() { + return prometheusReporterProperties.getStep(); + } - /** - * Returning null is fine since we override all the methods from PrometheusConfig. - */ - @Override - public String get(String key) { - return null; // Nothing to see here, move along. - } + /** Returning null is fine since we override all the methods from PrometheusConfig. */ + @Override + public String get(String key) { + return null; // Nothing to see here, move along. + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/PrometheusConfiguration.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/PrometheusConfiguration.java index c5d9ed53ff..38eca0a4f6 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/PrometheusConfiguration.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/PrometheusConfiguration.java @@ -14,53 +14,60 @@ @EnableConfigurationProperties(MicrometerRegistryProperties.class) public class PrometheusConfiguration { - @Bean - @ConditionalOnMissingBean - public PrometheusMeterRegistry micrometerRegistry(MicrometerRegistryProperties properties, - PrometheusConfig prometheusConfig) { - return new PrometheusMeterRegistryFactory(properties, - prometheusConfig, "hermes-management").provide(); - } + @Bean + @ConditionalOnMissingBean + public PrometheusMeterRegistry micrometerRegistry( + MicrometerRegistryProperties properties, PrometheusConfig prometheusConfig) { + return new PrometheusMeterRegistryFactory(properties, prometheusConfig, "hermes-management") + .provide(); + } - @Bean - @ConditionalOnMissingBean - public PrometheusConfig prometheusConfig(PrometheusProperties properties) { - return new PrometheusConfigAdapter(properties); - } + @Bean + @ConditionalOnMissingBean + public PrometheusConfig prometheusConfig(PrometheusProperties properties) { + return new PrometheusConfigAdapter(properties); + } - public static class PrometheusMeterRegistryFactory { - private final MicrometerRegistryProperties parameters; - private final PrometheusConfig prometheusConfig; - private final String prefix; + public static class PrometheusMeterRegistryFactory { + private final MicrometerRegistryProperties parameters; + private final PrometheusConfig prometheusConfig; + private final String prefix; - public PrometheusMeterRegistryFactory(MicrometerRegistryProperties properties, - PrometheusConfig prometheusConfig, - String prefix) { - this.parameters = properties; - this.prometheusConfig = prometheusConfig; - this.prefix = prefix + "_"; - } + public PrometheusMeterRegistryFactory( + MicrometerRegistryProperties properties, PrometheusConfig prometheusConfig, String prefix) { + this.parameters = properties; + this.prometheusConfig = prometheusConfig; + this.prefix = prefix + "_"; + } - public PrometheusMeterRegistry provide() { - PrometheusMeterRegistry meterRegistry = new PrometheusMeterRegistry(prometheusConfig); - applyFilters(meterRegistry); - return meterRegistry; - } + public PrometheusMeterRegistry provide() { + PrometheusMeterRegistry meterRegistry = new PrometheusMeterRegistry(prometheusConfig); + applyFilters(meterRegistry); + return meterRegistry; + } - private void applyFilters(PrometheusMeterRegistry meterRegistry) { - meterRegistry.config().meterFilter(new MeterFilter() { + private void applyFilters(PrometheusMeterRegistry meterRegistry) { + meterRegistry + .config() + .meterFilter( + new MeterFilter() { @Override public Meter.Id map(Meter.Id id) { - return id.withName(prefix + id.getName()); + return id.withName(prefix + id.getName()); } @Override - public DistributionStatisticConfig configure(Meter.Id id, DistributionStatisticConfig config) { - return DistributionStatisticConfig.builder() - .percentiles(parameters.getPercentiles().stream().mapToDouble(Double::doubleValue).toArray() - ).build().merge(config); + public DistributionStatisticConfig configure( + Meter.Id id, DistributionStatisticConfig config) { + return DistributionStatisticConfig.builder() + .percentiles( + parameters.getPercentiles().stream() + .mapToDouble(Double::doubleValue) + .toArray()) + .build() + .merge(config); } - }); - } + }); } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/PrometheusMonitoringClientProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/PrometheusMonitoringClientProperties.java index cdf4ce2cdb..1e6ba49dfd 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/PrometheusMonitoringClientProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/PrometheusMonitoringClientProperties.java @@ -1,31 +1,31 @@ package pl.allegro.tech.hermes.management.config; public class PrometheusMonitoringClientProperties extends ExternalMonitoringClientProperties { - private String consumersMetricsPrefix = "hermes_consumers"; - private String frontendMetricsPrefix = "hermes_frontend"; - private String additionalFilters = ""; + private String consumersMetricsPrefix = "hermes_consumers"; + private String frontendMetricsPrefix = "hermes_frontend"; + private String additionalFilters = ""; - public String getConsumersMetricsPrefix() { - return consumersMetricsPrefix; - } + public String getConsumersMetricsPrefix() { + return consumersMetricsPrefix; + } - public void setConsumersMetricsPrefix(String consumersMetricsPrefix) { - this.consumersMetricsPrefix = consumersMetricsPrefix; - } + public void setConsumersMetricsPrefix(String consumersMetricsPrefix) { + this.consumersMetricsPrefix = consumersMetricsPrefix; + } - public String getFrontendMetricsPrefix() { - return frontendMetricsPrefix; - } + public String getFrontendMetricsPrefix() { + return frontendMetricsPrefix; + } - public void setFrontendMetricsPrefix(String frontendMetricsPrefix) { - this.frontendMetricsPrefix = frontendMetricsPrefix; - } + public void setFrontendMetricsPrefix(String frontendMetricsPrefix) { + this.frontendMetricsPrefix = frontendMetricsPrefix; + } - public String getAdditionalFilters() { - return additionalFilters; - } + public String getAdditionalFilters() { + return additionalFilters; + } - public void setAdditionalFilters(String additionalFilters) { - this.additionalFilters = additionalFilters; - } + public void setAdditionalFilters(String additionalFilters) { + this.additionalFilters = additionalFilters; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/PrometheusProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/PrometheusProperties.java index 1dcebc1eaf..c7da900f77 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/PrometheusProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/PrometheusProperties.java @@ -1,28 +1,27 @@ package pl.allegro.tech.hermes.management.config; -import org.springframework.boot.context.properties.ConfigurationProperties; - import java.time.Duration; +import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties(prefix = "metrics.prometheus") public class PrometheusProperties { - private Duration step = Duration.ofMinutes(1); - private boolean descriptions = true; + private Duration step = Duration.ofMinutes(1); + private boolean descriptions = true; - public Duration getStep() { - return this.step; - } + public Duration getStep() { + return this.step; + } - public void setStep(Duration step) { - this.step = step; - } + public void setStep(Duration step) { + this.step = step; + } - public boolean getDescriptions() { - return this.descriptions; - } + public boolean getDescriptions() { + return this.descriptions; + } - public void setDescriptions(boolean descriptions) { - this.descriptions = descriptions; - } + public void setDescriptions(boolean descriptions) { + this.descriptions = descriptions; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ReadinessConfiguration.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ReadinessConfiguration.java index 9a5a9d1984..d6906b3532 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ReadinessConfiguration.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/ReadinessConfiguration.java @@ -1,5 +1,6 @@ package pl.allegro.tech.hermes.management.config; +import java.util.List; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import pl.allegro.tech.hermes.management.config.storage.StorageClustersProperties; @@ -8,18 +9,18 @@ import pl.allegro.tech.hermes.management.domain.readiness.DatacenterReadinessRepository; import pl.allegro.tech.hermes.management.domain.readiness.ReadinessService; -import java.util.List; - @Configuration public class ReadinessConfiguration { - @Bean - ReadinessService readinessService(MultiDatacenterRepositoryCommandExecutor commandExecutor, - DatacenterReadinessRepository readinessRepository, - StorageClustersProperties storageClustersProperties) { - List datacenters = storageClustersProperties.getClusters().stream() - .map(StorageProperties::getDatacenter) - .toList(); - return new ReadinessService(commandExecutor, readinessRepository, datacenters); - } + @Bean + ReadinessService readinessService( + MultiDatacenterRepositoryCommandExecutor commandExecutor, + DatacenterReadinessRepository readinessRepository, + StorageClustersProperties storageClustersProperties) { + List datacenters = + storageClustersProperties.getClusters().stream() + .map(StorageProperties::getDatacenter) + .toList(); + return new ReadinessService(commandExecutor, readinessRepository, datacenters); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SchemaCacheProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SchemaCacheProperties.java index d54a05483c..9dcf228d77 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SchemaCacheProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SchemaCacheProperties.java @@ -5,31 +5,31 @@ @ConfigurationProperties(prefix = "schema.cache") public class SchemaCacheProperties { - private int poolSize = 2; - private int refreshAfterWriteMinutes = 10; - private int expireAfterWriteMinutes = 60 * 24; + private int poolSize = 2; + private int refreshAfterWriteMinutes = 10; + private int expireAfterWriteMinutes = 60 * 24; - public int getPoolSize() { - return poolSize; - } + public int getPoolSize() { + return poolSize; + } - public void setPoolSize(int poolSize) { - this.poolSize = poolSize; - } + public void setPoolSize(int poolSize) { + this.poolSize = poolSize; + } - public int getRefreshAfterWriteMinutes() { - return refreshAfterWriteMinutes; - } + public int getRefreshAfterWriteMinutes() { + return refreshAfterWriteMinutes; + } - public void setRefreshAfterWriteMinutes(int refreshAfterWriteMinutes) { - this.refreshAfterWriteMinutes = refreshAfterWriteMinutes; - } + public void setRefreshAfterWriteMinutes(int refreshAfterWriteMinutes) { + this.refreshAfterWriteMinutes = refreshAfterWriteMinutes; + } - public int getExpireAfterWriteMinutes() { - return expireAfterWriteMinutes; - } + public int getExpireAfterWriteMinutes() { + return expireAfterWriteMinutes; + } - public void setExpireAfterWriteMinutes(int expireAfterWriteMinutes) { - this.expireAfterWriteMinutes = expireAfterWriteMinutes; - } + public void setExpireAfterWriteMinutes(int expireAfterWriteMinutes) { + this.expireAfterWriteMinutes = expireAfterWriteMinutes; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SchemaRepositoryConfiguration.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SchemaRepositoryConfiguration.java index ccbd520dc1..601972e1af 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SchemaRepositoryConfiguration.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SchemaRepositoryConfiguration.java @@ -1,9 +1,12 @@ package pl.allegro.tech.hermes.management.config; +import static pl.allegro.tech.hermes.schema.SubjectNamingStrategy.qualifiedName; + import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.jakarta.rs.json.JacksonJsonProvider; import jakarta.ws.rs.client.Client; import jakarta.ws.rs.client.ClientBuilder; +import java.net.URI; import org.apache.avro.Schema; import org.glassfish.jersey.client.ClientConfig; import org.glassfish.jersey.client.ClientProperties; @@ -28,66 +31,70 @@ import pl.allegro.tech.hermes.schema.resolver.DefaultSchemaRepositoryInstanceResolver; import pl.allegro.tech.hermes.schema.resolver.SchemaRepositoryInstanceResolver; -import java.net.URI; - -import static pl.allegro.tech.hermes.schema.SubjectNamingStrategy.qualifiedName; - @Configuration @EnableConfigurationProperties({SchemaRepositoryProperties.class}) public class SchemaRepositoryConfiguration { - @Autowired - @Lazy - TopicService topicService; - - @Autowired - private SchemaRepositoryProperties schemaRepositoryProperties; + @Autowired @Lazy TopicService topicService; - @Bean(name = "schemaRepositoryClient") - public Client schemaRepositoryClient(ObjectMapper mapper) { - ClientConfig config = new ClientConfig() - .property(ClientProperties.CONNECT_TIMEOUT, schemaRepositoryProperties.getConnectionTimeoutMillis()) - .property(ClientProperties.READ_TIMEOUT, schemaRepositoryProperties.getSocketTimeoutMillis()) - .register(new JacksonJsonProvider(mapper)); + @Autowired private SchemaRepositoryProperties schemaRepositoryProperties; - return ClientBuilder.newClient(config); - } + @Bean(name = "schemaRepositoryClient") + public Client schemaRepositoryClient(ObjectMapper mapper) { + ClientConfig config = + new ClientConfig() + .property( + ClientProperties.CONNECT_TIMEOUT, + schemaRepositoryProperties.getConnectionTimeoutMillis()) + .property( + ClientProperties.READ_TIMEOUT, schemaRepositoryProperties.getSocketTimeoutMillis()) + .register(new JacksonJsonProvider(mapper)); - @Bean - public SubjectNamingStrategy subjectNamingStrategy(KafkaClustersProperties kafkaClustersProperties) { - return qualifiedName - .withNamespacePrefixIf( - schemaRepositoryProperties.isSubjectNamespaceEnabled(), - new SubjectNamingStrategy.Namespace(kafkaClustersProperties.getDefaultNamespace(), - kafkaClustersProperties.getNamespaceSeparator())) - .withValueSuffixIf(schemaRepositoryProperties.isSubjectSuffixEnabled()); - } + return ClientBuilder.newClient(config); + } - @Bean - @ConditionalOnMissingBean(RawSchemaClient.class) - public RawSchemaClient schemaRegistryRawSchemaClient( - SchemaRepositoryInstanceResolver schemaRepositoryInstanceResolver, - ObjectMapper objectMapper, - SubjectNamingStrategy subjectNamingStrategy - ) { - return new SchemaRegistryRawSchemaClient(schemaRepositoryInstanceResolver, objectMapper, - schemaRepositoryProperties.isValidationEnabled(), schemaRepositoryProperties.getDeleteSchemaPathSuffix(), - subjectNamingStrategy); - } + @Bean + public SubjectNamingStrategy subjectNamingStrategy( + KafkaClustersProperties kafkaClustersProperties) { + return qualifiedName + .withNamespacePrefixIf( + schemaRepositoryProperties.isSubjectNamespaceEnabled(), + new SubjectNamingStrategy.Namespace( + kafkaClustersProperties.getDefaultNamespace(), + kafkaClustersProperties.getNamespaceSeparator())) + .withValueSuffixIf(schemaRepositoryProperties.isSubjectSuffixEnabled()); + } - @Bean - @ConditionalOnMissingBean(SchemaRepositoryInstanceResolver.class) - public SchemaRepositoryInstanceResolver defaultSchemaRepositoryInstanceResolver(@Qualifier("schemaRepositoryClient") Client client) { - return new DefaultSchemaRepositoryInstanceResolver(client, URI.create(schemaRepositoryProperties.getServerUrl())); - } + @Bean + @ConditionalOnMissingBean(RawSchemaClient.class) + public RawSchemaClient schemaRegistryRawSchemaClient( + SchemaRepositoryInstanceResolver schemaRepositoryInstanceResolver, + ObjectMapper objectMapper, + SubjectNamingStrategy subjectNamingStrategy) { + return new SchemaRegistryRawSchemaClient( + schemaRepositoryInstanceResolver, + objectMapper, + schemaRepositoryProperties.isValidationEnabled(), + schemaRepositoryProperties.getDeleteSchemaPathSuffix(), + subjectNamingStrategy); + } - @Bean - public SchemaRepository aggregateSchemaRepository(RawSchemaClient rawSchemaClient) { - SchemaVersionsRepository versionsRepository = new DirectSchemaVersionsRepository(rawSchemaClient); - CompiledSchemaRepository avroSchemaRepository = new DirectCompiledSchemaRepository<>( - rawSchemaClient, SchemaCompilersFactory.avroSchemaCompiler()); + @Bean + @ConditionalOnMissingBean(SchemaRepositoryInstanceResolver.class) + public SchemaRepositoryInstanceResolver defaultSchemaRepositoryInstanceResolver( + @Qualifier("schemaRepositoryClient") Client client) { + return new DefaultSchemaRepositoryInstanceResolver( + client, URI.create(schemaRepositoryProperties.getServerUrl())); + } - return new SchemaRepository(versionsRepository, avroSchemaRepository); - } + @Bean + public SchemaRepository aggregateSchemaRepository(RawSchemaClient rawSchemaClient) { + SchemaVersionsRepository versionsRepository = + new DirectSchemaVersionsRepository(rawSchemaClient); + CompiledSchemaRepository avroSchemaRepository = + new DirectCompiledSchemaRepository<>( + rawSchemaClient, SchemaCompilersFactory.avroSchemaCompiler()); + return new SchemaRepository(versionsRepository, avroSchemaRepository); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SchemaRepositoryProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SchemaRepositoryProperties.java index 085cfac8c9..4df89b9b92 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SchemaRepositoryProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SchemaRepositoryProperties.java @@ -5,72 +5,72 @@ @ConfigurationProperties(prefix = "schema.repository") public class SchemaRepositoryProperties { - private String serverUrl = "http://localhost:8888/"; + private String serverUrl = "http://localhost:8888/"; - private boolean validationEnabled = false; + private boolean validationEnabled = false; - private int connectionTimeoutMillis = 1000; + private int connectionTimeoutMillis = 1000; - private int socketTimeoutMillis = 3000; - private String deleteSchemaPathSuffix = "versions"; + private int socketTimeoutMillis = 3000; + private String deleteSchemaPathSuffix = "versions"; - private boolean subjectSuffixEnabled = false; + private boolean subjectSuffixEnabled = false; - private boolean subjectNamespaceEnabled = false; + private boolean subjectNamespaceEnabled = false; - public String getServerUrl() { - return serverUrl; - } + public String getServerUrl() { + return serverUrl; + } - public void setServerUrl(String serverUrl) { - this.serverUrl = serverUrl; - } + public void setServerUrl(String serverUrl) { + this.serverUrl = serverUrl; + } - public boolean isValidationEnabled() { - return validationEnabled; - } + public boolean isValidationEnabled() { + return validationEnabled; + } - public void setValidationEnabled(boolean validationEnabled) { - this.validationEnabled = validationEnabled; - } + public void setValidationEnabled(boolean validationEnabled) { + this.validationEnabled = validationEnabled; + } - public int getConnectionTimeoutMillis() { - return connectionTimeoutMillis; - } + public int getConnectionTimeoutMillis() { + return connectionTimeoutMillis; + } - public void setConnectionTimeoutMillis(int connectionTimeoutMillis) { - this.connectionTimeoutMillis = connectionTimeoutMillis; - } + public void setConnectionTimeoutMillis(int connectionTimeoutMillis) { + this.connectionTimeoutMillis = connectionTimeoutMillis; + } - public int getSocketTimeoutMillis() { - return socketTimeoutMillis; - } + public int getSocketTimeoutMillis() { + return socketTimeoutMillis; + } - public String getDeleteSchemaPathSuffix() { - return deleteSchemaPathSuffix; - } + public String getDeleteSchemaPathSuffix() { + return deleteSchemaPathSuffix; + } - public void setDeleteSchemaPathSuffix(String deleteSchemaPathSuffix) { - this.deleteSchemaPathSuffix = deleteSchemaPathSuffix; - } + public void setDeleteSchemaPathSuffix(String deleteSchemaPathSuffix) { + this.deleteSchemaPathSuffix = deleteSchemaPathSuffix; + } - public void setSocketTimeoutMillis(int socketTimeoutMillis) { - this.socketTimeoutMillis = socketTimeoutMillis; - } + public void setSocketTimeoutMillis(int socketTimeoutMillis) { + this.socketTimeoutMillis = socketTimeoutMillis; + } - public boolean isSubjectSuffixEnabled() { - return subjectSuffixEnabled; - } + public boolean isSubjectSuffixEnabled() { + return subjectSuffixEnabled; + } - public void setSubjectSuffixEnabled(boolean subjectSuffixEnabled) { - this.subjectSuffixEnabled = subjectSuffixEnabled; - } + public void setSubjectSuffixEnabled(boolean subjectSuffixEnabled) { + this.subjectSuffixEnabled = subjectSuffixEnabled; + } - public boolean isSubjectNamespaceEnabled() { - return subjectNamespaceEnabled; - } + public boolean isSubjectNamespaceEnabled() { + return subjectNamespaceEnabled; + } - public void setSubjectNamespaceEnabled(boolean subjectNamespaceEnabled) { - this.subjectNamespaceEnabled = subjectNamespaceEnabled; - } + public void setSubjectNamespaceEnabled(boolean subjectNamespaceEnabled) { + this.subjectNamespaceEnabled = subjectNamespaceEnabled; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SubscriptionConfiguration.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SubscriptionConfiguration.java index 9ea4e1b0fb..c90f380386 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SubscriptionConfiguration.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SubscriptionConfiguration.java @@ -1,5 +1,8 @@ package pl.allegro.tech.hermes.management.config; +import static java.util.stream.Collectors.toList; + +import java.util.List; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -14,52 +17,50 @@ import pl.allegro.tech.hermes.management.domain.subscription.validator.SubscriptionValidator; import pl.allegro.tech.hermes.management.domain.topic.TopicService; -import java.util.List; - -import static java.util.stream.Collectors.toList; - @Configuration @EnableConfigurationProperties(SubscriptionProperties.class) public class SubscriptionConfiguration { - @Bean - public SubscriptionValidator subscriptionValidator(OwnerIdValidator ownerIdValidator, - ApiPreconditions apiPreconditions, - TopicService topicService, - SubscriptionRepository subscriptionRepository, - List endpointAddressValidators, - EndpointOwnershipValidator endpointOwnershipValidator, - SubscriptionProperties subscriptionProperties) { - return new SubscriptionValidator( - ownerIdValidator, - apiPreconditions, - topicService, - subscriptionRepository, - endpointAddressValidators, - endpointOwnershipValidator, - createListOfSubscribersWithAccessToAnyTopic(subscriptionProperties) - ); - } + @Bean + public SubscriptionValidator subscriptionValidator( + OwnerIdValidator ownerIdValidator, + ApiPreconditions apiPreconditions, + TopicService topicService, + SubscriptionRepository subscriptionRepository, + List endpointAddressValidators, + EndpointOwnershipValidator endpointOwnershipValidator, + SubscriptionProperties subscriptionProperties) { + return new SubscriptionValidator( + ownerIdValidator, + apiPreconditions, + topicService, + subscriptionRepository, + endpointAddressValidators, + endpointOwnershipValidator, + createListOfSubscribersWithAccessToAnyTopic(subscriptionProperties)); + } - private List createListOfSubscribersWithAccessToAnyTopic( - SubscriptionProperties subscriptionProperties - ) { - return subscriptionProperties.getSubscribersWithAccessToAnyTopic().stream() - .map(subscriber -> new SubscriberWithAccessToAnyTopic( - subscriber.getOwnerSource(), - subscriber.getOwnerId(), - subscriber.getProtocols()) - ) - .collect(toList()); - } + private List createListOfSubscribersWithAccessToAnyTopic( + SubscriptionProperties subscriptionProperties) { + return subscriptionProperties.getSubscribersWithAccessToAnyTopic().stream() + .map( + subscriber -> + new SubscriberWithAccessToAnyTopic( + subscriber.getOwnerSource(), + subscriber.getOwnerId(), + subscriber.getProtocols())) + .collect(toList()); + } - @Bean - public EndpointOwnershipValidator defaultEndpointOwnershipValidator() { - return new NoOpEndpointOwnershipValidator(); - } + @Bean + public EndpointOwnershipValidator defaultEndpointOwnershipValidator() { + return new NoOpEndpointOwnershipValidator(); + } - @Bean - public EndpointAddressValidator endpointAddressFormatValidator(SubscriptionProperties subscriptionProperties) { - return new EndpointAddressFormatValidator(subscriptionProperties.getAdditionalEndpointProtocols()); - } + @Bean + public EndpointAddressValidator endpointAddressFormatValidator( + SubscriptionProperties subscriptionProperties) { + return new EndpointAddressFormatValidator( + subscriptionProperties.getAdditionalEndpointProtocols()); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SubscriptionHealthConfiguration.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SubscriptionHealthConfiguration.java index a8c176c362..7918c413b3 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SubscriptionHealthConfiguration.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SubscriptionHealthConfiguration.java @@ -1,6 +1,7 @@ package pl.allegro.tech.hermes.management.config; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.concurrent.Executors; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; @@ -26,105 +27,107 @@ import pl.allegro.tech.hermes.management.infrastructure.kafka.MultiDCAwareService; import pl.allegro.tech.hermes.tracker.management.LogRepository; -import java.util.concurrent.Executors; - @Configuration @EnableConfigurationProperties({SubscriptionHealthProperties.class}) public class SubscriptionHealthConfiguration { - private static final DisabledIndicator DISABLED_INDICATOR = new DisabledIndicator(); + private static final DisabledIndicator DISABLED_INDICATOR = new DisabledIndicator(); - @Autowired - private SubscriptionHealthProperties subscriptionHealthProperties; + @Autowired private SubscriptionHealthProperties subscriptionHealthProperties; - @Bean - public SubscriptionHealthProblemIndicator laggingIndicator() { - if (subscriptionHealthProperties.isLaggingIndicatorEnabled()) { - return new LaggingIndicator(subscriptionHealthProperties.getMaxLagInSeconds()); - } - return DISABLED_INDICATOR; + @Bean + public SubscriptionHealthProblemIndicator laggingIndicator() { + if (subscriptionHealthProperties.isLaggingIndicatorEnabled()) { + return new LaggingIndicator(subscriptionHealthProperties.getMaxLagInSeconds()); } + return DISABLED_INDICATOR; + } - @Bean - public SubscriptionHealthProblemIndicator unreachableIndicator() { - if (subscriptionHealthProperties.isUnreachableIndicatorEnabled()) { - return new UnreachableIndicator( - subscriptionHealthProperties.getMaxOtherErrorsRatio(), - subscriptionHealthProperties.getMinSubscriptionRateForReliableMetrics()); - } - return DISABLED_INDICATOR; + @Bean + public SubscriptionHealthProblemIndicator unreachableIndicator() { + if (subscriptionHealthProperties.isUnreachableIndicatorEnabled()) { + return new UnreachableIndicator( + subscriptionHealthProperties.getMaxOtherErrorsRatio(), + subscriptionHealthProperties.getMinSubscriptionRateForReliableMetrics()); } + return DISABLED_INDICATOR; + } - @Bean - public SubscriptionHealthProblemIndicator timingOutIndicator() { - if (subscriptionHealthProperties.isTimingOutIndicatorEnabled()) { - return new TimingOutIndicator( - subscriptionHealthProperties.getMaxTimeoutsRatio(), - subscriptionHealthProperties.getMinSubscriptionRateForReliableMetrics()); - } - return DISABLED_INDICATOR; + @Bean + public SubscriptionHealthProblemIndicator timingOutIndicator() { + if (subscriptionHealthProperties.isTimingOutIndicatorEnabled()) { + return new TimingOutIndicator( + subscriptionHealthProperties.getMaxTimeoutsRatio(), + subscriptionHealthProperties.getMinSubscriptionRateForReliableMetrics()); } + return DISABLED_INDICATOR; + } - @Bean - public SubscriptionHealthProblemIndicator malfunctioningIndicator() { - if (subscriptionHealthProperties.isMalfunctioningIndicatorEnabled()) { - return new MalfunctioningIndicator( - subscriptionHealthProperties.getMax5xxErrorsRatio(), - subscriptionHealthProperties.getMinSubscriptionRateForReliableMetrics()); - } - return DISABLED_INDICATOR; + @Bean + public SubscriptionHealthProblemIndicator malfunctioningIndicator() { + if (subscriptionHealthProperties.isMalfunctioningIndicatorEnabled()) { + return new MalfunctioningIndicator( + subscriptionHealthProperties.getMax5xxErrorsRatio(), + subscriptionHealthProperties.getMinSubscriptionRateForReliableMetrics()); } + return DISABLED_INDICATOR; + } - @Bean - public SubscriptionHealthProblemIndicator receivingMalformedMessagesIndicator() { - if (subscriptionHealthProperties.isReceivingMalformedMessagesIndicatorEnabled()) { - return new ReceivingMalformedMessagesIndicator( - subscriptionHealthProperties.getMax4xxErrorsRatio(), - subscriptionHealthProperties.getMinSubscriptionRateForReliableMetrics()); - } - return DISABLED_INDICATOR; + @Bean + public SubscriptionHealthProblemIndicator receivingMalformedMessagesIndicator() { + if (subscriptionHealthProperties.isReceivingMalformedMessagesIndicatorEnabled()) { + return new ReceivingMalformedMessagesIndicator( + subscriptionHealthProperties.getMax4xxErrorsRatio(), + subscriptionHealthProperties.getMinSubscriptionRateForReliableMetrics()); } + return DISABLED_INDICATOR; + } - @Bean - public SubscriptionService subscriptionService(SubscriptionRepository subscriptionRepository, - SubscriptionOwnerCache subscriptionOwnerCache, - TopicService topicService, - SubscriptionMetricsRepository metricsRepository, - SubscriptionHealthChecker subscriptionHealthChecker, - LogRepository logRepository, - SubscriptionValidator subscriptionValidator, - Auditor auditor, - MultiDatacenterRepositoryCommandExecutor multiDcExecutor, - MultiDCAwareService multiDCAwareService, - RepositoryManager repositoryManager, - SubscriptionHealthProperties subscriptionHealthProperties, - SubscriptionRemover subscriptionRemover) { - return new SubscriptionService( - subscriptionRepository, - subscriptionOwnerCache, - topicService, - metricsRepository, - subscriptionHealthChecker, - logRepository, - subscriptionValidator, - auditor, - multiDcExecutor, - multiDCAwareService, - repositoryManager, - Executors.newFixedThreadPool( - subscriptionHealthProperties.getThreads(), - new ThreadFactoryBuilder().setNameFormat("subscription-health-check-executor-%d").build() - ), - subscriptionHealthProperties.getTimeoutMillis(), - subscriptionRemover - ); - } + @Bean + public SubscriptionService subscriptionService( + SubscriptionRepository subscriptionRepository, + SubscriptionOwnerCache subscriptionOwnerCache, + TopicService topicService, + SubscriptionMetricsRepository metricsRepository, + SubscriptionHealthChecker subscriptionHealthChecker, + LogRepository logRepository, + SubscriptionValidator subscriptionValidator, + Auditor auditor, + MultiDatacenterRepositoryCommandExecutor multiDcExecutor, + MultiDCAwareService multiDCAwareService, + RepositoryManager repositoryManager, + SubscriptionHealthProperties subscriptionHealthProperties, + SubscriptionRemover subscriptionRemover) { + return new SubscriptionService( + subscriptionRepository, + subscriptionOwnerCache, + topicService, + metricsRepository, + subscriptionHealthChecker, + logRepository, + subscriptionValidator, + auditor, + multiDcExecutor, + multiDCAwareService, + repositoryManager, + Executors.newFixedThreadPool( + subscriptionHealthProperties.getThreads(), + new ThreadFactoryBuilder() + .setNameFormat("subscription-health-check-executor-%d") + .build()), + subscriptionHealthProperties.getTimeoutMillis(), + subscriptionRemover); + } - @Bean - public SubscriptionRemover subscriptionRemover(Auditor auditor, - MultiDatacenterRepositoryCommandExecutor multiDatacenterRepositoryCommandExecutor, - SubscriptionOwnerCache subscriptionOwnerCache, - SubscriptionRepository subscriptionRepository) { - return new SubscriptionRemover(auditor, multiDatacenterRepositoryCommandExecutor, - subscriptionOwnerCache, subscriptionRepository); - } + @Bean + public SubscriptionRemover subscriptionRemover( + Auditor auditor, + MultiDatacenterRepositoryCommandExecutor multiDatacenterRepositoryCommandExecutor, + SubscriptionOwnerCache subscriptionOwnerCache, + SubscriptionRepository subscriptionRepository) { + return new SubscriptionRemover( + auditor, + multiDatacenterRepositoryCommandExecutor, + subscriptionOwnerCache, + subscriptionRepository); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SubscriptionHealthProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SubscriptionHealthProperties.java index bfb677cbd0..1d23217ea6 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SubscriptionHealthProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SubscriptionHealthProperties.java @@ -4,121 +4,123 @@ @ConfigurationProperties(prefix = "subscription-health") public class SubscriptionHealthProperties { - private int maxLagInSeconds = 600; - private double maxOtherErrorsRatio = 0.5; - private double maxTimeoutsRatio = 0.1; - private double max5xxErrorsRatio = 0.1; - private double max4xxErrorsRatio = 0.1; - private double minSubscriptionRateForReliableMetrics = 2.0; - private boolean laggingIndicatorEnabled = true; - private boolean malfunctioningIndicatorEnabled = true; - private boolean receivingMalformedMessagesIndicatorEnabled = true; - private boolean timingOutIndicatorEnabled = true; - private boolean unreachableIndicatorEnabled = true; - private long timeoutMillis = 8000; - private int threads = 16; - - public int getMaxLagInSeconds() { - return maxLagInSeconds; - } - - public void setMaxLagInSeconds(int maxLagInSeconds) { - this.maxLagInSeconds = maxLagInSeconds; - } - - public double getMaxOtherErrorsRatio() { - return maxOtherErrorsRatio; - } - - public void setMaxOtherErrorsRatio(double maxOtherErrorsRatio) { - this.maxOtherErrorsRatio = maxOtherErrorsRatio; - } - - public double getMaxTimeoutsRatio() { - return maxTimeoutsRatio; - } - - public void setMaxTimeoutsRatio(double maxTimeoutsRatio) { - this.maxTimeoutsRatio = maxTimeoutsRatio; - } - - public double getMax5xxErrorsRatio() { - return max5xxErrorsRatio; - } - - public void setMax5xxErrorsRatio(double max5xxErrorsRatio) { - this.max5xxErrorsRatio = max5xxErrorsRatio; - } - - public double getMax4xxErrorsRatio() { - return max4xxErrorsRatio; - } - - public void setMax4xxErrorsRatio(double max4xxErrorsRatio) { - this.max4xxErrorsRatio = max4xxErrorsRatio; - } - - public double getMinSubscriptionRateForReliableMetrics() { - return minSubscriptionRateForReliableMetrics; - } - - public void setMinSubscriptionRateForReliableMetrics(double minSubscriptionRateForReliableMetrics) { - this.minSubscriptionRateForReliableMetrics = minSubscriptionRateForReliableMetrics; - } - - public boolean isLaggingIndicatorEnabled() { - return laggingIndicatorEnabled; - } - - public void setLaggingIndicatorEnabled(boolean laggingIndicatorEnabled) { - this.laggingIndicatorEnabled = laggingIndicatorEnabled; - } - - public boolean isMalfunctioningIndicatorEnabled() { - return malfunctioningIndicatorEnabled; - } - - public void setMalfunctioningIndicatorEnabled(boolean malfunctioningIndicatorEnabled) { - this.malfunctioningIndicatorEnabled = malfunctioningIndicatorEnabled; - } - - public boolean isReceivingMalformedMessagesIndicatorEnabled() { - return receivingMalformedMessagesIndicatorEnabled; - } - - public void setReceivingMalformedMessagesIndicatorEnabled(boolean receivingMalformedMessagesIndicatorEnabled) { - this.receivingMalformedMessagesIndicatorEnabled = receivingMalformedMessagesIndicatorEnabled; - } - - public boolean isTimingOutIndicatorEnabled() { - return timingOutIndicatorEnabled; - } - - public void setTimingOutIndicatorEnabled(boolean timingOutIndicatorEnabled) { - this.timingOutIndicatorEnabled = timingOutIndicatorEnabled; - } - - public boolean isUnreachableIndicatorEnabled() { - return unreachableIndicatorEnabled; - } - - public void setUnreachableIndicatorEnabled(boolean unreachableIndicatorEnabled) { - this.unreachableIndicatorEnabled = unreachableIndicatorEnabled; - } - - public long getTimeoutMillis() { - return timeoutMillis; - } - - public void setTimeoutMillis(long timeoutMillis) { - this.timeoutMillis = timeoutMillis; - } - - public int getThreads() { - return threads; - } - - public void setThreads(int threads) { - this.threads = threads; - } + private int maxLagInSeconds = 600; + private double maxOtherErrorsRatio = 0.5; + private double maxTimeoutsRatio = 0.1; + private double max5xxErrorsRatio = 0.1; + private double max4xxErrorsRatio = 0.1; + private double minSubscriptionRateForReliableMetrics = 2.0; + private boolean laggingIndicatorEnabled = true; + private boolean malfunctioningIndicatorEnabled = true; + private boolean receivingMalformedMessagesIndicatorEnabled = true; + private boolean timingOutIndicatorEnabled = true; + private boolean unreachableIndicatorEnabled = true; + private long timeoutMillis = 8000; + private int threads = 16; + + public int getMaxLagInSeconds() { + return maxLagInSeconds; + } + + public void setMaxLagInSeconds(int maxLagInSeconds) { + this.maxLagInSeconds = maxLagInSeconds; + } + + public double getMaxOtherErrorsRatio() { + return maxOtherErrorsRatio; + } + + public void setMaxOtherErrorsRatio(double maxOtherErrorsRatio) { + this.maxOtherErrorsRatio = maxOtherErrorsRatio; + } + + public double getMaxTimeoutsRatio() { + return maxTimeoutsRatio; + } + + public void setMaxTimeoutsRatio(double maxTimeoutsRatio) { + this.maxTimeoutsRatio = maxTimeoutsRatio; + } + + public double getMax5xxErrorsRatio() { + return max5xxErrorsRatio; + } + + public void setMax5xxErrorsRatio(double max5xxErrorsRatio) { + this.max5xxErrorsRatio = max5xxErrorsRatio; + } + + public double getMax4xxErrorsRatio() { + return max4xxErrorsRatio; + } + + public void setMax4xxErrorsRatio(double max4xxErrorsRatio) { + this.max4xxErrorsRatio = max4xxErrorsRatio; + } + + public double getMinSubscriptionRateForReliableMetrics() { + return minSubscriptionRateForReliableMetrics; + } + + public void setMinSubscriptionRateForReliableMetrics( + double minSubscriptionRateForReliableMetrics) { + this.minSubscriptionRateForReliableMetrics = minSubscriptionRateForReliableMetrics; + } + + public boolean isLaggingIndicatorEnabled() { + return laggingIndicatorEnabled; + } + + public void setLaggingIndicatorEnabled(boolean laggingIndicatorEnabled) { + this.laggingIndicatorEnabled = laggingIndicatorEnabled; + } + + public boolean isMalfunctioningIndicatorEnabled() { + return malfunctioningIndicatorEnabled; + } + + public void setMalfunctioningIndicatorEnabled(boolean malfunctioningIndicatorEnabled) { + this.malfunctioningIndicatorEnabled = malfunctioningIndicatorEnabled; + } + + public boolean isReceivingMalformedMessagesIndicatorEnabled() { + return receivingMalformedMessagesIndicatorEnabled; + } + + public void setReceivingMalformedMessagesIndicatorEnabled( + boolean receivingMalformedMessagesIndicatorEnabled) { + this.receivingMalformedMessagesIndicatorEnabled = receivingMalformedMessagesIndicatorEnabled; + } + + public boolean isTimingOutIndicatorEnabled() { + return timingOutIndicatorEnabled; + } + + public void setTimingOutIndicatorEnabled(boolean timingOutIndicatorEnabled) { + this.timingOutIndicatorEnabled = timingOutIndicatorEnabled; + } + + public boolean isUnreachableIndicatorEnabled() { + return unreachableIndicatorEnabled; + } + + public void setUnreachableIndicatorEnabled(boolean unreachableIndicatorEnabled) { + this.unreachableIndicatorEnabled = unreachableIndicatorEnabled; + } + + public long getTimeoutMillis() { + return timeoutMillis; + } + + public void setTimeoutMillis(long timeoutMillis) { + this.timeoutMillis = timeoutMillis; + } + + public int getThreads() { + return threads; + } + + public void setThreads(int threads) { + this.threads = threads; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SubscriptionProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SubscriptionProperties.java index 01cfc1b4f9..26d17e67bb 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SubscriptionProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/SubscriptionProperties.java @@ -1,90 +1,92 @@ package pl.allegro.tech.hermes.management.config; -import org.springframework.boot.context.properties.ConfigurationProperties; - import java.util.ArrayList; import java.util.List; +import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties("subscription") public class SubscriptionProperties { - private List additionalEndpointProtocols = new ArrayList<>(); + private List additionalEndpointProtocols = new ArrayList<>(); - private List subscribersWithAccessToAnyTopic = new ArrayList<>(); + private List subscribersWithAccessToAnyTopic = new ArrayList<>(); - private int intervalBetweenCheckinIfOffsetsMovedInMillis = 50; + private int intervalBetweenCheckinIfOffsetsMovedInMillis = 50; - private int offsetsMovedTimeoutInSeconds = 30; + private int offsetsMovedTimeoutInSeconds = 30; - private boolean createConsumerGroupManuallyEnabled = true; + private boolean createConsumerGroupManuallyEnabled = true; - public List getAdditionalEndpointProtocols() { - return additionalEndpointProtocols; - } + public List getAdditionalEndpointProtocols() { + return additionalEndpointProtocols; + } - public void setAdditionalEndpointProtocols(List additionalEndpointProtocols) { - this.additionalEndpointProtocols = additionalEndpointProtocols; - } + public void setAdditionalEndpointProtocols(List additionalEndpointProtocols) { + this.additionalEndpointProtocols = additionalEndpointProtocols; + } - public int getIntervalBetweenCheckinIfOffsetsMovedInMillis() { - return intervalBetweenCheckinIfOffsetsMovedInMillis; - } + public int getIntervalBetweenCheckinIfOffsetsMovedInMillis() { + return intervalBetweenCheckinIfOffsetsMovedInMillis; + } - public void setIntervalBetweenCheckinIfOffsetsMovedInMillis(int intervalBetweenCheckinIfOffsetsMovedInMillis) { - this.intervalBetweenCheckinIfOffsetsMovedInMillis = intervalBetweenCheckinIfOffsetsMovedInMillis; - } + public void setIntervalBetweenCheckinIfOffsetsMovedInMillis( + int intervalBetweenCheckinIfOffsetsMovedInMillis) { + this.intervalBetweenCheckinIfOffsetsMovedInMillis = + intervalBetweenCheckinIfOffsetsMovedInMillis; + } - public int getOffsetsMovedTimeoutInSeconds() { - return offsetsMovedTimeoutInSeconds; - } + public int getOffsetsMovedTimeoutInSeconds() { + return offsetsMovedTimeoutInSeconds; + } - public void setOffsetsMovedTimeoutInSeconds(int offsetsMovedTimeoutInSeconds) { - this.offsetsMovedTimeoutInSeconds = offsetsMovedTimeoutInSeconds; - } + public void setOffsetsMovedTimeoutInSeconds(int offsetsMovedTimeoutInSeconds) { + this.offsetsMovedTimeoutInSeconds = offsetsMovedTimeoutInSeconds; + } - public boolean isCreateConsumerGroupManuallyEnabled() { - return createConsumerGroupManuallyEnabled; - } + public boolean isCreateConsumerGroupManuallyEnabled() { + return createConsumerGroupManuallyEnabled; + } - public void setCreateConsumerGroupManuallyEnabled(boolean createConsumerGroupManuallyEnabled) { - this.createConsumerGroupManuallyEnabled = createConsumerGroupManuallyEnabled; - } + public void setCreateConsumerGroupManuallyEnabled(boolean createConsumerGroupManuallyEnabled) { + this.createConsumerGroupManuallyEnabled = createConsumerGroupManuallyEnabled; + } - public List getSubscribersWithAccessToAnyTopic() { - return subscribersWithAccessToAnyTopic; - } + public List getSubscribersWithAccessToAnyTopic() { + return subscribersWithAccessToAnyTopic; + } - public void setSubscribersWithAccessToAnyTopic(List subscribersWithAccessToAnyTopic) { - this.subscribersWithAccessToAnyTopic = subscribersWithAccessToAnyTopic; - } + public void setSubscribersWithAccessToAnyTopic( + List subscribersWithAccessToAnyTopic) { + this.subscribersWithAccessToAnyTopic = subscribersWithAccessToAnyTopic; + } - public static class SubscriberProperties { - private String ownerSource; - private String ownerId; - private List protocols = new ArrayList<>(); + public static class SubscriberProperties { + private String ownerSource; + private String ownerId; + private List protocols = new ArrayList<>(); - public String getOwnerSource() { - return ownerSource; - } + public String getOwnerSource() { + return ownerSource; + } - public void setOwnerSource(String ownerSource) { - this.ownerSource = ownerSource; - } + public void setOwnerSource(String ownerSource) { + this.ownerSource = ownerSource; + } - public String getOwnerId() { - return ownerId; - } + public String getOwnerId() { + return ownerId; + } - public void setOwnerId(String ownerId) { - this.ownerId = ownerId; - } + public void setOwnerId(String ownerId) { + this.ownerId = ownerId; + } - public List getProtocols() { - return protocols; - } + public List getProtocols() { + return protocols; + } - public void setProtocols(List protocols) { - this.protocols = protocols; - } + public void setProtocols(List protocols) { + this.protocols = protocols; } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/TopicProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/TopicProperties.java index 77dd98a1a7..725982233d 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/TopicProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/TopicProperties.java @@ -1,176 +1,174 @@ package pl.allegro.tech.hermes.management.config; -import org.springframework.boot.context.properties.ConfigurationProperties; -import pl.allegro.tech.hermes.api.ContentType; -import pl.allegro.tech.hermes.api.TopicLabel; - import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Set; +import org.springframework.boot.context.properties.ConfigurationProperties; +import pl.allegro.tech.hermes.api.ContentType; +import pl.allegro.tech.hermes.api.TopicLabel; @ConfigurationProperties(prefix = "topic") public class TopicProperties { - private int replicationFactor = 1; - - private int partitions = 10; - - private boolean allowRemoval = false; - - private boolean removeSchema = false; - - private List allowedContentTypes = Arrays.asList(ContentType.AVRO, ContentType.JSON); - - private Set allowedTopicLabels = Collections.emptySet(); + private int replicationFactor = 1; - private boolean uncleanLeaderElectionEnabled = false; + private int partitions = 10; - private int touchDelayInSeconds = 120; + private boolean allowRemoval = false; - private boolean touchSchedulerEnabled = true; + private boolean removeSchema = false; - private int subscriptionsAssignmentsCompletedTimeoutSeconds = 30; + private List allowedContentTypes = Arrays.asList(ContentType.AVRO, ContentType.JSON); - private boolean defaultSchemaIdAwareSerializationEnabled = false; + private Set allowedTopicLabels = Collections.emptySet(); - private boolean defaultFallbackToRemoteDatacenterEnabled = false; + private boolean uncleanLeaderElectionEnabled = false; - private boolean avroContentTypeMetadataRequired = true; + private int touchDelayInSeconds = 120; - /** - *

Introduced in Kafka 0.11.0.0 mechanism of splitting oversized batches does not respect configuration of maximum - * message size which broker can accept. It can cause an infinite loop of resending the same records in one batch. - * To avoid the issue this parameter should be greater than or equal to maximum size of request that the producer - * can send (parameter kafka.producer.max.request.size). Note that if you change this setting, it is necessary to - * manually update max.message.bytes for existing topics on broker side.

- * - *

For more information see:

- * - */ - private int maxMessageSize = 1024 * 1024; + private boolean touchSchedulerEnabled = true; - public int getReplicationFactor() { - return replicationFactor; - } + private int subscriptionsAssignmentsCompletedTimeoutSeconds = 30; - public void setReplicationFactor(int replicationFactor) { - this.replicationFactor = replicationFactor; - } + private boolean defaultSchemaIdAwareSerializationEnabled = false; - public int getPartitions() { - return partitions; - } + private boolean defaultFallbackToRemoteDatacenterEnabled = false; - public void setPartitions(int partitions) { - this.partitions = partitions; - } + private boolean avroContentTypeMetadataRequired = true; - public boolean isAllowRemoval() { - return allowRemoval; - } + /** + * Introduced in Kafka 0.11.0.0 mechanism of splitting oversized batches does not respect + * configuration of maximum message size which broker can accept. It can cause an infinite loop of + * resending the same records in one batch. To avoid the issue this parameter should be greater + * than or equal to maximum size of request that the producer can send (parameter + * kafka.producer.max.request.size). Note that if you change this setting, it is necessary to + * manually update max.message.bytes for existing topics on broker side. + * + *

For more information see: + * + *

+ */ + private int maxMessageSize = 1024 * 1024; - public void setAllowRemoval(boolean allowRemoval) { - this.allowRemoval = allowRemoval; - } + public int getReplicationFactor() { + return replicationFactor; + } - public boolean isRemoveSchema() { - return removeSchema; - } + public void setReplicationFactor(int replicationFactor) { + this.replicationFactor = replicationFactor; + } - public void setRemoveSchema(boolean removeSchema) { - this.removeSchema = removeSchema; - } + public int getPartitions() { + return partitions; + } - public List getAllowedContentTypes() { - return allowedContentTypes; - } + public void setPartitions(int partitions) { + this.partitions = partitions; + } - public void setAllowedContentTypes(List allowedContentTypes) { - this.allowedContentTypes = allowedContentTypes; - } + public boolean isAllowRemoval() { + return allowRemoval; + } - public Set getAllowedTopicLabels() { - return allowedTopicLabels; - } + public void setAllowRemoval(boolean allowRemoval) { + this.allowRemoval = allowRemoval; + } - public void setAllowedTopicLabels(Set allowedTopicLabels) { - this.allowedTopicLabels = allowedTopicLabels; - } + public boolean isRemoveSchema() { + return removeSchema; + } - public boolean isUncleanLeaderElectionEnabled() { - return uncleanLeaderElectionEnabled; - } + public void setRemoveSchema(boolean removeSchema) { + this.removeSchema = removeSchema; + } - public void setUncleanLeaderElectionEnabled(boolean uncleanLeaderElectionEnabled) { - this.uncleanLeaderElectionEnabled = uncleanLeaderElectionEnabled; - } + public List getAllowedContentTypes() { + return allowedContentTypes; + } - public int getTouchDelayInSeconds() { - return touchDelayInSeconds; - } + public void setAllowedContentTypes(List allowedContentTypes) { + this.allowedContentTypes = allowedContentTypes; + } - public void setTouchDelayInSeconds(int touchDelayInSeconds) { - this.touchDelayInSeconds = touchDelayInSeconds; - } - - public boolean isTouchSchedulerEnabled() { - return touchSchedulerEnabled; - } - - public void setTouchSchedulerEnabled(boolean touchSchedulerEnabled) { - this.touchSchedulerEnabled = touchSchedulerEnabled; - } - - public int getSubscriptionsAssignmentsCompletedTimeoutSeconds() { - return subscriptionsAssignmentsCompletedTimeoutSeconds; - } - - public void setSubscriptionsAssignmentsCompletedTimeoutSeconds(int subscriptionsAssignmentsCompletedTimeoutSeconds) { - this.subscriptionsAssignmentsCompletedTimeoutSeconds = subscriptionsAssignmentsCompletedTimeoutSeconds; - } - - public int getMaxMessageSize() { - return maxMessageSize; - } - - public void setMaxMessageSize(int maxMessageSize) { - this.maxMessageSize = maxMessageSize; - } - - public boolean isDefaultSchemaIdAwareSerializationEnabled() { - return defaultSchemaIdAwareSerializationEnabled; - } - - public void setDefaultSchemaIdAwareSerializationEnabled(boolean defaultSchemaIdAwareSerializationEnabled) { - this.defaultSchemaIdAwareSerializationEnabled = defaultSchemaIdAwareSerializationEnabled; - } + public Set getAllowedTopicLabels() { + return allowedTopicLabels; + } - public boolean isDefaultFallbackToRemoteDatacenterEnabled() { - return defaultFallbackToRemoteDatacenterEnabled; - } + public void setAllowedTopicLabels(Set allowedTopicLabels) { + this.allowedTopicLabels = allowedTopicLabels; + } - public void setDefaultFallbackToRemoteDatacenterEnabled(boolean defaultFallbackToRemoteDatacenterEnabled) { - this.defaultFallbackToRemoteDatacenterEnabled = defaultFallbackToRemoteDatacenterEnabled; - } + public boolean isUncleanLeaderElectionEnabled() { + return uncleanLeaderElectionEnabled; + } - public boolean isAvroContentTypeMetadataRequired() { - return avroContentTypeMetadataRequired; - } + public void setUncleanLeaderElectionEnabled(boolean uncleanLeaderElectionEnabled) { + this.uncleanLeaderElectionEnabled = uncleanLeaderElectionEnabled; + } - public void setAvroContentTypeMetadataRequired(boolean avroContentTypeMetadataRequired) { - this.avroContentTypeMetadataRequired = avroContentTypeMetadataRequired; - } + public int getTouchDelayInSeconds() { + return touchDelayInSeconds; + } + public void setTouchDelayInSeconds(int touchDelayInSeconds) { + this.touchDelayInSeconds = touchDelayInSeconds; + } + + public boolean isTouchSchedulerEnabled() { + return touchSchedulerEnabled; + } + + public void setTouchSchedulerEnabled(boolean touchSchedulerEnabled) { + this.touchSchedulerEnabled = touchSchedulerEnabled; + } + + public int getSubscriptionsAssignmentsCompletedTimeoutSeconds() { + return subscriptionsAssignmentsCompletedTimeoutSeconds; + } + + public void setSubscriptionsAssignmentsCompletedTimeoutSeconds( + int subscriptionsAssignmentsCompletedTimeoutSeconds) { + this.subscriptionsAssignmentsCompletedTimeoutSeconds = + subscriptionsAssignmentsCompletedTimeoutSeconds; + } + + public int getMaxMessageSize() { + return maxMessageSize; + } + + public void setMaxMessageSize(int maxMessageSize) { + this.maxMessageSize = maxMessageSize; + } + + public boolean isDefaultSchemaIdAwareSerializationEnabled() { + return defaultSchemaIdAwareSerializationEnabled; + } + + public void setDefaultSchemaIdAwareSerializationEnabled( + boolean defaultSchemaIdAwareSerializationEnabled) { + this.defaultSchemaIdAwareSerializationEnabled = defaultSchemaIdAwareSerializationEnabled; + } + + public boolean isDefaultFallbackToRemoteDatacenterEnabled() { + return defaultFallbackToRemoteDatacenterEnabled; + } + + public void setDefaultFallbackToRemoteDatacenterEnabled( + boolean defaultFallbackToRemoteDatacenterEnabled) { + this.defaultFallbackToRemoteDatacenterEnabled = defaultFallbackToRemoteDatacenterEnabled; + } + + public boolean isAvroContentTypeMetadataRequired() { + return avroContentTypeMetadataRequired; + } + + public void setAvroContentTypeMetadataRequired(boolean avroContentTypeMetadataRequired) { + this.avroContentTypeMetadataRequired = avroContentTypeMetadataRequired; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/console/ConsoleConfigProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/console/ConsoleConfigProperties.java index f1f97eb740..ff77fe9982 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/console/ConsoleConfigProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/console/ConsoleConfigProperties.java @@ -1,63 +1,64 @@ package pl.allegro.tech.hermes.management.config.console; -import org.springframework.boot.context.properties.ConfigurationProperties; - import java.time.Duration; +import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties(prefix = "console.config") public class ConsoleConfigProperties { - private String location = "console/config-local.json"; - private ConfigurationType type = ConfigurationType.SPRING_CONFIG; - private HttpClientProperties httpClient = new HttpClientProperties(); + private String location = "console/config-local.json"; + private ConfigurationType type = ConfigurationType.SPRING_CONFIG; + private HttpClientProperties httpClient = new HttpClientProperties(); - public String getLocation() { - return location; - } + public String getLocation() { + return location; + } - public void setLocation(String location) { - this.location = location; - } + public void setLocation(String location) { + this.location = location; + } - public ConfigurationType getType() { - return type; - } + public ConfigurationType getType() { + return type; + } - public void setType(ConfigurationType type) { - this.type = type; - } + public void setType(ConfigurationType type) { + this.type = type; + } - public HttpClientProperties getHttpClient() { - return httpClient; - } - - public void setHttpClient(HttpClientProperties httpClient) { - this.httpClient = httpClient; - } + public HttpClientProperties getHttpClient() { + return httpClient; + } - public static class HttpClientProperties { + public void setHttpClient(HttpClientProperties httpClient) { + this.httpClient = httpClient; + } - private Duration connectTimeout = Duration.ofMillis(500); - private Duration readTimeout = Duration.ofSeconds(3); + public static class HttpClientProperties { - public Duration getConnectTimeout() { - return connectTimeout; - } + private Duration connectTimeout = Duration.ofMillis(500); + private Duration readTimeout = Duration.ofSeconds(3); - public void setConnectTimeout(Duration connectTimeout) { - this.connectTimeout = connectTimeout; - } + public Duration getConnectTimeout() { + return connectTimeout; + } - public Duration getReadTimeout() { - return readTimeout; - } + public void setConnectTimeout(Duration connectTimeout) { + this.connectTimeout = connectTimeout; + } - public void setReadTimeout(Duration readTimeout) { - this.readTimeout = readTimeout; - } + public Duration getReadTimeout() { + return readTimeout; } - public enum ConfigurationType { - CLASSPATH_RESOURCE, HTTP_RESOURCE, SPRING_CONFIG + public void setReadTimeout(Duration readTimeout) { + this.readTimeout = readTimeout; } + } + + public enum ConfigurationType { + CLASSPATH_RESOURCE, + HTTP_RESOURCE, + SPRING_CONFIG + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/console/ConsoleConfiguration.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/console/ConsoleConfiguration.java index e9f589414e..fc2ea5b350 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/console/ConsoleConfiguration.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/console/ConsoleConfiguration.java @@ -1,6 +1,7 @@ package pl.allegro.tech.hermes.management.config.console; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.concurrent.TimeUnit; import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder; import org.apache.hc.core5.http.io.SocketConfig; @@ -16,49 +17,51 @@ import pl.allegro.tech.hermes.management.infrastructure.console.HttpConsoleConfigurationRepository; import pl.allegro.tech.hermes.management.infrastructure.console.SpringConfigConsoleConfigurationRepository; -import java.util.concurrent.TimeUnit; - @Configuration @EnableConfigurationProperties({ConsoleConfigProperties.class, ConsoleProperties.class}) public class ConsoleConfiguration { - @Bean - FilterRegistrationBean frontendRoutesFilter() { - FilterRegistrationBean registrationBean = new FilterRegistrationBean<>(); - registrationBean.setFilter(new FrontendRoutesFilter()); - return registrationBean; - } - - @Bean - ConsoleConfigurationRepository consoleConfigurationRepository( - ConsoleConfigProperties properties, ObjectMapper objectMapper, ConsoleProperties consoleProperties) { - switch (properties.getType()) { - case CLASSPATH_RESOURCE: - return new ClasspathFileConsoleConfigurationRepository(properties); - case HTTP_RESOURCE: - return httpConsoleConfigurationRepository(properties); - case SPRING_CONFIG: - return new SpringConfigConsoleConfigurationRepository(objectMapper, consoleProperties); - default: - throw new IllegalArgumentException("Unsupported console config type"); - } - } + @Bean + FilterRegistrationBean frontendRoutesFilter() { + FilterRegistrationBean registrationBean = new FilterRegistrationBean<>(); + registrationBean.setFilter(new FrontendRoutesFilter()); + return registrationBean; + } - private ConsoleConfigurationRepository httpConsoleConfigurationRepository(ConsoleConfigProperties properties) { - var httpClientProperties = properties.getHttpClient(); - var socketConfig = SocketConfig.custom() - .setSoTimeout((int) httpClientProperties.getReadTimeout().toMillis(), TimeUnit.MILLISECONDS) - .build(); - var connectionManager = PoolingHttpClientConnectionManagerBuilder.create() - .setDefaultSocketConfig(socketConfig) - .build(); - var client = HttpClientBuilder.create() - .setConnectionManager(connectionManager) - .build(); - HttpComponentsClientHttpRequestFactory requestFactory = new HttpComponentsClientHttpRequestFactory(client); - requestFactory.setConnectTimeout(properties.getHttpClient().getConnectTimeout()); - RestTemplate restTemplate = new RestTemplate(requestFactory); - return new HttpConsoleConfigurationRepository(properties, restTemplate); + @Bean + ConsoleConfigurationRepository consoleConfigurationRepository( + ConsoleConfigProperties properties, + ObjectMapper objectMapper, + ConsoleProperties consoleProperties) { + switch (properties.getType()) { + case CLASSPATH_RESOURCE: + return new ClasspathFileConsoleConfigurationRepository(properties); + case HTTP_RESOURCE: + return httpConsoleConfigurationRepository(properties); + case SPRING_CONFIG: + return new SpringConfigConsoleConfigurationRepository(objectMapper, consoleProperties); + default: + throw new IllegalArgumentException("Unsupported console config type"); } + } + private ConsoleConfigurationRepository httpConsoleConfigurationRepository( + ConsoleConfigProperties properties) { + var httpClientProperties = properties.getHttpClient(); + var socketConfig = + SocketConfig.custom() + .setSoTimeout( + (int) httpClientProperties.getReadTimeout().toMillis(), TimeUnit.MILLISECONDS) + .build(); + var connectionManager = + PoolingHttpClientConnectionManagerBuilder.create() + .setDefaultSocketConfig(socketConfig) + .build(); + var client = HttpClientBuilder.create().setConnectionManager(connectionManager).build(); + HttpComponentsClientHttpRequestFactory requestFactory = + new HttpComponentsClientHttpRequestFactory(client); + requestFactory.setConnectTimeout(properties.getHttpClient().getConnectTimeout()); + RestTemplate restTemplate = new RestTemplate(requestFactory); + return new HttpConsoleConfigurationRepository(properties, restTemplate); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/console/ConsoleProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/console/ConsoleProperties.java index e6b5435a4c..c36fe397b9 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/console/ConsoleProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/console/ConsoleProperties.java @@ -1,891 +1,887 @@ package pl.allegro.tech.hermes.management.config.console; import com.google.common.collect.Lists; -import org.springframework.boot.context.properties.ConfigurationProperties; - import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import org.springframework.boot.context.properties.ConfigurationProperties; /** - * This class could have a much better structure, however it has this one due to compatibility with old JSON config file format. + * This class could have a much better structure, however it has this one due to compatibility with + * old JSON config file format. */ @ConfigurationProperties(prefix = "console") public class ConsoleProperties { - private Console console = new Console(); - private Dashboard dashboard = new Dashboard(); - private Hermes hermes = new Hermes(); - private Metrics metrics = new Metrics(); - private Auth auth = new Auth(); - private Owner owner = new Owner(); - private TopicView topic = new TopicView(); - private SubscriptionView subscription = new SubscriptionView(); - private ConsistencyView consistency = new ConsistencyView(); - private GroupView group = new GroupView(); - private Costs costs = new Costs(); + private Console console = new Console(); + private Dashboard dashboard = new Dashboard(); + private Hermes hermes = new Hermes(); + private Metrics metrics = new Metrics(); + private Auth auth = new Auth(); + private Owner owner = new Owner(); + private TopicView topic = new TopicView(); + private SubscriptionView subscription = new SubscriptionView(); + private ConsistencyView consistency = new ConsistencyView(); + private GroupView group = new GroupView(); + private Costs costs = new Costs(); + + public Dashboard getDashboard() { + return dashboard; + } + + public void setDashboard(Dashboard dashboard) { + this.dashboard = dashboard; + } + + public Hermes getHermes() { + return hermes; + } + + public void setHermes(Hermes hermes) { + this.hermes = hermes; + } + + public Metrics getMetrics() { + return metrics; + } + + public void setMetrics(Metrics metrics) { + this.metrics = metrics; + } + + public Auth getAuth() { + return auth; + } + + public void setAuth(Auth auth) { + this.auth = auth; + } + + public Console getConsole() { + return console; + } + + public void setConsole(Console console) { + this.console = console; + } + + public ConsistencyView getConsistency() { + return consistency; + } + + public void setConsistency(ConsistencyView consistency) { + this.consistency = consistency; + } + + public GroupView getGroup() { + return group; + } + + public void setGroup(GroupView group) { + this.group = group; + } + + public Costs getCosts() { + return costs; + } - public Dashboard getDashboard() { - return dashboard; + public void setCosts(Costs costs) { + this.costs = costs; + } + + public static final class Console { + private String title = "hermes console"; + private String contactLink = ""; + private String environmentName = "LOCAL"; + + private boolean criticalEnvironment = false; + + public String getTitle() { + return title; } - public void setDashboard(Dashboard dashboard) { - this.dashboard = dashboard; + public void setTitle(String title) { + this.title = title; } - public Hermes getHermes() { - return hermes; + public String getContactLink() { + return contactLink; } - public void setHermes(Hermes hermes) { - this.hermes = hermes; + public void setContactLink(String contactLink) { + this.contactLink = contactLink; } - public Metrics getMetrics() { - return metrics; + public boolean isCriticalEnvironment() { + return criticalEnvironment; } - public void setMetrics(Metrics metrics) { - this.metrics = metrics; + public void setCriticalEnvironment(boolean isCriticalEnvironment) { + this.criticalEnvironment = isCriticalEnvironment; } - public Auth getAuth() { - return auth; + public String getEnvironmentName() { + return environmentName; } - public void setAuth(Auth auth) { - this.auth = auth; + public void setEnvironmentName(String environmentName) { + this.environmentName = environmentName; } + } + + public Owner getOwner() { + return owner; + } + + public void setOwner(Owner owner) { + this.owner = owner; + } + + public TopicView getTopic() { + return topic; + } + + public void setTopic(TopicView topic) { + this.topic = topic; + } + + public SubscriptionView getSubscription() { + return subscription; + } + + public void setSubscription(SubscriptionView subscription) { + this.subscription = subscription; + } - public Console getConsole() { - return console; + public static final class Dashboard { + private String metrics = "http://localhost:8082"; + private String docs = "http://hermes-pubsub.rtfd.org"; + + public String getMetrics() { + return metrics; } - public void setConsole(Console console) { - this.console = console; + public void setMetrics(String metrics) { + this.metrics = metrics; } - public ConsistencyView getConsistency() { - return consistency; + public String getDocs() { + return docs; } - public void setConsistency(ConsistencyView consistency) { - this.consistency = consistency; + public void setDocs(String docs) { + this.docs = docs; } + } + + public static final class Hermes { + private Discovery discovery = new Discovery(); - public GroupView getGroup() { - return group; + public Discovery getDiscovery() { + return discovery; } - public void setGroup(GroupView group) { - this.group = group; + public void setDiscovery(Discovery discovery) { + this.discovery = discovery; } + } - public Costs getCosts() { - return costs; + public static final class Discovery { + private String type = "simple"; + private SimpleDiscovery simple = new SimpleDiscovery(); + + public String getType() { + return type; } - public void setCosts(Costs costs) { - this.costs = costs; + public void setType(String type) { + this.type = type; } - public static final class Console { - private String title = "hermes console"; - private String contactLink = ""; - private String environmentName = "LOCAL"; + public SimpleDiscovery getSimple() { + return simple; + } - private boolean criticalEnvironment = false; + public void setSimple(SimpleDiscovery simple) { + this.simple = simple; + } + } - public String getTitle() { - return title; - } + public static final class SimpleDiscovery { + private String url = ""; - public void setTitle(String title) { - this.title = title; - } + public String getUrl() { + return url; + } - public String getContactLink() { - return contactLink; - } + public void setUrl(String url) { + this.url = url; + } + } - public void setContactLink(String contactLink) { - this.contactLink = contactLink; - } + public static final class Metrics { + private boolean fetchingDashboardUrlEnabled = false; - public boolean isCriticalEnvironment() { - return criticalEnvironment; - } + public boolean isFetchingDashboardUrlEnabled() { + return fetchingDashboardUrlEnabled; + } - public void setCriticalEnvironment(boolean isCriticalEnvironment) { - this.criticalEnvironment = isCriticalEnvironment; - } + public void setFetchingDashboardUrlEnabled(boolean fetchingDashboardUrlEnabled) { + this.fetchingDashboardUrlEnabled = fetchingDashboardUrlEnabled; + } + } - public String getEnvironmentName() { - return environmentName; - } + public static final class Auth { + private OAuth oauth = new OAuth(); + private AuthHeaders headers = new AuthHeaders(); - public void setEnvironmentName(String environmentName) { - this.environmentName = environmentName; - } + public OAuth getOauth() { + return oauth; } - public Owner getOwner() { - return owner; + public void setOauth(OAuth oauth) { + this.oauth = oauth; } - public void setOwner(Owner owner) { - this.owner = owner; + public AuthHeaders getHeaders() { + return headers; } - public TopicView getTopic() { - return topic; + public void setHeaders(AuthHeaders headers) { + this.headers = headers; } + } + + public static final class OAuth { + private boolean enabled = false; + private String url = "localhost:8092"; + private String authorizationEndpoint = "/auth/oauth/authorize"; + private String tokenEndpoint = "/auth/oauth/token"; + private String clientId = "hermes"; + private String scope = "hermes"; - public void setTopic(TopicView topic) { - this.topic = topic; + public boolean isEnabled() { + return enabled; } - public SubscriptionView getSubscription() { - return subscription; + public void setEnabled(boolean enabled) { + this.enabled = enabled; } - public void setSubscription(SubscriptionView subscription) { - this.subscription = subscription; + public String getUrl() { + return url; } - public static final class Dashboard { - private String metrics = "http://localhost:8082"; - private String docs = "http://hermes-pubsub.rtfd.org"; + public void setUrl(String url) { + this.url = url; + } - public String getMetrics() { - return metrics; - } + public String getAuthorizationEndpoint() { + return authorizationEndpoint; + } - public void setMetrics(String metrics) { - this.metrics = metrics; - } + public void setAuthorizationEndpoint(String authorizationEndpoint) { + this.authorizationEndpoint = authorizationEndpoint; + } - public String getDocs() { - return docs; - } + public String getTokenEndpoint() { + return tokenEndpoint; + } - public void setDocs(String docs) { - this.docs = docs; - } + public void setTokenEndpoint(String tokenEndpoint) { + this.tokenEndpoint = tokenEndpoint; } - public static final class Hermes { - private Discovery discovery = new Discovery(); + public String getClientId() { + return clientId; + } - public Discovery getDiscovery() { - return discovery; - } + public void setClientId(String clientId) { + this.clientId = clientId; + } - public void setDiscovery(Discovery discovery) { - this.discovery = discovery; - } + public String getScope() { + return scope; } - public static final class Discovery { - private String type = "simple"; - private SimpleDiscovery simple = new SimpleDiscovery(); + public void setScope(String scope) { + this.scope = scope; + } + } - public String getType() { - return type; - } + public static final class AuthHeaders { + private boolean enabled = false; + private String adminHeader = "Hermes-Admin-Password"; - public void setType(String type) { - this.type = type; - } + public boolean isEnabled() { + return enabled; + } - public SimpleDiscovery getSimple() { - return simple; - } + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } - public void setSimple(SimpleDiscovery simple) { - this.simple = simple; - } + public String getAdminHeader() { + return adminHeader; } - public static final class SimpleDiscovery { - private String url = ""; + public void setAdminHeader(String adminHeader) { + this.adminHeader = adminHeader; + } + } - public String getUrl() { - return url; - } + public static final class TopicView { + private boolean messagePreviewEnabled = true; + private boolean offlineClientsEnabled = false; + private boolean authEnabled = true; + private DefaultTopicView defaults = new DefaultTopicView(); + private String buttonsExtension = ""; + private boolean removeSchema = false; + private boolean schemaIdAwareSerializationEnabled = false; + private boolean avroContentTypeMetadataRequired = true; + private List contentTypes = + Lists.newArrayList( + new TopicContentType("AVRO", "AVRO"), new TopicContentType("JSON", "JSON")); + private boolean readOnlyModeEnabled = false; + private Set allowedTopicLabels = Collections.emptySet(); + private List retentionUnits = + Lists.newArrayList(new RetentionUnit("DAYS", "DAYS")); + private boolean offlineRetransmissionEnabled = false; + private String offlineRetransmissionDescription = "Offline retransmission"; - public void setUrl(String url) { - this.url = url; - } + public boolean isMessagePreviewEnabled() { + return messagePreviewEnabled; } - public static final class Metrics { - private boolean fetchingDashboardUrlEnabled = false; + public void setMessagePreviewEnabled(boolean messagePreviewEnabled) { + this.messagePreviewEnabled = messagePreviewEnabled; + } - public boolean isFetchingDashboardUrlEnabled() { - return fetchingDashboardUrlEnabled; - } + public boolean isOfflineClientsEnabled() { + return offlineClientsEnabled; + } - public void setFetchingDashboardUrlEnabled(boolean fetchingDashboardUrlEnabled) { - this.fetchingDashboardUrlEnabled = fetchingDashboardUrlEnabled; - } + public void setOfflineClientsEnabled(boolean offlineClientsEnabled) { + this.offlineClientsEnabled = offlineClientsEnabled; } - public static final class Auth { - private OAuth oauth = new OAuth(); - private AuthHeaders headers = new AuthHeaders(); + public boolean isAuthEnabled() { + return authEnabled; + } - public OAuth getOauth() { - return oauth; - } + public void setAuthEnabled(boolean authEnabled) { + this.authEnabled = authEnabled; + } - public void setOauth(OAuth oauth) { - this.oauth = oauth; - } + public DefaultTopicView getDefaults() { + return defaults; + } - public AuthHeaders getHeaders() { - return headers; - } + public void setDefaults(DefaultTopicView defaults) { + this.defaults = defaults; + } - public void setHeaders(AuthHeaders headers) { - this.headers = headers; - } + public List getContentTypes() { + return contentTypes; } - public static final class OAuth { - private boolean enabled = false; - private String url = "localhost:8092"; - private String authorizationEndpoint = "/auth/oauth/authorize"; - private String tokenEndpoint = "/auth/oauth/token"; - private String clientId = "hermes"; - private String scope = "hermes"; + public void setContentTypes(List contentTypes) { + this.contentTypes = contentTypes; + } - public boolean isEnabled() { - return enabled; - } + public Set getAllowedTopicLabels() { + return allowedTopicLabels; + } - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } + public void setAllowedTopicLabels(Set allowedTopicLabels) { + this.allowedTopicLabels = allowedTopicLabels; + } - public String getUrl() { - return url; - } + public String getButtonsExtension() { + return buttonsExtension; + } - public void setUrl(String url) { - this.url = url; - } + public void setButtonsExtension(String buttonsExtension) { + this.buttonsExtension = buttonsExtension; + } - public String getAuthorizationEndpoint() { - return authorizationEndpoint; - } + public boolean isRemoveSchema() { + return removeSchema; + } - public void setAuthorizationEndpoint(String authorizationEndpoint) { - this.authorizationEndpoint = authorizationEndpoint; - } + public void setRemoveSchema(boolean removeSchema) { + this.removeSchema = removeSchema; + } - public String getTokenEndpoint() { - return tokenEndpoint; - } + public boolean isSchemaIdAwareSerializationEnabled() { + return schemaIdAwareSerializationEnabled; + } - public void setTokenEndpoint(String tokenEndpoint) { - this.tokenEndpoint = tokenEndpoint; - } + public void setSchemaIdAwareSerializationEnabled(boolean schemaIdAwareSerializationEnabled) { + this.schemaIdAwareSerializationEnabled = schemaIdAwareSerializationEnabled; + } - public String getClientId() { - return clientId; - } + public boolean isAvroContentTypeMetadataRequired() { + return avroContentTypeMetadataRequired; + } - public void setClientId(String clientId) { - this.clientId = clientId; - } + public void setAvroContentTypeMetadataRequired(boolean avroContentTypeMetadataRequired) { + this.avroContentTypeMetadataRequired = avroContentTypeMetadataRequired; + } - public String getScope() { - return scope; - } + public boolean isReadOnlyModeEnabled() { + return readOnlyModeEnabled; + } - public void setScope(String scope) { - this.scope = scope; - } - } - - public static final class AuthHeaders { - private boolean enabled = false; - private String adminHeader = "Hermes-Admin-Password"; - - public boolean isEnabled() { - return enabled; - } - - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } - - public String getAdminHeader() { - return adminHeader; - } - - public void setAdminHeader(String adminHeader) { - this.adminHeader = adminHeader; - } - } - - public static final class TopicView { - private boolean messagePreviewEnabled = true; - private boolean offlineClientsEnabled = false; - private boolean authEnabled = true; - private DefaultTopicView defaults = new DefaultTopicView(); - private String buttonsExtension = ""; - private boolean removeSchema = false; - private boolean schemaIdAwareSerializationEnabled = false; - private boolean avroContentTypeMetadataRequired = true; - private List contentTypes = Lists.newArrayList( - new TopicContentType("AVRO", "AVRO"), - new TopicContentType("JSON", "JSON") - ); - private boolean readOnlyModeEnabled = false; - private Set allowedTopicLabels = Collections.emptySet(); - private List retentionUnits = Lists.newArrayList( - new RetentionUnit("DAYS", "DAYS") - ); - private boolean offlineRetransmissionEnabled = false; - private String offlineRetransmissionDescription = "Offline retransmission"; - - public boolean isMessagePreviewEnabled() { - return messagePreviewEnabled; - } - - public void setMessagePreviewEnabled(boolean messagePreviewEnabled) { - this.messagePreviewEnabled = messagePreviewEnabled; - } - - public boolean isOfflineClientsEnabled() { - return offlineClientsEnabled; - } - - public void setOfflineClientsEnabled(boolean offlineClientsEnabled) { - this.offlineClientsEnabled = offlineClientsEnabled; - } - - public boolean isAuthEnabled() { - return authEnabled; - } - - public void setAuthEnabled(boolean authEnabled) { - this.authEnabled = authEnabled; - } - - public DefaultTopicView getDefaults() { - return defaults; - } - - public void setDefaults(DefaultTopicView defaults) { - this.defaults = defaults; - } - - public List getContentTypes() { - return contentTypes; - } - - public void setContentTypes(List contentTypes) { - this.contentTypes = contentTypes; - } - - public Set getAllowedTopicLabels() { - return allowedTopicLabels; - } - - public void setAllowedTopicLabels(Set allowedTopicLabels) { - this.allowedTopicLabels = allowedTopicLabels; - } - - public String getButtonsExtension() { - return buttonsExtension; - } - - public void setButtonsExtension(String buttonsExtension) { - this.buttonsExtension = buttonsExtension; - } - - public boolean isRemoveSchema() { - return removeSchema; - } - - public void setRemoveSchema(boolean removeSchema) { - this.removeSchema = removeSchema; - } - - public boolean isSchemaIdAwareSerializationEnabled() { - return schemaIdAwareSerializationEnabled; - } - - public void setSchemaIdAwareSerializationEnabled(boolean schemaIdAwareSerializationEnabled) { - this.schemaIdAwareSerializationEnabled = schemaIdAwareSerializationEnabled; - } - - public boolean isAvroContentTypeMetadataRequired() { - return avroContentTypeMetadataRequired; - } - - public void setAvroContentTypeMetadataRequired(boolean avroContentTypeMetadataRequired) { - this.avroContentTypeMetadataRequired = avroContentTypeMetadataRequired; - } - - public boolean isReadOnlyModeEnabled() { - return readOnlyModeEnabled; - } - - public void setReadOnlyModeEnabled(boolean readOnlyModeEnabled) { - this.readOnlyModeEnabled = readOnlyModeEnabled; - } - - public List getRetentionUnits() { - return retentionUnits; - } + public void setReadOnlyModeEnabled(boolean readOnlyModeEnabled) { + this.readOnlyModeEnabled = readOnlyModeEnabled; + } - public void setRetentionUnits(List retentionUnits) { - this.retentionUnits = retentionUnits; - } + public List getRetentionUnits() { + return retentionUnits; + } - public boolean getOfflineRetransmissionEnabled() { - return offlineRetransmissionEnabled; - } + public void setRetentionUnits(List retentionUnits) { + this.retentionUnits = retentionUnits; + } - public void setOfflineRetransmissionEnabled(boolean offlineRetransmissionEnabled) { - this.offlineRetransmissionEnabled = offlineRetransmissionEnabled; - } + public boolean getOfflineRetransmissionEnabled() { + return offlineRetransmissionEnabled; + } - public String getOfflineRetransmissionDescription() { - return offlineRetransmissionDescription; - } + public void setOfflineRetransmissionEnabled(boolean offlineRetransmissionEnabled) { + this.offlineRetransmissionEnabled = offlineRetransmissionEnabled; + } - public void setOfflineRetransmissionDescription(String offlineRetransmissionDescription) { - this.offlineRetransmissionDescription = offlineRetransmissionDescription; - } + public String getOfflineRetransmissionDescription() { + return offlineRetransmissionDescription; } - public static final class DefaultTopicView { - private String ack = "LEADER"; - private String contentType = "JSON"; - private RetentionTime retentionTime = new RetentionTime(); - private DefaultOfflineStorageView offlineStorage = new DefaultOfflineStorageView(); + public void setOfflineRetransmissionDescription(String offlineRetransmissionDescription) { + this.offlineRetransmissionDescription = offlineRetransmissionDescription; + } + } - public String getAck() { - return ack; - } + public static final class DefaultTopicView { + private String ack = "LEADER"; + private String contentType = "JSON"; + private RetentionTime retentionTime = new RetentionTime(); + private DefaultOfflineStorageView offlineStorage = new DefaultOfflineStorageView(); - public void setAck(String ack) { - this.ack = ack; - } + public String getAck() { + return ack; + } - public String getContentType() { - return contentType; - } + public void setAck(String ack) { + this.ack = ack; + } - public void setContentType(String contentType) { - this.contentType = contentType; - } + public String getContentType() { + return contentType; + } - public RetentionTime getRetentionTime() { - return retentionTime; - } + public void setContentType(String contentType) { + this.contentType = contentType; + } - public void setRetentionTime(RetentionTime retentionTime) { - this.retentionTime = retentionTime; - } + public RetentionTime getRetentionTime() { + return retentionTime; + } - public DefaultOfflineStorageView getOfflineStorage() { - return offlineStorage; - } + public void setRetentionTime(RetentionTime retentionTime) { + this.retentionTime = retentionTime; + } - public void setOfflineStorage(DefaultOfflineStorageView offlineStorage) { - this.offlineStorage = offlineStorage; - } + public DefaultOfflineStorageView getOfflineStorage() { + return offlineStorage; } - public static final class DefaultOfflineStorageView { - private boolean enabled = false; - private RetentionTime retentionTime = new RetentionTime(); + public void setOfflineStorage(DefaultOfflineStorageView offlineStorage) { + this.offlineStorage = offlineStorage; + } + } - public boolean isEnabled() { - return enabled; - } + public static final class DefaultOfflineStorageView { + private boolean enabled = false; + private RetentionTime retentionTime = new RetentionTime(); - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } - public RetentionTime getRetentionTime() { - return retentionTime; - } + public RetentionTime getRetentionTime() { + return retentionTime; + } - public void setRetentionTime(RetentionTime retentionTime) { - this.retentionTime = retentionTime; - } + public void setRetentionTime(RetentionTime retentionTime) { + this.retentionTime = retentionTime; } + } - public static final class RetentionTime { - private int duration = 1; - private TimeUnit retentionUnit = TimeUnit.DAYS; + public static final class RetentionTime { + private int duration = 1; + private TimeUnit retentionUnit = TimeUnit.DAYS; - public int getDuration() { - return duration; - } + public int getDuration() { + return duration; + } - public void setDuration(int duration) { - this.duration = duration; - } + public void setDuration(int duration) { + this.duration = duration; + } - public TimeUnit getRetentionUnit() { - return retentionUnit; - } + public TimeUnit getRetentionUnit() { + return retentionUnit; + } - public void setRetentionUnit(TimeUnit retentionUnit) { - this.retentionUnit = retentionUnit; - } + public void setRetentionUnit(TimeUnit retentionUnit) { + this.retentionUnit = retentionUnit; } + } - public static final class TopicContentType { - private String value = ""; - private String label = ""; + public static final class TopicContentType { + private String value = ""; + private String label = ""; - public TopicContentType() { - } + public TopicContentType() {} - public TopicContentType(String value, String label) { - this.value = value; - this.label = label; - } + public TopicContentType(String value, String label) { + this.value = value; + this.label = label; + } - public String getValue() { - return value; - } + public String getValue() { + return value; + } - public void setValue(String value) { - this.value = value; - } + public void setValue(String value) { + this.value = value; + } - public String getLabel() { - return label; - } + public String getLabel() { + return label; + } - public void setLabel(String label) { - this.label = label; - } + public void setLabel(String label) { + this.label = label; } + } - public static final class RetentionUnit { - private String value = ""; - private String label = ""; + public static final class RetentionUnit { + private String value = ""; + private String label = ""; - public RetentionUnit() { - } + public RetentionUnit() {} - public RetentionUnit(String value, String label) { - this.value = value; - this.label = label; - } + public RetentionUnit(String value, String label) { + this.value = value; + this.label = label; + } - public String getValue() { - return value; - } + public String getValue() { + return value; + } - public void setValue(String value) { - this.value = value; - } + public void setValue(String value) { + this.value = value; + } - public String getLabel() { - return label; - } + public String getLabel() { + return label; + } - public void setLabel(String label) { - this.label = label; - } + public void setLabel(String label) { + this.label = label; } + } - public static final class Owner { - private List sources = Lists.newArrayList( - new OwnerSource("Plaintext", "Name of the owner")); + public static final class Owner { + private List sources = + Lists.newArrayList(new OwnerSource("Plaintext", "Name of the owner")); - public List getSources() { - return sources; - } + public List getSources() { + return sources; + } - public void setSources(List sources) { - this.sources = sources; - } + public void setSources(List sources) { + this.sources = sources; } + } - public static final class OwnerSource { - private String name = ""; - private String placeholder = ""; + public static final class OwnerSource { + private String name = ""; + private String placeholder = ""; - public OwnerSource() { - } + public OwnerSource() {} - public OwnerSource(String name, String placeholder) { - this.name = name; - this.placeholder = placeholder; - } + public OwnerSource(String name, String placeholder) { + this.name = name; + this.placeholder = placeholder; + } - public String getName() { - return name; - } + public String getName() { + return name; + } - public void setName(String name) { - this.name = name; - } + public void setName(String name) { + this.name = name; + } - public String getPlaceholder() { - return placeholder; - } + public String getPlaceholder() { + return placeholder; + } - public void setPlaceholder(String placeholder) { - this.placeholder = placeholder; - } + public void setPlaceholder(String placeholder) { + this.placeholder = placeholder; } + } - public static final class SubscriptionView { - private Map endpointAddressResolverMetadata = new HashMap<>(); - private boolean showHeadersFilter = false; - private boolean showFixedHeaders = false; - private int requestTimeoutWarningThreshold = 1001; - private DefaultSubscriptionView defaults = new DefaultSubscriptionView(); - private List deliveryTypes = Lists.newArrayList( - new SubscriptionDeliveryType("SERIAL", "SERIAL"), - new SubscriptionDeliveryType("BATCH", "BATCH") - ); + public static final class SubscriptionView { + private Map endpointAddressResolverMetadata = + new HashMap<>(); + private boolean showHeadersFilter = false; + private boolean showFixedHeaders = false; + private int requestTimeoutWarningThreshold = 1001; + private DefaultSubscriptionView defaults = new DefaultSubscriptionView(); + private List deliveryTypes = + Lists.newArrayList( + new SubscriptionDeliveryType("SERIAL", "SERIAL"), + new SubscriptionDeliveryType("BATCH", "BATCH")); - public Map getEndpointAddressResolverMetadata() { - return endpointAddressResolverMetadata; - } + public Map getEndpointAddressResolverMetadata() { + return endpointAddressResolverMetadata; + } - public void setEndpointAddressResolverMetadata(Map endpointAddressResolverMetadata) { - this.endpointAddressResolverMetadata = endpointAddressResolverMetadata; - } + public void setEndpointAddressResolverMetadata( + Map endpointAddressResolverMetadata) { + this.endpointAddressResolverMetadata = endpointAddressResolverMetadata; + } - public boolean isShowHeadersFilter() { - return showHeadersFilter; - } + public boolean isShowHeadersFilter() { + return showHeadersFilter; + } - public void setShowHeadersFilter(boolean showHeadersFilter) { - this.showHeadersFilter = showHeadersFilter; - } + public void setShowHeadersFilter(boolean showHeadersFilter) { + this.showHeadersFilter = showHeadersFilter; + } - public DefaultSubscriptionView getDefaults() { - return defaults; - } + public DefaultSubscriptionView getDefaults() { + return defaults; + } - public void setDefaults(DefaultSubscriptionView defaults) { - this.defaults = defaults; - } + public void setDefaults(DefaultSubscriptionView defaults) { + this.defaults = defaults; + } - public List getDeliveryTypes() { - return deliveryTypes; - } + public List getDeliveryTypes() { + return deliveryTypes; + } - public void setDeliveryTypes(List deliveryTypes) { - this.deliveryTypes = deliveryTypes; - } + public void setDeliveryTypes(List deliveryTypes) { + this.deliveryTypes = deliveryTypes; + } - public boolean isShowFixedHeaders() { - return showFixedHeaders; - } + public boolean isShowFixedHeaders() { + return showFixedHeaders; + } - public void setShowFixedHeaders(boolean showFixedHeaders) { - this.showFixedHeaders = showFixedHeaders; - } + public void setShowFixedHeaders(boolean showFixedHeaders) { + this.showFixedHeaders = showFixedHeaders; + } - public int getRequestTimeoutWarningThreshold() { - return requestTimeoutWarningThreshold; - } + public int getRequestTimeoutWarningThreshold() { + return requestTimeoutWarningThreshold; + } - public void setRequestTimeoutWarningThreshold(int requestTimeoutWarningThreshold) { - this.requestTimeoutWarningThreshold = requestTimeoutWarningThreshold; - } + public void setRequestTimeoutWarningThreshold(int requestTimeoutWarningThreshold) { + this.requestTimeoutWarningThreshold = requestTimeoutWarningThreshold; } + } - public static final class GroupView { - private boolean nonAdminCreationEnabled = false; + public static final class GroupView { + private boolean nonAdminCreationEnabled = false; - public boolean isNonAdminCreationEnabled() { - return nonAdminCreationEnabled; - } + public boolean isNonAdminCreationEnabled() { + return nonAdminCreationEnabled; + } - public void setNonAdminCreationEnabled(boolean nonAdminCreationEnabled) { - this.nonAdminCreationEnabled = nonAdminCreationEnabled; - } + public void setNonAdminCreationEnabled(boolean nonAdminCreationEnabled) { + this.nonAdminCreationEnabled = nonAdminCreationEnabled; } + } - public static final class EndpointAddressResolverMetadata { - private String title; - private String type; - private String hint; + public static final class EndpointAddressResolverMetadata { + private String title; + private String type; + private String hint; - public String getTitle() { - return title; - } + public String getTitle() { + return title; + } - public void setTitle(String title) { - this.title = title; - } + public void setTitle(String title) { + this.title = title; + } - public String getType() { - return type; - } + public String getType() { + return type; + } - public void setType(String type) { - this.type = type; - } + public void setType(String type) { + this.type = type; + } - public String getHint() { - return hint; - } + public String getHint() { + return hint; + } - public void setHint(String hint) { - this.hint = hint; - } + public void setHint(String hint) { + this.hint = hint; } + } - public static final class DefaultSubscriptionView { - private SubscriptionPolicy subscriptionPolicy = new SubscriptionPolicy(); - private String deliveryType = "SERIAL"; + public static final class DefaultSubscriptionView { + private SubscriptionPolicy subscriptionPolicy = new SubscriptionPolicy(); + private String deliveryType = "SERIAL"; - public SubscriptionPolicy getSubscriptionPolicy() { - return subscriptionPolicy; - } + public SubscriptionPolicy getSubscriptionPolicy() { + return subscriptionPolicy; + } - public void setSubscriptionPolicy(SubscriptionPolicy subscriptionPolicy) { - this.subscriptionPolicy = subscriptionPolicy; - } + public void setSubscriptionPolicy(SubscriptionPolicy subscriptionPolicy) { + this.subscriptionPolicy = subscriptionPolicy; + } - public String getDeliveryType() { - return deliveryType; - } + public String getDeliveryType() { + return deliveryType; + } - public void setDeliveryType(String deliveryType) { - this.deliveryType = deliveryType; - } + public void setDeliveryType(String deliveryType) { + this.deliveryType = deliveryType; } + } - public static final class SubscriptionPolicy { - private int messageTtl = 3600; - private int requestTimeout = 1000; + public static final class SubscriptionPolicy { + private int messageTtl = 3600; + private int requestTimeout = 1000; - public int getMessageTtl() { - return messageTtl; - } + public int getMessageTtl() { + return messageTtl; + } - public void setMessageTtl(int messageTtl) { - this.messageTtl = messageTtl; - } + public void setMessageTtl(int messageTtl) { + this.messageTtl = messageTtl; + } - public int getRequestTimeout() { - return requestTimeout; - } + public int getRequestTimeout() { + return requestTimeout; + } - public void setRequestTimeout(int requestTimeout) { - this.requestTimeout = requestTimeout; - } + public void setRequestTimeout(int requestTimeout) { + this.requestTimeout = requestTimeout; } + } - public static final class SubscriptionDeliveryType { - private String value = ""; - private String label = ""; + public static final class SubscriptionDeliveryType { + private String value = ""; + private String label = ""; - public SubscriptionDeliveryType() { - } + public SubscriptionDeliveryType() {} - public SubscriptionDeliveryType(String value, String label) { - this.value = value; - this.label = label; - } + public SubscriptionDeliveryType(String value, String label) { + this.value = value; + this.label = label; + } - public String getValue() { - return value; - } + public String getValue() { + return value; + } - public void setValue(String value) { - this.value = value; - } + public void setValue(String value) { + this.value = value; + } - public String getLabel() { - return label; - } + public String getLabel() { + return label; + } - public void setLabel(String label) { - this.label = label; - } + public void setLabel(String label) { + this.label = label; } + } - public static final class ConsistencyView { - private int maxGroupBatchSize = 10; + public static final class ConsistencyView { + private int maxGroupBatchSize = 10; - public int getMaxGroupBatchSize() { - return maxGroupBatchSize; - } + public int getMaxGroupBatchSize() { + return maxGroupBatchSize; + } - public void setMaxGroupBatchSize(int maxGroupBatchSize) { - this.maxGroupBatchSize = maxGroupBatchSize; - } + public void setMaxGroupBatchSize(int maxGroupBatchSize) { + this.maxGroupBatchSize = maxGroupBatchSize; } + } - public static final class Costs { - private boolean enabled = false; - private String globalDetailsUrl = ""; - private String topicIframeUrl = ""; - private String topicDetailsUrl = ""; - private String subscriptionIframeUrl = ""; - private String subscriptionDetailsUrl = ""; + public static final class Costs { + private boolean enabled = false; + private String globalDetailsUrl = ""; + private String topicIframeUrl = ""; + private String topicDetailsUrl = ""; + private String subscriptionIframeUrl = ""; + private String subscriptionDetailsUrl = ""; - public boolean isEnabled() { - return enabled; - } + public boolean isEnabled() { + return enabled; + } - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } - public String getTopicIframeUrl() { - return topicIframeUrl; - } + public String getTopicIframeUrl() { + return topicIframeUrl; + } - public void setTopicIframeUrl(String topicIframeUrl) { - this.topicIframeUrl = topicIframeUrl; - } + public void setTopicIframeUrl(String topicIframeUrl) { + this.topicIframeUrl = topicIframeUrl; + } - public String getGlobalDetailsUrl() { - return globalDetailsUrl; - } + public String getGlobalDetailsUrl() { + return globalDetailsUrl; + } - public void setGlobalDetailsUrl(String globalDetailsUrl) { - this.globalDetailsUrl = globalDetailsUrl; - } + public void setGlobalDetailsUrl(String globalDetailsUrl) { + this.globalDetailsUrl = globalDetailsUrl; + } - public String getTopicDetailsUrl() { - return topicDetailsUrl; - } + public String getTopicDetailsUrl() { + return topicDetailsUrl; + } - public void setTopicDetailsUrl(String topicDetailsUrl) { - this.topicDetailsUrl = topicDetailsUrl; - } + public void setTopicDetailsUrl(String topicDetailsUrl) { + this.topicDetailsUrl = topicDetailsUrl; + } - public String getSubscriptionIframeUrl() { - return subscriptionIframeUrl; - } + public String getSubscriptionIframeUrl() { + return subscriptionIframeUrl; + } - public void setSubscriptionIframeUrl(String subscriptionIframeUrl) { - this.subscriptionIframeUrl = subscriptionIframeUrl; - } + public void setSubscriptionIframeUrl(String subscriptionIframeUrl) { + this.subscriptionIframeUrl = subscriptionIframeUrl; + } - public String getSubscriptionDetailsUrl() { - return subscriptionDetailsUrl; - } + public String getSubscriptionDetailsUrl() { + return subscriptionDetailsUrl; + } - public void setSubscriptionDetailsUrl(String subscriptionDetailsUrl) { - this.subscriptionDetailsUrl = subscriptionDetailsUrl; - } + public void setSubscriptionDetailsUrl(String subscriptionDetailsUrl) { + this.subscriptionDetailsUrl = subscriptionDetailsUrl; } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/kafka/KafkaClustersProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/kafka/KafkaClustersProperties.java index 6ba7cc8bb4..695a7d2e93 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/kafka/KafkaClustersProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/kafka/KafkaClustersProperties.java @@ -1,38 +1,36 @@ package pl.allegro.tech.hermes.management.config.kafka; - -import org.springframework.boot.context.properties.ConfigurationProperties; - import java.util.ArrayList; import java.util.List; +import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties(prefix = "kafka") public class KafkaClustersProperties { - private List clusters = new ArrayList<>(); - private String defaultNamespace = ""; - private String namespaceSeparator = "_"; + private List clusters = new ArrayList<>(); + private String defaultNamespace = ""; + private String namespaceSeparator = "_"; - public List getClusters() { - return clusters; - } + public List getClusters() { + return clusters; + } - public void setClusters(List clusters) { - this.clusters = clusters; - } + public void setClusters(List clusters) { + this.clusters = clusters; + } - public String getDefaultNamespace() { - return defaultNamespace; - } + public String getDefaultNamespace() { + return defaultNamespace; + } - public void setDefaultNamespace(String defaultNamespace) { - this.defaultNamespace = defaultNamespace; - } + public void setDefaultNamespace(String defaultNamespace) { + this.defaultNamespace = defaultNamespace; + } - public String getNamespaceSeparator() { - return namespaceSeparator; - } + public String getNamespaceSeparator() { + return namespaceSeparator; + } - public void setNamespaceSeparator(String namespaceSeparator) { - this.namespaceSeparator = namespaceSeparator; - } + public void setNamespaceSeparator(String namespaceSeparator) { + this.namespaceSeparator = namespaceSeparator; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/kafka/KafkaConfiguration.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/kafka/KafkaConfiguration.java index 0c50ef5970..a09e205432 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/kafka/KafkaConfiguration.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/kafka/KafkaConfiguration.java @@ -1,5 +1,19 @@ package pl.allegro.tech.hermes.management.config.kafka; +import static java.time.Duration.ofMillis; +import static java.time.Duration.ofSeconds; +import static java.util.stream.Collectors.toList; +import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG; +import static org.apache.kafka.clients.CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL; +import static org.apache.kafka.clients.CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG; +import static org.apache.kafka.clients.CommonClientConfigs.SECURITY_PROTOCOL_CONFIG; +import static org.apache.kafka.common.config.SaslConfigs.SASL_JAAS_CONFIG; +import static org.apache.kafka.common.config.SaslConfigs.SASL_MECHANISM; + +import java.time.Clock; +import java.util.List; +import java.util.Properties; +import java.util.stream.Collectors; import org.apache.kafka.clients.admin.AdminClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; @@ -34,152 +48,162 @@ import pl.allegro.tech.hermes.schema.SchemaRepository; import tech.allegro.schema.json2avro.converter.JsonAvroConverter; -import java.time.Clock; -import java.util.List; -import java.util.Properties; -import java.util.stream.Collectors; - -import static java.time.Duration.ofMillis; -import static java.time.Duration.ofSeconds; -import static java.util.stream.Collectors.toList; -import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG; -import static org.apache.kafka.clients.CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL; -import static org.apache.kafka.clients.CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG; -import static org.apache.kafka.clients.CommonClientConfigs.SECURITY_PROTOCOL_CONFIG; -import static org.apache.kafka.common.config.SaslConfigs.SASL_JAAS_CONFIG; -import static org.apache.kafka.common.config.SaslConfigs.SASL_MECHANISM; - @Configuration @EnableConfigurationProperties(KafkaClustersProperties.class) public class KafkaConfiguration implements MultipleDcKafkaNamesMappersFactory { - @Autowired - KafkaClustersProperties kafkaClustersProperties; - - @Autowired - TopicProperties topicProperties; - - @Autowired - SubscriptionProperties subscriptionProperties; - - @Autowired - CompositeMessageContentWrapper compositeMessageContentWrapper; - - @Autowired - ZookeeperRepositoryManager zookeeperRepositoryManager; - - @Autowired - MultiDatacenterRepositoryCommandExecutor multiDcExecutor; - - @Bean - MultiDCAwareService multiDCAwareService(KafkaNamesMappers kafkaNamesMappers, SchemaRepository schemaRepository, - Clock clock, JsonAvroConverter jsonAvroConverter) { - List> repositories = - zookeeperRepositoryManager.getRepositories(SubscriptionOffsetChangeIndicator.class); - - List clusters = kafkaClustersProperties.getClusters().stream().map(kafkaProperties -> { - KafkaNamesMapper kafkaNamesMapper = kafkaNamesMappers.getMapper(kafkaProperties.getQualifiedClusterName()); - AdminClient brokerAdminClient = brokerAdminClient(kafkaProperties); - BrokerStorage storage = brokersStorage(brokerAdminClient); - BrokerTopicManagement brokerTopicManagement = - new KafkaBrokerTopicManagement(topicProperties, brokerAdminClient, kafkaNamesMapper, kafkaProperties.getDatacenter()); - KafkaConsumerPool consumerPool = kafkaConsumersPool(kafkaProperties, storage, kafkaProperties.getBrokerList()); - KafkaRawMessageReader kafkaRawMessageReader = - new KafkaRawMessageReader(consumerPool, kafkaProperties.getKafkaConsumer().getPollTimeoutMillis()); - SubscriptionOffsetChangeIndicator subscriptionOffsetChangeIndicator = getRepository(repositories, kafkaProperties); - KafkaRetransmissionService retransmissionService = new KafkaRetransmissionService( - storage, - subscriptionOffsetChangeIndicator, - consumerPool, - kafkaNamesMapper - ); - KafkaSingleMessageReader messageReader = - new KafkaSingleMessageReader(kafkaRawMessageReader, schemaRepository, jsonAvroConverter); - return new BrokersClusterService(kafkaProperties.getQualifiedClusterName(), messageReader, - retransmissionService, brokerTopicManagement, kafkaNamesMapper, - new OffsetsAvailableChecker(consumerPool, storage), new LogEndOffsetChecker(consumerPool), - brokerAdminClient, createConsumerGroupManager(kafkaProperties, kafkaNamesMapper), - createKafkaConsumerManager(kafkaProperties, kafkaNamesMapper)); - }).collect(toList()); - - return new MultiDCAwareService( - clusters, - clock, - ofMillis(subscriptionProperties.getIntervalBetweenCheckinIfOffsetsMovedInMillis()), - ofSeconds(subscriptionProperties.getOffsetsMovedTimeoutInSeconds()), - multiDcExecutor); + @Autowired KafkaClustersProperties kafkaClustersProperties; + + @Autowired TopicProperties topicProperties; + + @Autowired SubscriptionProperties subscriptionProperties; + + @Autowired CompositeMessageContentWrapper compositeMessageContentWrapper; + + @Autowired ZookeeperRepositoryManager zookeeperRepositoryManager; + + @Autowired MultiDatacenterRepositoryCommandExecutor multiDcExecutor; + + @Bean + MultiDCAwareService multiDCAwareService( + KafkaNamesMappers kafkaNamesMappers, + SchemaRepository schemaRepository, + Clock clock, + JsonAvroConverter jsonAvroConverter) { + List> repositories = + zookeeperRepositoryManager.getRepositories(SubscriptionOffsetChangeIndicator.class); + + List clusters = + kafkaClustersProperties.getClusters().stream() + .map( + kafkaProperties -> { + KafkaNamesMapper kafkaNamesMapper = + kafkaNamesMappers.getMapper(kafkaProperties.getQualifiedClusterName()); + AdminClient brokerAdminClient = brokerAdminClient(kafkaProperties); + BrokerStorage storage = brokersStorage(brokerAdminClient); + BrokerTopicManagement brokerTopicManagement = + new KafkaBrokerTopicManagement( + topicProperties, + brokerAdminClient, + kafkaNamesMapper, + kafkaProperties.getDatacenter()); + KafkaConsumerPool consumerPool = + kafkaConsumersPool(kafkaProperties, storage, kafkaProperties.getBrokerList()); + KafkaRawMessageReader kafkaRawMessageReader = + new KafkaRawMessageReader( + consumerPool, kafkaProperties.getKafkaConsumer().getPollTimeoutMillis()); + SubscriptionOffsetChangeIndicator subscriptionOffsetChangeIndicator = + getRepository(repositories, kafkaProperties); + KafkaRetransmissionService retransmissionService = + new KafkaRetransmissionService( + storage, + subscriptionOffsetChangeIndicator, + consumerPool, + kafkaNamesMapper); + KafkaSingleMessageReader messageReader = + new KafkaSingleMessageReader( + kafkaRawMessageReader, schemaRepository, jsonAvroConverter); + return new BrokersClusterService( + kafkaProperties.getQualifiedClusterName(), + messageReader, + retransmissionService, + brokerTopicManagement, + kafkaNamesMapper, + new OffsetsAvailableChecker(consumerPool, storage), + new LogEndOffsetChecker(consumerPool), + brokerAdminClient, + createConsumerGroupManager(kafkaProperties, kafkaNamesMapper), + createKafkaConsumerManager(kafkaProperties, kafkaNamesMapper)); + }) + .collect(toList()); + + return new MultiDCAwareService( + clusters, + clock, + ofMillis(subscriptionProperties.getIntervalBetweenCheckinIfOffsetsMovedInMillis()), + ofSeconds(subscriptionProperties.getOffsetsMovedTimeoutInSeconds()), + multiDcExecutor); + } + + private ConsumerGroupManager createConsumerGroupManager( + KafkaProperties kafkaProperties, KafkaNamesMapper kafkaNamesMapper) { + return subscriptionProperties.isCreateConsumerGroupManuallyEnabled() + ? new KafkaConsumerGroupManager( + kafkaNamesMapper, + kafkaProperties.getQualifiedClusterName(), + kafkaProperties.getBrokerList(), + kafkaProperties) + : new NoOpConsumerGroupManager(); + } + + private KafkaConsumerManager createKafkaConsumerManager( + KafkaProperties kafkaProperties, KafkaNamesMapper kafkaNamesMapper) { + return new KafkaConsumerManager( + kafkaProperties, kafkaNamesMapper, kafkaProperties.getBrokerList()); + } + + private SubscriptionOffsetChangeIndicator getRepository( + List> repositories, + KafkaProperties kafkaProperties) { + if (repositories.size() == 1) { + return repositories.get(0).getRepository(); } - - private ConsumerGroupManager createConsumerGroupManager(KafkaProperties kafkaProperties, KafkaNamesMapper kafkaNamesMapper) { - return subscriptionProperties.isCreateConsumerGroupManuallyEnabled() - ? new KafkaConsumerGroupManager(kafkaNamesMapper, kafkaProperties.getQualifiedClusterName(), - kafkaProperties.getBrokerList(), kafkaProperties) - : new NoOpConsumerGroupManager(); - } - - private KafkaConsumerManager createKafkaConsumerManager(KafkaProperties kafkaProperties, - KafkaNamesMapper kafkaNamesMapper) { - return new KafkaConsumerManager(kafkaProperties, kafkaNamesMapper, kafkaProperties.getBrokerList()); - } - - private SubscriptionOffsetChangeIndicator getRepository( - List> repositories, - KafkaProperties kafkaProperties) { - if (repositories.size() == 1) { - return repositories.get(0).getRepository(); - } - return repositories.stream() - .filter(repository -> kafkaProperties.getDatacenter().equals(repository.getDatacenterName())) - .findFirst().orElseThrow(() -> - new IllegalArgumentException( - String.format("Kafka cluster dc name '%s' not matched with Zookeeper dc names: %s", - kafkaProperties.getDatacenter(), - repositories.stream() - .map(DatacenterBoundRepositoryHolder::getDatacenterName) - .collect(Collectors.joining(",")) - ) - ) - ) - .getRepository(); - } - - @Bean - @ConditionalOnMissingBean - KafkaNamesMappers kafkaNameMappers() { - return createDefaultKafkaNamesMapper(kafkaClustersProperties); - } - - private BrokerStorage brokersStorage(AdminClient kafkaAdminClient) { - return new KafkaBrokerStorage(kafkaAdminClient); - } - - private KafkaConsumerPool kafkaConsumersPool(KafkaProperties kafkaProperties, BrokerStorage brokerStorage, - String configuredBootstrapServers) { - KafkaConsumerPoolConfig config = new KafkaConsumerPoolConfig( - kafkaProperties.getKafkaConsumer().getCacheExpirationSeconds(), - kafkaProperties.getKafkaConsumer().getBufferSizeBytes(), - kafkaProperties.getKafkaConsumer().getFetchMaxWaitMillis(), - kafkaProperties.getKafkaConsumer().getFetchMinBytes(), - kafkaProperties.getKafkaConsumer().getNamePrefix(), - kafkaProperties.getKafkaConsumer().getConsumerGroupName(), - kafkaProperties.getAuthentication().isEnabled(), - kafkaProperties.getAuthentication().getMechanism(), - kafkaProperties.getAuthentication().getProtocol(), - kafkaProperties.getAuthentication().getJaasConfig()); - - return new KafkaConsumerPool(config, brokerStorage, configuredBootstrapServers); - } - - private AdminClient brokerAdminClient(KafkaProperties kafkaProperties) { - Properties props = new Properties(); - props.put(BOOTSTRAP_SERVERS_CONFIG, kafkaProperties.getBrokerList()); - props.put(SECURITY_PROTOCOL_CONFIG, DEFAULT_SECURITY_PROTOCOL); - props.put(REQUEST_TIMEOUT_MS_CONFIG, kafkaProperties.getKafkaServerRequestTimeoutMillis()); - if (kafkaProperties.getAuthentication().isEnabled()) { - props.put(SASL_MECHANISM, kafkaProperties.getAuthentication().getMechanism()); - props.put(SECURITY_PROTOCOL_CONFIG, kafkaProperties.getAuthentication().getProtocol()); - props.put(SASL_JAAS_CONFIG, kafkaProperties.getAuthentication().getJaasConfig()); - } - return AdminClient.create(props); + return repositories.stream() + .filter( + repository -> kafkaProperties.getDatacenter().equals(repository.getDatacenterName())) + .findFirst() + .orElseThrow( + () -> + new IllegalArgumentException( + String.format( + "Kafka cluster dc name '%s' not matched with Zookeeper dc names: %s", + kafkaProperties.getDatacenter(), + repositories.stream() + .map(DatacenterBoundRepositoryHolder::getDatacenterName) + .collect(Collectors.joining(","))))) + .getRepository(); + } + + @Bean + @ConditionalOnMissingBean + KafkaNamesMappers kafkaNameMappers() { + return createDefaultKafkaNamesMapper(kafkaClustersProperties); + } + + private BrokerStorage brokersStorage(AdminClient kafkaAdminClient) { + return new KafkaBrokerStorage(kafkaAdminClient); + } + + private KafkaConsumerPool kafkaConsumersPool( + KafkaProperties kafkaProperties, + BrokerStorage brokerStorage, + String configuredBootstrapServers) { + KafkaConsumerPoolConfig config = + new KafkaConsumerPoolConfig( + kafkaProperties.getKafkaConsumer().getCacheExpirationSeconds(), + kafkaProperties.getKafkaConsumer().getBufferSizeBytes(), + kafkaProperties.getKafkaConsumer().getFetchMaxWaitMillis(), + kafkaProperties.getKafkaConsumer().getFetchMinBytes(), + kafkaProperties.getKafkaConsumer().getNamePrefix(), + kafkaProperties.getKafkaConsumer().getConsumerGroupName(), + kafkaProperties.getAuthentication().isEnabled(), + kafkaProperties.getAuthentication().getMechanism(), + kafkaProperties.getAuthentication().getProtocol(), + kafkaProperties.getAuthentication().getJaasConfig()); + + return new KafkaConsumerPool(config, brokerStorage, configuredBootstrapServers); + } + + private AdminClient brokerAdminClient(KafkaProperties kafkaProperties) { + Properties props = new Properties(); + props.put(BOOTSTRAP_SERVERS_CONFIG, kafkaProperties.getBrokerList()); + props.put(SECURITY_PROTOCOL_CONFIG, DEFAULT_SECURITY_PROTOCOL); + props.put(REQUEST_TIMEOUT_MS_CONFIG, kafkaProperties.getKafkaServerRequestTimeoutMillis()); + if (kafkaProperties.getAuthentication().isEnabled()) { + props.put(SASL_MECHANISM, kafkaProperties.getAuthentication().getMechanism()); + props.put(SECURITY_PROTOCOL_CONFIG, kafkaProperties.getAuthentication().getProtocol()); + props.put(SASL_JAAS_CONFIG, kafkaProperties.getAuthentication().getJaasConfig()); } + return AdminClient.create(props); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/kafka/KafkaNamesMappers.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/kafka/KafkaNamesMappers.java index 2e26a2586a..25c51e8aa5 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/kafka/KafkaNamesMappers.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/kafka/KafkaNamesMappers.java @@ -1,19 +1,18 @@ package pl.allegro.tech.hermes.management.config.kafka; import com.google.common.collect.ImmutableMap; -import pl.allegro.tech.hermes.common.kafka.KafkaNamesMapper; - import java.util.Map; +import pl.allegro.tech.hermes.common.kafka.KafkaNamesMapper; public class KafkaNamesMappers { - private final Map mappers; + private final Map mappers; - public KafkaNamesMappers(Map mappers) { - this.mappers = ImmutableMap.copyOf(mappers); - } + public KafkaNamesMappers(Map mappers) { + this.mappers = ImmutableMap.copyOf(mappers); + } - public KafkaNamesMapper getMapper(String clusterName) { - return mappers.get(clusterName); - } -} \ No newline at end of file + public KafkaNamesMapper getMapper(String clusterName) { + return mappers.get(clusterName); + } +} diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/kafka/KafkaProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/kafka/KafkaProperties.java index 23463ec806..6f9c1ba09a 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/kafka/KafkaProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/kafka/KafkaProperties.java @@ -5,250 +5,250 @@ public class KafkaProperties implements KafkaParameters { - private String datacenter = "datacenter"; + private String datacenter = "datacenter"; - private String clusterName = "primary"; + private String clusterName = "primary"; - private String brokerList = "localhost:9092"; + private String brokerList = "localhost:9092"; - private int kafkaServerRequestTimeoutMillis = 3000; + private int kafkaServerRequestTimeoutMillis = 3000; - private int sessionTimeoutMillis = 10000; + private int sessionTimeoutMillis = 10000; - private int connectionTimeoutMillis = 3000; + private int connectionTimeoutMillis = 3000; - private int maxInflight = 10; + private int maxInflight = 10; - private int retryTimes = 3; + private int retryTimes = 3; - private int retrySleepMillis = 1000; + private int retrySleepMillis = 1000; - private String offsetsStorage = "kafka"; + private String offsetsStorage = "kafka"; - private boolean dualCommitEnabled = false; + private boolean dualCommitEnabled = false; - private String namespace = ""; + private String namespace = ""; - private KafkaConsumer kafkaConsumer = new KafkaConsumer(); + private KafkaConsumer kafkaConsumer = new KafkaConsumer(); - private KafkaAuthenticationProperties authentication = new KafkaAuthenticationProperties(); + private KafkaAuthenticationProperties authentication = new KafkaAuthenticationProperties(); - @Override - public boolean isAuthenticationEnabled() { - return authentication.isEnabled(); - } - - @Override - public String getAuthenticationMechanism() { - return authentication.getMechanism(); - } - - @Override - public String getAuthenticationProtocol() { - return authentication.getProtocol(); - } - - @Override - public String getBrokerList() { - return brokerList; - } - - @Override - public String getJaasConfig() { - authentication.getJaasConfig(); - return null; - } - - public static final class KafkaConsumer { - - private int cacheExpirationSeconds = 60; + @Override + public boolean isAuthenticationEnabled() { + return authentication.isEnabled(); + } - private int bufferSizeBytes = 64 * 1024; + @Override + public String getAuthenticationMechanism() { + return authentication.getMechanism(); + } - private int timeoutMillis = 5000; + @Override + public String getAuthenticationProtocol() { + return authentication.getProtocol(); + } - private String namePrefix = "offsetChecker"; + @Override + public String getBrokerList() { + return brokerList; + } - private int pollTimeoutMillis = 50; + @Override + public String getJaasConfig() { + authentication.getJaasConfig(); + return null; + } - private final int fetchMaxWaitMillis = 30; + public static final class KafkaConsumer { - private final int fetchMinBytes = 1; + private int cacheExpirationSeconds = 60; - private String consumerGroupName = "RETRANSMISSION_GROUP"; + private int bufferSizeBytes = 64 * 1024; - public int getCacheExpirationSeconds() { - return cacheExpirationSeconds; - } + private int timeoutMillis = 5000; - public void setCacheExpirationSeconds(int cacheExpirationSeconds) { - this.cacheExpirationSeconds = cacheExpirationSeconds; - } + private String namePrefix = "offsetChecker"; - public int getBufferSizeBytes() { - return bufferSizeBytes; - } + private int pollTimeoutMillis = 50; - public void setBufferSizeBytes(int bufferSizeBytes) { - this.bufferSizeBytes = bufferSizeBytes; - } + private final int fetchMaxWaitMillis = 30; - public int getTimeoutMillis() { - return timeoutMillis; - } + private final int fetchMinBytes = 1; - public void setTimeoutMillis(int timeoutMillis) { - this.timeoutMillis = timeoutMillis; - } + private String consumerGroupName = "RETRANSMISSION_GROUP"; - public String getNamePrefix() { - return namePrefix; - } - - public void setNamePrefix(String namePrefix) { - this.namePrefix = namePrefix; - } - - public int getPollTimeoutMillis() { - return pollTimeoutMillis; - } - - public void setPollTimeoutMillis(int pollTimeoutMillis) { - this.pollTimeoutMillis = pollTimeoutMillis; - } - - public String getConsumerGroupName() { - return consumerGroupName; - } - - public void setConsumerGroupName(String consumerGroupName) { - this.consumerGroupName = consumerGroupName; - } - - public int getFetchMaxWaitMillis() { - return fetchMaxWaitMillis; - } - - public int getFetchMinBytes() { - return fetchMinBytes; - } + public int getCacheExpirationSeconds() { + return cacheExpirationSeconds; } - public int getSessionTimeoutMillis() { - return sessionTimeoutMillis; + public void setCacheExpirationSeconds(int cacheExpirationSeconds) { + this.cacheExpirationSeconds = cacheExpirationSeconds; } - public void setSessionTimeoutMillis(int sessionTimeoutMillis) { - this.sessionTimeoutMillis = sessionTimeoutMillis; + public int getBufferSizeBytes() { + return bufferSizeBytes; } - public int getConnectionTimeoutMillis() { - return connectionTimeoutMillis; + public void setBufferSizeBytes(int bufferSizeBytes) { + this.bufferSizeBytes = bufferSizeBytes; } - public void setConnectionTimeoutMillis(int connectionTimeoutMillis) { - this.connectionTimeoutMillis = connectionTimeoutMillis; + public int getTimeoutMillis() { + return timeoutMillis; } - public int getRetryTimes() { - return retryTimes; + public void setTimeoutMillis(int timeoutMillis) { + this.timeoutMillis = timeoutMillis; } - public void setRetryTimes(int retryTimes) { - this.retryTimes = retryTimes; + public String getNamePrefix() { + return namePrefix; } - public int getRetrySleepMillis() { - return retrySleepMillis; + public void setNamePrefix(String namePrefix) { + this.namePrefix = namePrefix; } - public void setRetrySleepMillis(int retrySleepMillis) { - this.retrySleepMillis = retrySleepMillis; + public int getPollTimeoutMillis() { + return pollTimeoutMillis; } - public KafkaConsumer getKafkaConsumer() { - return kafkaConsumer; + public void setPollTimeoutMillis(int pollTimeoutMillis) { + this.pollTimeoutMillis = pollTimeoutMillis; } - public void setKafkaConsumer(KafkaConsumer kafkaConsumer) { - this.kafkaConsumer = kafkaConsumer; + public String getConsumerGroupName() { + return consumerGroupName; } - public KafkaAuthenticationProperties getAuthentication() { - return authentication; + public void setConsumerGroupName(String consumerGroupName) { + this.consumerGroupName = consumerGroupName; } - @Deprecated - public void setSasl(KafkaAuthenticationProperties sasl) { - this.authentication = sasl; + public int getFetchMaxWaitMillis() { + return fetchMaxWaitMillis; } - public void setAuthentication(KafkaAuthenticationProperties authentication) { - this.authentication = authentication; + public int getFetchMinBytes() { + return fetchMinBytes; } + } - public String getDatacenter() { - return datacenter; - } + public int getSessionTimeoutMillis() { + return sessionTimeoutMillis; + } - public void setDatacenter(String datacenter) { - this.datacenter = datacenter; - } + public void setSessionTimeoutMillis(int sessionTimeoutMillis) { + this.sessionTimeoutMillis = sessionTimeoutMillis; + } - public void setClusterName(String clusterName) { - this.clusterName = clusterName; - } + public int getConnectionTimeoutMillis() { + return connectionTimeoutMillis; + } - public String getQualifiedClusterName() { - return clusterName + "-" + datacenter; - } + public void setConnectionTimeoutMillis(int connectionTimeoutMillis) { + this.connectionTimeoutMillis = connectionTimeoutMillis; + } - public String getOffsetsStorage() { - return offsetsStorage; - } + public int getRetryTimes() { + return retryTimes; + } - public void setOffsetsStorage(String offsetsStorage) { - this.offsetsStorage = offsetsStorage; - } + public void setRetryTimes(int retryTimes) { + this.retryTimes = retryTimes; + } - public boolean isDualCommitEnabled() { - return dualCommitEnabled; - } + public int getRetrySleepMillis() { + return retrySleepMillis; + } - public void setDualCommitEnabled(boolean dualCommitEnabled) { - this.dualCommitEnabled = dualCommitEnabled; - } + public void setRetrySleepMillis(int retrySleepMillis) { + this.retrySleepMillis = retrySleepMillis; + } - public String getNamespace() { - return namespace; - } + public KafkaConsumer getKafkaConsumer() { + return kafkaConsumer; + } - public void setNamespace(String namespace) { - this.namespace = namespace; - } + public void setKafkaConsumer(KafkaConsumer kafkaConsumer) { + this.kafkaConsumer = kafkaConsumer; + } - public int getMaxInflight() { - return maxInflight; - } + public KafkaAuthenticationProperties getAuthentication() { + return authentication; + } - public void setMaxInflight(int maxInflight) { - this.maxInflight = maxInflight; - } + @Deprecated + public void setSasl(KafkaAuthenticationProperties sasl) { + this.authentication = sasl; + } - @Deprecated - public void setBootstrapKafkaServer(String bootstrapKafkaServer) { - this.brokerList = bootstrapKafkaServer; - } + public void setAuthentication(KafkaAuthenticationProperties authentication) { + this.authentication = authentication; + } - public void setBrokerList(String brokerList) { - this.brokerList = brokerList; - } + public String getDatacenter() { + return datacenter; + } - public int getKafkaServerRequestTimeoutMillis() { - return kafkaServerRequestTimeoutMillis; - } + public void setDatacenter(String datacenter) { + this.datacenter = datacenter; + } - public void setKafkaServerRequestTimeoutMillis(int kafkaServerRequestTimeoutMillis) { - this.kafkaServerRequestTimeoutMillis = kafkaServerRequestTimeoutMillis; - } + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + public String getQualifiedClusterName() { + return clusterName + "-" + datacenter; + } + + public String getOffsetsStorage() { + return offsetsStorage; + } + + public void setOffsetsStorage(String offsetsStorage) { + this.offsetsStorage = offsetsStorage; + } + + public boolean isDualCommitEnabled() { + return dualCommitEnabled; + } + + public void setDualCommitEnabled(boolean dualCommitEnabled) { + this.dualCommitEnabled = dualCommitEnabled; + } + + public String getNamespace() { + return namespace; + } + + public void setNamespace(String namespace) { + this.namespace = namespace; + } + + public int getMaxInflight() { + return maxInflight; + } + + public void setMaxInflight(int maxInflight) { + this.maxInflight = maxInflight; + } + + @Deprecated + public void setBootstrapKafkaServer(String bootstrapKafkaServer) { + this.brokerList = bootstrapKafkaServer; + } + + public void setBrokerList(String brokerList) { + this.brokerList = brokerList; + } + + public int getKafkaServerRequestTimeoutMillis() { + return kafkaServerRequestTimeoutMillis; + } + + public void setKafkaServerRequestTimeoutMillis(int kafkaServerRequestTimeoutMillis) { + this.kafkaServerRequestTimeoutMillis = kafkaServerRequestTimeoutMillis; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/kafka/MultipleDcKafkaNamesMappersFactory.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/kafka/MultipleDcKafkaNamesMappersFactory.java index 9f23ad0186..6bd2e853ef 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/kafka/MultipleDcKafkaNamesMappersFactory.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/kafka/MultipleDcKafkaNamesMappersFactory.java @@ -1,32 +1,42 @@ package pl.allegro.tech.hermes.management.config.kafka; -import pl.allegro.tech.hermes.common.kafka.KafkaNamesMapper; -import pl.allegro.tech.hermes.common.kafka.NamespaceKafkaNamesMapper; +import static java.util.stream.Collectors.toMap; import java.util.Map; import java.util.function.Function; - -import static java.util.stream.Collectors.toMap; +import pl.allegro.tech.hermes.common.kafka.KafkaNamesMapper; +import pl.allegro.tech.hermes.common.kafka.NamespaceKafkaNamesMapper; public interface MultipleDcKafkaNamesMappersFactory { - default KafkaNamesMappers createDefaultKafkaNamesMapper(KafkaClustersProperties clustersProperties) { - return createKafkaNamesMapper(clustersProperties, - namespace -> new NamespaceKafkaNamesMapper(namespace, clustersProperties.getNamespaceSeparator())); - } - - default KafkaNamesMappers createKafkaNamesMapper(KafkaClustersProperties clustersProperties, - Function factoryFunction) { - Map mappers = clustersProperties.getClusters().stream() - .filter(c -> c.getNamespace().isEmpty()) - .collect(toMap(KafkaProperties::getQualifiedClusterName, - kafkaProperties -> factoryFunction.apply(clustersProperties.getDefaultNamespace()))); - - mappers.putAll(clustersProperties.getClusters().stream() - .filter(c -> !c.getNamespace().isEmpty()) - .collect(toMap(KafkaProperties::getQualifiedClusterName, - kafkaProperties -> factoryFunction.apply(kafkaProperties.getNamespace())))); - - return new KafkaNamesMappers(mappers); - } + default KafkaNamesMappers createDefaultKafkaNamesMapper( + KafkaClustersProperties clustersProperties) { + return createKafkaNamesMapper( + clustersProperties, + namespace -> + new NamespaceKafkaNamesMapper(namespace, clustersProperties.getNamespaceSeparator())); + } + + default KafkaNamesMappers createKafkaNamesMapper( + KafkaClustersProperties clustersProperties, + Function factoryFunction) { + Map mappers = + clustersProperties.getClusters().stream() + .filter(c -> c.getNamespace().isEmpty()) + .collect( + toMap( + KafkaProperties::getQualifiedClusterName, + kafkaProperties -> + factoryFunction.apply(clustersProperties.getDefaultNamespace()))); + + mappers.putAll( + clustersProperties.getClusters().stream() + .filter(c -> !c.getNamespace().isEmpty()) + .collect( + toMap( + KafkaProperties::getQualifiedClusterName, + kafkaProperties -> factoryFunction.apply(kafkaProperties.getNamespace())))); + + return new KafkaNamesMappers(mappers); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/storage/DefaultZookeeperGroupRepositoryFactory.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/storage/DefaultZookeeperGroupRepositoryFactory.java index 9483f0364b..cb1330a5e3 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/storage/DefaultZookeeperGroupRepositoryFactory.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/storage/DefaultZookeeperGroupRepositoryFactory.java @@ -7,8 +7,9 @@ import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; public class DefaultZookeeperGroupRepositoryFactory implements ZookeeperGroupRepositoryFactory { - @Override - public GroupRepository create(CuratorFramework zookeeper, ObjectMapper mapper, ZookeeperPaths paths) { - return new ZookeeperGroupRepository(zookeeper, mapper, paths); - } + @Override + public GroupRepository create( + CuratorFramework zookeeper, ObjectMapper mapper, ZookeeperPaths paths) { + return new ZookeeperGroupRepository(zookeeper, mapper, paths); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/storage/StorageAuthorizationProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/storage/StorageAuthorizationProperties.java index 90a05251a9..457d829be6 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/storage/StorageAuthorizationProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/storage/StorageAuthorizationProperties.java @@ -5,31 +5,31 @@ @ConfigurationProperties(prefix = "authorization") public class StorageAuthorizationProperties { - private String scheme; - private String user; - private String password; + private String scheme; + private String user; + private String password; - public String getScheme() { - return scheme; - } + public String getScheme() { + return scheme; + } - public void setScheme(String scheme) { - this.scheme = scheme; - } + public void setScheme(String scheme) { + this.scheme = scheme; + } - public String getUser() { - return user; - } + public String getUser() { + return user; + } - public void setUser(String user) { - this.user = user; - } + public void setUser(String user) { + this.user = user; + } - public String getPassword() { - return password; - } + public String getPassword() { + return password; + } - public void setPassword(String password) { - this.password = password; - } + public void setPassword(String password) { + this.password = password; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/storage/StorageClustersProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/storage/StorageClustersProperties.java index 0244233f7c..f2175fafa7 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/storage/StorageClustersProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/storage/StorageClustersProperties.java @@ -1,133 +1,130 @@ package pl.allegro.tech.hermes.management.config.storage; +import java.util.ArrayList; +import java.util.List; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.NestedConfigurationProperty; import pl.allegro.tech.hermes.infrastructure.dc.DcNameSource; -import java.util.ArrayList; -import java.util.List; - @ConfigurationProperties(prefix = "storage") public class StorageClustersProperties { - private String pathPrefix = "/hermes"; - private int retryTimes = 3; - private int retrySleep = 1000; - private int sharedCountersExpiration = 72; - private DcNameSource datacenterNameSource; - private String datacenterNameSourceEnv = "DC"; - private boolean transactional = true; - private int maxConcurrentOperations = 100; - private String connectionString = "localhost:2181"; - private int sessionTimeout = 10000; - private int connectTimeout = 1000; - private List clusters = new ArrayList<>(); - - @NestedConfigurationProperty - private StorageAuthorizationProperties authorization; - - public String getPathPrefix() { - return pathPrefix; - } - - public void setPathPrefix(String pathPrefix) { - this.pathPrefix = pathPrefix; - } - - public int getRetryTimes() { - return retryTimes; - } - - public void setRetryTimes(int retryTimes) { - this.retryTimes = retryTimes; - } - - public int getRetrySleep() { - return retrySleep; - } - - public void setRetrySleep(int retrySleep) { - this.retrySleep = retrySleep; - } - - public int getSharedCountersExpiration() { - return sharedCountersExpiration; - } - - public void setSharedCountersExpiration(int sharedCountersExpiration) { - this.sharedCountersExpiration = sharedCountersExpiration; - } - - public StorageAuthorizationProperties getAuthorization() { - return authorization; - } - - public void setAuthorization(StorageAuthorizationProperties authorization) { - this.authorization = authorization; - } - - public List getClusters() { - return clusters; - } - - public void setClusters(List clusters) { - this.clusters = clusters; - } - - public DcNameSource getDatacenterNameSource() { - return datacenterNameSource; - } - - public void setDatacenterNameSource(DcNameSource datacenterNameSource) { - this.datacenterNameSource = datacenterNameSource; - } - - public String getDatacenterNameSourceEnv() { - return datacenterNameSourceEnv; - } - - public void setDatacenterNameSourceEnv(String datacenterNameSourceEnv) { - this.datacenterNameSourceEnv = datacenterNameSourceEnv; - } - - public boolean isTransactional() { - return transactional; - } - - public void setTransactional(boolean transactional) { - this.transactional = transactional; - } - - public int getMaxConcurrentOperations() { - return maxConcurrentOperations; - } - - public void setMaxConcurrentOperations(int maxConcurrentOperations) { - this.maxConcurrentOperations = maxConcurrentOperations; - } - - public String getConnectionString() { - return connectionString; - } - - public void setConnectionString(String connectionString) { - this.connectionString = connectionString; - } - - public int getSessionTimeout() { - return sessionTimeout; - } - - public void setSessionTimeout(int sessionTimeout) { - this.sessionTimeout = sessionTimeout; - } - - public int getConnectTimeout() { - return connectTimeout; - } + private String pathPrefix = "/hermes"; + private int retryTimes = 3; + private int retrySleep = 1000; + private int sharedCountersExpiration = 72; + private DcNameSource datacenterNameSource; + private String datacenterNameSourceEnv = "DC"; + private boolean transactional = true; + private int maxConcurrentOperations = 100; + private String connectionString = "localhost:2181"; + private int sessionTimeout = 10000; + private int connectTimeout = 1000; + private List clusters = new ArrayList<>(); + + @NestedConfigurationProperty private StorageAuthorizationProperties authorization; + + public String getPathPrefix() { + return pathPrefix; + } + + public void setPathPrefix(String pathPrefix) { + this.pathPrefix = pathPrefix; + } + + public int getRetryTimes() { + return retryTimes; + } + + public void setRetryTimes(int retryTimes) { + this.retryTimes = retryTimes; + } + + public int getRetrySleep() { + return retrySleep; + } + + public void setRetrySleep(int retrySleep) { + this.retrySleep = retrySleep; + } + + public int getSharedCountersExpiration() { + return sharedCountersExpiration; + } + + public void setSharedCountersExpiration(int sharedCountersExpiration) { + this.sharedCountersExpiration = sharedCountersExpiration; + } + + public StorageAuthorizationProperties getAuthorization() { + return authorization; + } + + public void setAuthorization(StorageAuthorizationProperties authorization) { + this.authorization = authorization; + } + + public List getClusters() { + return clusters; + } + + public void setClusters(List clusters) { + this.clusters = clusters; + } + + public DcNameSource getDatacenterNameSource() { + return datacenterNameSource; + } + + public void setDatacenterNameSource(DcNameSource datacenterNameSource) { + this.datacenterNameSource = datacenterNameSource; + } + + public String getDatacenterNameSourceEnv() { + return datacenterNameSourceEnv; + } + + public void setDatacenterNameSourceEnv(String datacenterNameSourceEnv) { + this.datacenterNameSourceEnv = datacenterNameSourceEnv; + } + + public boolean isTransactional() { + return transactional; + } + + public void setTransactional(boolean transactional) { + this.transactional = transactional; + } + + public int getMaxConcurrentOperations() { + return maxConcurrentOperations; + } + + public void setMaxConcurrentOperations(int maxConcurrentOperations) { + this.maxConcurrentOperations = maxConcurrentOperations; + } + + public String getConnectionString() { + return connectionString; + } + + public void setConnectionString(String connectionString) { + this.connectionString = connectionString; + } + + public int getSessionTimeout() { + return sessionTimeout; + } + + public void setSessionTimeout(int sessionTimeout) { + this.sessionTimeout = sessionTimeout; + } - public void setConnectTimeout(int connectTimeout) { - this.connectTimeout = connectTimeout; - } + public int getConnectTimeout() { + return connectTimeout; + } + public void setConnectTimeout(int connectTimeout) { + this.connectTimeout = connectTimeout; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/storage/StorageConfiguration.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/storage/StorageConfiguration.java index 9bf60b1c79..ef050ca2d5 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/storage/StorageConfiguration.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/storage/StorageConfiguration.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.management.config.storage; +import static org.slf4j.LoggerFactory.getLogger; + import com.fasterxml.jackson.databind.ObjectMapper; import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Autowired; @@ -40,133 +42,139 @@ import pl.allegro.tech.hermes.management.infrastructure.zookeeper.ZookeeperClientManager; import pl.allegro.tech.hermes.management.infrastructure.zookeeper.ZookeeperRepositoryManager; -import static org.slf4j.LoggerFactory.getLogger; - @Configuration @EnableConfigurationProperties(StorageClustersProperties.class) public class StorageConfiguration { - private static final Logger logger = getLogger(StorageConfiguration.class); - - @Autowired - StorageClustersProperties storageClustersProperties; - - @Autowired - ObjectMapper objectMapper; - - @Bean - DatacenterNameProvider dcNameProvider() { - if (storageClustersProperties.getDatacenterNameSource() == DcNameSource.ENV) { - return new EnvironmentVariableDatacenterNameProvider(storageClustersProperties.getDatacenterNameSourceEnv()); - } else { - return new DefaultDatacenterNameProvider(); - } - } - - @Bean(initMethod = "start", destroyMethod = "stop") - ZookeeperClientManager clientManager() { - return new ZookeeperClientManager(storageClustersProperties, dcNameProvider()); - } - - @Bean - ZookeeperGroupRepositoryFactory zookeeperGroupRepositoryFactory() { - return new DefaultZookeeperGroupRepositoryFactory(); - } - - @Bean(initMethod = "start") - ZookeeperRepositoryManager repositoryManager(ZookeeperGroupRepositoryFactory zookeeperGroupRepositoryFactory) { - return new ZookeeperRepositoryManager(clientManager(), dcNameProvider(), objectMapper, - zookeeperPaths(), zookeeperGroupRepositoryFactory); - } - - @Bean - ZookeeperPaths zookeeperPaths() { - return new ZookeeperPaths(storageClustersProperties.getPathPrefix()); - } - - @Bean - MultiDatacenterRepositoryCommandExecutor multiDcRepositoryCommandExecutor( - ZookeeperGroupRepositoryFactory zookeeperGroupRepositoryFactory, - ModeService modeService - ) { - return new MultiDatacenterRepositoryCommandExecutor( - repositoryManager(zookeeperGroupRepositoryFactory), - storageClustersProperties.isTransactional(), - modeService - ); - } - - @Bean - SummedSharedCounter summedSharedCounter(ZookeeperClientManager manager) { - return new SummedSharedCounter( - manager.getClients(), - storageClustersProperties.getSharedCountersExpiration(), - storageClustersProperties.getRetrySleep(), - storageClustersProperties.getRetryTimes() - ); - } - - @Bean - GroupRepository groupRepository() { - ZookeeperClient localClient = clientManager().getLocalClient(); - return new ZookeeperGroupRepository(localClient.getCuratorFramework(), objectMapper, zookeeperPaths()); - } - - @Bean - CredentialsRepository credentialsRepository() { - ZookeeperClient localClient = clientManager().getLocalClient(); - return new ZookeeperCredentialsRepository(localClient.getCuratorFramework(), objectMapper, zookeeperPaths()); - } - - @Bean - TopicRepository topicRepository() { - ZookeeperClient localClient = clientManager().getLocalClient(); - return new ZookeeperTopicRepository(localClient.getCuratorFramework(), objectMapper, zookeeperPaths(), - groupRepository()); - } - - @Bean - SubscriptionRepository subscriptionRepository() { - ZookeeperClient localClient = clientManager().getLocalClient(); - return new ZookeeperSubscriptionRepository(localClient.getCuratorFramework(), objectMapper, zookeeperPaths(), - topicRepository()); - } - - @Bean - OAuthProviderRepository oAuthProviderRepository() { - ZookeeperClient localClient = clientManager().getLocalClient(); - return new ZookeeperOAuthProviderRepository(localClient.getCuratorFramework(), objectMapper, zookeeperPaths()); - } - - @Bean - MessagePreviewRepository messagePreviewRepository() { - ZookeeperClient localClient = clientManager().getLocalClient(); - return new ZookeeperMessagePreviewRepository(localClient.getCuratorFramework(), objectMapper, zookeeperPaths()); - } - - @Bean - TopicBlacklistRepository topicBlacklistRepository() { - ZookeeperClient localClient = clientManager().getLocalClient(); - return new ZookeeperTopicBlacklistRepository(localClient.getCuratorFramework(), objectMapper, zookeeperPaths()); - } - - @Bean - WorkloadConstraintsRepository workloadConstraintsRepository() { - ZookeeperClient localClient = clientManager().getLocalClient(); - return new ZookeeperWorkloadConstraintsRepository(localClient.getCuratorFramework(), objectMapper, zookeeperPaths()); - } - - @Bean - @Primary - @Qualifier("zookeeperOfflineRetransmissionRepository") - OfflineRetransmissionRepository zookeeperOfflineRetransmissionRepository() { - ZookeeperClient localClient = clientManager().getLocalClient(); - return new ZookeeperOfflineRetransmissionRepository(localClient.getCuratorFramework(), objectMapper, zookeeperPaths()); - } - - @Bean - DatacenterReadinessRepository readinessRepository() { - ZookeeperClient localClient = clientManager().getLocalClient(); - return new ZookeeperDatacenterReadinessRepository(localClient.getCuratorFramework(), objectMapper, zookeeperPaths()); - } + private static final Logger logger = getLogger(StorageConfiguration.class); + + @Autowired StorageClustersProperties storageClustersProperties; + + @Autowired ObjectMapper objectMapper; + + @Bean + DatacenterNameProvider dcNameProvider() { + if (storageClustersProperties.getDatacenterNameSource() == DcNameSource.ENV) { + return new EnvironmentVariableDatacenterNameProvider( + storageClustersProperties.getDatacenterNameSourceEnv()); + } else { + return new DefaultDatacenterNameProvider(); + } + } + + @Bean(initMethod = "start", destroyMethod = "stop") + ZookeeperClientManager clientManager() { + return new ZookeeperClientManager(storageClustersProperties, dcNameProvider()); + } + + @Bean + ZookeeperGroupRepositoryFactory zookeeperGroupRepositoryFactory() { + return new DefaultZookeeperGroupRepositoryFactory(); + } + + @Bean(initMethod = "start") + ZookeeperRepositoryManager repositoryManager( + ZookeeperGroupRepositoryFactory zookeeperGroupRepositoryFactory) { + return new ZookeeperRepositoryManager( + clientManager(), + dcNameProvider(), + objectMapper, + zookeeperPaths(), + zookeeperGroupRepositoryFactory); + } + + @Bean + ZookeeperPaths zookeeperPaths() { + return new ZookeeperPaths(storageClustersProperties.getPathPrefix()); + } + + @Bean + MultiDatacenterRepositoryCommandExecutor multiDcRepositoryCommandExecutor( + ZookeeperGroupRepositoryFactory zookeeperGroupRepositoryFactory, ModeService modeService) { + return new MultiDatacenterRepositoryCommandExecutor( + repositoryManager(zookeeperGroupRepositoryFactory), + storageClustersProperties.isTransactional(), + modeService); + } + + @Bean + SummedSharedCounter summedSharedCounter(ZookeeperClientManager manager) { + return new SummedSharedCounter( + manager.getClients(), + storageClustersProperties.getSharedCountersExpiration(), + storageClustersProperties.getRetrySleep(), + storageClustersProperties.getRetryTimes()); + } + + @Bean + GroupRepository groupRepository() { + ZookeeperClient localClient = clientManager().getLocalClient(); + return new ZookeeperGroupRepository( + localClient.getCuratorFramework(), objectMapper, zookeeperPaths()); + } + + @Bean + CredentialsRepository credentialsRepository() { + ZookeeperClient localClient = clientManager().getLocalClient(); + return new ZookeeperCredentialsRepository( + localClient.getCuratorFramework(), objectMapper, zookeeperPaths()); + } + + @Bean + TopicRepository topicRepository() { + ZookeeperClient localClient = clientManager().getLocalClient(); + return new ZookeeperTopicRepository( + localClient.getCuratorFramework(), objectMapper, zookeeperPaths(), groupRepository()); + } + + @Bean + SubscriptionRepository subscriptionRepository() { + ZookeeperClient localClient = clientManager().getLocalClient(); + return new ZookeeperSubscriptionRepository( + localClient.getCuratorFramework(), objectMapper, zookeeperPaths(), topicRepository()); + } + + @Bean + OAuthProviderRepository oAuthProviderRepository() { + ZookeeperClient localClient = clientManager().getLocalClient(); + return new ZookeeperOAuthProviderRepository( + localClient.getCuratorFramework(), objectMapper, zookeeperPaths()); + } + + @Bean + MessagePreviewRepository messagePreviewRepository() { + ZookeeperClient localClient = clientManager().getLocalClient(); + return new ZookeeperMessagePreviewRepository( + localClient.getCuratorFramework(), objectMapper, zookeeperPaths()); + } + + @Bean + TopicBlacklistRepository topicBlacklistRepository() { + ZookeeperClient localClient = clientManager().getLocalClient(); + return new ZookeeperTopicBlacklistRepository( + localClient.getCuratorFramework(), objectMapper, zookeeperPaths()); + } + + @Bean + WorkloadConstraintsRepository workloadConstraintsRepository() { + ZookeeperClient localClient = clientManager().getLocalClient(); + return new ZookeeperWorkloadConstraintsRepository( + localClient.getCuratorFramework(), objectMapper, zookeeperPaths()); + } + + @Bean + @Primary + @Qualifier("zookeeperOfflineRetransmissionRepository") + OfflineRetransmissionRepository zookeeperOfflineRetransmissionRepository() { + ZookeeperClient localClient = clientManager().getLocalClient(); + return new ZookeeperOfflineRetransmissionRepository( + localClient.getCuratorFramework(), objectMapper, zookeeperPaths()); + } + + @Bean + DatacenterReadinessRepository readinessRepository() { + ZookeeperClient localClient = clientManager().getLocalClient(); + return new ZookeeperDatacenterReadinessRepository( + localClient.getCuratorFramework(), objectMapper, zookeeperPaths()); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/storage/StorageProperties.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/storage/StorageProperties.java index 03990431e9..2ff15c277f 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/storage/StorageProperties.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/storage/StorageProperties.java @@ -1,50 +1,49 @@ package pl.allegro.tech.hermes.management.config.storage; - public class StorageProperties { - private String datacenter = "dc"; - private String clusterName = "zk"; - private String connectionString = "localhost:2181"; - private int sessionTimeout = 10000; - private int connectTimeout = 1000; - - public String getConnectionString() { - return connectionString; - } - - public void setConnectionString(String connectionString) { - this.connectionString = connectionString; - } - - public int getSessionTimeout() { - return sessionTimeout; - } - - public void setSessionTimeout(int sessionTimeout) { - this.sessionTimeout = sessionTimeout; - } - - public int getConnectTimeout() { - return connectTimeout; - } - - public void setConnectTimeout(int connectTimeout) { - this.connectTimeout = connectTimeout; - } - - public String getDatacenter() { - return datacenter; - } - - public void setDatacenter(String datacenter) { - this.datacenter = datacenter; - } - - public String getClusterName() { - return clusterName; - } - - public void setClusterName(String clusterName) { - this.clusterName = clusterName; - } + private String datacenter = "dc"; + private String clusterName = "zk"; + private String connectionString = "localhost:2181"; + private int sessionTimeout = 10000; + private int connectTimeout = 1000; + + public String getConnectionString() { + return connectionString; + } + + public void setConnectionString(String connectionString) { + this.connectionString = connectionString; + } + + public int getSessionTimeout() { + return sessionTimeout; + } + + public void setSessionTimeout(int sessionTimeout) { + this.sessionTimeout = sessionTimeout; + } + + public int getConnectTimeout() { + return connectTimeout; + } + + public void setConnectTimeout(int connectTimeout) { + this.connectTimeout = connectTimeout; + } + + public String getDatacenter() { + return datacenter; + } + + public void setDatacenter(String datacenter) { + this.datacenter = datacenter; + } + + public String getClusterName() { + return clusterName; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/storage/ZookeeperGroupRepositoryFactory.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/storage/ZookeeperGroupRepositoryFactory.java index 50b8b965a5..b50445b48e 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/storage/ZookeeperGroupRepositoryFactory.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/config/storage/ZookeeperGroupRepositoryFactory.java @@ -6,5 +6,5 @@ import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; public interface ZookeeperGroupRepositoryFactory { - GroupRepository create(CuratorFramework zookeeper, ObjectMapper mapper, ZookeeperPaths paths); + GroupRepository create(CuratorFramework zookeeper, ObjectMapper mapper, ZookeeperPaths paths); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/Auditor.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/Auditor.java index e9cf18116d..eb6b23ff73 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/Auditor.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/Auditor.java @@ -5,37 +5,33 @@ public interface Auditor { - default void beforeObjectCreation(String username, Object createdObject) { - } + default void beforeObjectCreation(String username, Object createdObject) {} - default void beforeObjectCreation(String username, Anonymizable createdObject) { - beforeObjectCreation(username, (Object) createdObject.anonymize()); - } + default void beforeObjectCreation(String username, Anonymizable createdObject) { + beforeObjectCreation(username, (Object) createdObject.anonymize()); + } - default void beforeObjectRemoval(String username, String removedObjectType, String removedObjectName) { - } + default void beforeObjectRemoval( + String username, String removedObjectType, String removedObjectName) {} - default void beforeObjectUpdate(String username, String objectClassName, Object objectName, PatchData patchData) { - } + default void beforeObjectUpdate( + String username, String objectClassName, Object objectName, PatchData patchData) {} - default void objectCreated(String username, Object createdObject) { - } + default void objectCreated(String username, Object createdObject) {} - default void objectCreated(String username, Anonymizable createdObject) { - objectCreated(username, (Object) createdObject.anonymize()); - } + default void objectCreated(String username, Anonymizable createdObject) { + objectCreated(username, (Object) createdObject.anonymize()); + } - default void objectRemoved(String username, Object removedObject) { - } + default void objectRemoved(String username, Object removedObject) {} - default void objectRemoved(String username, Anonymizable removedObject) { - objectRemoved(username, (Object) removedObject.anonymize()); - } + default void objectRemoved(String username, Anonymizable removedObject) { + objectRemoved(username, (Object) removedObject.anonymize()); + } - default void objectUpdated(String username, Object oldObject, Object newObject) { - } + default void objectUpdated(String username, Object oldObject, Object newObject) {} - default void objectUpdated(String username, Anonymizable oldObject, Anonymizable newObject) { - objectUpdated(username, (Object) oldObject.anonymize(), (Object) newObject.anonymize()); - } + default void objectUpdated(String username, Anonymizable oldObject, Anonymizable newObject) { + objectUpdated(username, (Object) oldObject.anonymize(), (Object) newObject.anonymize()); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/GroupNameIsNotAllowedException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/GroupNameIsNotAllowedException.java index b41d1d0330..181696ff4d 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/GroupNameIsNotAllowedException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/GroupNameIsNotAllowedException.java @@ -3,12 +3,12 @@ import pl.allegro.tech.hermes.api.ErrorCode; public class GroupNameIsNotAllowedException extends ManagementException { - public GroupNameIsNotAllowedException(String message) { - super(message); - } + public GroupNameIsNotAllowedException(String message) { + super(message); + } - @Override - public ErrorCode getCode() { - return ErrorCode.GROUP_NAME_IS_INVALID; - } + @Override + public ErrorCode getCode() { + return ErrorCode.GROUP_NAME_IS_INVALID; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/ManagementException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/ManagementException.java index 725e071a86..e540451b02 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/ManagementException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/ManagementException.java @@ -4,18 +4,17 @@ public abstract class ManagementException extends RuntimeException { - public ManagementException(Throwable t) { - super(t); - } + public ManagementException(Throwable t) { + super(t); + } - public ManagementException(String message) { - super(message); - } + public ManagementException(String message) { + super(message); + } - public ManagementException(String message, Throwable cause) { - super(message, cause); - } - - public abstract ErrorCode getCode(); + public ManagementException(String message, Throwable cause) { + super(message, cause); + } + public abstract ErrorCode getCode(); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/MetricsDashboardUrl.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/MetricsDashboardUrl.java index 1725089a5f..c134303ea9 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/MetricsDashboardUrl.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/MetricsDashboardUrl.java @@ -2,39 +2,38 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.Objects; public class MetricsDashboardUrl { - private final String url; + private final String url; - @JsonCreator - public MetricsDashboardUrl(@JsonProperty("url") String url) { - this.url = url; - } + @JsonCreator + public MetricsDashboardUrl(@JsonProperty("url") String url) { + this.url = url; + } - public String getUrl() { - return url; - } + public String getUrl() { + return url; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } - if (o == null || getClass() != o.getClass()) { - return false; - } + if (o == null || getClass() != o.getClass()) { + return false; + } - MetricsDashboardUrl that = (MetricsDashboardUrl) o; + MetricsDashboardUrl that = (MetricsDashboardUrl) o; - return Objects.equals(url, that.url); - } + return Objects.equals(url, that.url); + } - @Override - public int hashCode() { - return Objects.hash(url); - } + @Override + public int hashCode() { + return Objects.hash(url); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/MetricsDashboardUrlService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/MetricsDashboardUrlService.java index 91a04ac1a7..d4d5142e4d 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/MetricsDashboardUrlService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/MetricsDashboardUrlService.java @@ -2,7 +2,7 @@ public interface MetricsDashboardUrlService { - String getUrlForTopic(String topic); + String getUrlForTopic(String topic); - String getUrlForSubscription(String topic, String subscription); + String getUrlForSubscription(String topic, String subscription); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/PermissionDeniedException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/PermissionDeniedException.java index 20e28aa49c..6d3affd7ea 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/PermissionDeniedException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/PermissionDeniedException.java @@ -3,12 +3,12 @@ import pl.allegro.tech.hermes.api.ErrorCode; public class PermissionDeniedException extends ManagementException { - public PermissionDeniedException(String message) { - super(message); - } + public PermissionDeniedException(String message) { + super(message); + } - @Override - public ErrorCode getCode() { - return ErrorCode.PERMISSION_DENIED; - } + @Override + public ErrorCode getCode() { + return ErrorCode.PERMISSION_DENIED; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/auth/RequestUser.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/auth/RequestUser.java index 4a9b121843..939890dfa1 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/auth/RequestUser.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/auth/RequestUser.java @@ -4,9 +4,9 @@ public interface RequestUser { - String getUsername(); + String getUsername(); - boolean isAdmin(); + boolean isAdmin(); - boolean isOwner(OwnerId ownerId); + boolean isOwner(OwnerId ownerId); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/blacklist/NotUnblacklistedException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/blacklist/NotUnblacklistedException.java index 510b8cd51f..5334ae963b 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/blacklist/NotUnblacklistedException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/blacklist/NotUnblacklistedException.java @@ -1,22 +1,22 @@ package pl.allegro.tech.hermes.management.domain.blacklist; +import static pl.allegro.tech.hermes.api.ErrorCode.TOPIC_NOT_UNBLACKLISTED; + import pl.allegro.tech.hermes.api.ErrorCode; import pl.allegro.tech.hermes.common.exception.HermesException; -import static pl.allegro.tech.hermes.api.ErrorCode.TOPIC_NOT_UNBLACKLISTED; - public class NotUnblacklistedException extends HermesException { - @Override - public ErrorCode getCode() { - return TOPIC_NOT_UNBLACKLISTED; - } + @Override + public ErrorCode getCode() { + return TOPIC_NOT_UNBLACKLISTED; + } - public NotUnblacklistedException(String qualifiedTopicName, Throwable cause) { - super("Topic not unblacklisted: " + qualifiedTopicName, cause); - } + public NotUnblacklistedException(String qualifiedTopicName, Throwable cause) { + super("Topic not unblacklisted: " + qualifiedTopicName, cause); + } - public NotUnblacklistedException(String qualifiedTopicName) { - this(qualifiedTopicName, null); - } + public NotUnblacklistedException(String qualifiedTopicName) { + this(qualifiedTopicName, null); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/blacklist/TopicBlacklistRepository.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/blacklist/TopicBlacklistRepository.java index ae62045264..63dc586869 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/blacklist/TopicBlacklistRepository.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/blacklist/TopicBlacklistRepository.java @@ -4,11 +4,11 @@ public interface TopicBlacklistRepository { - void add(String qualifiedTopicName); + void add(String qualifiedTopicName); - void remove(String qualifiedTopicName); + void remove(String qualifiedTopicName); - boolean isBlacklisted(String qualifiedTopicName); + boolean isBlacklisted(String qualifiedTopicName); - List list(); + List list(); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/blacklist/TopicBlacklistService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/blacklist/TopicBlacklistService.java index 3e5350ecc5..2d775897ff 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/blacklist/TopicBlacklistService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/blacklist/TopicBlacklistService.java @@ -1,5 +1,6 @@ package pl.allegro.tech.hermes.management.domain.blacklist; +import java.util.List; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import pl.allegro.tech.hermes.management.domain.auth.RequestUser; @@ -7,35 +8,35 @@ import pl.allegro.tech.hermes.management.domain.blacklist.commands.RemoveTopicFromBlacklistRepositoryCommand; import pl.allegro.tech.hermes.management.domain.dc.MultiDatacenterRepositoryCommandExecutor; -import java.util.List; - @Component public class TopicBlacklistService { - private final TopicBlacklistRepository repository; - private final MultiDatacenterRepositoryCommandExecutor multiDcExecutor; - - @Autowired - public TopicBlacklistService(TopicBlacklistRepository repository, - MultiDatacenterRepositoryCommandExecutor multiDcExecutor) { - this.repository = repository; - this.multiDcExecutor = multiDcExecutor; - } - - public void blacklist(String qualifiedTopicName, RequestUser blacklistRequester) { - multiDcExecutor.executeByUser(new AddTopicToBlacklistRepositoryCommand(qualifiedTopicName), blacklistRequester); - } - - public void unblacklist(String qualifiedTopicName, RequestUser unblacklistRequester) { - multiDcExecutor.executeByUser(new RemoveTopicFromBlacklistRepositoryCommand(qualifiedTopicName), unblacklistRequester); - } - - public boolean isBlacklisted(String qualifiedTopicName) { - return repository.isBlacklisted(qualifiedTopicName); - } - - public List list() { - return repository.list(); - } - + private final TopicBlacklistRepository repository; + private final MultiDatacenterRepositoryCommandExecutor multiDcExecutor; + + @Autowired + public TopicBlacklistService( + TopicBlacklistRepository repository, + MultiDatacenterRepositoryCommandExecutor multiDcExecutor) { + this.repository = repository; + this.multiDcExecutor = multiDcExecutor; + } + + public void blacklist(String qualifiedTopicName, RequestUser blacklistRequester) { + multiDcExecutor.executeByUser( + new AddTopicToBlacklistRepositoryCommand(qualifiedTopicName), blacklistRequester); + } + + public void unblacklist(String qualifiedTopicName, RequestUser unblacklistRequester) { + multiDcExecutor.executeByUser( + new RemoveTopicFromBlacklistRepositoryCommand(qualifiedTopicName), unblacklistRequester); + } + + public boolean isBlacklisted(String qualifiedTopicName) { + return repository.isBlacklisted(qualifiedTopicName); + } + + public List list() { + return repository.list(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/blacklist/commands/AddTopicToBlacklistRepositoryCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/blacklist/commands/AddTopicToBlacklistRepositoryCommand.java index d4259545d8..76692ba517 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/blacklist/commands/AddTopicToBlacklistRepositoryCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/blacklist/commands/AddTopicToBlacklistRepositoryCommand.java @@ -4,34 +4,36 @@ import pl.allegro.tech.hermes.management.domain.dc.DatacenterBoundRepositoryHolder; import pl.allegro.tech.hermes.management.domain.dc.RepositoryCommand; -public class AddTopicToBlacklistRepositoryCommand extends RepositoryCommand { - - private final String qualifiedTopicName; - - public AddTopicToBlacklistRepositoryCommand(String qualifiedTopicName) { - this.qualifiedTopicName = qualifiedTopicName; - } - - @Override - public void backup(DatacenterBoundRepositoryHolder holder) {} - - @Override - public void execute(DatacenterBoundRepositoryHolder holder) { - holder.getRepository().add(qualifiedTopicName); - } - - @Override - public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) { - holder.getRepository().remove(qualifiedTopicName); - } - - @Override - public Class getRepositoryType() { - return TopicBlacklistRepository.class; - } - - @Override - public String toString() { - return "AddTopicToBlacklist(" + qualifiedTopicName + ")"; - } +public class AddTopicToBlacklistRepositoryCommand + extends RepositoryCommand { + + private final String qualifiedTopicName; + + public AddTopicToBlacklistRepositoryCommand(String qualifiedTopicName) { + this.qualifiedTopicName = qualifiedTopicName; + } + + @Override + public void backup(DatacenterBoundRepositoryHolder holder) {} + + @Override + public void execute(DatacenterBoundRepositoryHolder holder) { + holder.getRepository().add(qualifiedTopicName); + } + + @Override + public void rollback( + DatacenterBoundRepositoryHolder holder, Exception exception) { + holder.getRepository().remove(qualifiedTopicName); + } + + @Override + public Class getRepositoryType() { + return TopicBlacklistRepository.class; + } + + @Override + public String toString() { + return "AddTopicToBlacklist(" + qualifiedTopicName + ")"; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/blacklist/commands/RemoveTopicFromBlacklistRepositoryCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/blacklist/commands/RemoveTopicFromBlacklistRepositoryCommand.java index 1ca0ef3367..a69ec307fe 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/blacklist/commands/RemoveTopicFromBlacklistRepositoryCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/blacklist/commands/RemoveTopicFromBlacklistRepositoryCommand.java @@ -4,38 +4,40 @@ import pl.allegro.tech.hermes.management.domain.dc.DatacenterBoundRepositoryHolder; import pl.allegro.tech.hermes.management.domain.dc.RepositoryCommand; -public class RemoveTopicFromBlacklistRepositoryCommand extends RepositoryCommand { - private final String qualifiedTopicName; - private boolean exists = false; - - public RemoveTopicFromBlacklistRepositoryCommand(String qualifiedTopicName) { - this.qualifiedTopicName = qualifiedTopicName; +public class RemoveTopicFromBlacklistRepositoryCommand + extends RepositoryCommand { + private final String qualifiedTopicName; + private boolean exists = false; + + public RemoveTopicFromBlacklistRepositoryCommand(String qualifiedTopicName) { + this.qualifiedTopicName = qualifiedTopicName; + } + + @Override + public void backup(DatacenterBoundRepositoryHolder holder) { + exists = holder.getRepository().isBlacklisted(qualifiedTopicName); + } + + @Override + public void execute(DatacenterBoundRepositoryHolder holder) { + holder.getRepository().remove(qualifiedTopicName); + } + + @Override + public void rollback( + DatacenterBoundRepositoryHolder holder, Exception exception) { + if (exists) { + holder.getRepository().add(qualifiedTopicName); } + } - @Override - public void backup(DatacenterBoundRepositoryHolder holder) { - exists = holder.getRepository().isBlacklisted(qualifiedTopicName); - } + @Override + public Class getRepositoryType() { + return TopicBlacklistRepository.class; + } - @Override - public void execute(DatacenterBoundRepositoryHolder holder) { - holder.getRepository().remove(qualifiedTopicName); - } - - @Override - public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) { - if (exists) { - holder.getRepository().add(qualifiedTopicName); - } - } - - @Override - public Class getRepositoryType() { - return TopicBlacklistRepository.class; - } - - @Override - public String toString() { - return "RemoveTopicFromBlacklist(" + qualifiedTopicName + ")"; - } + @Override + public String toString() { + return "RemoveTopicFromBlacklist(" + qualifiedTopicName + ")"; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/clients/AllTopicClientsService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/clients/AllTopicClientsService.java index 1930721a3f..beefbc050f 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/clients/AllTopicClientsService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/clients/AllTopicClientsService.java @@ -1,10 +1,9 @@ package pl.allegro.tech.hermes.management.domain.clients; -import pl.allegro.tech.hermes.api.TopicName; - import java.util.List; +import pl.allegro.tech.hermes.api.TopicName; public interface AllTopicClientsService { - List getAllClientsByTopic(TopicName topicName); + List getAllClientsByTopic(TopicName topicName); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/clients/DefaultAllTopicClientsService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/clients/DefaultAllTopicClientsService.java index 40a137a1ca..0cc6a08490 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/clients/DefaultAllTopicClientsService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/clients/DefaultAllTopicClientsService.java @@ -1,22 +1,23 @@ package pl.allegro.tech.hermes.management.domain.clients; -import pl.allegro.tech.hermes.api.TopicName; -import pl.allegro.tech.hermes.domain.subscription.SubscriptionRepository; - import java.util.List; import java.util.stream.Collectors; +import pl.allegro.tech.hermes.api.TopicName; +import pl.allegro.tech.hermes.domain.subscription.SubscriptionRepository; public class DefaultAllTopicClientsService implements AllTopicClientsService { - private final SubscriptionRepository subscriptionRepository; + private final SubscriptionRepository subscriptionRepository; - public DefaultAllTopicClientsService(SubscriptionRepository subscriptionRepository) { - this.subscriptionRepository = subscriptionRepository; - } + public DefaultAllTopicClientsService(SubscriptionRepository subscriptionRepository) { + this.subscriptionRepository = subscriptionRepository; + } - @Override - public List getAllClientsByTopic(TopicName topicName) { - return subscriptionRepository.listSubscriptions(topicName) - .stream().map(subscription -> subscription.getOwner().getId()).distinct().collect(Collectors.toList()); - } + @Override + public List getAllClientsByTopic(TopicName topicName) { + return subscriptionRepository.listSubscriptions(topicName).stream() + .map(subscription -> subscription.getOwner().getId()) + .distinct() + .collect(Collectors.toList()); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/clients/IframeSource.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/clients/IframeSource.java index 96e83c6ae5..e611e4228a 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/clients/IframeSource.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/clients/IframeSource.java @@ -2,36 +2,35 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.Objects; public final class IframeSource { - private final String source; + private final String source; - @JsonCreator - public IframeSource(@JsonProperty("source") String source) { - this.source = source; - } + @JsonCreator + public IframeSource(@JsonProperty("source") String source) { + this.source = source; + } - public String getSource() { - return source; - } + public String getSource() { + return source; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - IframeSource that = (IframeSource) o; - return Objects.equals(source, that.source); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(source); + if (o == null || getClass() != o.getClass()) { + return false; } + IframeSource that = (IframeSource) o; + return Objects.equals(source, that.source); + } + + @Override + public int hashCode() { + return Objects.hash(source); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/clients/OfflineClientsService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/clients/OfflineClientsService.java index 4be43c29e7..a5f5225a50 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/clients/OfflineClientsService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/clients/OfflineClientsService.java @@ -3,5 +3,5 @@ import pl.allegro.tech.hermes.api.TopicName; public interface OfflineClientsService { - String getIframeSource(TopicName topic); + String getIframeSource(TopicName topic); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/consistency/ConsistencyCheckingException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/consistency/ConsistencyCheckingException.java index 279c2fd1e4..a25508af61 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/consistency/ConsistencyCheckingException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/consistency/ConsistencyCheckingException.java @@ -2,7 +2,7 @@ public class ConsistencyCheckingException extends RuntimeException { - ConsistencyCheckingException(String message, Throwable th) { - super(message, th); - } + ConsistencyCheckingException(String message, Throwable th) { + super(message, th); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/consistency/DcConsistencyService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/consistency/DcConsistencyService.java index cb20828a4f..3f193cf672 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/consistency/DcConsistencyService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/consistency/DcConsistencyService.java @@ -1,9 +1,29 @@ package pl.allegro.tech.hermes.management.domain.consistency; +import static java.util.Collections.emptyList; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toSet; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.util.concurrent.ThreadFactoryBuilder; import jakarta.annotation.PreDestroy; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.Function; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -27,320 +47,318 @@ import pl.allegro.tech.hermes.management.domain.dc.DatacenterBoundRepositoryHolder; import pl.allegro.tech.hermes.management.domain.dc.RepositoryManager; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.BiConsumer; -import java.util.function.Consumer; -import java.util.function.Function; - -import static java.util.Collections.emptyList; -import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toSet; - @Component public class DcConsistencyService { - private static final Logger logger = LoggerFactory.getLogger(DcConsistencyService.class); + private static final Logger logger = LoggerFactory.getLogger(DcConsistencyService.class); - private final ExecutorService executor; - private final ScheduledExecutorService scheduler; - private final List> groupRepositories; - private final List> topicRepositories; - private final List> subscriptionRepositories; - private final ObjectMapper objectMapper; - private final AtomicBoolean isStorageConsistent = new AtomicBoolean(true); + private final ExecutorService executor; + private final ScheduledExecutorService scheduler; + private final List> groupRepositories; + private final List> topicRepositories; + private final List> + subscriptionRepositories; + private final ObjectMapper objectMapper; + private final AtomicBoolean isStorageConsistent = new AtomicBoolean(true); - public DcConsistencyService(RepositoryManager repositoryManager, - ObjectMapper objectMapper, - ConsistencyCheckerProperties properties, - MetricsFacade metricsFacade) { - this.groupRepositories = repositoryManager.getRepositories(GroupRepository.class); - this.topicRepositories = repositoryManager.getRepositories(TopicRepository.class); - this.subscriptionRepositories = repositoryManager.getRepositories(SubscriptionRepository.class); - this.objectMapper = objectMapper; - this.executor = Executors.newFixedThreadPool( - properties.getThreadPoolSize(), - new ThreadFactoryBuilder() - .setNameFormat("consistency-checker-%d") - .build() - ); - this.scheduler = Executors.newSingleThreadScheduledExecutor( - new ThreadFactoryBuilder() - .setNameFormat("consistency-checker-scheduler-%d") - .build() - ); - if (properties.isPeriodicCheckEnabled()) { - scheduler.scheduleAtFixedRate(this::reportConsistency, - properties.getInitialRefreshDelay().getSeconds(), - properties.getRefreshInterval().getSeconds(), - TimeUnit.SECONDS); - metricsFacade.consistency().registerStorageConsistencyGauge(isStorageConsistent, isConsistent -> isConsistent.get() ? 1 : 0); - } + public DcConsistencyService( + RepositoryManager repositoryManager, + ObjectMapper objectMapper, + ConsistencyCheckerProperties properties, + MetricsFacade metricsFacade) { + this.groupRepositories = repositoryManager.getRepositories(GroupRepository.class); + this.topicRepositories = repositoryManager.getRepositories(TopicRepository.class); + this.subscriptionRepositories = repositoryManager.getRepositories(SubscriptionRepository.class); + this.objectMapper = objectMapper; + this.executor = + Executors.newFixedThreadPool( + properties.getThreadPoolSize(), + new ThreadFactoryBuilder().setNameFormat("consistency-checker-%d").build()); + this.scheduler = + Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder().setNameFormat("consistency-checker-scheduler-%d").build()); + if (properties.isPeriodicCheckEnabled()) { + scheduler.scheduleAtFixedRate( + this::reportConsistency, + properties.getInitialRefreshDelay().getSeconds(), + properties.getRefreshInterval().getSeconds(), + TimeUnit.SECONDS); + metricsFacade + .consistency() + .registerStorageConsistencyGauge( + isStorageConsistent, isConsistent -> isConsistent.get() ? 1 : 0); } + } - @PreDestroy - public void stop() { - executor.shutdown(); - scheduler.shutdown(); - } + @PreDestroy + public void stop() { + executor.shutdown(); + scheduler.shutdown(); + } - private void reportConsistency() { - long start = System.currentTimeMillis(); - Set groups = listAllGroupNames(); - List inconsistentGroups = listInconsistentGroups(groups); - long durationSeconds = (System.currentTimeMillis() - start) / 1000; - logger.info("Consistency check finished in {}s, number of inconsistent groups: {}", durationSeconds, inconsistentGroups.size()); - isStorageConsistent.set(inconsistentGroups.isEmpty()); - } + private void reportConsistency() { + long start = System.currentTimeMillis(); + Set groups = listAllGroupNames(); + List inconsistentGroups = listInconsistentGroups(groups); + long durationSeconds = (System.currentTimeMillis() - start) / 1000; + logger.info( + "Consistency check finished in {}s, number of inconsistent groups: {}", + durationSeconds, + inconsistentGroups.size()); + isStorageConsistent.set(inconsistentGroups.isEmpty()); + } - public List listInconsistentGroups(Set groupNames) { - List inconsistentGroups = new ArrayList<>(); - for (MetadataCopies copies : listCopiesOfGroups(groupNames)) { - List inconsistentMetadata = findInconsistentMetadata(copies); - List inconsistentTopics = listInconsistentTopics(copies.getId()); - if (!inconsistentMetadata.isEmpty() || !inconsistentTopics.isEmpty()) { - inconsistentGroups.add(new InconsistentGroup(copies.getId(), inconsistentMetadata, inconsistentTopics)); - } - } - return inconsistentGroups; + public List listInconsistentGroups(Set groupNames) { + List inconsistentGroups = new ArrayList<>(); + for (MetadataCopies copies : listCopiesOfGroups(groupNames)) { + List inconsistentMetadata = findInconsistentMetadata(copies); + List inconsistentTopics = listInconsistentTopics(copies.getId()); + if (!inconsistentMetadata.isEmpty() || !inconsistentTopics.isEmpty()) { + inconsistentGroups.add( + new InconsistentGroup(copies.getId(), inconsistentMetadata, inconsistentTopics)); + } } + return inconsistentGroups; + } - public void syncGroup(String groupName, String primaryDatacenter) { - sync(groupRepositories, primaryDatacenter, - repo -> repo.groupExists(groupName), - repo -> { - try { - return Optional.of(repo.getGroupDetails(groupName)); - } catch (GroupNotExistsException ignored) { - return Optional.empty(); - } - }, - GroupRepository::createGroup, - GroupRepository::updateGroup, - repo -> repo.removeGroup(groupName) - ); - } + public void syncGroup(String groupName, String primaryDatacenter) { + sync( + groupRepositories, + primaryDatacenter, + repo -> repo.groupExists(groupName), + repo -> { + try { + return Optional.of(repo.getGroupDetails(groupName)); + } catch (GroupNotExistsException ignored) { + return Optional.empty(); + } + }, + GroupRepository::createGroup, + GroupRepository::updateGroup, + repo -> repo.removeGroup(groupName)); + } - public void syncTopic(TopicName topicName, String primaryDatacenter) { - sync(topicRepositories, primaryDatacenter, - repo -> repo.topicExists(topicName), - repo -> { - try { - return Optional.of(repo.getTopicDetails(topicName)); - } catch (TopicNotExistsException ignored) { - return Optional.empty(); - } - }, - TopicRepository::createTopic, - TopicRepository::updateTopic, - repo -> repo.removeTopic(topicName) - ); - } + public void syncTopic(TopicName topicName, String primaryDatacenter) { + sync( + topicRepositories, + primaryDatacenter, + repo -> repo.topicExists(topicName), + repo -> { + try { + return Optional.of(repo.getTopicDetails(topicName)); + } catch (TopicNotExistsException ignored) { + return Optional.empty(); + } + }, + TopicRepository::createTopic, + TopicRepository::updateTopic, + repo -> repo.removeTopic(topicName)); + } - public void syncSubscription(SubscriptionName subscriptionName, String primaryDatacenter) { - sync(subscriptionRepositories, primaryDatacenter, - repo -> repo.subscriptionExists(subscriptionName.getTopicName(), subscriptionName.getName()), - repo -> { - try { - return Optional.of(repo.getSubscriptionDetails(subscriptionName)); - } catch (SubscriptionNotExistsException ignored) { - return Optional.empty(); - } - }, - SubscriptionRepository::createSubscription, - SubscriptionRepository::updateSubscription, - repo -> repo.removeSubscription(subscriptionName.getTopicName(), subscriptionName.getName()) - ); - } + public void syncSubscription(SubscriptionName subscriptionName, String primaryDatacenter) { + sync( + subscriptionRepositories, + primaryDatacenter, + repo -> + repo.subscriptionExists(subscriptionName.getTopicName(), subscriptionName.getName()), + repo -> { + try { + return Optional.of(repo.getSubscriptionDetails(subscriptionName)); + } catch (SubscriptionNotExistsException ignored) { + return Optional.empty(); + } + }, + SubscriptionRepository::createSubscription, + SubscriptionRepository::updateSubscription, + repo -> + repo.removeSubscription(subscriptionName.getTopicName(), subscriptionName.getName())); + } - private void sync(List> repositories, - String sourceOfTruthDatacenter, - Function exists, - Function> get, - BiConsumer create, - BiConsumer update, - Consumer delete - ) { - var request = partition(repositories, sourceOfTruthDatacenter); - var primaryRepository = request.primaryHolder.getRepository(); - Optional primary = get.apply(primaryRepository); - var primaryPresent = primary.isPresent(); + private void sync( + List> repositories, + String sourceOfTruthDatacenter, + Function exists, + Function> get, + BiConsumer create, + BiConsumer update, + Consumer delete) { + var request = partition(repositories, sourceOfTruthDatacenter); + var primaryRepository = request.primaryHolder.getRepository(); + Optional primary = get.apply(primaryRepository); + var primaryPresent = primary.isPresent(); - for (var holder : request.replicaHolders) { - var repository = holder.getRepository(); - var replicaPresent = exists.apply(repository); + for (var holder : request.replicaHolders) { + var repository = holder.getRepository(); + var replicaPresent = exists.apply(repository); - if (primaryPresent && replicaPresent) { - update.accept(repository, primary.get()); - } else if (primaryPresent) { - create.accept(repository, primary.get()); - } else if (replicaPresent) { - delete.accept(repository); - } - } + if (primaryPresent && replicaPresent) { + update.accept(repository, primary.get()); + } else if (primaryPresent) { + create.accept(repository, primary.get()); + } else if (replicaPresent) { + delete.accept(repository); + } } + } - private List listCopiesOfGroups(Set groupNames) { - Map>> futuresPerDatacenter = new HashMap<>(); - for (DatacenterBoundRepositoryHolder repositoryHolder : groupRepositories) { - Future> future = executor.submit(() -> listGroups(repositoryHolder.getRepository(), groupNames)); - futuresPerDatacenter.put(repositoryHolder.getDatacenterName(), future); - } - return listCopies(futuresPerDatacenter, Group::getGroupName); + private List listCopiesOfGroups(Set groupNames) { + Map>> futuresPerDatacenter = new HashMap<>(); + for (DatacenterBoundRepositoryHolder repositoryHolder : groupRepositories) { + Future> future = + executor.submit(() -> listGroups(repositoryHolder.getRepository(), groupNames)); + futuresPerDatacenter.put(repositoryHolder.getDatacenterName(), future); } + return listCopies(futuresPerDatacenter, Group::getGroupName); + } - private List listGroups(GroupRepository repository, Set groupNames) { - List groups = new ArrayList<>(); - for (String groupName : groupNames) { - try { - Group group = repository.getGroupDetails(groupName); - groups.add(group); - } catch (GroupNotExistsException e) { - // ignore - } - } - return groups; + private List listGroups(GroupRepository repository, Set groupNames) { + List groups = new ArrayList<>(); + for (String groupName : groupNames) { + try { + Group group = repository.getGroupDetails(groupName); + groups.add(group); + } catch (GroupNotExistsException e) { + // ignore + } } + return groups; + } - private List listInconsistentTopics(String group) { - List inconsistentTopics = new ArrayList<>(); - for (MetadataCopies copies : listCopiesOfTopicsFromGroup(group)) { - List inconsistentMetadata = findInconsistentMetadata(copies); - List inconsistentSubscriptions = listInconsistentSubscriptions(copies.getId()); - if (!inconsistentMetadata.isEmpty() || !inconsistentSubscriptions.isEmpty()) { - inconsistentTopics.add(new InconsistentTopic(copies.getId(), inconsistentMetadata, inconsistentSubscriptions)); - } - } - return inconsistentTopics; + private List listInconsistentTopics(String group) { + List inconsistentTopics = new ArrayList<>(); + for (MetadataCopies copies : listCopiesOfTopicsFromGroup(group)) { + List inconsistentMetadata = findInconsistentMetadata(copies); + List inconsistentSubscriptions = + listInconsistentSubscriptions(copies.getId()); + if (!inconsistentMetadata.isEmpty() || !inconsistentSubscriptions.isEmpty()) { + inconsistentTopics.add( + new InconsistentTopic(copies.getId(), inconsistentMetadata, inconsistentSubscriptions)); + } } + return inconsistentTopics; + } - private List listCopiesOfTopicsFromGroup(String group) { - Map>> futuresPerDatacenter = new HashMap<>(); - for (DatacenterBoundRepositoryHolder repositoryHolder : topicRepositories) { - Future> future = executor.submit(() -> listTopics(repositoryHolder.getRepository(), group)); - futuresPerDatacenter.put(repositoryHolder.getDatacenterName(), future); - } - return listCopies(futuresPerDatacenter, Topic::getQualifiedName); + private List listCopiesOfTopicsFromGroup(String group) { + Map>> futuresPerDatacenter = new HashMap<>(); + for (DatacenterBoundRepositoryHolder repositoryHolder : topicRepositories) { + Future> future = + executor.submit(() -> listTopics(repositoryHolder.getRepository(), group)); + futuresPerDatacenter.put(repositoryHolder.getDatacenterName(), future); } + return listCopies(futuresPerDatacenter, Topic::getQualifiedName); + } - private List listTopics(TopicRepository topicRepository, String group) { - try { - return topicRepository.listTopics(group); - } catch (GroupNotExistsException e) { - return emptyList(); - } + private List listTopics(TopicRepository topicRepository, String group) { + try { + return topicRepository.listTopics(group); + } catch (GroupNotExistsException e) { + return emptyList(); } + } - private List listInconsistentSubscriptions(String topic) { - return listCopiesOfSubscriptionsFromTopic(topic).stream() - .filter(copies -> !copies.areAllEqual()) - .map(copies -> new InconsistentSubscription(copies.getId(), findInconsistentMetadata(copies))) - .collect(toList()); - } + private List listInconsistentSubscriptions(String topic) { + return listCopiesOfSubscriptionsFromTopic(topic).stream() + .filter(copies -> !copies.areAllEqual()) + .map( + copies -> + new InconsistentSubscription(copies.getId(), findInconsistentMetadata(copies))) + .collect(toList()); + } - private List listCopiesOfSubscriptionsFromTopic(String topic) { - Map>> futuresPerDatacenter = new HashMap<>(); - for (DatacenterBoundRepositoryHolder repositoryHolder : subscriptionRepositories) { - Future> future = executor.submit( - () -> listSubscriptions(repositoryHolder.getRepository(), topic) - ); - futuresPerDatacenter.put(repositoryHolder.getDatacenterName(), future); - } - return listCopies(futuresPerDatacenter, subscription -> subscription.getQualifiedName().getQualifiedName()); + private List listCopiesOfSubscriptionsFromTopic(String topic) { + Map>> futuresPerDatacenter = new HashMap<>(); + for (DatacenterBoundRepositoryHolder repositoryHolder : + subscriptionRepositories) { + Future> future = + executor.submit(() -> listSubscriptions(repositoryHolder.getRepository(), topic)); + futuresPerDatacenter.put(repositoryHolder.getDatacenterName(), future); } + return listCopies( + futuresPerDatacenter, subscription -> subscription.getQualifiedName().getQualifiedName()); + } - private List listSubscriptions(SubscriptionRepository subscriptionRepository, String topic) { - try { - return subscriptionRepository.listSubscriptions(TopicName.fromQualifiedName(topic)); - } catch (TopicNotExistsException e) { - return emptyList(); - } + private List listSubscriptions( + SubscriptionRepository subscriptionRepository, String topic) { + try { + return subscriptionRepository.listSubscriptions(TopicName.fromQualifiedName(topic)); + } catch (TopicNotExistsException e) { + return emptyList(); } + } - private List listCopies(Map>> futuresPerDatacenter, Function idResolver) { - Map copiesPerId = new HashMap<>(); - Set datacenters = futuresPerDatacenter.keySet(); - for (Map.Entry>> entry : futuresPerDatacenter.entrySet()) { - List entities = resolveFuture(entry.getValue()); - String datacenter = entry.getKey(); - for (T entity : entities) { - String id = idResolver.apply(entity); - MetadataCopies copies = copiesPerId.getOrDefault(id, new MetadataCopies(id, datacenters)); - copies.put(datacenter, entity); - copiesPerId.put(id, copies); - } - } - return new ArrayList<>(copiesPerId.values()); + private List listCopies( + Map>> futuresPerDatacenter, Function idResolver) { + Map copiesPerId = new HashMap<>(); + Set datacenters = futuresPerDatacenter.keySet(); + for (Map.Entry>> entry : futuresPerDatacenter.entrySet()) { + List entities = resolveFuture(entry.getValue()); + String datacenter = entry.getKey(); + for (T entity : entities) { + String id = idResolver.apply(entity); + MetadataCopies copies = copiesPerId.getOrDefault(id, new MetadataCopies(id, datacenters)); + copies.put(datacenter, entity); + copiesPerId.put(id, copies); + } } + return new ArrayList<>(copiesPerId.values()); + } - private List findInconsistentMetadata(MetadataCopies copies) { - if (copies.areAllEqual()) { - return emptyList(); - } - return copies.getCopyPerDatacenter().entrySet().stream() - .map(entry -> mapToInconsistentMetadata(entry.getKey(), entry.getValue())) - .collect(toList()); + private List findInconsistentMetadata(MetadataCopies copies) { + if (copies.areAllEqual()) { + return emptyList(); } + return copies.getCopyPerDatacenter().entrySet().stream() + .map(entry -> mapToInconsistentMetadata(entry.getKey(), entry.getValue())) + .collect(toList()); + } - private InconsistentMetadata mapToInconsistentMetadata(String id, Object content) { - try { - if (content == null) { - return new InconsistentMetadata(id, null); - } - return new InconsistentMetadata(id, objectMapper.writeValueAsString(content)); - } catch (JsonProcessingException e) { - throw new ConsistencyCheckingException("Metadata serialization failed", e); - } + private InconsistentMetadata mapToInconsistentMetadata(String id, Object content) { + try { + if (content == null) { + return new InconsistentMetadata(id, null); + } + return new InconsistentMetadata(id, objectMapper.writeValueAsString(content)); + } catch (JsonProcessingException e) { + throw new ConsistencyCheckingException("Metadata serialization failed", e); } + } - public Set listAllGroupNames() { - List>> results = new ArrayList<>(); - for (DatacenterBoundRepositoryHolder repositoryHolder : groupRepositories) { - Future> submit = executor.submit(() -> repositoryHolder.getRepository().listGroupNames()); - results.add(submit); - } - return results.stream().map(this::resolveFuture) - .flatMap(Collection::stream) - .collect(toSet()); + public Set listAllGroupNames() { + List>> results = new ArrayList<>(); + for (DatacenterBoundRepositoryHolder repositoryHolder : groupRepositories) { + Future> submit = + executor.submit(() -> repositoryHolder.getRepository().listGroupNames()); + results.add(submit); } + return results.stream().map(this::resolveFuture).flatMap(Collection::stream).collect(toSet()); + } - private T resolveFuture(Future future) { - try { - return future.get(); - } catch (Exception e) { - throw new ConsistencyCheckingException("Fetching metadata failed", e); - } + private T resolveFuture(Future future) { + try { + return future.get(); + } catch (Exception e) { + throw new ConsistencyCheckingException("Fetching metadata failed", e); } + } - private record DatacenterRepositoryHolderSyncRequest( - DatacenterBoundRepositoryHolder primaryHolder, - List> replicaHolders - ) { - } + private record DatacenterRepositoryHolderSyncRequest( + DatacenterBoundRepositoryHolder primaryHolder, + List> replicaHolders) {} - private DatacenterRepositoryHolderSyncRequest partition(List> repositoryHolders, String primaryDatacenter) { - List> replicas = new ArrayList<>(); - DatacenterBoundRepositoryHolder primary = null; - for (DatacenterBoundRepositoryHolder repositoryHolder : repositoryHolders) { - if (repositoryHolder.getDatacenterName().equals(primaryDatacenter)) { - primary = repositoryHolder; - } else { - replicas.add(repositoryHolder); - } - } - if (primary == null) { - throw new SynchronizationException("Source of truth datacenter not found: " + primaryDatacenter); - } - return new DatacenterRepositoryHolderSyncRequest<>(primary, replicas); + private DatacenterRepositoryHolderSyncRequest partition( + List> repositoryHolders, String primaryDatacenter) { + List> replicas = new ArrayList<>(); + DatacenterBoundRepositoryHolder primary = null; + for (DatacenterBoundRepositoryHolder repositoryHolder : repositoryHolders) { + if (repositoryHolder.getDatacenterName().equals(primaryDatacenter)) { + primary = repositoryHolder; + } else { + replicas.add(repositoryHolder); + } + } + if (primary == null) { + throw new SynchronizationException( + "Source of truth datacenter not found: " + primaryDatacenter); } + return new DatacenterRepositoryHolderSyncRequest<>(primary, replicas); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/consistency/KafkaHermesConsistencyService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/consistency/KafkaHermesConsistencyService.java index 171de26179..cf6310554f 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/consistency/KafkaHermesConsistencyService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/consistency/KafkaHermesConsistencyService.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.management.domain.consistency; +import static java.util.Arrays.asList; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -8,54 +13,56 @@ import pl.allegro.tech.hermes.management.domain.topic.TopicService; import pl.allegro.tech.hermes.management.infrastructure.kafka.MultiDCAwareService; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import static java.util.Arrays.asList; - @Component public class KafkaHermesConsistencyService { - private static final Logger logger = LoggerFactory.getLogger(KafkaHermesConsistencyService.class); - - private static final String AVRO_SUFFIX = "_avro"; - private static final List IGNORED_TOPIC = asList("__consumer_offsets"); - private final TopicService topicService; - private final MultiDCAwareService multiDCAwareService; - private final KafkaClustersProperties kafkaClustersProperties; - - public KafkaHermesConsistencyService( - TopicService topicService, - MultiDCAwareService multiDCAwareService, - KafkaClustersProperties kafkaClustersProperties) { - this.topicService = topicService; - this.kafkaClustersProperties = kafkaClustersProperties; - this.multiDCAwareService = multiDCAwareService; - } - - public Set listInconsistentTopics() { - List topicsFromHermes = topicService.listQualifiedTopicNames(); - - return multiDCAwareService.listTopicFromAllDC() - .stream() - .filter(topic -> !IGNORED_TOPIC.contains(topic) && !topicsFromHermes - .contains(mapToHermesFormat(topic))) - .collect(Collectors.toSet()); - } - - public void removeTopic(String topicName, RequestUser requestUser) { - logger.info("Removing topic {} only on brokers. Requested by {}", topicName, requestUser.getUsername()); - multiDCAwareService.removeTopicByName(topicName); - logger.info("Successfully removed topic {} on brokers. Requested by {}", topicName, requestUser.getUsername()); - } - - private String mapToHermesFormat(String topic) { - String prefix = kafkaClustersProperties.getDefaultNamespace() + kafkaClustersProperties.getNamespaceSeparator(); - - int beginIndex = topic.startsWith(prefix) ? prefix.length() : 0; - int endIndex = topic.endsWith(AVRO_SUFFIX) ? topic.length() - AVRO_SUFFIX.length() : topic.length(); - - return topic.substring(beginIndex, endIndex); - } + private static final Logger logger = LoggerFactory.getLogger(KafkaHermesConsistencyService.class); + + private static final String AVRO_SUFFIX = "_avro"; + private static final List IGNORED_TOPIC = asList("__consumer_offsets"); + private final TopicService topicService; + private final MultiDCAwareService multiDCAwareService; + private final KafkaClustersProperties kafkaClustersProperties; + + public KafkaHermesConsistencyService( + TopicService topicService, + MultiDCAwareService multiDCAwareService, + KafkaClustersProperties kafkaClustersProperties) { + this.topicService = topicService; + this.kafkaClustersProperties = kafkaClustersProperties; + this.multiDCAwareService = multiDCAwareService; + } + + public Set listInconsistentTopics() { + List topicsFromHermes = topicService.listQualifiedTopicNames(); + + return multiDCAwareService.listTopicFromAllDC().stream() + .filter( + topic -> + !IGNORED_TOPIC.contains(topic) + && !topicsFromHermes.contains(mapToHermesFormat(topic))) + .collect(Collectors.toSet()); + } + + public void removeTopic(String topicName, RequestUser requestUser) { + logger.info( + "Removing topic {} only on brokers. Requested by {}", topicName, requestUser.getUsername()); + multiDCAwareService.removeTopicByName(topicName); + logger.info( + "Successfully removed topic {} on brokers. Requested by {}", + topicName, + requestUser.getUsername()); + } + + private String mapToHermesFormat(String topic) { + String prefix = + kafkaClustersProperties.getDefaultNamespace() + + kafkaClustersProperties.getNamespaceSeparator(); + + int beginIndex = topic.startsWith(prefix) ? prefix.length() : 0; + int endIndex = + topic.endsWith(AVRO_SUFFIX) ? topic.length() - AVRO_SUFFIX.length() : topic.length(); + + return topic.substring(beginIndex, endIndex); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/consistency/MetadataCopies.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/consistency/MetadataCopies.java index 991e10a111..1270fa4008 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/consistency/MetadataCopies.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/consistency/MetadataCopies.java @@ -5,30 +5,30 @@ import java.util.Set; class MetadataCopies { - private final String id; - private final Map copyPerDatacenter = new HashMap<>(); + private final String id; + private final Map copyPerDatacenter = new HashMap<>(); - MetadataCopies(String id, Set datacenters) { - this.id = id; - datacenters.forEach(dc -> copyPerDatacenter.put(dc, null)); - } + MetadataCopies(String id, Set datacenters) { + this.id = id; + datacenters.forEach(dc -> copyPerDatacenter.put(dc, null)); + } - void put(String datacenter, Object value) { - if (!copyPerDatacenter.containsKey(datacenter)) { - throw new IllegalArgumentException("Invalid datacenter: " + datacenter); - } - copyPerDatacenter.put(datacenter, value); + void put(String datacenter, Object value) { + if (!copyPerDatacenter.containsKey(datacenter)) { + throw new IllegalArgumentException("Invalid datacenter: " + datacenter); } + copyPerDatacenter.put(datacenter, value); + } - boolean areAllEqual() { - return copyPerDatacenter.values().stream().distinct().count() == 1; - } + boolean areAllEqual() { + return copyPerDatacenter.values().stream().distinct().count() == 1; + } - String getId() { - return id; - } + String getId() { + return id; + } - Map getCopyPerDatacenter() { - return copyPerDatacenter; - } + Map getCopyPerDatacenter() { + return copyPerDatacenter; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/consistency/SynchronizationException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/consistency/SynchronizationException.java index fb7a461e2f..53a5f52d8e 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/consistency/SynchronizationException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/consistency/SynchronizationException.java @@ -1,7 +1,7 @@ package pl.allegro.tech.hermes.management.domain.consistency; public class SynchronizationException extends RuntimeException { - public SynchronizationException(String message) { - super(message); - } + public SynchronizationException(String message) { + super(message); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/console/ConsoleConfigurationRepository.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/console/ConsoleConfigurationRepository.java index 4c9e54430c..bc4b544162 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/console/ConsoleConfigurationRepository.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/console/ConsoleConfigurationRepository.java @@ -2,5 +2,5 @@ public interface ConsoleConfigurationRepository { - String getConfiguration(); + String getConfiguration(); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/console/ConsoleService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/console/ConsoleService.java index f715d3c0b4..60411abb1d 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/console/ConsoleService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/console/ConsoleService.java @@ -5,17 +5,17 @@ @Service public class ConsoleService { - private ConsoleConfigurationRepository repository; + private ConsoleConfigurationRepository repository; - public ConsoleService(ConsoleConfigurationRepository repository) { - this.repository = repository; - } + public ConsoleService(ConsoleConfigurationRepository repository) { + this.repository = repository; + } - public String getConfiguration() { - return "var config = " + repository.getConfiguration(); - } + public String getConfiguration() { + return "var config = " + repository.getConfiguration(); + } - public String getConfigurationJson() { - return repository.getConfiguration(); - } + public String getConfigurationJson() { + return repository.getConfiguration(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/credentials/CredentialsService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/credentials/CredentialsService.java index 2584bd379b..ed179e9219 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/credentials/CredentialsService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/credentials/CredentialsService.java @@ -13,26 +13,28 @@ @Component public class CredentialsService { - private static final Logger logger = LoggerFactory.getLogger(CredentialsService.class); + private static final Logger logger = LoggerFactory.getLogger(CredentialsService.class); - private final ZookeeperPaths paths; - private final MultiDatacenterRepositoryCommandExecutor multiDcExecutor; - private final CredentialsRepository credentialsRepository; + private final ZookeeperPaths paths; + private final MultiDatacenterRepositoryCommandExecutor multiDcExecutor; + private final CredentialsRepository credentialsRepository; - @Autowired - public CredentialsService(ZookeeperPaths paths, - MultiDatacenterRepositoryCommandExecutor multiDcExecutor, - CredentialsRepository credentialsRepository) { - this.paths = paths; - this.multiDcExecutor = multiDcExecutor; - this.credentialsRepository = credentialsRepository; - } + @Autowired + public CredentialsService( + ZookeeperPaths paths, + MultiDatacenterRepositoryCommandExecutor multiDcExecutor, + CredentialsRepository credentialsRepository) { + this.paths = paths; + this.multiDcExecutor = multiDcExecutor; + this.credentialsRepository = credentialsRepository; + } - public NodePassword readAdminPassword() { - return credentialsRepository.readAdminPassword(); - } + public NodePassword readAdminPassword() { + return credentialsRepository.readAdminPassword(); + } - public void overwriteAdminPassword(String password) { - multiDcExecutor.execute(new UpdateCredentialsRepositoryCommand(credentialsRepository, password)); - } + public void overwriteAdminPassword(String password) { + multiDcExecutor.execute( + new UpdateCredentialsRepositoryCommand(credentialsRepository, password)); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/credentials/commands/UpdateCredentialsRepositoryCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/credentials/commands/UpdateCredentialsRepositoryCommand.java index f8c30cc7fc..a432c8fd79 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/credentials/commands/UpdateCredentialsRepositoryCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/credentials/commands/UpdateCredentialsRepositoryCommand.java @@ -6,29 +6,28 @@ public class UpdateCredentialsRepositoryCommand extends RepositoryCommand { - private final CredentialsRepository repository; - private final String password; - - public UpdateCredentialsRepositoryCommand(CredentialsRepository repository, String password) { - this.repository = repository; - this.password = password; - } - - @Override - public void backup(DatacenterBoundRepositoryHolder holder) { - } - - @Override - public void execute(DatacenterBoundRepositoryHolder holder) { - repository.overwriteAdminPassword(password); - } - - @Override - public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) { - } - - @Override - public Class getRepositoryType() { - return CredentialsRepository.class; - } + private final CredentialsRepository repository; + private final String password; + + public UpdateCredentialsRepositoryCommand(CredentialsRepository repository, String password) { + this.repository = repository; + this.password = password; + } + + @Override + public void backup(DatacenterBoundRepositoryHolder holder) {} + + @Override + public void execute(DatacenterBoundRepositoryHolder holder) { + repository.overwriteAdminPassword(password); + } + + @Override + public void rollback( + DatacenterBoundRepositoryHolder holder, Exception exception) {} + + @Override + public Class getRepositoryType() { + return CredentialsRepository.class; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/dc/DatacenterBoundQueryResult.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/dc/DatacenterBoundQueryResult.java index 3de8d6ce46..90bcb23f0c 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/dc/DatacenterBoundQueryResult.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/dc/DatacenterBoundQueryResult.java @@ -1,20 +1,19 @@ package pl.allegro.tech.hermes.management.domain.dc; public class DatacenterBoundQueryResult { - private final String datacenterName; - private final T result; + private final String datacenterName; + private final T result; - public DatacenterBoundQueryResult(T result, String datacenterName) { - this.datacenterName = datacenterName; - this.result = result; - } + public DatacenterBoundQueryResult(T result, String datacenterName) { + this.datacenterName = datacenterName; + this.result = result; + } - public String getDatacenterName() { - return datacenterName; - } - - public T getResult() { - return result; - } + public String getDatacenterName() { + return datacenterName; + } + public T getResult() { + return result; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/dc/DatacenterBoundRepositoryHolder.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/dc/DatacenterBoundRepositoryHolder.java index 7a1b5639bc..9711499305 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/dc/DatacenterBoundRepositoryHolder.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/dc/DatacenterBoundRepositoryHolder.java @@ -2,19 +2,19 @@ public class DatacenterBoundRepositoryHolder { - private final R repository; - private final String datacenterName; + private final R repository; + private final String datacenterName; - public DatacenterBoundRepositoryHolder(R repository, String datacenterName) { - this.repository = repository; - this.datacenterName = datacenterName; - } + public DatacenterBoundRepositoryHolder(R repository, String datacenterName) { + this.repository = repository; + this.datacenterName = datacenterName; + } - public R getRepository() { - return repository; - } + public R getRepository() { + return repository; + } - public String getDatacenterName() { - return datacenterName; - } + public String getDatacenterName() { + return datacenterName; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/dc/ExceptionWrapper.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/dc/ExceptionWrapper.java index 8428d72928..caef4c84a3 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/dc/ExceptionWrapper.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/dc/ExceptionWrapper.java @@ -4,12 +4,12 @@ import pl.allegro.tech.hermes.common.exception.InternalProcessingException; class ExceptionWrapper { - static RuntimeException wrapInInternalProcessingExceptionIfNeeded(Exception toWrap, - String command, - String dcName) { - if (toWrap instanceof HermesException) { - return (HermesException) toWrap; - } - return new InternalProcessingException("Execution of command '" + command + "' failed on DC '" + dcName + "'.", toWrap); + static RuntimeException wrapInInternalProcessingExceptionIfNeeded( + Exception toWrap, String command, String dcName) { + if (toWrap instanceof HermesException) { + return (HermesException) toWrap; } + return new InternalProcessingException( + "Execution of command '" + command + "' failed on DC '" + dcName + "'.", toWrap); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/dc/MultiDatacenterRepositoryCommandExecutor.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/dc/MultiDatacenterRepositoryCommandExecutor.java index 2648407fa4..b343bd06ee 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/dc/MultiDatacenterRepositoryCommandExecutor.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/dc/MultiDatacenterRepositoryCommandExecutor.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.management.domain.dc; +import java.util.ArrayList; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.common.exception.InternalProcessingException; @@ -7,91 +9,126 @@ import pl.allegro.tech.hermes.management.domain.auth.RequestUser; import pl.allegro.tech.hermes.management.domain.mode.ModeService; -import java.util.ArrayList; -import java.util.List; - - public class MultiDatacenterRepositoryCommandExecutor { - private static final Logger logger = LoggerFactory.getLogger(MultiDatacenterRepositoryCommandExecutor.class); + private static final Logger logger = + LoggerFactory.getLogger(MultiDatacenterRepositoryCommandExecutor.class); - private final RepositoryManager repositoryManager; - private final boolean rollbackEnabled; - private final ModeService modeService; + private final RepositoryManager repositoryManager; + private final boolean rollbackEnabled; + private final ModeService modeService; - public MultiDatacenterRepositoryCommandExecutor(RepositoryManager repositoryManager, boolean rollbackEnabled, ModeService modeService) { - this.repositoryManager = repositoryManager; - this.rollbackEnabled = rollbackEnabled; - this.modeService = modeService; - } + public MultiDatacenterRepositoryCommandExecutor( + RepositoryManager repositoryManager, boolean rollbackEnabled, ModeService modeService) { + this.repositoryManager = repositoryManager; + this.rollbackEnabled = rollbackEnabled; + this.modeService = modeService; + } - public void executeByUser(RepositoryCommand command, RequestUser requestUser) { - if (requestUser.isAdmin() && modeService.isReadOnlyEnabled()) { - execute(command, false, false); - } else { - execute(command); - } + public void executeByUser(RepositoryCommand command, RequestUser requestUser) { + if (requestUser.isAdmin() && modeService.isReadOnlyEnabled()) { + execute(command, false, false); + } else { + execute(command); } + } + + public void execute(RepositoryCommand command) { + execute(command, rollbackEnabled, true); + } - public void execute(RepositoryCommand command) { - execute(command, rollbackEnabled, true); + private void execute( + RepositoryCommand command, + boolean isRollbackEnabled, + boolean shouldStopExecutionOnFailure) { + if (isRollbackEnabled) { + backup(command); } - private void execute(RepositoryCommand command, boolean isRollbackEnabled, boolean shouldStopExecutionOnFailure) { + List> repoHolders = + repositoryManager.getRepositories(command.getRepositoryType()); + List> executedRepoHolders = new ArrayList<>(); + + for (DatacenterBoundRepositoryHolder repoHolder : repoHolders) { + long start = System.currentTimeMillis(); + try { + executedRepoHolders.add(repoHolder); + logger.info( + "Executing repository command: {} in ZK dc: {}", + command, + repoHolder.getDatacenterName()); + command.execute(repoHolder); + logger.info( + "Successfully executed repository command: {} in ZK dc: {} in: {} ms", + command, + repoHolder.getDatacenterName(), + System.currentTimeMillis() - start); + } catch (RepositoryNotAvailableException e) { + logger.warn("Execute failed with an RepositoryNotAvailableException error", e); if (isRollbackEnabled) { - backup(command); + rollback(executedRepoHolders, command, e); } - - List> repoHolders = repositoryManager.getRepositories(command.getRepositoryType()); - List> executedRepoHolders = new ArrayList<>(); - - for (DatacenterBoundRepositoryHolder repoHolder : repoHolders) { - long start = System.currentTimeMillis(); - try { - executedRepoHolders.add(repoHolder); - logger.info("Executing repository command: {} in ZK dc: {}", command, repoHolder.getDatacenterName()); - command.execute(repoHolder); - logger.info("Successfully executed repository command: {} in ZK dc: {} in: {} ms", command, repoHolder.getDatacenterName(), System.currentTimeMillis() - start); - } catch (RepositoryNotAvailableException e) { - logger.warn("Execute failed with an RepositoryNotAvailableException error", e); - if (isRollbackEnabled) { - rollback(executedRepoHolders, command, e); - } - if (shouldStopExecutionOnFailure) { - throw ExceptionWrapper.wrapInInternalProcessingExceptionIfNeeded(e, command.toString(), repoHolder.getDatacenterName()); - } - } catch (Exception e) { - logger.warn("Failed to execute repository command: {} in ZK dc: {} in: {} ms", command, repoHolder.getDatacenterName(), System.currentTimeMillis() - start, e); - if (isRollbackEnabled) { - rollback(executedRepoHolders, command, e); - } - throw ExceptionWrapper.wrapInInternalProcessingExceptionIfNeeded(e, command.toString(), repoHolder.getDatacenterName()); - } + if (shouldStopExecutionOnFailure) { + throw ExceptionWrapper.wrapInInternalProcessingExceptionIfNeeded( + e, command.toString(), repoHolder.getDatacenterName()); } - } - - private void rollback(List> repoHolders, RepositoryCommand command, Exception exception) { - long start = System.currentTimeMillis(); - for (DatacenterBoundRepositoryHolder repoHolder : repoHolders) { - logger.info("Executing rollback of repository command: {} in ZK dc: {}", command, repoHolder.getDatacenterName()); - try { - command.rollback(repoHolder, exception); - logger.info("Successfully executed rollback of repository command: {} in ZK dc: {} in: {} ms", command, repoHolder.getDatacenterName(), System.currentTimeMillis() - start); - } catch (Exception e) { - logger.error("Rollback procedure failed for command {} on DC {}", command, repoHolder.getDatacenterName(), e); - } + } catch (Exception e) { + logger.warn( + "Failed to execute repository command: {} in ZK dc: {} in: {} ms", + command, + repoHolder.getDatacenterName(), + System.currentTimeMillis() - start, + e); + if (isRollbackEnabled) { + rollback(executedRepoHolders, command, e); } + throw ExceptionWrapper.wrapInInternalProcessingExceptionIfNeeded( + e, command.toString(), repoHolder.getDatacenterName()); + } } + } - private void backup(RepositoryCommand command) { - DatacenterBoundRepositoryHolder repoHolder = repositoryManager.getLocalRepository(command.getRepositoryType()); - try { - logger.debug("Creating backup for command: {}", command); - command.backup(repoHolder); - } catch (Exception e) { - throw new InternalProcessingException("Backup procedure for command '" + command - + "' failed on DC '" + repoHolder.getDatacenterName() + "'.", e); - } + private void rollback( + List> repoHolders, + RepositoryCommand command, + Exception exception) { + long start = System.currentTimeMillis(); + for (DatacenterBoundRepositoryHolder repoHolder : repoHolders) { + logger.info( + "Executing rollback of repository command: {} in ZK dc: {}", + command, + repoHolder.getDatacenterName()); + try { + command.rollback(repoHolder, exception); + logger.info( + "Successfully executed rollback of repository command: {} in ZK dc: {} in: {} ms", + command, + repoHolder.getDatacenterName(), + System.currentTimeMillis() - start); + } catch (Exception e) { + logger.error( + "Rollback procedure failed for command {} on DC {}", + command, + repoHolder.getDatacenterName(), + e); + } } + } + private void backup(RepositoryCommand command) { + DatacenterBoundRepositoryHolder repoHolder = + repositoryManager.getLocalRepository(command.getRepositoryType()); + try { + logger.debug("Creating backup for command: {}", command); + command.backup(repoHolder); + } catch (Exception e) { + throw new InternalProcessingException( + "Backup procedure for command '" + + command + + "' failed on DC '" + + repoHolder.getDatacenterName() + + "'.", + e); + } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/dc/RepositoryCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/dc/RepositoryCommand.java index 1b2d899137..d0017abb51 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/dc/RepositoryCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/dc/RepositoryCommand.java @@ -2,12 +2,11 @@ public abstract class RepositoryCommand { - public abstract void backup(DatacenterBoundRepositoryHolder holder); + public abstract void backup(DatacenterBoundRepositoryHolder holder); - public abstract void execute(DatacenterBoundRepositoryHolder holder); + public abstract void execute(DatacenterBoundRepositoryHolder holder); - public abstract void rollback(DatacenterBoundRepositoryHolder holder, Exception exception); - - public abstract Class getRepositoryType(); + public abstract void rollback(DatacenterBoundRepositoryHolder holder, Exception exception); + public abstract Class getRepositoryType(); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/dc/RepositoryManager.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/dc/RepositoryManager.java index 07915686f4..4c9ef97f93 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/dc/RepositoryManager.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/dc/RepositoryManager.java @@ -4,7 +4,7 @@ public interface RepositoryManager { - DatacenterBoundRepositoryHolder getLocalRepository(Class repositoryType); + DatacenterBoundRepositoryHolder getLocalRepository(Class repositoryType); - List> getRepositories(Class repositoryType); + List> getRepositories(Class repositoryType); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/filtering/FilteringService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/filtering/FilteringService.java index 09cb1148f9..14554f16c7 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/filtering/FilteringService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/filtering/FilteringService.java @@ -1,5 +1,13 @@ package pl.allegro.tech.hermes.management.domain.filtering; +import static pl.allegro.tech.hermes.api.ContentType.AVRO; +import static pl.allegro.tech.hermes.api.MessageFiltersVerificationResult.VerificationStatus.ERROR; +import static pl.allegro.tech.hermes.api.MessageFiltersVerificationResult.VerificationStatus.MATCHED; +import static pl.allegro.tech.hermes.api.MessageFiltersVerificationResult.VerificationStatus.NOT_MATCHED; + +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.avro.Schema; import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.stereotype.Component; @@ -16,80 +24,71 @@ import tech.allegro.schema.json2avro.converter.AvroConversionException; import tech.allegro.schema.json2avro.converter.JsonAvroConverter; -import java.util.Objects; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static pl.allegro.tech.hermes.api.ContentType.AVRO; -import static pl.allegro.tech.hermes.api.MessageFiltersVerificationResult.VerificationStatus.ERROR; -import static pl.allegro.tech.hermes.api.MessageFiltersVerificationResult.VerificationStatus.MATCHED; -import static pl.allegro.tech.hermes.api.MessageFiltersVerificationResult.VerificationStatus.NOT_MATCHED; - @Component public class FilteringService { - private final FilterChainFactory filterChainFactory; - private final SchemaRepository schemaRepository; - private final TopicService topicService; - private final JsonAvroConverter jsonAvroConverter; + private final FilterChainFactory filterChainFactory; + private final SchemaRepository schemaRepository; + private final TopicService topicService; + private final JsonAvroConverter jsonAvroConverter; - public FilteringService(FilterChainFactory filterChainFactory, - SchemaRepository schemaRepository, - TopicService topicService, - JsonAvroConverter jsonAvroConverter) { - this.filterChainFactory = filterChainFactory; - this.schemaRepository = schemaRepository; - this.topicService = topicService; - this.jsonAvroConverter = jsonAvroConverter; - } - - public MessageFiltersVerificationResult verify(MessageFiltersVerificationInput verification, TopicName topicName) { - Topic topic = topicService.getTopicDetails(topicName); - CompiledSchema avroSchema = getLatestAvroSchemaIfExists(topic); + public FilteringService( + FilterChainFactory filterChainFactory, + SchemaRepository schemaRepository, + TopicService topicService, + JsonAvroConverter jsonAvroConverter) { + this.filterChainFactory = filterChainFactory; + this.schemaRepository = schemaRepository; + this.topicService = topicService; + this.jsonAvroConverter = jsonAvroConverter; + } - byte[] messageContent; - try { - messageContent = getBytes(verification.getMessage(), topic, avroSchema); - } catch (AvroConversionException e) { - return new MessageFiltersVerificationResult(ERROR, createErrorMessage(e)); - } + public MessageFiltersVerificationResult verify( + MessageFiltersVerificationInput verification, TopicName topicName) { + Topic topic = topicService.getTopicDetails(topicName); + CompiledSchema avroSchema = getLatestAvroSchemaIfExists(topic); - MessageForFiltersVerification message = new MessageForFiltersVerification(messageContent, topic.getContentType(), avroSchema); - FilterChain filterChain = filterChainFactory.create(verification.getFilters()); - FilterResult result = filterChain.apply(message); - return toMessageFiltersVerificationResult(result); + byte[] messageContent; + try { + messageContent = getBytes(verification.getMessage(), topic, avroSchema); + } catch (AvroConversionException e) { + return new MessageFiltersVerificationResult(ERROR, createErrorMessage(e)); } - private CompiledSchema getLatestAvroSchemaIfExists(Topic topic) { - if (AVRO.equals(topic.getContentType())) { - return schemaRepository.getLatestAvroSchema(topic); - } - return null; - } + MessageForFiltersVerification message = + new MessageForFiltersVerification(messageContent, topic.getContentType(), avroSchema); + FilterChain filterChain = filterChainFactory.create(verification.getFilters()); + FilterResult result = filterChain.apply(message); + return toMessageFiltersVerificationResult(result); + } - private byte[] getBytes(byte[] message, Topic topic, CompiledSchema avroSchema) { - switch (topic.getContentType()) { - case JSON: - return message; - case AVRO: - return jsonAvroConverter.convertToAvro(message, avroSchema.getSchema()); - default: - throw new IllegalArgumentException(); - } + private CompiledSchema getLatestAvroSchemaIfExists(Topic topic) { + if (AVRO.equals(topic.getContentType())) { + return schemaRepository.getLatestAvroSchema(topic); } + return null; + } - private MessageFiltersVerificationResult toMessageFiltersVerificationResult(FilterResult result) { - return new MessageFiltersVerificationResult( - result.isFiltered() ? NOT_MATCHED : MATCHED, - result.getCause() - .map(this::createErrorMessage) - .orElse(null) - ); + private byte[] getBytes(byte[] message, Topic topic, CompiledSchema avroSchema) { + switch (topic.getContentType()) { + case JSON: + return message; + case AVRO: + return jsonAvroConverter.convertToAvro(message, avroSchema.getSchema()); + default: + throw new IllegalArgumentException(); } + } - private String createErrorMessage(Throwable th) { - Throwable rootCause = ExceptionUtils.getRootCause(th); - return Stream.of(th.getMessage(), (rootCause != null ? rootCause.getMessage() : null)) - .filter(Objects::nonNull) - .collect(Collectors.joining("\n")); - } + private MessageFiltersVerificationResult toMessageFiltersVerificationResult(FilterResult result) { + return new MessageFiltersVerificationResult( + result.isFiltered() ? NOT_MATCHED : MATCHED, + result.getCause().map(this::createErrorMessage).orElse(null)); + } + + private String createErrorMessage(Throwable th) { + Throwable rootCause = ExceptionUtils.getRootCause(th); + return Stream.of(th.getMessage(), (rootCause != null ? rootCause.getMessage() : null)) + .filter(Objects::nonNull) + .collect(Collectors.joining("\n")); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/filtering/MessageForFiltersVerification.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/filtering/MessageForFiltersVerification.java index 7a5da2cb91..4144972ebb 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/filtering/MessageForFiltersVerification.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/filtering/MessageForFiltersVerification.java @@ -1,42 +1,42 @@ package pl.allegro.tech.hermes.management.domain.filtering; +import java.util.Collections; +import java.util.Map; +import java.util.Optional; import org.apache.avro.Schema; import pl.allegro.tech.hermes.api.ContentType; import pl.allegro.tech.hermes.domain.filtering.FilterableMessage; import pl.allegro.tech.hermes.schema.CompiledSchema; -import java.util.Collections; -import java.util.Map; -import java.util.Optional; - class MessageForFiltersVerification implements FilterableMessage { - private final byte[] data; - private final ContentType contentType; - private final CompiledSchema schema; - - MessageForFiltersVerification(byte[] data, ContentType contentType, CompiledSchema schema) { - this.data = data; - this.contentType = contentType; - this.schema = schema; - } - - @Override - public ContentType getContentType() { - return contentType; - } - - @Override - public Map getExternalMetadata() { - return Collections.emptyMap(); - } - - @Override - public byte[] getData() { - return data; - } - - @Override - public Optional> getSchema() { - return Optional.ofNullable(schema); - } + private final byte[] data; + private final ContentType contentType; + private final CompiledSchema schema; + + MessageForFiltersVerification( + byte[] data, ContentType contentType, CompiledSchema schema) { + this.data = data; + this.contentType = contentType; + this.schema = schema; + } + + @Override + public ContentType getContentType() { + return contentType; + } + + @Override + public Map getExternalMetadata() { + return Collections.emptyMap(); + } + + @Override + public byte[] getData() { + return data; + } + + @Override + public Optional> getSchema() { + return Optional.ofNullable(schema); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/group/GroupNameValidator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/group/GroupNameValidator.java index a243fe5347..87e0239d02 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/group/GroupNameValidator.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/group/GroupNameValidator.java @@ -1,20 +1,20 @@ package pl.allegro.tech.hermes.management.domain.group; -import pl.allegro.tech.hermes.management.domain.GroupNameIsNotAllowedException; - import java.util.regex.Pattern; +import pl.allegro.tech.hermes.management.domain.GroupNameIsNotAllowedException; class GroupNameValidator { - private final Pattern allowedPattern; + private final Pattern allowedPattern; - public GroupNameValidator(String allowedRegex) { - this.allowedPattern = Pattern.compile(allowedRegex); - } + public GroupNameValidator(String allowedRegex) { + this.allowedPattern = Pattern.compile(allowedRegex); + } - public void requireValid(String groupName) { - if (!allowedPattern.matcher(groupName).matches()) { - throw new GroupNameIsNotAllowedException(String.format("Group name should match pattern %s", allowedPattern)); - } + public void requireValid(String groupName) { + if (!allowedPattern.matcher(groupName).matches()) { + throw new GroupNameIsNotAllowedException( + String.format("Group name should match pattern %s", allowedPattern)); } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/group/GroupService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/group/GroupService.java index f5f38540d4..5ee724f80d 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/group/GroupService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/group/GroupService.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.management.domain.group; +import java.util.List; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -19,77 +21,71 @@ import pl.allegro.tech.hermes.management.domain.group.commands.RemoveGroupRepositoryCommand; import pl.allegro.tech.hermes.management.domain.group.commands.UpdateGroupRepositoryCommand; -import java.util.List; -import java.util.stream.Collectors; - @Component public class GroupService { - private static final Logger logger = LoggerFactory.getLogger(GroupService.class); + private static final Logger logger = LoggerFactory.getLogger(GroupService.class); - private final GroupRepository groupRepository; - private final Auditor auditor; - private final MultiDatacenterRepositoryCommandExecutor multiDcExecutor; - private final GroupValidator validator; + private final GroupRepository groupRepository; + private final Auditor auditor; + private final MultiDatacenterRepositoryCommandExecutor multiDcExecutor; + private final GroupValidator validator; - @Autowired - public GroupService(GroupRepository groupRepository, - Auditor auditor, - MultiDatacenterRepositoryCommandExecutor multiDcExecutor, - GroupValidator validator) { - this.groupRepository = groupRepository; - this.auditor = auditor; - this.multiDcExecutor = multiDcExecutor; - this.validator = validator; - } + @Autowired + public GroupService( + GroupRepository groupRepository, + Auditor auditor, + MultiDatacenterRepositoryCommandExecutor multiDcExecutor, + GroupValidator validator) { + this.groupRepository = groupRepository; + this.auditor = auditor; + this.multiDcExecutor = multiDcExecutor; + this.validator = validator; + } - public List listGroups() { - return groupRepository.listGroups(); - } + public List listGroups() { + return groupRepository.listGroups(); + } - public List listGroupNames() { - return groupRepository.listGroupNames(); - } + public List listGroupNames() { + return groupRepository.listGroupNames(); + } - public Group getGroupDetails(String groupName) { - return groupRepository.getGroupDetails(groupName); - } + public Group getGroupDetails(String groupName) { + return groupRepository.getGroupDetails(groupName); + } - public void createGroup(Group group, - RequestUser createdBy, - CreatorRights creatorRights) { - validator.checkCreation(group, creatorRights); - multiDcExecutor.executeByUser(new CreateGroupRepositoryCommand(group), createdBy); - auditor.objectCreated(createdBy.getUsername(), group); - } + public void createGroup(Group group, RequestUser createdBy, CreatorRights creatorRights) { + validator.checkCreation(group, creatorRights); + multiDcExecutor.executeByUser(new CreateGroupRepositoryCommand(group), createdBy); + auditor.objectCreated(createdBy.getUsername(), group); + } - public void removeGroup(String groupName, RequestUser removedBy) { - multiDcExecutor.executeByUser(new RemoveGroupRepositoryCommand(groupName), removedBy); - auditor.objectRemoved(removedBy.getUsername(), Group.from(groupName)); - } + public void removeGroup(String groupName, RequestUser removedBy) { + multiDcExecutor.executeByUser(new RemoveGroupRepositoryCommand(groupName), removedBy); + auditor.objectRemoved(removedBy.getUsername(), Group.from(groupName)); + } - public void checkGroupExists(String groupName) { - if (!groupRepository.groupExists(groupName)) { - throw new GroupNotExistsException(groupName); - } + public void checkGroupExists(String groupName) { + if (!groupRepository.groupExists(groupName)) { + throw new GroupNotExistsException(groupName); } + } - public void updateGroup(String groupName, PatchData patch, RequestUser modifiedBy) { - try { - Group retrieved = groupRepository.getGroupDetails(groupName); - Group modified = Patch.apply(retrieved, patch); - multiDcExecutor.executeByUser(new UpdateGroupRepositoryCommand(modified), modifiedBy); - groupRepository.updateGroup(modified); - auditor.objectUpdated(modifiedBy.getUsername(), retrieved, modified); - } catch (MalformedDataException exception) { - logger.warn("Problem with reading details of group {}.", groupName); - throw exception; - } + public void updateGroup(String groupName, PatchData patch, RequestUser modifiedBy) { + try { + Group retrieved = groupRepository.getGroupDetails(groupName); + Group modified = Patch.apply(retrieved, patch); + multiDcExecutor.executeByUser(new UpdateGroupRepositoryCommand(modified), modifiedBy); + groupRepository.updateGroup(modified); + auditor.objectUpdated(modifiedBy.getUsername(), retrieved, modified); + } catch (MalformedDataException exception) { + logger.warn("Problem with reading details of group {}.", groupName); + throw exception; } + } - public List queryGroup(Query query) { - return query - .filter(listGroups()) - .collect(Collectors.toList()); - } + public List queryGroup(Query query) { + return query.filter(listGroups()).collect(Collectors.toList()); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/group/GroupValidator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/group/GroupValidator.java index 1277ebdd67..de1d878f71 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/group/GroupValidator.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/group/GroupValidator.java @@ -11,24 +11,24 @@ @Component public class GroupValidator { - private final GroupRepository repository; + private final GroupRepository repository; - private final GroupNameValidator groupNameValidator; + private final GroupNameValidator groupNameValidator; - public GroupValidator(GroupRepository repository, GroupProperties groupProperties) { - this.repository = repository; - this.groupNameValidator = new GroupNameValidator(groupProperties.getAllowedGroupNameRegex()); - } + public GroupValidator(GroupRepository repository, GroupProperties groupProperties) { + this.repository = repository; + this.groupNameValidator = new GroupNameValidator(groupProperties.getAllowedGroupNameRegex()); + } - public void checkCreation(Group toCheck, CreatorRights creatorRights) { - groupNameValidator.requireValid(toCheck.getGroupName()); + public void checkCreation(Group toCheck, CreatorRights creatorRights) { + groupNameValidator.requireValid(toCheck.getGroupName()); - if (!creatorRights.allowedToCreate(toCheck)) { - throw new PermissionDeniedException("You are not allowed to create groups"); - } + if (!creatorRights.allowedToCreate(toCheck)) { + throw new PermissionDeniedException("You are not allowed to create groups"); + } - if (repository.groupExists(toCheck.getGroupName())) { - throw new GroupAlreadyExistsException(toCheck.getGroupName()); - } + if (repository.groupExists(toCheck.getGroupName())) { + throw new GroupAlreadyExistsException(toCheck.getGroupName()); } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/group/commands/CreateGroupRepositoryCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/group/commands/CreateGroupRepositoryCommand.java index cfea032d3d..4241f1db80 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/group/commands/CreateGroupRepositoryCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/group/commands/CreateGroupRepositoryCommand.java @@ -7,37 +7,38 @@ public class CreateGroupRepositoryCommand extends RepositoryCommand { - private final Group group; - private boolean exists; - - public CreateGroupRepositoryCommand(Group group) { - this.group = group; + private final Group group; + private boolean exists; + + public CreateGroupRepositoryCommand(Group group) { + this.group = group; + } + + @Override + public void backup(DatacenterBoundRepositoryHolder holder) { + exists = holder.getRepository().groupExists(group.getGroupName()); + } + + @Override + public void execute(DatacenterBoundRepositoryHolder holder) { + holder.getRepository().createGroup(group); + } + + @Override + public void rollback( + DatacenterBoundRepositoryHolder holder, Exception exception) { + if (!exists) { + holder.getRepository().removeGroup(group.getGroupName()); } + } - @Override - public void backup(DatacenterBoundRepositoryHolder holder) { - exists = holder.getRepository().groupExists(group.getGroupName()); - } + @Override + public Class getRepositoryType() { + return GroupRepository.class; + } - @Override - public void execute(DatacenterBoundRepositoryHolder holder) { - holder.getRepository().createGroup(group); - } - - @Override - public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) { - if (!exists) { - holder.getRepository().removeGroup(group.getGroupName()); - } - } - - @Override - public Class getRepositoryType() { - return GroupRepository.class; - } - - @Override - public String toString() { - return "CreateGroup(" + group.getGroupName() + ")"; - } + @Override + public String toString() { + return "CreateGroup(" + group.getGroupName() + ")"; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/group/commands/RemoveGroupRepositoryCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/group/commands/RemoveGroupRepositoryCommand.java index c4714744db..31856817c7 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/group/commands/RemoveGroupRepositoryCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/group/commands/RemoveGroupRepositoryCommand.java @@ -7,36 +7,37 @@ public class RemoveGroupRepositoryCommand extends RepositoryCommand { - private final String groupName; - - private Group backup; - - public RemoveGroupRepositoryCommand(String groupName) { - this.groupName = groupName; - } - - @Override - public void backup(DatacenterBoundRepositoryHolder holder) { - backup = holder.getRepository().getGroupDetails(groupName); - } - - @Override - public void execute(DatacenterBoundRepositoryHolder holder) { - holder.getRepository().removeGroup(groupName); - } - - @Override - public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) { - holder.getRepository().createGroup(backup); - } - - @Override - public Class getRepositoryType() { - return GroupRepository.class; - } - - @Override - public String toString() { - return "RemoveGroup(" + groupName + ")"; - } + private final String groupName; + + private Group backup; + + public RemoveGroupRepositoryCommand(String groupName) { + this.groupName = groupName; + } + + @Override + public void backup(DatacenterBoundRepositoryHolder holder) { + backup = holder.getRepository().getGroupDetails(groupName); + } + + @Override + public void execute(DatacenterBoundRepositoryHolder holder) { + holder.getRepository().removeGroup(groupName); + } + + @Override + public void rollback( + DatacenterBoundRepositoryHolder holder, Exception exception) { + holder.getRepository().createGroup(backup); + } + + @Override + public Class getRepositoryType() { + return GroupRepository.class; + } + + @Override + public String toString() { + return "RemoveGroup(" + groupName + ")"; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/group/commands/UpdateGroupRepositoryCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/group/commands/UpdateGroupRepositoryCommand.java index 62fa38600c..14c2177d90 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/group/commands/UpdateGroupRepositoryCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/group/commands/UpdateGroupRepositoryCommand.java @@ -7,36 +7,37 @@ public class UpdateGroupRepositoryCommand extends RepositoryCommand { - private final Group group; - - private Group backup; - - public UpdateGroupRepositoryCommand(Group group) { - this.group = group; - } - - @Override - public void backup(DatacenterBoundRepositoryHolder holder) { - backup = holder.getRepository().getGroupDetails(group.getGroupName()); - } - - @Override - public void execute(DatacenterBoundRepositoryHolder holder) { - holder.getRepository().updateGroup(group); - } - - @Override - public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) { - holder.getRepository().updateGroup(backup); - } - - @Override - public Class getRepositoryType() { - return GroupRepository.class; - } - - @Override - public String toString() { - return "UpdateGroup(" + group.getGroupName() + ")"; - } + private final Group group; + + private Group backup; + + public UpdateGroupRepositoryCommand(Group group) { + this.group = group; + } + + @Override + public void backup(DatacenterBoundRepositoryHolder holder) { + backup = holder.getRepository().getGroupDetails(group.getGroupName()); + } + + @Override + public void execute(DatacenterBoundRepositoryHolder holder) { + holder.getRepository().updateGroup(group); + } + + @Override + public void rollback( + DatacenterBoundRepositoryHolder holder, Exception exception) { + holder.getRepository().updateGroup(backup); + } + + @Override + public Class getRepositoryType() { + return GroupRepository.class; + } + + @Override + public String toString() { + return "UpdateGroup(" + group.getGroupName() + ")"; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/health/CouldNotResolveHostNameException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/health/CouldNotResolveHostNameException.java index 048ee4f27d..fa1065eb28 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/health/CouldNotResolveHostNameException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/health/CouldNotResolveHostNameException.java @@ -1,7 +1,7 @@ package pl.allegro.tech.hermes.management.domain.health; class CouldNotResolveHostNameException extends RuntimeException { - CouldNotResolveHostNameException(Throwable cause) { - super(cause); - } + CouldNotResolveHostNameException(Throwable cause) { + super(cause); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/health/HealthCheckScheduler.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/health/HealthCheckScheduler.java index 7958986dc6..b9920af761 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/health/HealthCheckScheduler.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/health/HealthCheckScheduler.java @@ -4,6 +4,9 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import io.micrometer.core.instrument.MeterRegistry; import jakarta.annotation.PostConstruct; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -12,64 +15,60 @@ import pl.allegro.tech.hermes.management.domain.mode.ModeService; import pl.allegro.tech.hermes.management.infrastructure.zookeeper.ZookeeperClientManager; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - @Component public class HealthCheckScheduler { - private static final Logger logger = LoggerFactory.getLogger(HealthCheckScheduler.class); + private static final Logger logger = LoggerFactory.getLogger(HealthCheckScheduler.class); - private final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor( - new ThreadFactoryBuilder().setNameFormat("storage-health-check-scheduler-%d").build() - ); - private final ZookeeperClientManager zookeeperClientManager; - private final ZookeeperPaths zookeeperPaths; - private final NodeDataProvider nodeDataProvider; - private final ObjectMapper objectMapper; - private final ModeService modeService; - private final MeterRegistry meterRegistry; - private final Long periodSeconds; - private final boolean enabled; + private final ScheduledExecutorService executorService = + Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder().setNameFormat("storage-health-check-scheduler-%d").build()); + private final ZookeeperClientManager zookeeperClientManager; + private final ZookeeperPaths zookeeperPaths; + private final NodeDataProvider nodeDataProvider; + private final ObjectMapper objectMapper; + private final ModeService modeService; + private final MeterRegistry meterRegistry; + private final Long periodSeconds; + private final boolean enabled; - public HealthCheckScheduler(ZookeeperClientManager zookeeperClientManager, - ZookeeperPaths zookeeperPaths, - NodeDataProvider nodeDataProvider, - ObjectMapper objectMapper, - ModeService modeService, - MeterRegistry meterRegistry, - @Value("${management.health.periodSeconds:30}") Long periodSeconds, - @Value("${management.health.enabled:false}") boolean enabled) { - this.zookeeperClientManager = zookeeperClientManager; - this.zookeeperPaths = zookeeperPaths; - this.nodeDataProvider = nodeDataProvider; - this.objectMapper = objectMapper; - this.modeService = modeService; - this.meterRegistry = meterRegistry; - this.periodSeconds = periodSeconds; - this.enabled = enabled; - } + public HealthCheckScheduler( + ZookeeperClientManager zookeeperClientManager, + ZookeeperPaths zookeeperPaths, + NodeDataProvider nodeDataProvider, + ObjectMapper objectMapper, + ModeService modeService, + MeterRegistry meterRegistry, + @Value("${management.health.periodSeconds:30}") Long periodSeconds, + @Value("${management.health.enabled:false}") boolean enabled) { + this.zookeeperClientManager = zookeeperClientManager; + this.zookeeperPaths = zookeeperPaths; + this.nodeDataProvider = nodeDataProvider; + this.objectMapper = objectMapper; + this.modeService = modeService; + this.meterRegistry = meterRegistry; + this.periodSeconds = periodSeconds; + this.enabled = enabled; + } - @PostConstruct - public void scheduleHealthCheck() { - if (enabled) { - logger.info("Starting the storage health check scheduler"); - String healthCheckPath = zookeeperPaths.nodeHealthPathForManagementHost( - nodeDataProvider.getHostname(), - nodeDataProvider.getServerPort() - ); - HealthCheckTask healthCheckTask = new HealthCheckTask( - zookeeperClientManager.getClients(), - healthCheckPath, - objectMapper, - modeService, - meterRegistry - ); - executorService.scheduleAtFixedRate(healthCheckTask, 0, periodSeconds, TimeUnit.SECONDS); - } else { - logger.info("Storage health check is disabled"); - modeService.setMode(ModeService.ManagementMode.READ_WRITE); - } + @PostConstruct + public void scheduleHealthCheck() { + if (enabled) { + logger.info("Starting the storage health check scheduler"); + String healthCheckPath = + zookeeperPaths.nodeHealthPathForManagementHost( + nodeDataProvider.getHostname(), nodeDataProvider.getServerPort()); + HealthCheckTask healthCheckTask = + new HealthCheckTask( + zookeeperClientManager.getClients(), + healthCheckPath, + objectMapper, + modeService, + meterRegistry); + executorService.scheduleAtFixedRate(healthCheckTask, 0, periodSeconds, TimeUnit.SECONDS); + } else { + logger.info("Storage health check is disabled"); + modeService.setMode(ModeService.ManagementMode.READ_WRITE); } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/health/HealthCheckTask.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/health/HealthCheckTask.java index 4a80112c0c..c89d9c109b 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/health/HealthCheckTask.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/health/HealthCheckTask.java @@ -2,69 +2,74 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.micrometer.core.instrument.MeterRegistry; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import pl.allegro.tech.hermes.management.domain.mode.ModeService; -import pl.allegro.tech.hermes.management.infrastructure.zookeeper.ZookeeperClient; - import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.Collection; import java.util.List; import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import pl.allegro.tech.hermes.management.domain.mode.ModeService; +import pl.allegro.tech.hermes.management.infrastructure.zookeeper.ZookeeperClient; class HealthCheckTask implements Runnable { - private static final Logger logger = LoggerFactory.getLogger(HealthCheckTask.class); + private static final Logger logger = LoggerFactory.getLogger(HealthCheckTask.class); - private final Collection zookeeperClients; - private final String healthCheckPath; - private final ObjectMapper objectMapper; - private final ModeService modeService; - private final MeterRegistry meterRegistry; + private final Collection zookeeperClients; + private final String healthCheckPath; + private final ObjectMapper objectMapper; + private final ModeService modeService; + private final MeterRegistry meterRegistry; - HealthCheckTask(Collection zookeeperClients, String healthCheckPath, ObjectMapper objectMapper, - ModeService modeService, MeterRegistry meterRegistry) { - this.zookeeperClients = zookeeperClients; - this.healthCheckPath = healthCheckPath; - this.objectMapper = objectMapper; - this.modeService = modeService; - this.meterRegistry = meterRegistry; - } + HealthCheckTask( + Collection zookeeperClients, + String healthCheckPath, + ObjectMapper objectMapper, + ModeService modeService, + MeterRegistry meterRegistry) { + this.zookeeperClients = zookeeperClients; + this.healthCheckPath = healthCheckPath; + this.objectMapper = objectMapper; + this.modeService = modeService; + this.meterRegistry = meterRegistry; + } - @Override - public void run() { - final List healthCheckResults = zookeeperClients.stream() - .map(this::doHealthCheck) - .collect(Collectors.toList()); - updateMode(healthCheckResults); - } + @Override + public void run() { + final List healthCheckResults = + zookeeperClients.stream().map(this::doHealthCheck).collect(Collectors.toList()); + updateMode(healthCheckResults); + } - private HealthCheckResult doHealthCheck(ZookeeperClient zookeeperClient) { - final String timestamp = LocalDateTime.now().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); - try { - zookeeperClient.ensureEphemeralNodeExists(healthCheckPath); - zookeeperClient.getCuratorFramework() - .setData() - .forPath(healthCheckPath, objectMapper.writeValueAsBytes(timestamp)); - meterRegistry.counter("storage-health-check.successful").increment(); - return HealthCheckResult.HEALTHY; - } catch (Exception e) { - meterRegistry.counter("storage-health-check.failed").increment(); - logger.error("Storage health check failed for datacenter {}", zookeeperClient.getDatacenterName(), e); - return HealthCheckResult.UNHEALTHY; - } + private HealthCheckResult doHealthCheck(ZookeeperClient zookeeperClient) { + final String timestamp = LocalDateTime.now().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); + try { + zookeeperClient.ensureEphemeralNodeExists(healthCheckPath); + zookeeperClient + .getCuratorFramework() + .setData() + .forPath(healthCheckPath, objectMapper.writeValueAsBytes(timestamp)); + meterRegistry.counter("storage-health-check.successful").increment(); + return HealthCheckResult.HEALTHY; + } catch (Exception e) { + meterRegistry.counter("storage-health-check.failed").increment(); + logger.error( + "Storage health check failed for datacenter {}", zookeeperClient.getDatacenterName(), e); + return HealthCheckResult.UNHEALTHY; } + } - private void updateMode(List healthCheckResults) { - if (healthCheckResults.contains(HealthCheckResult.UNHEALTHY)) { - modeService.setMode(ModeService.ManagementMode.READ_ONLY); - } else { - modeService.setMode(ModeService.ManagementMode.READ_WRITE); - } + private void updateMode(List healthCheckResults) { + if (healthCheckResults.contains(HealthCheckResult.UNHEALTHY)) { + modeService.setMode(ModeService.ManagementMode.READ_ONLY); + } else { + modeService.setMode(ModeService.ManagementMode.READ_WRITE); } + } - private enum HealthCheckResult { - HEALTHY, UNHEALTHY - } + private enum HealthCheckResult { + HEALTHY, + UNHEALTHY + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/health/NodeDataProvider.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/health/NodeDataProvider.java index 5c0517c5d1..1cd5506171 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/health/NodeDataProvider.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/health/NodeDataProvider.java @@ -1,29 +1,28 @@ package pl.allegro.tech.hermes.management.domain.health; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - import java.net.InetAddress; import java.net.UnknownHostException; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; @Component class NodeDataProvider { - private final String serverPort; + private final String serverPort; - NodeDataProvider(@Value("${server.port}") String serverPort) { - this.serverPort = serverPort; - } + NodeDataProvider(@Value("${server.port}") String serverPort) { + this.serverPort = serverPort; + } - String getHostname() { - try { - return InetAddress.getLocalHost().getHostName(); - } catch (UnknownHostException e) { - throw new CouldNotResolveHostNameException(e); - } + String getHostname() { + try { + return InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + throw new CouldNotResolveHostNameException(e); } + } - String getServerPort() { - return serverPort; - } + String getServerPort() { + return serverPort; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/message/RetransmissionService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/message/RetransmissionService.java index 4e217dbf95..5ad8f6fc35 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/message/RetransmissionService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/message/RetransmissionService.java @@ -1,14 +1,13 @@ package pl.allegro.tech.hermes.management.domain.message; +import java.util.List; import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.common.kafka.offset.PartitionOffset; -import java.util.List; - public interface RetransmissionService { - List indicateOffsetChange(Topic topic, String subscription, String brokersClusterName, - long timestamp, boolean dryRun); + List indicateOffsetChange( + Topic topic, String subscription, String brokersClusterName, long timestamp, boolean dryRun); - boolean areOffsetsMoved(Topic topic, String subscriptionName, String brokersClusterName); + boolean areOffsetsMoved(Topic topic, String subscriptionName, String brokersClusterName); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/mode/ModeService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/mode/ModeService.java index 0c75cd6c3c..2e070fd479 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/mode/ModeService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/mode/ModeService.java @@ -5,45 +5,45 @@ @Component public class ModeService { - public static final String READ_WRITE = "readWrite"; - public static final String READ_ONLY = "readOnly"; - public static final String READ_ONLY_ADMIN = "readOnlyAdmin"; + public static final String READ_WRITE = "readWrite"; + public static final String READ_ONLY = "readOnly"; + public static final String READ_ONLY_ADMIN = "readOnlyAdmin"; - public enum ManagementMode { - READ_WRITE(ModeService.READ_WRITE), - READ_ONLY(ModeService.READ_ONLY), - READ_ONLY_ADMIN(ModeService.READ_ONLY_ADMIN); + public enum ManagementMode { + READ_WRITE(ModeService.READ_WRITE), + READ_ONLY(ModeService.READ_ONLY), + READ_ONLY_ADMIN(ModeService.READ_ONLY_ADMIN); - private final String text; + private final String text; - ManagementMode(String text) { - this.text = text; - } + ManagementMode(String text) { + this.text = text; + } - @Override - public String toString() { - return text; - } + @Override + public String toString() { + return text; } + } - private volatile ManagementMode mode = ManagementMode.READ_ONLY; + private volatile ManagementMode mode = ManagementMode.READ_ONLY; - public ManagementMode getMode() { - return mode; - } + public ManagementMode getMode() { + return mode; + } - public synchronized void setModeByAdmin(ManagementMode mode) { - this.mode = mode; - } + public synchronized void setModeByAdmin(ManagementMode mode) { + this.mode = mode; + } - public boolean isReadOnlyEnabled() { - return mode == ManagementMode.READ_ONLY || mode == ManagementMode.READ_ONLY_ADMIN; - } + public boolean isReadOnlyEnabled() { + return mode == ManagementMode.READ_ONLY || mode == ManagementMode.READ_ONLY_ADMIN; + } - public synchronized void setMode(ManagementMode newMode) { - /* READ_ONLY_ADMIN is a flag that can be changed only by admin */ - if (!mode.equals(ManagementMode.READ_ONLY_ADMIN)) { - this.mode = newMode; - } + public synchronized void setMode(ManagementMode newMode) { + /* READ_ONLY_ADMIN is a flag that can be changed only by admin */ + if (!mode.equals(ManagementMode.READ_ONLY_ADMIN)) { + this.mode = newMode; } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/oauth/OAuthProviderService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/oauth/OAuthProviderService.java index dedff9b5e6..10e84e114e 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/oauth/OAuthProviderService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/oauth/OAuthProviderService.java @@ -1,5 +1,6 @@ package pl.allegro.tech.hermes.management.domain.oauth; +import java.util.List; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import pl.allegro.tech.hermes.api.OAuthProvider; @@ -14,51 +15,55 @@ import pl.allegro.tech.hermes.management.domain.oauth.commands.RemoveOAuthProviderRepositoryCommand; import pl.allegro.tech.hermes.management.domain.oauth.commands.UpdateOAuthProviderRepositoryCommand; -import java.util.List; - @Component public class OAuthProviderService { - private final OAuthProviderRepository repository; - private final ApiPreconditions preconditions; - private final Auditor auditor; - private final MultiDatacenterRepositoryCommandExecutor multiDcExecutor; + private final OAuthProviderRepository repository; + private final ApiPreconditions preconditions; + private final Auditor auditor; + private final MultiDatacenterRepositoryCommandExecutor multiDcExecutor; - @Autowired - public OAuthProviderService(OAuthProviderRepository repository, ApiPreconditions preconditions, Auditor auditor, - MultiDatacenterRepositoryCommandExecutor multiDcExecutor) { - this.repository = repository; - this.preconditions = preconditions; - this.auditor = auditor; - this.multiDcExecutor = multiDcExecutor; - } + @Autowired + public OAuthProviderService( + OAuthProviderRepository repository, + ApiPreconditions preconditions, + Auditor auditor, + MultiDatacenterRepositoryCommandExecutor multiDcExecutor) { + this.repository = repository; + this.preconditions = preconditions; + this.auditor = auditor; + this.multiDcExecutor = multiDcExecutor; + } - public List listOAuthProviderNames() { - return repository.listOAuthProviderNames(); - } + public List listOAuthProviderNames() { + return repository.listOAuthProviderNames(); + } - public OAuthProvider getOAuthProviderDetails(String oAuthProviderName) { - return repository.getOAuthProviderDetails(oAuthProviderName).anonymize(); - } + public OAuthProvider getOAuthProviderDetails(String oAuthProviderName) { + return repository.getOAuthProviderDetails(oAuthProviderName).anonymize(); + } - public void createOAuthProvider(OAuthProvider oAuthProvider, RequestUser createdBy) { - preconditions.checkConstraints(oAuthProvider, false); - multiDcExecutor.executeByUser(new CreateOAuthProviderRepositoryCommand(oAuthProvider), createdBy); - auditor.objectCreated(createdBy.getUsername(), oAuthProvider); - } + public void createOAuthProvider(OAuthProvider oAuthProvider, RequestUser createdBy) { + preconditions.checkConstraints(oAuthProvider, false); + multiDcExecutor.executeByUser( + new CreateOAuthProviderRepositoryCommand(oAuthProvider), createdBy); + auditor.objectCreated(createdBy.getUsername(), oAuthProvider); + } - public void removeOAuthProvider(String oAuthProviderName, RequestUser removedBy) { - OAuthProvider oAuthProvider = repository.getOAuthProviderDetails(oAuthProviderName); - multiDcExecutor.executeByUser(new RemoveOAuthProviderRepositoryCommand(oAuthProviderName), removedBy); - auditor.objectRemoved(removedBy.getUsername(), oAuthProvider); - } + public void removeOAuthProvider(String oAuthProviderName, RequestUser removedBy) { + OAuthProvider oAuthProvider = repository.getOAuthProviderDetails(oAuthProviderName); + multiDcExecutor.executeByUser( + new RemoveOAuthProviderRepositoryCommand(oAuthProviderName), removedBy); + auditor.objectRemoved(removedBy.getUsername(), oAuthProvider); + } - public void updateOAuthProvider(String oAuthProviderName, PatchData patch, RequestUser updatedBy) { - OAuthProvider retrieved = repository.getOAuthProviderDetails(oAuthProviderName); - OAuthProvider updated = Patch.apply(retrieved, patch); - preconditions.checkConstraints(updated, false); + public void updateOAuthProvider( + String oAuthProviderName, PatchData patch, RequestUser updatedBy) { + OAuthProvider retrieved = repository.getOAuthProviderDetails(oAuthProviderName); + OAuthProvider updated = Patch.apply(retrieved, patch); + preconditions.checkConstraints(updated, false); - multiDcExecutor.executeByUser(new UpdateOAuthProviderRepositoryCommand(updated), updatedBy); - auditor.objectUpdated(updatedBy.getUsername(), retrieved, updated); - } + multiDcExecutor.executeByUser(new UpdateOAuthProviderRepositoryCommand(updated), updatedBy); + auditor.objectUpdated(updatedBy.getUsername(), retrieved, updated); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/oauth/commands/CreateOAuthProviderRepositoryCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/oauth/commands/CreateOAuthProviderRepositoryCommand.java index be440319d3..b8f034fd8e 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/oauth/commands/CreateOAuthProviderRepositoryCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/oauth/commands/CreateOAuthProviderRepositoryCommand.java @@ -5,34 +5,36 @@ import pl.allegro.tech.hermes.management.domain.dc.DatacenterBoundRepositoryHolder; import pl.allegro.tech.hermes.management.domain.dc.RepositoryCommand; -public class CreateOAuthProviderRepositoryCommand extends RepositoryCommand { - - private final OAuthProvider provider; - - public CreateOAuthProviderRepositoryCommand(OAuthProvider provider) { - this.provider = provider; - } - - @Override - public void backup(DatacenterBoundRepositoryHolder holder) {} - - @Override - public void execute(DatacenterBoundRepositoryHolder holder) { - holder.getRepository().createOAuthProvider(provider); - } - - @Override - public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) { - holder.getRepository().removeOAuthProvider(provider.getName()); - } - - @Override - public Class getRepositoryType() { - return OAuthProviderRepository.class; - } - - @Override - public String toString() { - return "CreateOAuthProvider(" + provider.getName() + ")"; - } +public class CreateOAuthProviderRepositoryCommand + extends RepositoryCommand { + + private final OAuthProvider provider; + + public CreateOAuthProviderRepositoryCommand(OAuthProvider provider) { + this.provider = provider; + } + + @Override + public void backup(DatacenterBoundRepositoryHolder holder) {} + + @Override + public void execute(DatacenterBoundRepositoryHolder holder) { + holder.getRepository().createOAuthProvider(provider); + } + + @Override + public void rollback( + DatacenterBoundRepositoryHolder holder, Exception exception) { + holder.getRepository().removeOAuthProvider(provider.getName()); + } + + @Override + public Class getRepositoryType() { + return OAuthProviderRepository.class; + } + + @Override + public String toString() { + return "CreateOAuthProvider(" + provider.getName() + ")"; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/oauth/commands/RemoveOAuthProviderRepositoryCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/oauth/commands/RemoveOAuthProviderRepositoryCommand.java index a17a725779..5a877f6d9f 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/oauth/commands/RemoveOAuthProviderRepositoryCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/oauth/commands/RemoveOAuthProviderRepositoryCommand.java @@ -5,38 +5,40 @@ import pl.allegro.tech.hermes.management.domain.dc.DatacenterBoundRepositoryHolder; import pl.allegro.tech.hermes.management.domain.dc.RepositoryCommand; -public class RemoveOAuthProviderRepositoryCommand extends RepositoryCommand { - - private final String providerName; - - private OAuthProvider backup; - - public RemoveOAuthProviderRepositoryCommand(String providerName) { - this.providerName = providerName; - } - - @Override - public void backup(DatacenterBoundRepositoryHolder holder) { - backup = holder.getRepository().getOAuthProviderDetails(providerName); - } - - @Override - public void execute(DatacenterBoundRepositoryHolder holder) { - holder.getRepository().removeOAuthProvider(providerName); - } - - @Override - public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) { - holder.getRepository().createOAuthProvider(backup); - } - - @Override - public Class getRepositoryType() { - return OAuthProviderRepository.class; - } - - @Override - public String toString() { - return "RemoveOAuthProvider(" + providerName + ")"; - } +public class RemoveOAuthProviderRepositoryCommand + extends RepositoryCommand { + + private final String providerName; + + private OAuthProvider backup; + + public RemoveOAuthProviderRepositoryCommand(String providerName) { + this.providerName = providerName; + } + + @Override + public void backup(DatacenterBoundRepositoryHolder holder) { + backup = holder.getRepository().getOAuthProviderDetails(providerName); + } + + @Override + public void execute(DatacenterBoundRepositoryHolder holder) { + holder.getRepository().removeOAuthProvider(providerName); + } + + @Override + public void rollback( + DatacenterBoundRepositoryHolder holder, Exception exception) { + holder.getRepository().createOAuthProvider(backup); + } + + @Override + public Class getRepositoryType() { + return OAuthProviderRepository.class; + } + + @Override + public String toString() { + return "RemoveOAuthProvider(" + providerName + ")"; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/oauth/commands/UpdateOAuthProviderRepositoryCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/oauth/commands/UpdateOAuthProviderRepositoryCommand.java index beeed66941..8d536ec0ba 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/oauth/commands/UpdateOAuthProviderRepositoryCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/oauth/commands/UpdateOAuthProviderRepositoryCommand.java @@ -5,38 +5,40 @@ import pl.allegro.tech.hermes.management.domain.dc.DatacenterBoundRepositoryHolder; import pl.allegro.tech.hermes.management.domain.dc.RepositoryCommand; -public class UpdateOAuthProviderRepositoryCommand extends RepositoryCommand { - - private final OAuthProvider provider; - - private OAuthProvider backup; - - public UpdateOAuthProviderRepositoryCommand(OAuthProvider provider) { - this.provider = provider; - } - - @Override - public void backup(DatacenterBoundRepositoryHolder holder) { - backup = holder.getRepository().getOAuthProviderDetails(provider.getName()); - } - - @Override - public void execute(DatacenterBoundRepositoryHolder holder) { - holder.getRepository().updateOAuthProvider(provider); - } - - @Override - public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) { - holder.getRepository().updateOAuthProvider(backup); - } - - @Override - public Class getRepositoryType() { - return OAuthProviderRepository.class; - } - - @Override - public String toString() { - return "UpdateOAuthProvider(" + provider.getName() + ")"; - } +public class UpdateOAuthProviderRepositoryCommand + extends RepositoryCommand { + + private final OAuthProvider provider; + + private OAuthProvider backup; + + public UpdateOAuthProviderRepositoryCommand(OAuthProvider provider) { + this.provider = provider; + } + + @Override + public void backup(DatacenterBoundRepositoryHolder holder) { + backup = holder.getRepository().getOAuthProviderDetails(provider.getName()); + } + + @Override + public void execute(DatacenterBoundRepositoryHolder holder) { + holder.getRepository().updateOAuthProvider(provider); + } + + @Override + public void rollback( + DatacenterBoundRepositoryHolder holder, Exception exception) { + holder.getRepository().updateOAuthProvider(backup); + } + + @Override + public Class getRepositoryType() { + return OAuthProviderRepository.class; + } + + @Override + public String toString() { + return "UpdateOAuthProvider(" + provider.getName() + ")"; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/owner/OwnerSource.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/owner/OwnerSource.java index 4769fb5c35..50da8f49d5 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/owner/OwnerSource.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/owner/OwnerSource.java @@ -1,45 +1,41 @@ package pl.allegro.tech.hermes.management.domain.owner; +import java.util.List; +import java.util.Optional; import pl.allegro.tech.hermes.api.ErrorCode; import pl.allegro.tech.hermes.api.Owner; import pl.allegro.tech.hermes.management.domain.ManagementException; -import java.util.List; -import java.util.Optional; - public interface OwnerSource { - String name(); + String name(); - boolean exists(String ownerId); + boolean exists(String ownerId); - Owner get(String id) throws OwnerNotFound; + Owner get(String id) throws OwnerNotFound; - default boolean isDeprecated() { - return false; - } + default boolean isDeprecated() { + return false; + } - /** - * Override if the implemented owner source supports autocompletion. - */ - default Optional autocompletion() { - return Optional.empty(); - } + /** Override if the implemented owner source supports autocompletion. */ + default Optional autocompletion() { + return Optional.empty(); + } - interface Autocompletion { - List ownersMatching(String searchString); - } + interface Autocompletion { + List ownersMatching(String searchString); + } - class OwnerNotFound extends ManagementException { + class OwnerNotFound extends ManagementException { - public OwnerNotFound(String source, String id) { - super("Owner of id '" + id + "' not found in source " + source); - } - - @Override - public ErrorCode getCode() { - return ErrorCode.OWNER_NOT_FOUND; - } + public OwnerNotFound(String source, String id) { + super("Owner of id '" + id + "' not found in source " + source); } + @Override + public ErrorCode getCode() { + return ErrorCode.OWNER_NOT_FOUND; + } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/owner/OwnerSourceNotFound.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/owner/OwnerSourceNotFound.java index 2be616ef2d..4f195fe506 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/owner/OwnerSourceNotFound.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/owner/OwnerSourceNotFound.java @@ -5,13 +5,12 @@ public class OwnerSourceNotFound extends ManagementException { - public OwnerSourceNotFound(String name) { - super("Owner source named '" + name + "' not found"); - } - - @Override - public ErrorCode getCode() { - return ErrorCode.OWNER_SOURCE_NOT_FOUND; - } + public OwnerSourceNotFound(String name) { + super("Owner source named '" + name + "' not found"); + } + @Override + public ErrorCode getCode() { + return ErrorCode.OWNER_SOURCE_NOT_FOUND; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/owner/OwnerSources.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/owner/OwnerSources.java index 9b7d24a9b7..e9c9e63457 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/owner/OwnerSources.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/owner/OwnerSources.java @@ -1,63 +1,61 @@ package pl.allegro.tech.hermes.management.domain.owner; import com.google.common.collect.ImmutableList; -import pl.allegro.tech.hermes.api.ErrorCode; -import pl.allegro.tech.hermes.common.exception.HermesException; - import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; +import pl.allegro.tech.hermes.api.ErrorCode; +import pl.allegro.tech.hermes.common.exception.HermesException; public class OwnerSources implements Iterable { - private final Map ownerSourcesByNames; + private final Map ownerSourcesByNames; - private final List ownerSources; + private final List ownerSources; - public OwnerSources(List ownerSources) { - if (ownerSources.isEmpty()) { - throw new IllegalArgumentException("At least one owner source must be configured"); - } + public OwnerSources(List ownerSources) { + if (ownerSources.isEmpty()) { + throw new IllegalArgumentException("At least one owner source must be configured"); + } - this.ownerSourcesByNames = ownerSources.stream().collect( + this.ownerSourcesByNames = + ownerSources.stream() + .collect( Collectors.toMap( - OwnerSource::name, - Function.identity(), - (a, b) -> { - throw new IllegalArgumentException("Duplicate owner source " + a.name()); - } - ) - ); - - this.ownerSources = ImmutableList.copyOf(ownerSources); - } + OwnerSource::name, + Function.identity(), + (a, b) -> { + throw new IllegalArgumentException("Duplicate owner source " + a.name()); + })); - public Optional getByName(String name) { - return Optional.ofNullable(ownerSourcesByNames.get(name)); - } + this.ownerSources = ImmutableList.copyOf(ownerSources); + } - public OwnerSource.Autocompletion getAutocompletionFor(String name) { - OwnerSource source = getByName(name).orElseThrow(() -> new OwnerSourceNotFound(name)); - return source.autocompletion().orElseThrow(() -> new AutocompleteNotSupportedException(source)); - } + public Optional getByName(String name) { + return Optional.ofNullable(ownerSourcesByNames.get(name)); + } - public Iterator iterator() { - return ownerSources.iterator(); - } + public OwnerSource.Autocompletion getAutocompletionFor(String name) { + OwnerSource source = getByName(name).orElseThrow(() -> new OwnerSourceNotFound(name)); + return source.autocompletion().orElseThrow(() -> new AutocompleteNotSupportedException(source)); + } - private static class AutocompleteNotSupportedException extends HermesException { + public Iterator iterator() { + return ownerSources.iterator(); + } - AutocompleteNotSupportedException(OwnerSource source) { - super("Owner source '" + source.name() + "' doesn't support autocomplete"); - } + private static class AutocompleteNotSupportedException extends HermesException { - @Override - public ErrorCode getCode() { - return ErrorCode.OWNER_SOURCE_DOESNT_SUPPORT_AUTOCOMPLETE; - } + AutocompleteNotSupportedException(OwnerSource source) { + super("Owner source '" + source.name() + "' doesn't support autocomplete"); + } + @Override + public ErrorCode getCode() { + return ErrorCode.OWNER_SOURCE_DOESNT_SUPPORT_AUTOCOMPLETE; } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/owner/PlaintextOwnerSource.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/owner/PlaintextOwnerSource.java index 279358164f..6c7ea8ccae 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/owner/PlaintextOwnerSource.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/owner/PlaintextOwnerSource.java @@ -8,22 +8,21 @@ @Order(PlaintextOwnerSource.ORDER) public class PlaintextOwnerSource implements OwnerSource { - public static final int ORDER = 0; - public static final String NAME = "Plaintext"; + public static final int ORDER = 0; + public static final String NAME = "Plaintext"; - @Override - public String name() { - return NAME; - } + @Override + public String name() { + return NAME; + } - @Override - public boolean exists(String ownerId) { - return true; - } - - @Override - public Owner get(String id) throws OwnerNotFound { - return new Owner(id, id); - } + @Override + public boolean exists(String ownerId) { + return true; + } + @Override + public Owner get(String id) throws OwnerNotFound { + return new Owner(id, id); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/owner/validator/OwnerIdValidationException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/owner/validator/OwnerIdValidationException.java index 5d0ce6ac00..03f5cc0722 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/owner/validator/OwnerIdValidationException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/owner/validator/OwnerIdValidationException.java @@ -5,13 +5,12 @@ public class OwnerIdValidationException extends ManagementException { - public OwnerIdValidationException(String message) { - super(message); - } - - @Override - public ErrorCode getCode() { - return ErrorCode.VALIDATION_ERROR; - } + public OwnerIdValidationException(String message) { + super(message); + } + @Override + public ErrorCode getCode() { + return ErrorCode.VALIDATION_ERROR; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/owner/validator/OwnerIdValidator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/owner/validator/OwnerIdValidator.java index 9d3d8d2469..af94db4e78 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/owner/validator/OwnerIdValidator.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/owner/validator/OwnerIdValidator.java @@ -9,19 +9,25 @@ @Component public class OwnerIdValidator { - private final OwnerSources ownerSources; + private final OwnerSources ownerSources; - @Autowired - public OwnerIdValidator(OwnerSources ownerSources) { - this.ownerSources = ownerSources; - } + @Autowired + public OwnerIdValidator(OwnerSources ownerSources) { + this.ownerSources = ownerSources; + } - public void check(OwnerId toCheck) { - OwnerSource source = ownerSources.getByName(toCheck.getSource()) - .orElseThrow(() -> new OwnerIdValidationException("Owner source '" + toCheck.getSource() + "' doesn't exist")); + public void check(OwnerId toCheck) { + OwnerSource source = + ownerSources + .getByName(toCheck.getSource()) + .orElseThrow( + () -> + new OwnerIdValidationException( + "Owner source '" + toCheck.getSource() + "' doesn't exist")); - if (!source.exists(toCheck.getId())) { - throw new OwnerIdValidationException("Owner '" + toCheck.getId() + "' doesn't exist in source " + toCheck.getSource()); - } + if (!source.exists(toCheck.getId())) { + throw new OwnerIdValidationException( + "Owner '" + toCheck.getId() + "' doesn't exist in source " + toCheck.getSource()); } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/readiness/DatacenterReadinessRepository.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/readiness/DatacenterReadinessRepository.java index d1270c3a14..f86fde1491 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/readiness/DatacenterReadinessRepository.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/readiness/DatacenterReadinessRepository.java @@ -1,12 +1,11 @@ package pl.allegro.tech.hermes.management.domain.readiness; -import pl.allegro.tech.hermes.api.DatacenterReadiness; - import java.util.List; +import pl.allegro.tech.hermes.api.DatacenterReadiness; public interface DatacenterReadinessRepository { - List getReadiness(); + List getReadiness(); - void setReadiness(List datacenterReadiness); + void setReadiness(List datacenterReadiness); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/readiness/ReadinessService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/readiness/ReadinessService.java index 30d144ac7a..a9d51f6555 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/readiness/ReadinessService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/readiness/ReadinessService.java @@ -1,7 +1,6 @@ package pl.allegro.tech.hermes.management.domain.readiness; -import pl.allegro.tech.hermes.api.DatacenterReadiness; -import pl.allegro.tech.hermes.management.domain.dc.MultiDatacenterRepositoryCommandExecutor; +import static pl.allegro.tech.hermes.api.DatacenterReadiness.ReadinessStatus.READY; import java.util.HashMap; import java.util.List; @@ -9,52 +8,52 @@ import java.util.Objects; import java.util.function.Function; import java.util.stream.Collectors; - -import static pl.allegro.tech.hermes.api.DatacenterReadiness.ReadinessStatus.READY; +import pl.allegro.tech.hermes.api.DatacenterReadiness; +import pl.allegro.tech.hermes.management.domain.dc.MultiDatacenterRepositoryCommandExecutor; public class ReadinessService { - private final MultiDatacenterRepositoryCommandExecutor commandExecutor; - private final DatacenterReadinessRepository readinessRepository; - private final List datacenters; - - public ReadinessService(MultiDatacenterRepositoryCommandExecutor commandExecutor, - DatacenterReadinessRepository readinessRepository, - List datacenters) { - this.commandExecutor = commandExecutor; - this.readinessRepository = readinessRepository; - this.datacenters = datacenters; - } - - public void setReady(DatacenterReadiness datacenterReadiness) { - Map current = getReadiness(); - Map toSave = new HashMap<>(); - for (String datacenter : datacenters) { - toSave.put(datacenter, current.get(datacenter)); - } - toSave.put(datacenterReadiness.getDatacenter(), datacenterReadiness); - List readiness = toSave.values().stream() - .filter(Objects::nonNull) - .toList(); - commandExecutor.execute(new SetReadinessCommand(readiness)); + private final MultiDatacenterRepositoryCommandExecutor commandExecutor; + private final DatacenterReadinessRepository readinessRepository; + private final List datacenters; + + public ReadinessService( + MultiDatacenterRepositoryCommandExecutor commandExecutor, + DatacenterReadinessRepository readinessRepository, + List datacenters) { + this.commandExecutor = commandExecutor; + this.readinessRepository = readinessRepository; + this.datacenters = datacenters; + } + + public void setReady(DatacenterReadiness datacenterReadiness) { + Map current = getReadiness(); + Map toSave = new HashMap<>(); + for (String datacenter : datacenters) { + toSave.put(datacenter, current.get(datacenter)); } - - public List getDatacentersReadiness() { - Map current = getReadiness(); - Map result = new HashMap<>(); - for (String datacenter : datacenters) { - DatacenterReadiness datacenterReadiness = current.get(datacenter); - if (datacenterReadiness == null) { - result.put(datacenter, new DatacenterReadiness(datacenter, READY)); - } else { - result.put(datacenter, datacenterReadiness); - } - } - return result.values().stream().toList(); + toSave.put(datacenterReadiness.getDatacenter(), datacenterReadiness); + List readiness = + toSave.values().stream().filter(Objects::nonNull).toList(); + commandExecutor.execute(new SetReadinessCommand(readiness)); + } + + public List getDatacentersReadiness() { + Map current = getReadiness(); + Map result = new HashMap<>(); + for (String datacenter : datacenters) { + DatacenterReadiness datacenterReadiness = current.get(datacenter); + if (datacenterReadiness == null) { + result.put(datacenter, new DatacenterReadiness(datacenter, READY)); + } else { + result.put(datacenter, datacenterReadiness); + } } + return result.values().stream().toList(); + } - private Map getReadiness() { - return readinessRepository.getReadiness().stream() - .collect(Collectors.toMap(DatacenterReadiness::getDatacenter, Function.identity())); - } + private Map getReadiness() { + return readinessRepository.getReadiness().stream() + .collect(Collectors.toMap(DatacenterReadiness::getDatacenter, Function.identity())); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/readiness/SetReadinessCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/readiness/SetReadinessCommand.java index 5dac6e9174..3077c43fa9 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/readiness/SetReadinessCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/readiness/SetReadinessCommand.java @@ -1,36 +1,36 @@ package pl.allegro.tech.hermes.management.domain.readiness; +import java.util.List; import pl.allegro.tech.hermes.api.DatacenterReadiness; import pl.allegro.tech.hermes.management.domain.dc.DatacenterBoundRepositoryHolder; import pl.allegro.tech.hermes.management.domain.dc.RepositoryCommand; -import java.util.List; - public class SetReadinessCommand extends RepositoryCommand { - private final List readiness; + private final List readiness; - public SetReadinessCommand(List readiness) { - this.readiness = readiness; - } + public SetReadinessCommand(List readiness) { + this.readiness = readiness; + } - @Override - public void backup(DatacenterBoundRepositoryHolder holder) { } + @Override + public void backup(DatacenterBoundRepositoryHolder holder) {} - @Override - public void execute(DatacenterBoundRepositoryHolder holder) { - holder.getRepository().setReadiness(readiness); - } + @Override + public void execute(DatacenterBoundRepositoryHolder holder) { + holder.getRepository().setReadiness(readiness); + } - @Override - public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) { } + @Override + public void rollback( + DatacenterBoundRepositoryHolder holder, Exception exception) {} - @Override - public Class getRepositoryType() { - return DatacenterReadinessRepository.class; - } + @Override + public Class getRepositoryType() { + return DatacenterReadinessRepository.class; + } - @Override - public String toString() { - return "SetReadinessCommand(" + readiness.toString() + ")"; - } + @Override + public String toString() { + return "SetReadinessCommand(" + readiness.toString() + ")"; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/CreateOfflineRetransmissionTaskCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/CreateOfflineRetransmissionTaskCommand.java index cfd2e40c61..6377d006fd 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/CreateOfflineRetransmissionTaskCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/CreateOfflineRetransmissionTaskCommand.java @@ -4,34 +4,36 @@ import pl.allegro.tech.hermes.management.domain.dc.DatacenterBoundRepositoryHolder; import pl.allegro.tech.hermes.management.domain.dc.RepositoryCommand; -class CreateOfflineRetransmissionTaskCommand extends RepositoryCommand { - private final OfflineRetransmissionTask task; - - CreateOfflineRetransmissionTaskCommand(OfflineRetransmissionTask task) { - this.task = task; - } - - @Override - public void backup(DatacenterBoundRepositoryHolder holder) { - } - - @Override - public void execute(DatacenterBoundRepositoryHolder holder) { - holder.getRepository().saveTask(task); - } - - @Override - public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) { - holder.getRepository().deleteTask(task.getTaskId()); - } - - @Override - public Class getRepositoryType() { - return OfflineRetransmissionRepository.class; - } - - @Override - public String toString() { - return "CreateOfflineRetransmissionTaskCommand(" + task.getTaskId() + ")"; - } +class CreateOfflineRetransmissionTaskCommand + extends RepositoryCommand { + private final OfflineRetransmissionTask task; + + CreateOfflineRetransmissionTaskCommand(OfflineRetransmissionTask task) { + this.task = task; + } + + @Override + public void backup(DatacenterBoundRepositoryHolder holder) {} + + @Override + public void execute(DatacenterBoundRepositoryHolder holder) { + holder.getRepository().saveTask(task); + } + + @Override + public void rollback( + DatacenterBoundRepositoryHolder holder, + Exception exception) { + holder.getRepository().deleteTask(task.getTaskId()); + } + + @Override + public Class getRepositoryType() { + return OfflineRetransmissionRepository.class; + } + + @Override + public String toString() { + return "CreateOfflineRetransmissionTaskCommand(" + task.getTaskId() + ")"; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/DcAwareOfflineRetransmissionRepository.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/DcAwareOfflineRetransmissionRepository.java index b627a4d097..c3f904e9d9 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/DcAwareOfflineRetransmissionRepository.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/DcAwareOfflineRetransmissionRepository.java @@ -1,32 +1,32 @@ package pl.allegro.tech.hermes.management.domain.retransmit; +import java.util.List; import pl.allegro.tech.hermes.api.OfflineRetransmissionTask; import pl.allegro.tech.hermes.management.domain.dc.MultiDatacenterRepositoryCommandExecutor; -import java.util.List; - public class DcAwareOfflineRetransmissionRepository implements OfflineRetransmissionRepository { - private final MultiDatacenterRepositoryCommandExecutor commandExecutor; - private final OfflineRetransmissionRepository offlineRetransmissionRepository; + private final MultiDatacenterRepositoryCommandExecutor commandExecutor; + private final OfflineRetransmissionRepository offlineRetransmissionRepository; - public DcAwareOfflineRetransmissionRepository(MultiDatacenterRepositoryCommandExecutor commandExecutor, - OfflineRetransmissionRepository offlineRetransmissionRepository) { - this.commandExecutor = commandExecutor; - this.offlineRetransmissionRepository = offlineRetransmissionRepository; - } + public DcAwareOfflineRetransmissionRepository( + MultiDatacenterRepositoryCommandExecutor commandExecutor, + OfflineRetransmissionRepository offlineRetransmissionRepository) { + this.commandExecutor = commandExecutor; + this.offlineRetransmissionRepository = offlineRetransmissionRepository; + } - @Override - public void saveTask(OfflineRetransmissionTask task) { - commandExecutor.execute(new CreateOfflineRetransmissionTaskCommand(task)); - } + @Override + public void saveTask(OfflineRetransmissionTask task) { + commandExecutor.execute(new CreateOfflineRetransmissionTaskCommand(task)); + } - @Override - public List getAllTasks() { - return offlineRetransmissionRepository.getAllTasks(); - } + @Override + public List getAllTasks() { + return offlineRetransmissionRepository.getAllTasks(); + } - @Override - public void deleteTask(String taskId) { - commandExecutor.execute(new DeleteOfflineRetransmissionTaskCommand(taskId)); - } + @Override + public void deleteTask(String taskId) { + commandExecutor.execute(new DeleteOfflineRetransmissionTaskCommand(taskId)); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/DeleteOfflineRetransmissionTaskCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/DeleteOfflineRetransmissionTaskCommand.java index 14d0f2029c..466ff67589 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/DeleteOfflineRetransmissionTaskCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/DeleteOfflineRetransmissionTaskCommand.java @@ -3,30 +3,29 @@ import pl.allegro.tech.hermes.management.domain.dc.DatacenterBoundRepositoryHolder; import pl.allegro.tech.hermes.management.domain.dc.RepositoryCommand; -class DeleteOfflineRetransmissionTaskCommand extends RepositoryCommand { - private final String taskId; - - DeleteOfflineRetransmissionTaskCommand(String taskId) { - this.taskId = taskId; - } - - @Override - public void backup(DatacenterBoundRepositoryHolder holder) { - - } - - @Override - public void execute(DatacenterBoundRepositoryHolder holder) { - holder.getRepository().deleteTask(taskId); - } - - @Override - public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) { - - } - - @Override - public Class getRepositoryType() { - return OfflineRetransmissionRepository.class; - } +class DeleteOfflineRetransmissionTaskCommand + extends RepositoryCommand { + private final String taskId; + + DeleteOfflineRetransmissionTaskCommand(String taskId) { + this.taskId = taskId; + } + + @Override + public void backup(DatacenterBoundRepositoryHolder holder) {} + + @Override + public void execute(DatacenterBoundRepositoryHolder holder) { + holder.getRepository().deleteTask(taskId); + } + + @Override + public void rollback( + DatacenterBoundRepositoryHolder holder, + Exception exception) {} + + @Override + public Class getRepositoryType() { + return OfflineRetransmissionRepository.class; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/OfflineRetransmissionRepository.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/OfflineRetransmissionRepository.java index deef876ce1..584d2aff3a 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/OfflineRetransmissionRepository.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/OfflineRetransmissionRepository.java @@ -1,13 +1,12 @@ package pl.allegro.tech.hermes.management.domain.retransmit; -import pl.allegro.tech.hermes.api.OfflineRetransmissionTask; - import java.util.List; +import pl.allegro.tech.hermes.api.OfflineRetransmissionTask; public interface OfflineRetransmissionRepository { - void saveTask(OfflineRetransmissionTask task); + void saveTask(OfflineRetransmissionTask task); - List getAllTasks(); + List getAllTasks(); - void deleteTask(String taskId); + void deleteTask(String taskId); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/OfflineRetransmissionService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/OfflineRetransmissionService.java index 0c7259da15..4cc6bfb061 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/OfflineRetransmissionService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/OfflineRetransmissionService.java @@ -1,5 +1,8 @@ package pl.allegro.tech.hermes.management.domain.retransmit; +import java.time.Instant; +import java.util.List; +import java.util.UUID; import pl.allegro.tech.hermes.api.OfflineRetransmissionRequest; import pl.allegro.tech.hermes.api.OfflineRetransmissionTask; import pl.allegro.tech.hermes.api.Topic; @@ -7,75 +10,74 @@ import pl.allegro.tech.hermes.common.exception.InternalProcessingException; import pl.allegro.tech.hermes.domain.topic.TopicRepository; -import java.time.Instant; -import java.util.List; -import java.util.UUID; - public class OfflineRetransmissionService { - private final OfflineRetransmissionRepository offlineRetransmissionRepository; - private final TopicRepository topicRepository; + private final OfflineRetransmissionRepository offlineRetransmissionRepository; + private final TopicRepository topicRepository; - public OfflineRetransmissionService(OfflineRetransmissionRepository offlineRetransmissionRepository, TopicRepository topicRepository) { - this.offlineRetransmissionRepository = offlineRetransmissionRepository; - this.topicRepository = topicRepository; - } + public OfflineRetransmissionService( + OfflineRetransmissionRepository offlineRetransmissionRepository, + TopicRepository topicRepository) { + this.offlineRetransmissionRepository = offlineRetransmissionRepository; + this.topicRepository = topicRepository; + } - public void validateRequest(OfflineRetransmissionRequest request) { - TopicName sourceTopicName = TopicName.fromQualifiedName(request.getSourceTopic().orElse(null)); - TopicName targetTopicName = TopicName.fromQualifiedName(request.getTargetTopic()); + public void validateRequest(OfflineRetransmissionRequest request) { + TopicName sourceTopicName = TopicName.fromQualifiedName(request.getSourceTopic().orElse(null)); + TopicName targetTopicName = TopicName.fromQualifiedName(request.getTargetTopic()); - ensureTopicsExist(sourceTopicName, targetTopicName); - ensureTimeRangeIsProper(request); - ensureTopicIsNotStoredOffline(targetTopicName); - } + ensureTopicsExist(sourceTopicName, targetTopicName); + ensureTimeRangeIsProper(request); + ensureTopicIsNotStoredOffline(targetTopicName); + } - public OfflineRetransmissionTask createTask(OfflineRetransmissionRequest request) { - return saveTask(request); - } + public OfflineRetransmissionTask createTask(OfflineRetransmissionRequest request) { + return saveTask(request); + } - public List getAllTasks() { - return offlineRetransmissionRepository.getAllTasks(); - } + public List getAllTasks() { + return offlineRetransmissionRepository.getAllTasks(); + } - public void deleteTask(String taskId) { - try { - offlineRetransmissionRepository.deleteTask(taskId); - } catch (InternalProcessingException ex) { - if (ex.getCause() instanceof OfflineRetransmissionValidationException) { - throw (OfflineRetransmissionValidationException) ex.getCause(); - } - throw ex; - } + public void deleteTask(String taskId) { + try { + offlineRetransmissionRepository.deleteTask(taskId); + } catch (InternalProcessingException ex) { + if (ex.getCause() instanceof OfflineRetransmissionValidationException) { + throw (OfflineRetransmissionValidationException) ex.getCause(); + } + throw ex; } + } - private void ensureTopicsExist(TopicName sourceTopicName, TopicName targetTopicName) { - if (sourceTopicName != null && !topicRepository.topicExists(sourceTopicName)) { - throw new OfflineRetransmissionValidationException("Source topic does not exist"); - } - - if (!topicRepository.topicExists(targetTopicName)) { - throw new OfflineRetransmissionValidationException("Target topic does not exist"); - } + private void ensureTopicsExist(TopicName sourceTopicName, TopicName targetTopicName) { + if (sourceTopicName != null && !topicRepository.topicExists(sourceTopicName)) { + throw new OfflineRetransmissionValidationException("Source topic does not exist"); } - private void ensureTimeRangeIsProper(OfflineRetransmissionRequest request) { - if (request.getStartTimestamp().isAfter(request.getEndTimestamp()) - || request.getStartTimestamp().equals(request.getEndTimestamp())) { - throw new OfflineRetransmissionValidationException("End timestamp must be greater than start timestamp"); - } + if (!topicRepository.topicExists(targetTopicName)) { + throw new OfflineRetransmissionValidationException("Target topic does not exist"); } + } - private void ensureTopicIsNotStoredOffline(TopicName targetTopicName) { - Topic targetTopic = topicRepository.getTopicDetails(targetTopicName); - if (targetTopic.getOfflineStorage().isEnabled()) { - throw new OfflineRetransmissionValidationException("Target topic must not be stored offline"); - } + private void ensureTimeRangeIsProper(OfflineRetransmissionRequest request) { + if (request.getStartTimestamp().isAfter(request.getEndTimestamp()) + || request.getStartTimestamp().equals(request.getEndTimestamp())) { + throw new OfflineRetransmissionValidationException( + "End timestamp must be greater than start timestamp"); } + } - private OfflineRetransmissionTask saveTask(OfflineRetransmissionRequest request) { - OfflineRetransmissionTask task = - new OfflineRetransmissionTask(UUID.randomUUID().toString(), request, Instant.now()); - offlineRetransmissionRepository.saveTask(task); - return task; + private void ensureTopicIsNotStoredOffline(TopicName targetTopicName) { + Topic targetTopic = topicRepository.getTopicDetails(targetTopicName); + if (targetTopic.getOfflineStorage().isEnabled()) { + throw new OfflineRetransmissionValidationException("Target topic must not be stored offline"); } + } + + private OfflineRetransmissionTask saveTask(OfflineRetransmissionRequest request) { + OfflineRetransmissionTask task = + new OfflineRetransmissionTask(UUID.randomUUID().toString(), request, Instant.now()); + offlineRetransmissionRepository.saveTask(task); + return task; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/OfflineRetransmissionValidationException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/OfflineRetransmissionValidationException.java index 99872510af..9bdf79e70a 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/OfflineRetransmissionValidationException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/OfflineRetransmissionValidationException.java @@ -5,12 +5,12 @@ public class OfflineRetransmissionValidationException extends HermesException { - public OfflineRetransmissionValidationException(String msg) { - super(msg); - } + public OfflineRetransmissionValidationException(String msg) { + super(msg); + } - @Override - public ErrorCode getCode() { - return ErrorCode.VALIDATION_ERROR; - } + @Override + public ErrorCode getCode() { + return ErrorCode.VALIDATION_ERROR; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/RetransmitCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/RetransmitCommand.java index 669f45aca0..b225ab0232 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/RetransmitCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/retransmit/RetransmitCommand.java @@ -7,28 +7,25 @@ public class RetransmitCommand extends RepositoryCommand { - private final SubscriptionName subscriptionName; + private final SubscriptionName subscriptionName; - public RetransmitCommand(SubscriptionName subscriptionName) { - this.subscriptionName = subscriptionName; - } + public RetransmitCommand(SubscriptionName subscriptionName) { + this.subscriptionName = subscriptionName; + } - @Override - public void backup(DatacenterBoundRepositoryHolder holder) { - } + @Override + public void backup(DatacenterBoundRepositoryHolder holder) {} - @Override - public void execute(DatacenterBoundRepositoryHolder holder) { - holder.getRepository().retransmit(subscriptionName); - } + @Override + public void execute(DatacenterBoundRepositoryHolder holder) { + holder.getRepository().retransmit(subscriptionName); + } - @Override - public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) { + @Override + public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) {} - } - - @Override - public Class getRepositoryType() { - return AdminTool.class; - } + @Override + public Class getRepositoryType() { + return AdminTool.class; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/ConsumerGroupManager.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/ConsumerGroupManager.java index 0a9eede110..9a8a99a1d3 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/ConsumerGroupManager.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/ConsumerGroupManager.java @@ -5,5 +5,5 @@ public interface ConsumerGroupManager { - void createConsumerGroup(Topic topic, Subscription subscription); + void createConsumerGroup(Topic topic, Subscription subscription); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/SubscriptionLagSource.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/SubscriptionLagSource.java index a67ce8984e..711da6806c 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/SubscriptionLagSource.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/SubscriptionLagSource.java @@ -4,5 +4,5 @@ import pl.allegro.tech.hermes.api.TopicName; public interface SubscriptionLagSource { - MetricLongValue getLag(TopicName topicName, String subscriptionName); + MetricLongValue getLag(TopicName topicName, String subscriptionName); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/SubscriptionMetricsRepository.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/SubscriptionMetricsRepository.java index 427fd1a1d2..d220e837c7 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/SubscriptionMetricsRepository.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/SubscriptionMetricsRepository.java @@ -6,7 +6,7 @@ public interface SubscriptionMetricsRepository { - SubscriptionMetrics loadMetrics(TopicName topicName, String subscriptionName); + SubscriptionMetrics loadMetrics(TopicName topicName, String subscriptionName); - PersistentSubscriptionMetrics loadZookeeperMetrics(TopicName topicName, String subscriptionName); + PersistentSubscriptionMetrics loadZookeeperMetrics(TopicName topicName, String subscriptionName); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/SubscriptionOwnerCache.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/SubscriptionOwnerCache.java index 9604d8e9ea..08d7bdb582 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/SubscriptionOwnerCache.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/SubscriptionOwnerCache.java @@ -5,6 +5,10 @@ import com.google.common.collect.Multimaps; import com.google.common.util.concurrent.ThreadFactoryBuilder; import jakarta.annotation.PreDestroy; +import java.util.Collection; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -15,69 +19,71 @@ import pl.allegro.tech.hermes.api.TopicName; import pl.allegro.tech.hermes.domain.subscription.SubscriptionRepository; -import java.util.Collection; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - @Component public class SubscriptionOwnerCache { - private static final Logger logger = LoggerFactory.getLogger(SubscriptionOwnerCache.class); + private static final Logger logger = LoggerFactory.getLogger(SubscriptionOwnerCache.class); - private final SubscriptionRepository subscriptionRepository; - private final ScheduledExecutorService scheduledExecutorService; + private final SubscriptionRepository subscriptionRepository; + private final ScheduledExecutorService scheduledExecutorService; - private Multimap cache = Multimaps.synchronizedMultimap(ArrayListMultimap.create()); + private Multimap cache = + Multimaps.synchronizedMultimap(ArrayListMultimap.create()); - public SubscriptionOwnerCache(SubscriptionRepository subscriptionRepository, - @Value("${subscriptionOwnerCache.refreshRateInSeconds}") int refreshRateInSeconds) { - this.subscriptionRepository = subscriptionRepository; - scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( - new ThreadFactoryBuilder() - .setNameFormat("subscription-owner-cache-%d") - .build()); - scheduledExecutorService.scheduleAtFixedRate(this::refillCache, 0, refreshRateInSeconds, TimeUnit.SECONDS); - } + public SubscriptionOwnerCache( + SubscriptionRepository subscriptionRepository, + @Value("${subscriptionOwnerCache.refreshRateInSeconds}") int refreshRateInSeconds) { + this.subscriptionRepository = subscriptionRepository; + scheduledExecutorService = + Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder().setNameFormat("subscription-owner-cache-%d").build()); + scheduledExecutorService.scheduleAtFixedRate( + this::refillCache, 0, refreshRateInSeconds, TimeUnit.SECONDS); + } - @PreDestroy - public void stop() { - scheduledExecutorService.shutdown(); - } + @PreDestroy + public void stop() { + scheduledExecutorService.shutdown(); + } - Collection get(OwnerId ownerId) { - return cache.get(ownerId); - } + Collection get(OwnerId ownerId) { + return cache.get(ownerId); + } - Collection getAll() { - return cache.values(); - } + Collection getAll() { + return cache.values(); + } - void onRemovedSubscription(String subscriptionName, TopicName topicName) { - cache.entries().removeIf(entry -> entry.getValue().equals(new SubscriptionName(subscriptionName, topicName))); - } + void onRemovedSubscription(String subscriptionName, TopicName topicName) { + cache + .entries() + .removeIf( + entry -> entry.getValue().equals(new SubscriptionName(subscriptionName, topicName))); + } - void onCreatedSubscription(Subscription subscription) { - cache.put(subscription.getOwner(), subscription.getQualifiedName()); - } + void onCreatedSubscription(Subscription subscription) { + cache.put(subscription.getOwner(), subscription.getQualifiedName()); + } - void onUpdatedSubscription(Subscription oldSubscription, Subscription newSubscription) { - cache.remove(oldSubscription.getOwner(), oldSubscription.getQualifiedName()); - cache.put(newSubscription.getOwner(), newSubscription.getQualifiedName()); - } + void onUpdatedSubscription(Subscription oldSubscription, Subscription newSubscription) { + cache.remove(oldSubscription.getOwner(), oldSubscription.getQualifiedName()); + cache.put(newSubscription.getOwner(), newSubscription.getQualifiedName()); + } - private void refillCache() { - try { - logger.info("Starting filling SubscriptionOwnerCache"); - long start = System.currentTimeMillis(); - Multimap cache = ArrayListMultimap.create(); - subscriptionRepository.listAllSubscriptions() - .forEach(subscription -> cache.put(subscription.getOwner(), subscription.getQualifiedName())); - this.cache = Multimaps.synchronizedMultimap(cache); - long end = System.currentTimeMillis(); - logger.info("SubscriptionOwnerCache filled. Took {}ms", end - start); - } catch (Exception e) { - logger.error("Error while filling SubscriptionOwnerCache", e); - } + private void refillCache() { + try { + logger.info("Starting filling SubscriptionOwnerCache"); + long start = System.currentTimeMillis(); + Multimap cache = ArrayListMultimap.create(); + subscriptionRepository + .listAllSubscriptions() + .forEach( + subscription -> cache.put(subscription.getOwner(), subscription.getQualifiedName())); + this.cache = Multimaps.synchronizedMultimap(cache); + long end = System.currentTimeMillis(); + logger.info("SubscriptionOwnerCache filled. Took {}ms", end - start); + } catch (Exception e) { + logger.error("Error while filling SubscriptionOwnerCache", e); } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/SubscriptionRemover.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/SubscriptionRemover.java index 016e18b35f..b83b57a72f 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/SubscriptionRemover.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/SubscriptionRemover.java @@ -1,5 +1,6 @@ package pl.allegro.tech.hermes.management.domain.subscription; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.Subscription; @@ -12,52 +13,58 @@ import pl.allegro.tech.hermes.management.domain.dc.MultiDatacenterRepositoryCommandExecutor; import pl.allegro.tech.hermes.management.domain.subscription.commands.RemoveSubscriptionRepositoryCommand; -import java.util.List; - public class SubscriptionRemover { - private static final Logger logger = LoggerFactory.getLogger(SubscriptionRemover.class); - private final Auditor auditor; - private final MultiDatacenterRepositoryCommandExecutor multiDcExecutor; - private final SubscriptionOwnerCache subscriptionOwnerCache; - private final SubscriptionRepository subscriptionRepository; - - - public SubscriptionRemover(Auditor auditor, - MultiDatacenterRepositoryCommandExecutor multiDcExecutor, - SubscriptionOwnerCache subscriptionOwnerCache, - SubscriptionRepository subscriptionRepository) { - this.auditor = auditor; - this.multiDcExecutor = multiDcExecutor; - this.subscriptionOwnerCache = subscriptionOwnerCache; - this.subscriptionRepository = subscriptionRepository; - } + private static final Logger logger = LoggerFactory.getLogger(SubscriptionRemover.class); + private final Auditor auditor; + private final MultiDatacenterRepositoryCommandExecutor multiDcExecutor; + private final SubscriptionOwnerCache subscriptionOwnerCache; + private final SubscriptionRepository subscriptionRepository; - public void removeSubscription(TopicName topicName, String subscriptionName, RequestUser removedBy) { - auditor.beforeObjectRemoval(removedBy.getUsername(), Subscription.class.getSimpleName(), subscriptionName); - Subscription subscription = subscriptionRepository.getSubscriptionDetails(topicName, subscriptionName); - multiDcExecutor.executeByUser(new RemoveSubscriptionRepositoryCommand(topicName, subscriptionName), removedBy); - auditor.objectRemoved(removedBy.getUsername(), subscription); - subscriptionOwnerCache.onRemovedSubscription(subscriptionName, topicName); - } + public SubscriptionRemover( + Auditor auditor, + MultiDatacenterRepositoryCommandExecutor multiDcExecutor, + SubscriptionOwnerCache subscriptionOwnerCache, + SubscriptionRepository subscriptionRepository) { + this.auditor = auditor; + this.multiDcExecutor = multiDcExecutor; + this.subscriptionOwnerCache = subscriptionOwnerCache; + this.subscriptionRepository = subscriptionRepository; + } - public void removeSubscriptionRelatedToTopic(Topic topic, RequestUser removedBy) { - List subscriptions = subscriptionRepository.listSubscriptions(topic.getName()); - ensureSubscriptionsHaveAutoRemove(subscriptions, topic.getName()); - logger.info("Removing subscriptions of topic: {}, subscriptions: {}", topic.getName(), subscriptions); - long start = System.currentTimeMillis(); - subscriptions.forEach(sub -> removeSubscription(topic.getName(), sub.getName(), removedBy)); - logger.info("Removed subscriptions of topic: {} in {} ms", topic.getName(), System.currentTimeMillis() - start); - } + public void removeSubscription( + TopicName topicName, String subscriptionName, RequestUser removedBy) { + auditor.beforeObjectRemoval( + removedBy.getUsername(), Subscription.class.getSimpleName(), subscriptionName); + Subscription subscription = + subscriptionRepository.getSubscriptionDetails(topicName, subscriptionName); + multiDcExecutor.executeByUser( + new RemoveSubscriptionRepositoryCommand(topicName, subscriptionName), removedBy); + auditor.objectRemoved(removedBy.getUsername(), subscription); + subscriptionOwnerCache.onRemovedSubscription(subscriptionName, topicName); + } + public void removeSubscriptionRelatedToTopic(Topic topic, RequestUser removedBy) { + List subscriptions = subscriptionRepository.listSubscriptions(topic.getName()); + ensureSubscriptionsHaveAutoRemove(subscriptions, topic.getName()); + logger.info( + "Removing subscriptions of topic: {}, subscriptions: {}", topic.getName(), subscriptions); + long start = System.currentTimeMillis(); + subscriptions.forEach(sub -> removeSubscription(topic.getName(), sub.getName(), removedBy)); + logger.info( + "Removed subscriptions of topic: {} in {} ms", + topic.getName(), + System.currentTimeMillis() - start); + } - private void ensureSubscriptionsHaveAutoRemove(List subscriptions, TopicName topicName) { - boolean anySubscriptionWithoutAutoRemove = subscriptions.stream() - .anyMatch(sub -> !sub.isAutoDeleteWithTopicEnabled()); + private void ensureSubscriptionsHaveAutoRemove( + List subscriptions, TopicName topicName) { + boolean anySubscriptionWithoutAutoRemove = + subscriptions.stream().anyMatch(sub -> !sub.isAutoDeleteWithTopicEnabled()); - if (anySubscriptionWithoutAutoRemove) { - logger.info("Cannot remove topic due to connected subscriptions"); - throw new TopicNotEmptyException(topicName); - } + if (anySubscriptionWithoutAutoRemove) { + logger.info("Cannot remove topic due to connected subscriptions"); + throw new TopicNotEmptyException(topicName); } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/SubscriptionService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/SubscriptionService.java index 4cbfeae6ec..c8d088e598 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/SubscriptionService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/SubscriptionService.java @@ -1,5 +1,23 @@ package pl.allegro.tech.hermes.management.domain.subscription; +import static java.util.stream.Collectors.toList; +import static pl.allegro.tech.hermes.api.SubscriptionHealth.Status; +import static pl.allegro.tech.hermes.api.TopicName.fromQualifiedName; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.commons.collections4.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,378 +54,411 @@ import pl.allegro.tech.hermes.management.infrastructure.kafka.MultiDCAwareService; import pl.allegro.tech.hermes.tracker.management.LogRepository; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static java.util.stream.Collectors.toList; -import static pl.allegro.tech.hermes.api.SubscriptionHealth.Status; -import static pl.allegro.tech.hermes.api.TopicName.fromQualifiedName; - public class SubscriptionService { - private static final Logger logger = LoggerFactory.getLogger(SubscriptionService.class); - - private static final int LAST_MESSAGE_COUNT = 100; - - private final SubscriptionRepository subscriptionRepository; - private final SubscriptionOwnerCache subscriptionOwnerCache; - private final TopicService topicService; - private final SubscriptionMetricsRepository metricsRepository; - private final SubscriptionHealthChecker subscriptionHealthChecker; - private final LogRepository logRepository; - private final SubscriptionValidator subscriptionValidator; - private final Auditor auditor; - private final MultiDatacenterRepositoryCommandExecutor multiDcExecutor; - private final MultiDCAwareService multiDCAwareService; - private final RepositoryManager repositoryManager; - private final long subscriptionHealthCheckTimeoutMillis; - private final ExecutorService subscriptionHealthCheckExecutorService; - private final SubscriptionRemover subscriptionRemover; - - - public SubscriptionService(SubscriptionRepository subscriptionRepository, - SubscriptionOwnerCache subscriptionOwnerCache, - TopicService topicService, - SubscriptionMetricsRepository metricsRepository, - SubscriptionHealthChecker subscriptionHealthChecker, - LogRepository logRepository, - SubscriptionValidator subscriptionValidator, - Auditor auditor, - MultiDatacenterRepositoryCommandExecutor multiDcExecutor, - MultiDCAwareService multiDCAwareService, - RepositoryManager repositoryManager, - ExecutorService unhealthyGetExecutorService, - long unhealthyGetTimeoutMillis, - SubscriptionRemover subscriptionRemover) { - this.subscriptionRepository = subscriptionRepository; - this.subscriptionOwnerCache = subscriptionOwnerCache; - this.topicService = topicService; - this.metricsRepository = metricsRepository; - this.subscriptionHealthChecker = subscriptionHealthChecker; - this.logRepository = logRepository; - this.subscriptionValidator = subscriptionValidator; - this.auditor = auditor; - this.multiDcExecutor = multiDcExecutor; - this.multiDCAwareService = multiDCAwareService; - this.repositoryManager = repositoryManager; - this.subscriptionHealthCheckExecutorService = unhealthyGetExecutorService; - this.subscriptionHealthCheckTimeoutMillis = unhealthyGetTimeoutMillis; - this.subscriptionRemover = subscriptionRemover; - } - - public List listSubscriptionNames(TopicName topicName) { - return subscriptionRepository.listSubscriptionNames(topicName); - } - - public List listTrackedSubscriptionNames(TopicName topicName) { - return listSubscriptions(topicName).stream() - .filter(Subscription::isTrackingEnabled) - .map(Subscription::getName) - .collect(toList()); - } - - public List listFilteredSubscriptionNames(TopicName topicName, Query query) { - return query.filter(listSubscriptions(topicName)) - .map(Subscription::getName) - .collect(toList()); - } - - public List listSubscriptions(TopicName topicName) { - return subscriptionRepository.listSubscriptions(topicName); - } - - public void createSubscription(Subscription subscription, RequestUser createdBy, String qualifiedTopicName) { - auditor.beforeObjectCreation(createdBy.getUsername(), subscription); - subscriptionValidator.checkCreation(subscription, createdBy); - - Topic topic = topicService.getTopicDetails(fromQualifiedName(qualifiedTopicName)); - multiDCAwareService.createConsumerGroups(topic, subscription); - - multiDcExecutor.executeByUser(new CreateSubscriptionRepositoryCommand(subscription), createdBy); - auditor.objectCreated(createdBy.getUsername(), subscription); - subscriptionOwnerCache.onCreatedSubscription(subscription); - } - - public Subscription getSubscriptionDetails(TopicName topicName, String subscriptionName) { - Subscription subscription = subscriptionRepository.getSubscriptionDetails(topicName, subscriptionName) - .anonymize(); - subscription.setState(getEffectiveState(topicName, subscriptionName)); - return subscription; - } - - private CompletableFuture> getSubscriptionDetails(Collection subscriptionNames) { - return CompletableFuture.supplyAsync(() -> - subscriptionRepository.getSubscriptionDetails(subscriptionNames), subscriptionHealthCheckExecutorService); - } - - private Subscription.State getEffectiveState(TopicName topicName, String subscriptionName) { - Set states = loadSubscriptionStatesFromAllDc(topicName, subscriptionName); - - if (states.size() > 1) { - logger.warn("Some states are out of sync: {}", states); - } - - if (states.contains(Subscription.State.ACTIVE)) { - return Subscription.State.ACTIVE; - } else if (states.contains(Subscription.State.SUSPENDED)) { - return Subscription.State.SUSPENDED; - } else { - return Subscription.State.PENDING; - } - } - - private Set loadSubscriptionStatesFromAllDc(TopicName topicName, String subscriptionName) { - List> holders = - repositoryManager.getRepositories(SubscriptionRepository.class); - Set states = new HashSet<>(); - for (DatacenterBoundRepositoryHolder holder : holders) { - try { - Subscription.State state = holder.getRepository().getSubscriptionDetails(topicName, subscriptionName) - .getState(); - states.add(state); - } catch (Exception e) { - logger.warn("Could not load state of subscription (topic: {}, name: {}) from DC {}.", - topicName, subscriptionName, holder.getDatacenterName()); - } - } - return states; - } - - public void removeSubscription(TopicName topicName, String subscriptionName, RequestUser removedBy) { - subscriptionRemover.removeSubscription(topicName, subscriptionName, removedBy); - } - - public void updateSubscription(TopicName topicName, - String subscriptionName, - PatchData patch, - RequestUser modifiedBy) { - auditor.beforeObjectUpdate(modifiedBy.getUsername(), Subscription.class.getSimpleName(), - new SubscriptionName(subscriptionName, topicName), patch); - - Subscription retrieved = subscriptionRepository.getSubscriptionDetails(topicName, subscriptionName); - Subscription.State oldState = retrieved.getState(); - Subscription updated = Patch.apply(retrieved, patch); - revertStateIfChangedToPending(updated, oldState); - subscriptionValidator.checkModification(updated, modifiedBy, retrieved); - subscriptionOwnerCache.onUpdatedSubscription(retrieved, updated); - - if (!retrieved.equals(updated)) { - multiDcExecutor.executeByUser(new UpdateSubscriptionRepositoryCommand(updated), modifiedBy); - auditor.objectUpdated(modifiedBy.getUsername(), retrieved, updated); - } - } - - private void revertStateIfChangedToPending(Subscription updated, Subscription.State oldState) { - if (updated.getState() == Subscription.State.PENDING) { - updated.setState(oldState); - } - } - - public void updateSubscriptionState(TopicName topicName, String subscriptionName, Subscription.State state, RequestUser modifiedBy) { - if (state != Subscription.State.PENDING) { - PatchData patchData = PatchData.patchData().set("state", state).build(); - auditor.beforeObjectUpdate( - modifiedBy.getUsername(), - Subscription.class.getSimpleName(), - new SubscriptionName(subscriptionName, topicName), - patchData); - Subscription retrieved = subscriptionRepository.getSubscriptionDetails(topicName, subscriptionName); - if (!retrieved.getState().equals(state)) { - Subscription updated = Patch.apply(retrieved, patchData); - multiDcExecutor.executeByUser(new UpdateSubscriptionRepositoryCommand(updated), modifiedBy); - auditor.objectUpdated(modifiedBy.getUsername(), retrieved, updated); - } - } - } - - public Subscription.State getSubscriptionState(TopicName topicName, String subscriptionName) { - return getSubscriptionDetails(topicName, subscriptionName).getState(); - } - - public SubscriptionMetrics getSubscriptionMetrics(TopicName topicName, String subscriptionName) { - subscriptionRepository.ensureSubscriptionExists(topicName, subscriptionName); - return metricsRepository.loadMetrics(topicName, subscriptionName); - } - - public PersistentSubscriptionMetrics getPersistentSubscriptionMetrics(TopicName topicName, String subscriptionName) { - subscriptionRepository.ensureSubscriptionExists(topicName, subscriptionName); - return metricsRepository.loadZookeeperMetrics(topicName, subscriptionName); - } - - public SubscriptionHealth getSubscriptionHealth(TopicName topicName, String subscriptionName) { - Subscription subscription = getSubscriptionDetails(topicName, subscriptionName); - return getHealth(subscription); - } - - public Optional getLatestUndeliveredMessage(TopicName topicName, String subscriptionName) { - List> holders = - repositoryManager.getRepositories(LastUndeliveredMessageReader.class); - List traces = new ArrayList<>(); - for (DatacenterBoundRepositoryHolder holder : holders) { - try { - holder.getRepository().last(topicName, subscriptionName).ifPresent(traces::add); - } catch (Exception e) { - logger.warn("Could not load latest undelivered message from DC: {}", holder.getDatacenterName()); - } - } - return traces.stream().max(Comparator.comparing(SentMessageTrace::getTimestamp)); - } - - public List getLatestUndeliveredMessagesTrackerLogs(TopicName topicName, String subscriptionName) { - return logRepository.getLastUndeliveredMessages(topicName.qualifiedName(), subscriptionName, LAST_MESSAGE_COUNT); - } - - public List getMessageStatus(String qualifiedTopicName, String subscriptionName, String messageId) { - return logRepository.getMessageStatus(qualifiedTopicName, subscriptionName, messageId); - } - - public List querySubscription(Query query) { - return query - .filter(getAllSubscriptions()) - .collect(toList()); - } - - public List querySubscriptionsMetrics(Query query) { - List filteredSubscriptions = query.filterNames(getAllSubscriptions()) - .collect(toList()); - return query.filter(getSubscriptionsMetrics(filteredSubscriptions)) - .collect(toList()); - } - - public List getAllSubscriptions() { - return topicService.getAllTopics() - .stream() - .map(Topic::getName) - .map(this::listSubscriptions) - .flatMap(List::stream) - .map(Subscription::anonymize) - .collect(toList()); - } - - public List getForOwnerId(OwnerId ownerId) { - Collection subscriptionNames = subscriptionOwnerCache.get(ownerId); - return subscriptionRepository.getSubscriptionDetails(subscriptionNames); - } - - public List getAllUnhealthy(boolean respectMonitoringSeverity, - List subscriptionNames, - List qualifiedTopicNames) { - return getUnhealthyList(subscriptionOwnerCache.getAll(), respectMonitoringSeverity, subscriptionNames, qualifiedTopicNames); - } - - public List getUnhealthyForOwner(OwnerId ownerId, - boolean respectMonitoringSeverity, - List subscriptionNames, - List qualifiedTopicNames) { - return getUnhealthyList(subscriptionOwnerCache.get(ownerId), respectMonitoringSeverity, subscriptionNames, qualifiedTopicNames); - } - - public SubscriptionStats getStats() { - List subscriptions = getAllSubscriptions(); - long trackingEnabledSubscriptionsCount = subscriptions.stream().filter(Subscription::isTrackingEnabled).count(); - long avroSubscriptionCount = subscriptions.stream().filter(s -> s.getContentType() == ContentType.AVRO).count(); - return new SubscriptionStats(subscriptions.size(), trackingEnabledSubscriptionsCount, avroSubscriptionCount); - } - - private List getUnhealthyList(Collection ownerSubscriptionNames, - boolean respectMonitoringSeverity, - List subscriptionNames, - List qualifiedTopicNames) { - try { - return getSubscriptionDetails(ownerSubscriptionNames) - .thenComposeAsync(ownerSubscriptions -> { - List> unhealthySubscriptions = filterSubscriptions( - ownerSubscriptions, - respectMonitoringSeverity, - subscriptionNames, - qualifiedTopicNames - ); - return CompletableFuture.allOf(unhealthySubscriptions.toArray(new CompletableFuture[0])).thenApply( - v -> unhealthySubscriptions.stream() - .map(CompletableFuture::join) - .filter(Objects::nonNull) - .collect(Collectors.toList()) - ); - }, - subscriptionHealthCheckExecutorService - ) - .get(subscriptionHealthCheckTimeoutMillis, TimeUnit.MILLISECONDS); - } catch (TimeoutException e) { - logger.error("Timeout occurred while fetching unhealthy subscriptions...", e); - throw new UnhealthySubscriptionGetException("Fetching unhealthy subscriptions timed out."); - } catch (Exception e) { - logger.error("Fetching unhealthy subscriptions failed...", e); - throw new UnhealthySubscriptionGetException("Fetching unhealthy subscriptions failed.", e); - } - } - - private List> filterSubscriptions(Collection subscriptions, - boolean respectMonitoringSeverity, - List subscriptionNames, - List qualifiedTopicNames) { - boolean shouldFilterBySubscriptionNames = CollectionUtils.isNotEmpty(subscriptionNames); - boolean shouldFilterByQualifiedTopicNames = CollectionUtils.isNotEmpty(qualifiedTopicNames); - - Stream subscriptionStream = subscriptions.stream() - .filter(s -> filterBySeverityMonitorFlag(respectMonitoringSeverity, s.isSeverityNotImportant())); - - if (shouldFilterBySubscriptionNames) { - subscriptionStream = subscriptionStream.filter(s -> filterBySubscriptionNames(subscriptionNames, s.getName())); - } - if (shouldFilterByQualifiedTopicNames) { - subscriptionStream = - subscriptionStream.filter(s -> filterByQualifiedTopicNames(qualifiedTopicNames, s.getQualifiedTopicName())); - } - - return subscriptionStream - .map(s -> CompletableFuture.supplyAsync(() -> getUnhealthy(s), subscriptionHealthCheckExecutorService)) - .collect(toList()); - } - - private boolean filterBySubscriptionNames(List subscriptionNames, String subscriptionName) { - return subscriptionNames.contains(subscriptionName); - } - - private boolean filterByQualifiedTopicNames(List qualifiedTopicNames, String qualifiedTopicName) { - return qualifiedTopicNames.contains(qualifiedTopicName); - } - - private boolean filterBySeverityMonitorFlag(boolean respectMonitoringSeverity, boolean isSeverityNotImportant) { - return !(respectMonitoringSeverity && isSeverityNotImportant); - } - - private SubscriptionHealth getHealth(Subscription subscription) { - TopicName topicName = subscription.getTopicName(); - TopicMetrics topicMetrics = topicService.getTopicMetrics(topicName); - SubscriptionMetrics subscriptionMetrics = getSubscriptionMetrics(topicName, subscription.getName()); - return subscriptionHealthChecker.checkHealth(subscription, topicMetrics, subscriptionMetrics); - } - - private UnhealthySubscription getUnhealthy(Subscription subscription) { - SubscriptionHealth subscriptionHealth = getHealth(subscription); - if (subscriptionHealth.getStatus() == Status.UNHEALTHY) { - return UnhealthySubscription.from(subscription, subscriptionHealth); - } else { - return null; - } - } - - private List getSubscriptionsMetrics(List subscriptions) { - return subscriptions.stream() - .map(s -> { - SubscriptionMetrics metrics = metricsRepository.loadMetrics(s.getTopicName(), s.getName()); - return SubscriptionNameWithMetrics.from(metrics, s.getName(), s.getQualifiedTopicName()); - }).collect(toList()); - } + private static final Logger logger = LoggerFactory.getLogger(SubscriptionService.class); + + private static final int LAST_MESSAGE_COUNT = 100; + + private final SubscriptionRepository subscriptionRepository; + private final SubscriptionOwnerCache subscriptionOwnerCache; + private final TopicService topicService; + private final SubscriptionMetricsRepository metricsRepository; + private final SubscriptionHealthChecker subscriptionHealthChecker; + private final LogRepository logRepository; + private final SubscriptionValidator subscriptionValidator; + private final Auditor auditor; + private final MultiDatacenterRepositoryCommandExecutor multiDcExecutor; + private final MultiDCAwareService multiDCAwareService; + private final RepositoryManager repositoryManager; + private final long subscriptionHealthCheckTimeoutMillis; + private final ExecutorService subscriptionHealthCheckExecutorService; + private final SubscriptionRemover subscriptionRemover; + + public SubscriptionService( + SubscriptionRepository subscriptionRepository, + SubscriptionOwnerCache subscriptionOwnerCache, + TopicService topicService, + SubscriptionMetricsRepository metricsRepository, + SubscriptionHealthChecker subscriptionHealthChecker, + LogRepository logRepository, + SubscriptionValidator subscriptionValidator, + Auditor auditor, + MultiDatacenterRepositoryCommandExecutor multiDcExecutor, + MultiDCAwareService multiDCAwareService, + RepositoryManager repositoryManager, + ExecutorService unhealthyGetExecutorService, + long unhealthyGetTimeoutMillis, + SubscriptionRemover subscriptionRemover) { + this.subscriptionRepository = subscriptionRepository; + this.subscriptionOwnerCache = subscriptionOwnerCache; + this.topicService = topicService; + this.metricsRepository = metricsRepository; + this.subscriptionHealthChecker = subscriptionHealthChecker; + this.logRepository = logRepository; + this.subscriptionValidator = subscriptionValidator; + this.auditor = auditor; + this.multiDcExecutor = multiDcExecutor; + this.multiDCAwareService = multiDCAwareService; + this.repositoryManager = repositoryManager; + this.subscriptionHealthCheckExecutorService = unhealthyGetExecutorService; + this.subscriptionHealthCheckTimeoutMillis = unhealthyGetTimeoutMillis; + this.subscriptionRemover = subscriptionRemover; + } + + public List listSubscriptionNames(TopicName topicName) { + return subscriptionRepository.listSubscriptionNames(topicName); + } + + public List listTrackedSubscriptionNames(TopicName topicName) { + return listSubscriptions(topicName).stream() + .filter(Subscription::isTrackingEnabled) + .map(Subscription::getName) + .collect(toList()); + } + + public List listFilteredSubscriptionNames( + TopicName topicName, Query query) { + return query.filter(listSubscriptions(topicName)).map(Subscription::getName).collect(toList()); + } + + public List listSubscriptions(TopicName topicName) { + return subscriptionRepository.listSubscriptions(topicName); + } + + public void createSubscription( + Subscription subscription, RequestUser createdBy, String qualifiedTopicName) { + auditor.beforeObjectCreation(createdBy.getUsername(), subscription); + subscriptionValidator.checkCreation(subscription, createdBy); + + Topic topic = topicService.getTopicDetails(fromQualifiedName(qualifiedTopicName)); + multiDCAwareService.createConsumerGroups(topic, subscription); + + multiDcExecutor.executeByUser(new CreateSubscriptionRepositoryCommand(subscription), createdBy); + auditor.objectCreated(createdBy.getUsername(), subscription); + subscriptionOwnerCache.onCreatedSubscription(subscription); + } + + public Subscription getSubscriptionDetails(TopicName topicName, String subscriptionName) { + Subscription subscription = + subscriptionRepository.getSubscriptionDetails(topicName, subscriptionName).anonymize(); + subscription.setState(getEffectiveState(topicName, subscriptionName)); + return subscription; + } + + private CompletableFuture> getSubscriptionDetails( + Collection subscriptionNames) { + return CompletableFuture.supplyAsync( + () -> subscriptionRepository.getSubscriptionDetails(subscriptionNames), + subscriptionHealthCheckExecutorService); + } + + private Subscription.State getEffectiveState(TopicName topicName, String subscriptionName) { + Set states = loadSubscriptionStatesFromAllDc(topicName, subscriptionName); + + if (states.size() > 1) { + logger.warn("Some states are out of sync: {}", states); + } + + if (states.contains(Subscription.State.ACTIVE)) { + return Subscription.State.ACTIVE; + } else if (states.contains(Subscription.State.SUSPENDED)) { + return Subscription.State.SUSPENDED; + } else { + return Subscription.State.PENDING; + } + } + + private Set loadSubscriptionStatesFromAllDc( + TopicName topicName, String subscriptionName) { + List> holders = + repositoryManager.getRepositories(SubscriptionRepository.class); + Set states = new HashSet<>(); + for (DatacenterBoundRepositoryHolder holder : holders) { + try { + Subscription.State state = + holder.getRepository().getSubscriptionDetails(topicName, subscriptionName).getState(); + states.add(state); + } catch (Exception e) { + logger.warn( + "Could not load state of subscription (topic: {}, name: {}) from DC {}.", + topicName, + subscriptionName, + holder.getDatacenterName()); + } + } + return states; + } + + public void removeSubscription( + TopicName topicName, String subscriptionName, RequestUser removedBy) { + subscriptionRemover.removeSubscription(topicName, subscriptionName, removedBy); + } + + public void updateSubscription( + TopicName topicName, String subscriptionName, PatchData patch, RequestUser modifiedBy) { + auditor.beforeObjectUpdate( + modifiedBy.getUsername(), + Subscription.class.getSimpleName(), + new SubscriptionName(subscriptionName, topicName), + patch); + + Subscription retrieved = + subscriptionRepository.getSubscriptionDetails(topicName, subscriptionName); + Subscription.State oldState = retrieved.getState(); + Subscription updated = Patch.apply(retrieved, patch); + revertStateIfChangedToPending(updated, oldState); + subscriptionValidator.checkModification(updated, modifiedBy, retrieved); + subscriptionOwnerCache.onUpdatedSubscription(retrieved, updated); + + if (!retrieved.equals(updated)) { + multiDcExecutor.executeByUser(new UpdateSubscriptionRepositoryCommand(updated), modifiedBy); + auditor.objectUpdated(modifiedBy.getUsername(), retrieved, updated); + } + } + + private void revertStateIfChangedToPending(Subscription updated, Subscription.State oldState) { + if (updated.getState() == Subscription.State.PENDING) { + updated.setState(oldState); + } + } + + public void updateSubscriptionState( + TopicName topicName, + String subscriptionName, + Subscription.State state, + RequestUser modifiedBy) { + if (state != Subscription.State.PENDING) { + PatchData patchData = PatchData.patchData().set("state", state).build(); + auditor.beforeObjectUpdate( + modifiedBy.getUsername(), + Subscription.class.getSimpleName(), + new SubscriptionName(subscriptionName, topicName), + patchData); + Subscription retrieved = + subscriptionRepository.getSubscriptionDetails(topicName, subscriptionName); + if (!retrieved.getState().equals(state)) { + Subscription updated = Patch.apply(retrieved, patchData); + multiDcExecutor.executeByUser(new UpdateSubscriptionRepositoryCommand(updated), modifiedBy); + auditor.objectUpdated(modifiedBy.getUsername(), retrieved, updated); + } + } + } + + public Subscription.State getSubscriptionState(TopicName topicName, String subscriptionName) { + return getSubscriptionDetails(topicName, subscriptionName).getState(); + } + + public SubscriptionMetrics getSubscriptionMetrics(TopicName topicName, String subscriptionName) { + subscriptionRepository.ensureSubscriptionExists(topicName, subscriptionName); + return metricsRepository.loadMetrics(topicName, subscriptionName); + } + + public PersistentSubscriptionMetrics getPersistentSubscriptionMetrics( + TopicName topicName, String subscriptionName) { + subscriptionRepository.ensureSubscriptionExists(topicName, subscriptionName); + return metricsRepository.loadZookeeperMetrics(topicName, subscriptionName); + } + + public SubscriptionHealth getSubscriptionHealth(TopicName topicName, String subscriptionName) { + Subscription subscription = getSubscriptionDetails(topicName, subscriptionName); + return getHealth(subscription); + } + + public Optional getLatestUndeliveredMessage( + TopicName topicName, String subscriptionName) { + List> holders = + repositoryManager.getRepositories(LastUndeliveredMessageReader.class); + List traces = new ArrayList<>(); + for (DatacenterBoundRepositoryHolder holder : holders) { + try { + holder.getRepository().last(topicName, subscriptionName).ifPresent(traces::add); + } catch (Exception e) { + logger.warn( + "Could not load latest undelivered message from DC: {}", holder.getDatacenterName()); + } + } + return traces.stream().max(Comparator.comparing(SentMessageTrace::getTimestamp)); + } + + public List getLatestUndeliveredMessagesTrackerLogs( + TopicName topicName, String subscriptionName) { + return logRepository.getLastUndeliveredMessages( + topicName.qualifiedName(), subscriptionName, LAST_MESSAGE_COUNT); + } + + public List getMessageStatus( + String qualifiedTopicName, String subscriptionName, String messageId) { + return logRepository.getMessageStatus(qualifiedTopicName, subscriptionName, messageId); + } + + public List querySubscription(Query query) { + return query.filter(getAllSubscriptions()).collect(toList()); + } + + public List querySubscriptionsMetrics( + Query query) { + List filteredSubscriptions = + query.filterNames(getAllSubscriptions()).collect(toList()); + return query.filter(getSubscriptionsMetrics(filteredSubscriptions)).collect(toList()); + } + + public List getAllSubscriptions() { + return topicService.getAllTopics().stream() + .map(Topic::getName) + .map(this::listSubscriptions) + .flatMap(List::stream) + .map(Subscription::anonymize) + .collect(toList()); + } + + public List getForOwnerId(OwnerId ownerId) { + Collection subscriptionNames = subscriptionOwnerCache.get(ownerId); + return subscriptionRepository.getSubscriptionDetails(subscriptionNames); + } + + public List getAllUnhealthy( + boolean respectMonitoringSeverity, + List subscriptionNames, + List qualifiedTopicNames) { + return getUnhealthyList( + subscriptionOwnerCache.getAll(), + respectMonitoringSeverity, + subscriptionNames, + qualifiedTopicNames); + } + + public List getUnhealthyForOwner( + OwnerId ownerId, + boolean respectMonitoringSeverity, + List subscriptionNames, + List qualifiedTopicNames) { + return getUnhealthyList( + subscriptionOwnerCache.get(ownerId), + respectMonitoringSeverity, + subscriptionNames, + qualifiedTopicNames); + } + + public SubscriptionStats getStats() { + List subscriptions = getAllSubscriptions(); + long trackingEnabledSubscriptionsCount = + subscriptions.stream().filter(Subscription::isTrackingEnabled).count(); + long avroSubscriptionCount = + subscriptions.stream().filter(s -> s.getContentType() == ContentType.AVRO).count(); + return new SubscriptionStats( + subscriptions.size(), trackingEnabledSubscriptionsCount, avroSubscriptionCount); + } + + private List getUnhealthyList( + Collection ownerSubscriptionNames, + boolean respectMonitoringSeverity, + List subscriptionNames, + List qualifiedTopicNames) { + try { + return getSubscriptionDetails(ownerSubscriptionNames) + .thenComposeAsync( + ownerSubscriptions -> { + List> unhealthySubscriptions = + filterSubscriptions( + ownerSubscriptions, + respectMonitoringSeverity, + subscriptionNames, + qualifiedTopicNames); + return CompletableFuture.allOf( + unhealthySubscriptions.toArray(new CompletableFuture[0])) + .thenApply( + v -> + unhealthySubscriptions.stream() + .map(CompletableFuture::join) + .filter(Objects::nonNull) + .collect(Collectors.toList())); + }, + subscriptionHealthCheckExecutorService) + .get(subscriptionHealthCheckTimeoutMillis, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + logger.error("Timeout occurred while fetching unhealthy subscriptions...", e); + throw new UnhealthySubscriptionGetException("Fetching unhealthy subscriptions timed out."); + } catch (Exception e) { + logger.error("Fetching unhealthy subscriptions failed...", e); + throw new UnhealthySubscriptionGetException("Fetching unhealthy subscriptions failed.", e); + } + } + + private List> filterSubscriptions( + Collection subscriptions, + boolean respectMonitoringSeverity, + List subscriptionNames, + List qualifiedTopicNames) { + boolean shouldFilterBySubscriptionNames = CollectionUtils.isNotEmpty(subscriptionNames); + boolean shouldFilterByQualifiedTopicNames = CollectionUtils.isNotEmpty(qualifiedTopicNames); + + Stream subscriptionStream = + subscriptions.stream() + .filter( + s -> + filterBySeverityMonitorFlag( + respectMonitoringSeverity, s.isSeverityNotImportant())); + + if (shouldFilterBySubscriptionNames) { + subscriptionStream = + subscriptionStream.filter(s -> filterBySubscriptionNames(subscriptionNames, s.getName())); + } + if (shouldFilterByQualifiedTopicNames) { + subscriptionStream = + subscriptionStream.filter( + s -> filterByQualifiedTopicNames(qualifiedTopicNames, s.getQualifiedTopicName())); + } + + return subscriptionStream + .map( + s -> + CompletableFuture.supplyAsync( + () -> getUnhealthy(s), subscriptionHealthCheckExecutorService)) + .collect(toList()); + } + + private boolean filterBySubscriptionNames( + List subscriptionNames, String subscriptionName) { + return subscriptionNames.contains(subscriptionName); + } + + private boolean filterByQualifiedTopicNames( + List qualifiedTopicNames, String qualifiedTopicName) { + return qualifiedTopicNames.contains(qualifiedTopicName); + } + + private boolean filterBySeverityMonitorFlag( + boolean respectMonitoringSeverity, boolean isSeverityNotImportant) { + return !(respectMonitoringSeverity && isSeverityNotImportant); + } + + private SubscriptionHealth getHealth(Subscription subscription) { + TopicName topicName = subscription.getTopicName(); + TopicMetrics topicMetrics = topicService.getTopicMetrics(topicName); + SubscriptionMetrics subscriptionMetrics = + getSubscriptionMetrics(topicName, subscription.getName()); + return subscriptionHealthChecker.checkHealth(subscription, topicMetrics, subscriptionMetrics); + } + + private UnhealthySubscription getUnhealthy(Subscription subscription) { + SubscriptionHealth subscriptionHealth = getHealth(subscription); + if (subscriptionHealth.getStatus() == Status.UNHEALTHY) { + return UnhealthySubscription.from(subscription, subscriptionHealth); + } else { + return null; + } + } + + private List getSubscriptionsMetrics( + List subscriptions) { + return subscriptions.stream() + .map( + s -> { + SubscriptionMetrics metrics = + metricsRepository.loadMetrics(s.getTopicName(), s.getName()); + return SubscriptionNameWithMetrics.from( + metrics, s.getName(), s.getQualifiedTopicName()); + }) + .collect(toList()); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/UnhealthySubscriptionGetException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/UnhealthySubscriptionGetException.java index 5537913596..f6d97ab707 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/UnhealthySubscriptionGetException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/UnhealthySubscriptionGetException.java @@ -5,16 +5,16 @@ public class UnhealthySubscriptionGetException extends ManagementException { - public UnhealthySubscriptionGetException(String message) { - super(message); - } + public UnhealthySubscriptionGetException(String message) { + super(message); + } - public UnhealthySubscriptionGetException(String message, Throwable cause) { - super(message, cause); - } + public UnhealthySubscriptionGetException(String message, Throwable cause) { + super(message, cause); + } - @Override - public ErrorCode getCode() { - return ErrorCode.OTHER; - } + @Override + public ErrorCode getCode() { + return ErrorCode.OTHER; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/commands/CreateSubscriptionRepositoryCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/commands/CreateSubscriptionRepositoryCommand.java index 728dda8d7c..d32c884cc7 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/commands/CreateSubscriptionRepositoryCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/commands/CreateSubscriptionRepositoryCommand.java @@ -8,36 +8,37 @@ public class CreateSubscriptionRepositoryCommand extends RepositoryCommand { - private final Subscription subscription; - - public CreateSubscriptionRepositoryCommand(Subscription subscription) { - this.subscription = subscription; - } - - @Override - public void backup(DatacenterBoundRepositoryHolder holder) {} - - @Override - public void execute(DatacenterBoundRepositoryHolder holder) { - holder.getRepository().createSubscription(subscription); - } - - @Override - public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) { - if (exception instanceof SubscriptionAlreadyExistsException) { - // prevents removal of already existing subscription - return; - } - holder.getRepository().removeSubscription(subscription.getTopicName(), subscription.getName()); - } - - @Override - public Class getRepositoryType() { - return SubscriptionRepository.class; - } - - @Override - public String toString() { - return "CreateSubscription(" + subscription.getQualifiedName() + ")"; + private final Subscription subscription; + + public CreateSubscriptionRepositoryCommand(Subscription subscription) { + this.subscription = subscription; + } + + @Override + public void backup(DatacenterBoundRepositoryHolder holder) {} + + @Override + public void execute(DatacenterBoundRepositoryHolder holder) { + holder.getRepository().createSubscription(subscription); + } + + @Override + public void rollback( + DatacenterBoundRepositoryHolder holder, Exception exception) { + if (exception instanceof SubscriptionAlreadyExistsException) { + // prevents removal of already existing subscription + return; } + holder.getRepository().removeSubscription(subscription.getTopicName(), subscription.getName()); + } + + @Override + public Class getRepositoryType() { + return SubscriptionRepository.class; + } + + @Override + public String toString() { + return "CreateSubscription(" + subscription.getQualifiedName() + ")"; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/commands/RemoveSubscriptionRepositoryCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/commands/RemoveSubscriptionRepositoryCommand.java index d1b346b3dc..f22f4fdc12 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/commands/RemoveSubscriptionRepositoryCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/commands/RemoveSubscriptionRepositoryCommand.java @@ -9,38 +9,41 @@ public class RemoveSubscriptionRepositoryCommand extends RepositoryCommand { - private final TopicName topicName; - private final String subscriptionName; - - private Subscription backup; - - public RemoveSubscriptionRepositoryCommand(TopicName topicName, String subscriptionName) { - this.topicName = topicName; - this.subscriptionName = subscriptionName; - } - - @Override - public void backup(DatacenterBoundRepositoryHolder holder) { - backup = holder.getRepository().getSubscriptionDetails(topicName, subscriptionName); - } - - @Override - public void execute(DatacenterBoundRepositoryHolder holder) { - holder.getRepository().removeSubscription(topicName, subscriptionName); - } - - @Override - public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) { - holder.getRepository().createSubscription(backup); - } - - @Override - public Class getRepositoryType() { - return SubscriptionRepository.class; - } - - @Override - public String toString() { - return "RemoveSubscription(" + new SubscriptionName(subscriptionName, topicName).getQualifiedName() + ")"; - } + private final TopicName topicName; + private final String subscriptionName; + + private Subscription backup; + + public RemoveSubscriptionRepositoryCommand(TopicName topicName, String subscriptionName) { + this.topicName = topicName; + this.subscriptionName = subscriptionName; + } + + @Override + public void backup(DatacenterBoundRepositoryHolder holder) { + backup = holder.getRepository().getSubscriptionDetails(topicName, subscriptionName); + } + + @Override + public void execute(DatacenterBoundRepositoryHolder holder) { + holder.getRepository().removeSubscription(topicName, subscriptionName); + } + + @Override + public void rollback( + DatacenterBoundRepositoryHolder holder, Exception exception) { + holder.getRepository().createSubscription(backup); + } + + @Override + public Class getRepositoryType() { + return SubscriptionRepository.class; + } + + @Override + public String toString() { + return "RemoveSubscription(" + + new SubscriptionName(subscriptionName, topicName).getQualifiedName() + + ")"; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/commands/UpdateSubscriptionRepositoryCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/commands/UpdateSubscriptionRepositoryCommand.java index c0ac52c3c2..98449f7771 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/commands/UpdateSubscriptionRepositoryCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/commands/UpdateSubscriptionRepositoryCommand.java @@ -6,36 +6,40 @@ import pl.allegro.tech.hermes.management.domain.dc.RepositoryCommand; public class UpdateSubscriptionRepositoryCommand extends RepositoryCommand { - private final Subscription subscription; - - private Subscription backup; - - public UpdateSubscriptionRepositoryCommand(Subscription subscription) { - this.subscription = subscription; - } - - @Override - public void backup(DatacenterBoundRepositoryHolder holder) { - backup = holder.getRepository().getSubscriptionDetails(subscription.getTopicName(), subscription.getName()); - } - - @Override - public void execute(DatacenterBoundRepositoryHolder holder) { - holder.getRepository().updateSubscription(subscription); - } - - @Override - public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) { - holder.getRepository().updateSubscription(backup); - } - - @Override - public Class getRepositoryType() { - return SubscriptionRepository.class; - } - - @Override - public String toString() { - return "UpdateSubscription(" + subscription.getQualifiedName() + ")"; - } + private final Subscription subscription; + + private Subscription backup; + + public UpdateSubscriptionRepositoryCommand(Subscription subscription) { + this.subscription = subscription; + } + + @Override + public void backup(DatacenterBoundRepositoryHolder holder) { + backup = + holder + .getRepository() + .getSubscriptionDetails(subscription.getTopicName(), subscription.getName()); + } + + @Override + public void execute(DatacenterBoundRepositoryHolder holder) { + holder.getRepository().updateSubscription(subscription); + } + + @Override + public void rollback( + DatacenterBoundRepositoryHolder holder, Exception exception) { + holder.getRepository().updateSubscription(backup); + } + + @Override + public Class getRepositoryType() { + return SubscriptionRepository.class; + } + + @Override + public String toString() { + return "UpdateSubscription(" + subscription.getQualifiedName() + ")"; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/SubscriptionHealthChecker.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/SubscriptionHealthChecker.java index 7439f76999..01c34692a1 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/SubscriptionHealthChecker.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/SubscriptionHealthChecker.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.management.domain.subscription.health; +import static java.util.stream.Collectors.toSet; +import static pl.allegro.tech.hermes.api.Subscription.State.SUSPENDED; + +import java.util.Optional; +import java.util.Set; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import pl.allegro.tech.hermes.api.Subscription; @@ -8,49 +13,50 @@ import pl.allegro.tech.hermes.api.SubscriptionMetrics; import pl.allegro.tech.hermes.api.TopicMetrics; -import java.util.Optional; -import java.util.Set; - -import static java.util.stream.Collectors.toSet; -import static pl.allegro.tech.hermes.api.Subscription.State.SUSPENDED; - @Component public class SubscriptionHealthChecker { - private final Set problemIndicators; - - @Autowired - public SubscriptionHealthChecker(Set problemIndicators) { - this.problemIndicators = problemIndicators; - } - - public SubscriptionHealth checkHealth(Subscription subscription, TopicMetrics topicMetrics, SubscriptionMetrics subscriptionMetrics) { - if (isSuspended(subscription)) { - return SubscriptionHealth.HEALTHY; - } else { - return getActiveSubscriptionHealth(subscription, topicMetrics, subscriptionMetrics); - } - } - - private boolean isSuspended(Subscription subscription) { - return subscription.getState() == SUSPENDED; - } - - private SubscriptionHealth getActiveSubscriptionHealth(Subscription subscription, - TopicMetrics topicMetrics, - SubscriptionMetrics subscriptionMetrics) { - return SubscriptionHealthContext.createIfAllMetricsExist(subscription, topicMetrics, subscriptionMetrics) - .map(healthContext -> { - Set healthProblems = getHealthProblems(healthContext); - return SubscriptionHealth.of(healthProblems); - }) - .orElse(SubscriptionHealth.NO_DATA); - } - - private Set getHealthProblems(SubscriptionHealthContext healthContext) { - return problemIndicators.stream() - .map(indicator -> indicator.getProblem(healthContext)) - .filter(Optional::isPresent) - .map(Optional::get) - .collect(toSet()); + private final Set problemIndicators; + + @Autowired + public SubscriptionHealthChecker(Set problemIndicators) { + this.problemIndicators = problemIndicators; + } + + public SubscriptionHealth checkHealth( + Subscription subscription, + TopicMetrics topicMetrics, + SubscriptionMetrics subscriptionMetrics) { + if (isSuspended(subscription)) { + return SubscriptionHealth.HEALTHY; + } else { + return getActiveSubscriptionHealth(subscription, topicMetrics, subscriptionMetrics); } + } + + private boolean isSuspended(Subscription subscription) { + return subscription.getState() == SUSPENDED; + } + + private SubscriptionHealth getActiveSubscriptionHealth( + Subscription subscription, + TopicMetrics topicMetrics, + SubscriptionMetrics subscriptionMetrics) { + return SubscriptionHealthContext.createIfAllMetricsExist( + subscription, topicMetrics, subscriptionMetrics) + .map( + healthContext -> { + Set healthProblems = getHealthProblems(healthContext); + return SubscriptionHealth.of(healthProblems); + }) + .orElse(SubscriptionHealth.NO_DATA); + } + + private Set getHealthProblems( + SubscriptionHealthContext healthContext) { + return problemIndicators.stream() + .map(indicator -> indicator.getProblem(healthContext)) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(toSet()); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/SubscriptionHealthContext.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/SubscriptionHealthContext.java index 17a79ecfd3..850c3b24d3 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/SubscriptionHealthContext.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/SubscriptionHealthContext.java @@ -1,92 +1,94 @@ package pl.allegro.tech.hermes.management.domain.subscription.health; +import java.util.Optional; import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.api.SubscriptionMetrics; import pl.allegro.tech.hermes.api.TopicMetrics; -import java.util.Optional; - public final class SubscriptionHealthContext { - private final Subscription subscription; - private final double topicRate; - private final double subscriptionRate; - private final double timeoutsRate; - private final double otherErrorsRate; - private final double code4xxErrorsRate; - private final double code5xxErrorsRate; - private final double batchRate; - private final long lag; + private final Subscription subscription; + private final double topicRate; + private final double subscriptionRate; + private final double timeoutsRate; + private final double otherErrorsRate; + private final double code4xxErrorsRate; + private final double code5xxErrorsRate; + private final double batchRate; + private final long lag; - private SubscriptionHealthContext(Subscription subscription, - TopicMetrics topicMetrics, - SubscriptionMetrics subscriptionMetrics) { - this.subscription = subscription; - this.topicRate = topicMetrics.getRate().toDouble(); - this.subscriptionRate = subscriptionMetrics.getRate().toDouble(); - this.timeoutsRate = subscriptionMetrics.getTimeouts().toDouble(); - this.otherErrorsRate = subscriptionMetrics.getOtherErrors().toDouble(); - this.code4xxErrorsRate = subscriptionMetrics.getCodes4xx().toDouble(); - this.code5xxErrorsRate = subscriptionMetrics.getCodes5xx().toDouble(); - this.batchRate = subscriptionMetrics.getBatchRate().toDouble(); - this.lag = subscriptionMetrics.getLag().toLong(); - } + private SubscriptionHealthContext( + Subscription subscription, + TopicMetrics topicMetrics, + SubscriptionMetrics subscriptionMetrics) { + this.subscription = subscription; + this.topicRate = topicMetrics.getRate().toDouble(); + this.subscriptionRate = subscriptionMetrics.getRate().toDouble(); + this.timeoutsRate = subscriptionMetrics.getTimeouts().toDouble(); + this.otherErrorsRate = subscriptionMetrics.getOtherErrors().toDouble(); + this.code4xxErrorsRate = subscriptionMetrics.getCodes4xx().toDouble(); + this.code5xxErrorsRate = subscriptionMetrics.getCodes5xx().toDouble(); + this.batchRate = subscriptionMetrics.getBatchRate().toDouble(); + this.lag = subscriptionMetrics.getLag().toLong(); + } - static Optional createIfAllMetricsExist(Subscription subscription, - TopicMetrics topicMetrics, - SubscriptionMetrics subscriptionMetrics) { - if (topicMetrics.getRate().isAvailable() - && subscriptionMetrics.getRate().isAvailable() - && subscriptionMetrics.getTimeouts().isAvailable() - && subscriptionMetrics.getOtherErrors().isAvailable() - && subscriptionMetrics.getCodes4xx().isAvailable() - && subscriptionMetrics.getCodes5xx().isAvailable() - && subscriptionMetrics.getBatchRate().isAvailable() - && subscriptionMetrics.getLag().isAvailable()) { - return Optional.of(new SubscriptionHealthContext(subscription, topicMetrics, subscriptionMetrics)); - } - return Optional.empty(); + static Optional createIfAllMetricsExist( + Subscription subscription, + TopicMetrics topicMetrics, + SubscriptionMetrics subscriptionMetrics) { + if (topicMetrics.getRate().isAvailable() + && subscriptionMetrics.getRate().isAvailable() + && subscriptionMetrics.getTimeouts().isAvailable() + && subscriptionMetrics.getOtherErrors().isAvailable() + && subscriptionMetrics.getCodes4xx().isAvailable() + && subscriptionMetrics.getCodes5xx().isAvailable() + && subscriptionMetrics.getBatchRate().isAvailable() + && subscriptionMetrics.getLag().isAvailable()) { + return Optional.of( + new SubscriptionHealthContext(subscription, topicMetrics, subscriptionMetrics)); } + return Optional.empty(); + } - public boolean subscriptionHasRetryOnError() { - if (subscription.isBatchSubscription()) { - return subscription.getBatchSubscriptionPolicy().isRetryClientErrors(); - } else { - return subscription.getSerialSubscriptionPolicy().isRetryClientErrors(); - } + public boolean subscriptionHasRetryOnError() { + if (subscription.isBatchSubscription()) { + return subscription.getBatchSubscriptionPolicy().isRetryClientErrors(); + } else { + return subscription.getSerialSubscriptionPolicy().isRetryClientErrors(); } + } - public double getSubscriptionRateRespectingDeliveryType() { - if (subscription.isBatchSubscription()) { - return batchRate; - } - return subscriptionRate; + public double getSubscriptionRateRespectingDeliveryType() { + if (subscription.isBatchSubscription()) { + return batchRate; } + return subscriptionRate; + } - public double getOtherErrorsRate() { - return otherErrorsRate; - } + public double getOtherErrorsRate() { + return otherErrorsRate; + } - public double getTimeoutsRate() { - return timeoutsRate; - } + public double getTimeoutsRate() { + return timeoutsRate; + } - public double getCode4xxErrorsRate() { - return code4xxErrorsRate; - } + public double getCode4xxErrorsRate() { + return code4xxErrorsRate; + } - public double getCode5xxErrorsRate() { - return code5xxErrorsRate; - } + public double getCode5xxErrorsRate() { + return code5xxErrorsRate; + } - public long getLag() { - return lag; - } + public long getLag() { + return lag; + } - public double getTopicRate() { - return topicRate; - } + public double getTopicRate() { + return topicRate; + } - public Subscription getSubscription() { - return subscription; - } + public Subscription getSubscription() { + return subscription; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/SubscriptionHealthProblemIndicator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/SubscriptionHealthProblemIndicator.java index dc33183951..dbca99ef62 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/SubscriptionHealthProblemIndicator.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/SubscriptionHealthProblemIndicator.java @@ -1,9 +1,8 @@ package pl.allegro.tech.hermes.management.domain.subscription.health; -import pl.allegro.tech.hermes.api.SubscriptionHealthProblem; - import java.util.Optional; +import pl.allegro.tech.hermes.api.SubscriptionHealthProblem; public interface SubscriptionHealthProblemIndicator { - Optional getProblem(SubscriptionHealthContext context); + Optional getProblem(SubscriptionHealthContext context); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/problem/DisabledIndicator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/problem/DisabledIndicator.java index 6de68151ca..c61a904b54 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/problem/DisabledIndicator.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/problem/DisabledIndicator.java @@ -1,15 +1,14 @@ package pl.allegro.tech.hermes.management.domain.subscription.health.problem; +import java.util.Optional; import pl.allegro.tech.hermes.api.SubscriptionHealthProblem; import pl.allegro.tech.hermes.management.domain.subscription.health.SubscriptionHealthContext; import pl.allegro.tech.hermes.management.domain.subscription.health.SubscriptionHealthProblemIndicator; -import java.util.Optional; - public class DisabledIndicator implements SubscriptionHealthProblemIndicator { - @Override - public Optional getProblem(SubscriptionHealthContext context) { - return Optional.empty(); - } + @Override + public Optional getProblem(SubscriptionHealthContext context) { + return Optional.empty(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/problem/LaggingIndicator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/problem/LaggingIndicator.java index 0cdfc89636..be5ba18a8d 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/problem/LaggingIndicator.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/problem/LaggingIndicator.java @@ -1,27 +1,27 @@ package pl.allegro.tech.hermes.management.domain.subscription.health.problem; +import static pl.allegro.tech.hermes.api.SubscriptionHealthProblem.lagging; + +import java.util.Optional; import pl.allegro.tech.hermes.api.SubscriptionHealthProblem; import pl.allegro.tech.hermes.management.domain.subscription.health.SubscriptionHealthContext; import pl.allegro.tech.hermes.management.domain.subscription.health.SubscriptionHealthProblemIndicator; -import java.util.Optional; - -import static pl.allegro.tech.hermes.api.SubscriptionHealthProblem.lagging; - public class LaggingIndicator implements SubscriptionHealthProblemIndicator { - private final int maxLagInSeconds; + private final int maxLagInSeconds; - public LaggingIndicator(int maxLagInSeconds) { - this.maxLagInSeconds = maxLagInSeconds; - } + public LaggingIndicator(int maxLagInSeconds) { + this.maxLagInSeconds = maxLagInSeconds; + } - @Override - public Optional getProblem(SubscriptionHealthContext context) { - long subscriptionLag = context.getLag(); - double topicRate = context.getTopicRate(); - if (topicRate > 0.0 && subscriptionLag > maxLagInSeconds * topicRate) { - return Optional.of(lagging(subscriptionLag, context.getSubscription().getQualifiedName().toString())); - } - return Optional.empty(); + @Override + public Optional getProblem(SubscriptionHealthContext context) { + long subscriptionLag = context.getLag(); + double topicRate = context.getTopicRate(); + if (topicRate > 0.0 && subscriptionLag > maxLagInSeconds * topicRate) { + return Optional.of( + lagging(subscriptionLag, context.getSubscription().getQualifiedName().toString())); } + return Optional.empty(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/problem/MalfunctioningIndicator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/problem/MalfunctioningIndicator.java index 2d0c946f43..99dbeb1f89 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/problem/MalfunctioningIndicator.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/problem/MalfunctioningIndicator.java @@ -1,37 +1,41 @@ package pl.allegro.tech.hermes.management.domain.subscription.health.problem; +import static pl.allegro.tech.hermes.api.SubscriptionHealthProblem.malfunctioning; + +import java.util.Optional; import pl.allegro.tech.hermes.api.SubscriptionHealthProblem; import pl.allegro.tech.hermes.management.domain.subscription.health.SubscriptionHealthContext; import pl.allegro.tech.hermes.management.domain.subscription.health.SubscriptionHealthProblemIndicator; -import java.util.Optional; - -import static pl.allegro.tech.hermes.api.SubscriptionHealthProblem.malfunctioning; - public class MalfunctioningIndicator implements SubscriptionHealthProblemIndicator { - private final double max5xxErrorsRatio; - private final double minSubscriptionRateForReliableMetrics; - - public MalfunctioningIndicator(double max5xxErrorsRatio, double minSubscriptionRateForReliableMetrics) { - this.max5xxErrorsRatio = max5xxErrorsRatio; - this.minSubscriptionRateForReliableMetrics = minSubscriptionRateForReliableMetrics; - } - - @Override - public Optional getProblem(SubscriptionHealthContext context) { - if (areSubscriptionMetricsReliable(context) && isCode5xxErrorsRateHigh(context)) { - return Optional.of(malfunctioning(context.getCode5xxErrorsRate(), context.getSubscription().getQualifiedName().toString())); - } - return Optional.empty(); - } - - private boolean areSubscriptionMetricsReliable(SubscriptionHealthContext context) { - return context.getSubscriptionRateRespectingDeliveryType() > minSubscriptionRateForReliableMetrics; - } - - private boolean isCode5xxErrorsRateHigh(SubscriptionHealthContext context) { - double code5xxErrorsRate = context.getCode5xxErrorsRate(); - double rate = context.getSubscriptionRateRespectingDeliveryType(); - return code5xxErrorsRate > max5xxErrorsRatio * rate; + private final double max5xxErrorsRatio; + private final double minSubscriptionRateForReliableMetrics; + + public MalfunctioningIndicator( + double max5xxErrorsRatio, double minSubscriptionRateForReliableMetrics) { + this.max5xxErrorsRatio = max5xxErrorsRatio; + this.minSubscriptionRateForReliableMetrics = minSubscriptionRateForReliableMetrics; + } + + @Override + public Optional getProblem(SubscriptionHealthContext context) { + if (areSubscriptionMetricsReliable(context) && isCode5xxErrorsRateHigh(context)) { + return Optional.of( + malfunctioning( + context.getCode5xxErrorsRate(), + context.getSubscription().getQualifiedName().toString())); } + return Optional.empty(); + } + + private boolean areSubscriptionMetricsReliable(SubscriptionHealthContext context) { + return context.getSubscriptionRateRespectingDeliveryType() + > minSubscriptionRateForReliableMetrics; + } + + private boolean isCode5xxErrorsRateHigh(SubscriptionHealthContext context) { + double code5xxErrorsRate = context.getCode5xxErrorsRate(); + double rate = context.getSubscriptionRateRespectingDeliveryType(); + return code5xxErrorsRate > max5xxErrorsRatio * rate; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/problem/ReceivingMalformedMessagesIndicator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/problem/ReceivingMalformedMessagesIndicator.java index ba539228c4..389eb8fcc4 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/problem/ReceivingMalformedMessagesIndicator.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/problem/ReceivingMalformedMessagesIndicator.java @@ -1,41 +1,43 @@ package pl.allegro.tech.hermes.management.domain.subscription.health.problem; +import static pl.allegro.tech.hermes.api.SubscriptionHealthProblem.receivingMalformedMessages; + +import java.util.Optional; import pl.allegro.tech.hermes.api.SubscriptionHealthProblem; import pl.allegro.tech.hermes.management.domain.subscription.health.SubscriptionHealthContext; import pl.allegro.tech.hermes.management.domain.subscription.health.SubscriptionHealthProblemIndicator; -import java.util.Optional; - -import static pl.allegro.tech.hermes.api.SubscriptionHealthProblem.receivingMalformedMessages; - public class ReceivingMalformedMessagesIndicator implements SubscriptionHealthProblemIndicator { - private final double max4xxErrorsRatio; - private final double minSubscriptionRateForReliableMetrics; - - public ReceivingMalformedMessagesIndicator(double max4xxErrorsRatio, double minSubscriptionRateForReliableMetrics) { - this.max4xxErrorsRatio = max4xxErrorsRatio; - this.minSubscriptionRateForReliableMetrics = minSubscriptionRateForReliableMetrics; - } - - @Override - public Optional getProblem(SubscriptionHealthContext context) { - if (context.subscriptionHasRetryOnError() - && areSubscriptionMetricsReliable(context) - && isCode4xxErrorsRateHigh(context)) { - return Optional.of( - receivingMalformedMessages(context.getCode4xxErrorsRate(), context.getSubscription().getQualifiedName().toString()) - ); - } - return Optional.empty(); - } - - private boolean areSubscriptionMetricsReliable(SubscriptionHealthContext context) { - return context.getSubscriptionRateRespectingDeliveryType() > minSubscriptionRateForReliableMetrics; - } - - private boolean isCode4xxErrorsRateHigh(SubscriptionHealthContext context) { - double code4xxErrorsRate = context.getCode4xxErrorsRate(); - double rate = context.getSubscriptionRateRespectingDeliveryType(); - return code4xxErrorsRate > max4xxErrorsRatio * rate; + private final double max4xxErrorsRatio; + private final double minSubscriptionRateForReliableMetrics; + + public ReceivingMalformedMessagesIndicator( + double max4xxErrorsRatio, double minSubscriptionRateForReliableMetrics) { + this.max4xxErrorsRatio = max4xxErrorsRatio; + this.minSubscriptionRateForReliableMetrics = minSubscriptionRateForReliableMetrics; + } + + @Override + public Optional getProblem(SubscriptionHealthContext context) { + if (context.subscriptionHasRetryOnError() + && areSubscriptionMetricsReliable(context) + && isCode4xxErrorsRateHigh(context)) { + return Optional.of( + receivingMalformedMessages( + context.getCode4xxErrorsRate(), + context.getSubscription().getQualifiedName().toString())); } + return Optional.empty(); + } + + private boolean areSubscriptionMetricsReliable(SubscriptionHealthContext context) { + return context.getSubscriptionRateRespectingDeliveryType() + > minSubscriptionRateForReliableMetrics; + } + + private boolean isCode4xxErrorsRateHigh(SubscriptionHealthContext context) { + double code4xxErrorsRate = context.getCode4xxErrorsRate(); + double rate = context.getSubscriptionRateRespectingDeliveryType(); + return code4xxErrorsRate > max4xxErrorsRatio * rate; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/problem/TimingOutIndicator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/problem/TimingOutIndicator.java index 293851a1b0..43ba4798ee 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/problem/TimingOutIndicator.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/problem/TimingOutIndicator.java @@ -1,36 +1,38 @@ package pl.allegro.tech.hermes.management.domain.subscription.health.problem; +import static pl.allegro.tech.hermes.api.SubscriptionHealthProblem.timingOut; + +import java.util.Optional; import pl.allegro.tech.hermes.api.SubscriptionHealthProblem; import pl.allegro.tech.hermes.management.domain.subscription.health.SubscriptionHealthContext; import pl.allegro.tech.hermes.management.domain.subscription.health.SubscriptionHealthProblemIndicator; -import java.util.Optional; - -import static pl.allegro.tech.hermes.api.SubscriptionHealthProblem.timingOut; - public class TimingOutIndicator implements SubscriptionHealthProblemIndicator { - private final double maxTimeoutsRatio; - private final double minSubscriptionRateForReliableMetrics; - - public TimingOutIndicator(double maxTimeoutsRatio, double minSubscriptionRateForReliableMetrics) { - this.maxTimeoutsRatio = maxTimeoutsRatio; - this.minSubscriptionRateForReliableMetrics = minSubscriptionRateForReliableMetrics; - } - - @Override - public Optional getProblem(SubscriptionHealthContext context) { - if (areSubscriptionMetricsReliable(context) && isTimeoutsRateHigh(context)) { - return Optional.of(timingOut(context.getTimeoutsRate(), context.getSubscription().getQualifiedName().toString())); - } - return Optional.empty(); - } - - private boolean areSubscriptionMetricsReliable(SubscriptionHealthContext context) { - return context.getSubscriptionRateRespectingDeliveryType() > minSubscriptionRateForReliableMetrics; - } - - private boolean isTimeoutsRateHigh(SubscriptionHealthContext context) { - double timeoutsRate = context.getTimeoutsRate(); - return timeoutsRate > maxTimeoutsRatio * context.getSubscriptionRateRespectingDeliveryType(); + private final double maxTimeoutsRatio; + private final double minSubscriptionRateForReliableMetrics; + + public TimingOutIndicator(double maxTimeoutsRatio, double minSubscriptionRateForReliableMetrics) { + this.maxTimeoutsRatio = maxTimeoutsRatio; + this.minSubscriptionRateForReliableMetrics = minSubscriptionRateForReliableMetrics; + } + + @Override + public Optional getProblem(SubscriptionHealthContext context) { + if (areSubscriptionMetricsReliable(context) && isTimeoutsRateHigh(context)) { + return Optional.of( + timingOut( + context.getTimeoutsRate(), context.getSubscription().getQualifiedName().toString())); } + return Optional.empty(); + } + + private boolean areSubscriptionMetricsReliable(SubscriptionHealthContext context) { + return context.getSubscriptionRateRespectingDeliveryType() + > minSubscriptionRateForReliableMetrics; + } + + private boolean isTimeoutsRateHigh(SubscriptionHealthContext context) { + double timeoutsRate = context.getTimeoutsRate(); + return timeoutsRate > maxTimeoutsRatio * context.getSubscriptionRateRespectingDeliveryType(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/problem/UnreachableIndicator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/problem/UnreachableIndicator.java index 795e92c6d7..0ea2fdb201 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/problem/UnreachableIndicator.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/health/problem/UnreachableIndicator.java @@ -1,36 +1,41 @@ package pl.allegro.tech.hermes.management.domain.subscription.health.problem; +import static pl.allegro.tech.hermes.api.SubscriptionHealthProblem.unreachable; + +import java.util.Optional; import pl.allegro.tech.hermes.api.SubscriptionHealthProblem; import pl.allegro.tech.hermes.management.domain.subscription.health.SubscriptionHealthContext; import pl.allegro.tech.hermes.management.domain.subscription.health.SubscriptionHealthProblemIndicator; -import java.util.Optional; - -import static pl.allegro.tech.hermes.api.SubscriptionHealthProblem.unreachable; - public class UnreachableIndicator implements SubscriptionHealthProblemIndicator { - private final double maxOtherErrorsRatio; - private final double minSubscriptionRateForReliableMetrics; - - public UnreachableIndicator(double maxOtherErrorsRatio, double minSubscriptionRateForReliableMetrics) { - this.maxOtherErrorsRatio = maxOtherErrorsRatio; - this.minSubscriptionRateForReliableMetrics = minSubscriptionRateForReliableMetrics; - } - - @Override - public Optional getProblem(SubscriptionHealthContext context) { - if (areSubscriptionMetricsReliable(context) && isOtherErrorsRateHigh(context)) { - return Optional.of(unreachable(context.getOtherErrorsRate(), context.getSubscription().getQualifiedName().toString())); - } - return Optional.empty(); - } - - private boolean areSubscriptionMetricsReliable(SubscriptionHealthContext context) { - return context.getSubscriptionRateRespectingDeliveryType() > minSubscriptionRateForReliableMetrics; - } - - private boolean isOtherErrorsRateHigh(SubscriptionHealthContext context) { - double otherErrorsRate = context.getOtherErrorsRate(); - return otherErrorsRate > maxOtherErrorsRatio * context.getSubscriptionRateRespectingDeliveryType(); + private final double maxOtherErrorsRatio; + private final double minSubscriptionRateForReliableMetrics; + + public UnreachableIndicator( + double maxOtherErrorsRatio, double minSubscriptionRateForReliableMetrics) { + this.maxOtherErrorsRatio = maxOtherErrorsRatio; + this.minSubscriptionRateForReliableMetrics = minSubscriptionRateForReliableMetrics; + } + + @Override + public Optional getProblem(SubscriptionHealthContext context) { + if (areSubscriptionMetricsReliable(context) && isOtherErrorsRateHigh(context)) { + return Optional.of( + unreachable( + context.getOtherErrorsRate(), + context.getSubscription().getQualifiedName().toString())); } + return Optional.empty(); + } + + private boolean areSubscriptionMetricsReliable(SubscriptionHealthContext context) { + return context.getSubscriptionRateRespectingDeliveryType() + > minSubscriptionRateForReliableMetrics; + } + + private boolean isOtherErrorsRateHigh(SubscriptionHealthContext context) { + double otherErrorsRate = context.getOtherErrorsRate(); + return otherErrorsRate + > maxOtherErrorsRatio * context.getSubscriptionRateRespectingDeliveryType(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/EndpointAddressFormatValidator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/EndpointAddressFormatValidator.java index 52fca17e35..4820b16f76 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/EndpointAddressFormatValidator.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/EndpointAddressFormatValidator.java @@ -1,52 +1,54 @@ package pl.allegro.tech.hermes.management.domain.subscription.validator; -import com.damnhandy.uri.template.UriTemplate; -import pl.allegro.tech.hermes.api.EndpointAddress; +import static java.util.Arrays.asList; +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toMap; +import com.damnhandy.uri.template.UriTemplate; import java.net.URI; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; - -import static java.util.Arrays.asList; -import static java.util.function.Function.identity; -import static java.util.stream.Collectors.toMap; +import pl.allegro.tech.hermes.api.EndpointAddress; public class EndpointAddressFormatValidator implements EndpointAddressValidator { - private final Set availableProtocol = new HashSet<>(); - - public EndpointAddressFormatValidator(List additionalEndpointProtocols) { - this.availableProtocol.addAll(additionalEndpointProtocols); - this.availableProtocol.add("http"); - this.availableProtocol.add("https"); - this.availableProtocol.add("jms"); - this.availableProtocol.add("googlepubsub"); - } - - @Override - public void check(EndpointAddress address) { - checkIfProtocolIsValid(address); - checkIfUriIsValid(address); - } - - private void checkIfProtocolIsValid(EndpointAddress address) { - if (!availableProtocol.contains(address.getProtocol())) { - throw new EndpointValidationException("Endpoint address has invalid format"); - } + private final Set availableProtocol = new HashSet<>(); + + public EndpointAddressFormatValidator(List additionalEndpointProtocols) { + this.availableProtocol.addAll(additionalEndpointProtocols); + this.availableProtocol.add("http"); + this.availableProtocol.add("https"); + this.availableProtocol.add("jms"); + this.availableProtocol.add("googlepubsub"); + } + + @Override + public void check(EndpointAddress address) { + checkIfProtocolIsValid(address); + checkIfUriIsValid(address); + } + + private void checkIfProtocolIsValid(EndpointAddress address) { + if (!availableProtocol.contains(address.getProtocol())) { + throw new EndpointValidationException("Endpoint address has invalid format"); } + } - private void checkIfUriIsValid(EndpointAddress address) { - UriTemplate template = UriTemplate.fromTemplate(address.getRawEndpoint()); - if (isInvalidHost(template)) { - throw new EndpointValidationException("Endpoint contains invalid chars in host name. Underscore is one of them."); - } + private void checkIfUriIsValid(EndpointAddress address) { + UriTemplate template = UriTemplate.fromTemplate(address.getRawEndpoint()); + if (isInvalidHost(template)) { + throw new EndpointValidationException( + "Endpoint contains invalid chars in host name. Underscore is one of them."); } + } - private boolean isInvalidHost(UriTemplate template) { - Map uriKeysWithEmptyValues = asList(template.getVariables()).stream().collect(toMap(identity(), v -> "empty")); + private boolean isInvalidHost(UriTemplate template) { + Map uriKeysWithEmptyValues = + asList(template.getVariables()).stream().collect(toMap(identity(), v -> "empty")); - //check if host is null due to bug in jdk https://bugs.java.com/bugdatabase/view_bug.do?bug_id=6587184 - return URI.create(template.expand(uriKeysWithEmptyValues)).getHost() == null; - } + // check if host is null due to bug in jdk + // https://bugs.java.com/bugdatabase/view_bug.do?bug_id=6587184 + return URI.create(template.expand(uriKeysWithEmptyValues)).getHost() == null; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/EndpointAddressValidator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/EndpointAddressValidator.java index 2f94c91012..28eb074892 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/EndpointAddressValidator.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/EndpointAddressValidator.java @@ -4,5 +4,5 @@ public interface EndpointAddressValidator { - void check(EndpointAddress address); + void check(EndpointAddress address); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/EndpointOwnershipValidator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/EndpointOwnershipValidator.java index 29acc1448f..f05ea1c1fb 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/EndpointOwnershipValidator.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/EndpointOwnershipValidator.java @@ -5,5 +5,5 @@ public interface EndpointOwnershipValidator { - void check(OwnerId owner, EndpointAddress endpoint); + void check(OwnerId owner, EndpointAddress endpoint); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/EndpointValidationException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/EndpointValidationException.java index d8a064a1d0..656610181c 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/EndpointValidationException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/EndpointValidationException.java @@ -5,12 +5,12 @@ public class EndpointValidationException extends ManagementException { - public EndpointValidationException(String message) { - super(message); - } + public EndpointValidationException(String message) { + super(message); + } - @Override - public ErrorCode getCode() { - return ErrorCode.VALIDATION_ERROR; - } + @Override + public ErrorCode getCode() { + return ErrorCode.VALIDATION_ERROR; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/MessageFilterTypeValidator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/MessageFilterTypeValidator.java index cfefee4f2b..7e65fe4c68 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/MessageFilterTypeValidator.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/MessageFilterTypeValidator.java @@ -1,63 +1,64 @@ package pl.allegro.tech.hermes.management.domain.subscription.validator; import com.google.common.base.Objects; -import pl.allegro.tech.hermes.api.ContentType; -import pl.allegro.tech.hermes.api.Subscription; -import pl.allegro.tech.hermes.api.Topic; - import java.util.Arrays; import java.util.HashSet; import java.util.Set; +import pl.allegro.tech.hermes.api.ContentType; +import pl.allegro.tech.hermes.api.Subscription; +import pl.allegro.tech.hermes.api.Topic; class MessageFilterTypeValidator { - private static final String ERROR_MESSAGE = "Message filter type %s doesn't match topic content type %s"; + private static final String ERROR_MESSAGE = + "Message filter type %s doesn't match topic content type %s"; + + private static final Set VALID_TYPES_COMBINATIONS = + new HashSet<>( + Arrays.asList( + new ContentTypeFilterTypePair(ContentType.JSON, "jsonpath"), + new ContentTypeFilterTypePair(ContentType.JSON, "header"), + new ContentTypeFilterTypePair(ContentType.AVRO, "avropath"), + new ContentTypeFilterTypePair(ContentType.AVRO, "header"))); + + void check(Subscription subscription, Topic topic) { + subscription.getFilters().stream() + .map(filter -> new ContentTypeFilterTypePair(topic.getContentType(), filter.getType())) + .forEach(this::checkTypeMaching); + } + + private void checkTypeMaching(ContentTypeFilterTypePair pair) { + if (!VALID_TYPES_COMBINATIONS.contains(pair)) { + throw new SubscriptionValidationException( + String.format(ERROR_MESSAGE, pair.filterType, pair.contentType)); + } + } - private static final Set VALID_TYPES_COMBINATIONS = new HashSet<>(Arrays.asList( - new ContentTypeFilterTypePair(ContentType.JSON, "jsonpath"), - new ContentTypeFilterTypePair(ContentType.JSON, "header"), - new ContentTypeFilterTypePair(ContentType.AVRO, "avropath"), - new ContentTypeFilterTypePair(ContentType.AVRO, "header") - )); + static class ContentTypeFilterTypePair { + private final ContentType contentType; + private final String filterType; - void check(Subscription subscription, Topic topic) { - subscription.getFilters() - .stream() - .map(filter -> new ContentTypeFilterTypePair(topic.getContentType(), filter.getType())) - .forEach(this::checkTypeMaching); + ContentTypeFilterTypePair(ContentType contentType, String filterType) { + this.contentType = contentType; + this.filterType = filterType; } - private void checkTypeMaching(ContentTypeFilterTypePair pair) { - if (!VALID_TYPES_COMBINATIONS.contains(pair)) { - throw new SubscriptionValidationException(String.format(ERROR_MESSAGE, pair.filterType, pair.contentType)); - } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ContentTypeFilterTypePair validPair = (ContentTypeFilterTypePair) o; + return contentType == validPair.contentType + && Objects.equal(filterType, validPair.filterType); } - static class ContentTypeFilterTypePair { - private final ContentType contentType; - private final String filterType; - - ContentTypeFilterTypePair(ContentType contentType, String filterType) { - this.contentType = contentType; - this.filterType = filterType; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ContentTypeFilterTypePair validPair = (ContentTypeFilterTypePair) o; - return contentType == validPair.contentType - && Objects.equal(filterType, validPair.filterType); - } - - @Override - public int hashCode() { - return Objects.hashCode(contentType, filterType); - } + @Override + public int hashCode() { + return Objects.hashCode(contentType, filterType); } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/NoOpEndpointOwnershipValidator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/NoOpEndpointOwnershipValidator.java index 0b2ce2636e..51af3e30d7 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/NoOpEndpointOwnershipValidator.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/NoOpEndpointOwnershipValidator.java @@ -5,8 +5,6 @@ public class NoOpEndpointOwnershipValidator implements EndpointOwnershipValidator { - @Override - public void check(OwnerId owner, EndpointAddress endpoint) { - - } + @Override + public void check(OwnerId owner, EndpointAddress endpoint) {} } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/SubscriberWithAccessToAnyTopic.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/SubscriberWithAccessToAnyTopic.java index f9b27c817b..5b27dce4bd 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/SubscriberWithAccessToAnyTopic.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/SubscriberWithAccessToAnyTopic.java @@ -1,20 +1,21 @@ package pl.allegro.tech.hermes.management.domain.subscription.validator; +import java.util.List; import pl.allegro.tech.hermes.api.OwnerId; import pl.allegro.tech.hermes.api.Subscription; -import java.util.List; - public class SubscriberWithAccessToAnyTopic { - private final OwnerId ownerId; - private final List protocols; + private final OwnerId ownerId; + private final List protocols; - public SubscriberWithAccessToAnyTopic(String ownerSource, String ownerId, List protocols) { - this.ownerId = new OwnerId(ownerSource, ownerId); - this.protocols = protocols; - } + public SubscriberWithAccessToAnyTopic( + String ownerSource, String ownerId, List protocols) { + this.ownerId = new OwnerId(ownerSource, ownerId); + this.protocols = protocols; + } - boolean matches(Subscription subscription) { - return ownerId.equals(subscription.getOwner()) && protocols.contains(subscription.getEndpoint().getProtocol()); - } + boolean matches(Subscription subscription) { + return ownerId.equals(subscription.getOwner()) + && protocols.contains(subscription.getEndpoint().getProtocol()); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/SubscriptionValidationException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/SubscriptionValidationException.java index d1f3b05d36..ca71c7bb75 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/SubscriptionValidationException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/SubscriptionValidationException.java @@ -5,12 +5,12 @@ public class SubscriptionValidationException extends ManagementException { - public SubscriptionValidationException(String message) { - super(message); - } + public SubscriptionValidationException(String message) { + super(message); + } - @Override - public ErrorCode getCode() { - return ErrorCode.VALIDATION_ERROR; - } + @Override + public ErrorCode getCode() { + return ErrorCode.VALIDATION_ERROR; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/SubscriptionValidator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/SubscriptionValidator.java index 095eecef1c..b907359505 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/SubscriptionValidator.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/subscription/validator/SubscriptionValidator.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.management.domain.subscription.validator; +import java.util.List; +import java.util.Objects; import pl.allegro.tech.hermes.api.Subscription; import pl.allegro.tech.hermes.api.SubscriptionPolicy; import pl.allegro.tech.hermes.api.Topic; @@ -11,142 +13,145 @@ import pl.allegro.tech.hermes.management.domain.owner.validator.OwnerIdValidator; import pl.allegro.tech.hermes.management.domain.topic.TopicService; -import java.util.List; -import java.util.Objects; - public class SubscriptionValidator { - private final OwnerIdValidator ownerIdValidator; - private final ApiPreconditions apiPreconditions; - private final MessageFilterTypeValidator messageFilterTypeValidator; - private final TopicService topicService; - private final SubscriptionRepository subscriptionRepository; - private final List endpointAddressValidators; - private final EndpointOwnershipValidator endpointOwnershipValidator; - private final List subscribersWithAccessToAnyTopic; - - public SubscriptionValidator(OwnerIdValidator ownerIdValidator, - ApiPreconditions apiPreconditions, - TopicService topicService, - SubscriptionRepository subscriptionRepository, - List endpointAddressValidators, - EndpointOwnershipValidator endpointOwnershipValidator, - List subscribersWithAccessToAnyTopic) { - this.ownerIdValidator = ownerIdValidator; - this.apiPreconditions = apiPreconditions; - this.messageFilterTypeValidator = new MessageFilterTypeValidator(); - this.topicService = topicService; - this.subscriptionRepository = subscriptionRepository; - this.endpointAddressValidators = endpointAddressValidators; - this.endpointOwnershipValidator = endpointOwnershipValidator; - this.subscribersWithAccessToAnyTopic = subscribersWithAccessToAnyTopic; + private final OwnerIdValidator ownerIdValidator; + private final ApiPreconditions apiPreconditions; + private final MessageFilterTypeValidator messageFilterTypeValidator; + private final TopicService topicService; + private final SubscriptionRepository subscriptionRepository; + private final List endpointAddressValidators; + private final EndpointOwnershipValidator endpointOwnershipValidator; + private final List subscribersWithAccessToAnyTopic; + + public SubscriptionValidator( + OwnerIdValidator ownerIdValidator, + ApiPreconditions apiPreconditions, + TopicService topicService, + SubscriptionRepository subscriptionRepository, + List endpointAddressValidators, + EndpointOwnershipValidator endpointOwnershipValidator, + List subscribersWithAccessToAnyTopic) { + this.ownerIdValidator = ownerIdValidator; + this.apiPreconditions = apiPreconditions; + this.messageFilterTypeValidator = new MessageFilterTypeValidator(); + this.topicService = topicService; + this.subscriptionRepository = subscriptionRepository; + this.endpointAddressValidators = endpointAddressValidators; + this.endpointOwnershipValidator = endpointOwnershipValidator; + this.subscribersWithAccessToAnyTopic = subscribersWithAccessToAnyTopic; + } + + public void checkCreation(Subscription toCheck, RequestUser createdBy) { + apiPreconditions.checkConstraints(toCheck, createdBy.isAdmin()); + checkOwner(toCheck); + checkEndpoint(toCheck); + checkPermissionsToManageSubscription(toCheck, createdBy); + ensureCreatedSubscriptionInflightIsValid(toCheck, createdBy); + Topic topic = topicService.getTopicDetails(toCheck.getTopicName()); + checkFilters(toCheck, topic); + checkIfSubscribingToTopicIsAllowed(toCheck, topic, createdBy); + if (subscriptionRepository.subscriptionExists(toCheck.getTopicName(), toCheck.getName())) { + throw new SubscriptionAlreadyExistsException(toCheck); } - - public void checkCreation(Subscription toCheck, RequestUser createdBy) { - apiPreconditions.checkConstraints(toCheck, createdBy.isAdmin()); - checkOwner(toCheck); - checkEndpoint(toCheck); - checkPermissionsToManageSubscription(toCheck, createdBy); - ensureCreatedSubscriptionInflightIsValid(toCheck, createdBy); - Topic topic = topicService.getTopicDetails(toCheck.getTopicName()); - checkFilters(toCheck, topic); - checkIfSubscribingToTopicIsAllowed(toCheck, topic, createdBy); - if (subscriptionRepository.subscriptionExists(toCheck.getTopicName(), toCheck.getName())) { - throw new SubscriptionAlreadyExistsException(toCheck); - } + } + + public void checkModification( + Subscription toCheck, RequestUser modifiedBy, Subscription previous) { + apiPreconditions.checkConstraints(toCheck, modifiedBy.isAdmin()); + checkOwner(toCheck); + checkEndpoint(toCheck); + checkPermissionsToManageSubscription(toCheck, modifiedBy); + ensureUpdatedSubscriptionInflightIsValid(previous, toCheck, modifiedBy); + Topic topic = topicService.getTopicDetails(toCheck.getTopicName()); + checkFilters(toCheck, topic); + if (!toCheck.getEndpoint().equals(previous.getEndpoint())) { + checkIfModifyingEndpointIsAllowed(toCheck, topic, modifiedBy); } - - public void checkModification(Subscription toCheck, RequestUser modifiedBy, Subscription previous) { - apiPreconditions.checkConstraints(toCheck, modifiedBy.isAdmin()); - checkOwner(toCheck); - checkEndpoint(toCheck); - checkPermissionsToManageSubscription(toCheck, modifiedBy); - ensureUpdatedSubscriptionInflightIsValid(previous, toCheck, modifiedBy); - Topic topic = topicService.getTopicDetails(toCheck.getTopicName()); - checkFilters(toCheck, topic); - if (!toCheck.getEndpoint().equals(previous.getEndpoint())) { - checkIfModifyingEndpointIsAllowed(toCheck, topic, modifiedBy); - } - subscriptionRepository.ensureSubscriptionExists(toCheck.getTopicName(), toCheck.getName()); + subscriptionRepository.ensureSubscriptionExists(toCheck.getTopicName(), toCheck.getName()); + } + + private void checkOwner(Subscription toCheck) { + ownerIdValidator.check(toCheck.getOwner()); + } + + private void checkEndpoint(Subscription toCheck) { + endpointAddressValidators.forEach(validator -> validator.check(toCheck.getEndpoint())); + endpointOwnershipValidator.check(toCheck.getOwner(), toCheck.getEndpoint()); + } + + private void checkFilters(Subscription toCheck, Topic topic) { + messageFilterTypeValidator.check(toCheck, topic); + } + + private void checkIfSubscribingToTopicIsAllowed( + Subscription toCheck, Topic topic, RequestUser requester) { + if (isSubscribingForbidden(toCheck, topic, requester)) { + throw new PermissionDeniedException( + "Subscribing to this topic has been restricted. Contact the topic owner to create a new subscription."); } + } - private void checkOwner(Subscription toCheck) { - ownerIdValidator.check(toCheck.getOwner()); + private void checkIfModifyingEndpointIsAllowed( + Subscription toCheck, Topic topic, RequestUser requester) { + if (isSubscribingForbidden(toCheck, topic, requester)) { + throw new PermissionDeniedException( + "Subscribing to this topic has been restricted. Contact the topic owner to modify the endpoint of this subscription."); } - - private void checkEndpoint(Subscription toCheck) { - endpointAddressValidators.forEach(validator -> validator.check(toCheck.getEndpoint())); - endpointOwnershipValidator.check(toCheck.getOwner(), toCheck.getEndpoint()); + } + + private boolean isSubscribingForbidden(Subscription toCheck, Topic topic, RequestUser requester) { + if (topic.isSubscribingRestricted()) { + boolean privilegedSubscriber = + subscribersWithAccessToAnyTopic.stream() + .anyMatch(subscriber -> subscriber.matches(toCheck)); + return !requester.isAdmin() && !requester.isOwner(topic.getOwner()) && !privilegedSubscriber; } + return false; + } - private void checkFilters(Subscription toCheck, Topic topic) { - messageFilterTypeValidator.check(toCheck, topic); + private void checkPermissionsToManageSubscription(Subscription toCheck, RequestUser requester) { + if (!requester.isAdmin() && !requester.isOwner(toCheck.getOwner())) { + throw new SubscriptionValidationException( + "Provide an owner that includes you, you would not be able to manage this subscription later"); } + } - private void checkIfSubscribingToTopicIsAllowed(Subscription toCheck, Topic topic, RequestUser requester) { - if (isSubscribingForbidden(toCheck, topic, requester)) { - throw new PermissionDeniedException( - "Subscribing to this topic has been restricted. Contact the topic owner to create a new subscription." - ); - } + private void ensureCreatedSubscriptionInflightIsValid( + Subscription subscription, RequestUser requester) { + if (requester.isAdmin()) { + return; } - - private void checkIfModifyingEndpointIsAllowed(Subscription toCheck, Topic topic, RequestUser requester) { - if (isSubscribingForbidden(toCheck, topic, requester)) { - throw new PermissionDeniedException( - "Subscribing to this topic has been restricted. Contact the topic owner to modify the endpoint of this subscription." - ); - } + SubscriptionPolicy subscriptionPolicy = subscription.getSerialSubscriptionPolicy(); + if (subscriptionPolicy == null) { + return; } - - private boolean isSubscribingForbidden(Subscription toCheck, Topic topic, RequestUser requester) { - if (topic.isSubscribingRestricted()) { - boolean privilegedSubscriber = subscribersWithAccessToAnyTopic.stream() - .anyMatch(subscriber -> subscriber.matches(toCheck)); - return !requester.isAdmin() && !requester.isOwner(topic.getOwner()) && !privilegedSubscriber; - } - return false; + if (subscriptionPolicy.getInflightSize() != null) { + throw new SubscriptionValidationException("Inflight size can't be set by non admin users"); } + } - private void checkPermissionsToManageSubscription(Subscription toCheck, RequestUser requester) { - if (!requester.isAdmin() && !requester.isOwner(toCheck.getOwner())) { - throw new SubscriptionValidationException( - "Provide an owner that includes you, you would not be able to manage this subscription later" - ); - } + private void ensureUpdatedSubscriptionInflightIsValid( + Subscription previous, Subscription updated, RequestUser requester) { + if (requester.isAdmin()) { + return; } - private void ensureCreatedSubscriptionInflightIsValid(Subscription subscription, RequestUser requester) { - if (requester.isAdmin()) { - return; - } - SubscriptionPolicy subscriptionPolicy = subscription.getSerialSubscriptionPolicy(); - if (subscriptionPolicy == null) { - return; - } - if (subscriptionPolicy.getInflightSize() != null) { - throw new SubscriptionValidationException( - "Inflight size can't be set by non admin users" - ); - } + SubscriptionPolicy updatedSubscriptionPolicy = updated.getSerialSubscriptionPolicy(); + if (updatedSubscriptionPolicy == null) { + return; } + Integer updatedInflight = updatedSubscriptionPolicy.getInflightSize(); - private void ensureUpdatedSubscriptionInflightIsValid(Subscription previous, Subscription updated, RequestUser requester) { - if (requester.isAdmin()) { - return; - } - - SubscriptionPolicy updatedSubscriptionPolicy = updated.getSerialSubscriptionPolicy(); - if (updatedSubscriptionPolicy == null) { - return; - } - Integer updatedInflight = updatedSubscriptionPolicy.getInflightSize(); - - SubscriptionPolicy previousSubscriptionPolicy = previous.getSerialSubscriptionPolicy(); - Integer previousInflight = previousSubscriptionPolicy == null ? null : previousSubscriptionPolicy.getInflightSize(); + SubscriptionPolicy previousSubscriptionPolicy = previous.getSerialSubscriptionPolicy(); + Integer previousInflight = + previousSubscriptionPolicy == null ? null : previousSubscriptionPolicy.getInflightSize(); - if (!Objects.equals(previousInflight, updatedInflight)) { - throw new SubscriptionValidationException(String.format("Inflight size can't be changed by non admin users. Changed from: %s, to: %s", previousInflight, updatedInflight)); - } + if (!Objects.equals(previousInflight, updatedInflight)) { + throw new SubscriptionValidationException( + String.format( + "Inflight size can't be changed by non admin users. Changed from: %s, to: %s", + previousInflight, updatedInflight)); } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/AssignmentsToSubscriptionsNotCompletedException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/AssignmentsToSubscriptionsNotCompletedException.java index 0e395fd11f..95511e9ae0 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/AssignmentsToSubscriptionsNotCompletedException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/AssignmentsToSubscriptionsNotCompletedException.java @@ -6,12 +6,13 @@ public class AssignmentsToSubscriptionsNotCompletedException extends ManagementException { - public AssignmentsToSubscriptionsNotCompletedException(Topic topic) { - super("Not all subscriptions for topic " + topic.getQualifiedName() + "have assigned consumers"); - } + public AssignmentsToSubscriptionsNotCompletedException(Topic topic) { + super( + "Not all subscriptions for topic " + topic.getQualifiedName() + "have assigned consumers"); + } - @Override - public ErrorCode getCode() { - return ErrorCode.UNABLE_TO_MOVE_OFFSETS_EXCEPTION; - } + @Override + public ErrorCode getCode() { + return ErrorCode.UNABLE_TO_MOVE_OFFSETS_EXCEPTION; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/BrokerTopicManagement.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/BrokerTopicManagement.java index d50889531d..dcd1e26880 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/BrokerTopicManagement.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/BrokerTopicManagement.java @@ -4,12 +4,11 @@ public interface BrokerTopicManagement { - void createTopic(Topic topic); + void createTopic(Topic topic); - void removeTopic(Topic topic); + void removeTopic(Topic topic); - void updateTopic(Topic topic); - - boolean topicExists(Topic topic); + void updateTopic(Topic topic); + boolean topicExists(Topic topic); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/CreatorRights.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/CreatorRights.java index 91c7d1043c..4f948f7db5 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/CreatorRights.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/CreatorRights.java @@ -4,6 +4,5 @@ public interface CreatorRights { - boolean allowedToManage(Topic topic); - + boolean allowedToManage(Topic topic); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/OffsetsNotAvailableException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/OffsetsNotAvailableException.java index 05605ea8a8..2379580b60 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/OffsetsNotAvailableException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/OffsetsNotAvailableException.java @@ -6,12 +6,13 @@ public class OffsetsNotAvailableException extends ManagementException { - public OffsetsNotAvailableException(Topic topic) { - super("Not all offsets related to hermes topic " + topic.getQualifiedName() + " were available."); - } + public OffsetsNotAvailableException(Topic topic) { + super( + "Not all offsets related to hermes topic " + topic.getQualifiedName() + " were available."); + } - @Override - public ErrorCode getCode() { - return ErrorCode.OFFSETS_NOT_AVAILABLE_EXCEPTION; - } + @Override + public ErrorCode getCode() { + return ErrorCode.OFFSETS_NOT_AVAILABLE_EXCEPTION; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/SingleMessageReader.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/SingleMessageReader.java index 40ce2c165a..0cdb00b5b1 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/SingleMessageReader.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/SingleMessageReader.java @@ -5,6 +5,5 @@ public interface SingleMessageReader { - String readMessageAsJson(Topic topic, KafkaTopic kafkaTopic, int partition, long offset); - + String readMessageAsJson(Topic topic, KafkaTopic kafkaTopic, int partition, long offset); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/SingleMessageReaderException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/SingleMessageReaderException.java index 788cef9f11..86954d476a 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/SingleMessageReaderException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/SingleMessageReaderException.java @@ -5,16 +5,16 @@ public class SingleMessageReaderException extends ManagementException { - public SingleMessageReaderException(String message) { - super(message); - } + public SingleMessageReaderException(String message) { + super(message); + } - public SingleMessageReaderException(String message, Throwable t) { - super(message, t); - } + public SingleMessageReaderException(String message, Throwable t) { + super(message, t); + } - @Override - public ErrorCode getCode() { - return ErrorCode.SINGLE_MESSAGE_READER_EXCEPTION; - } + @Override + public ErrorCode getCode() { + return ErrorCode.SINGLE_MESSAGE_READER_EXCEPTION; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/TopicContentTypeMigrationService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/TopicContentTypeMigrationService.java index e2e338f6c4..44117189d9 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/TopicContentTypeMigrationService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/TopicContentTypeMigrationService.java @@ -1,5 +1,11 @@ package pl.allegro.tech.hermes.management.domain.topic; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -10,87 +16,89 @@ import pl.allegro.tech.hermes.management.domain.auth.RequestUser; import pl.allegro.tech.hermes.management.infrastructure.kafka.MultiDCAwareService; -import java.time.Clock; -import java.time.Duration; -import java.time.Instant; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - @Component public class TopicContentTypeMigrationService { - private static final Logger logger = LoggerFactory.getLogger(TopicContentTypeMigrationService.class); - - private static final Duration CHECK_OFFSETS_AVAILABLE_TIMEOUT = Duration.ofSeconds(30); - private static final Duration INTERVAL_BETWEEN_OFFSETS_AVAILABLE_CHECK = Duration.ofMillis(500); - private static final Duration INTERVAL_BETWEEN_ASSIGNMENTS_COMPLETED_CHECK = Duration.ofMillis(500); - - private final SubscriptionRepository subscriptionRepository; - private final MultiDCAwareService multiDCAwareService; - private final Clock clock; - - @Autowired - public TopicContentTypeMigrationService(SubscriptionRepository subscriptionRepository, - MultiDCAwareService multiDCAwareService, - Clock clock) { - this.subscriptionRepository = subscriptionRepository; - this.multiDCAwareService = multiDCAwareService; - this.clock = clock; - } - - void notifySubscriptions(Topic topic, Instant beforeMigrationInstant, RequestUser requester) { - waitUntilOffsetsAvailableOnAllKafkaTopics(topic, CHECK_OFFSETS_AVAILABLE_TIMEOUT); - logger.info("Offsets available on all partitions of topic {}", topic.getQualifiedName()); - notSuspendedSubscriptionsForTopic(topic) - .map(Subscription::getName) - .forEach(sub -> notifySingleSubscription(topic, beforeMigrationInstant, sub, requester)); - } - - void waitUntilAllSubscriptionsHasConsumersAssigned(Topic topic, Duration assignmentCompletedTimeout) { - Instant abortAttemptsInstant = clock.instant().plus(assignmentCompletedTimeout); - - while (!allSubscriptionsHaveConsumersAssigned(topic)) { - if (clock.instant().isAfter(abortAttemptsInstant)) { - throw new AssignmentsToSubscriptionsNotCompletedException(topic); - } - sleep(INTERVAL_BETWEEN_ASSIGNMENTS_COMPLETED_CHECK); - } - } - - private void notifySingleSubscription(Topic topic, Instant beforeMigrationInstant, String subscriptionName, RequestUser requester) { - multiDCAwareService.retransmit(topic, subscriptionName, beforeMigrationInstant.toEpochMilli(), false, requester); - } - - private void waitUntilOffsetsAvailableOnAllKafkaTopics(Topic topic, Duration offsetsAvailableTimeout) { - Instant abortAttemptsInstant = clock.instant().plus(offsetsAvailableTimeout); - - while (!multiDCAwareService.areOffsetsAvailableOnAllKafkaTopics(topic)) { - if (clock.instant().isAfter(abortAttemptsInstant)) { - throw new OffsetsNotAvailableException(topic); - } - logger.info("Not all offsets related to hermes topic {} are available, will retry", topic.getQualifiedName()); - sleep(INTERVAL_BETWEEN_OFFSETS_AVAILABLE_CHECK); - } + private static final Logger logger = + LoggerFactory.getLogger(TopicContentTypeMigrationService.class); + + private static final Duration CHECK_OFFSETS_AVAILABLE_TIMEOUT = Duration.ofSeconds(30); + private static final Duration INTERVAL_BETWEEN_OFFSETS_AVAILABLE_CHECK = Duration.ofMillis(500); + private static final Duration INTERVAL_BETWEEN_ASSIGNMENTS_COMPLETED_CHECK = + Duration.ofMillis(500); + + private final SubscriptionRepository subscriptionRepository; + private final MultiDCAwareService multiDCAwareService; + private final Clock clock; + + @Autowired + public TopicContentTypeMigrationService( + SubscriptionRepository subscriptionRepository, + MultiDCAwareService multiDCAwareService, + Clock clock) { + this.subscriptionRepository = subscriptionRepository; + this.multiDCAwareService = multiDCAwareService; + this.clock = clock; + } + + void notifySubscriptions(Topic topic, Instant beforeMigrationInstant, RequestUser requester) { + waitUntilOffsetsAvailableOnAllKafkaTopics(topic, CHECK_OFFSETS_AVAILABLE_TIMEOUT); + logger.info("Offsets available on all partitions of topic {}", topic.getQualifiedName()); + notSuspendedSubscriptionsForTopic(topic) + .map(Subscription::getName) + .forEach(sub -> notifySingleSubscription(topic, beforeMigrationInstant, sub, requester)); + } + + void waitUntilAllSubscriptionsHasConsumersAssigned( + Topic topic, Duration assignmentCompletedTimeout) { + Instant abortAttemptsInstant = clock.instant().plus(assignmentCompletedTimeout); + + while (!allSubscriptionsHaveConsumersAssigned(topic)) { + if (clock.instant().isAfter(abortAttemptsInstant)) { + throw new AssignmentsToSubscriptionsNotCompletedException(topic); + } + sleep(INTERVAL_BETWEEN_ASSIGNMENTS_COMPLETED_CHECK); } - - private void sleep(Duration sleepDuration) { - try { - Thread.sleep(sleepDuration.toMillis()); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - - private boolean allSubscriptionsHaveConsumersAssigned(Topic topic) { - List notSuspendedSubscriptions = notSuspendedSubscriptionsForTopic(topic) - .collect(Collectors.toList()); - return multiDCAwareService.allSubscriptionsHaveConsumersAssigned(topic, notSuspendedSubscriptions); + } + + private void notifySingleSubscription( + Topic topic, Instant beforeMigrationInstant, String subscriptionName, RequestUser requester) { + multiDCAwareService.retransmit( + topic, subscriptionName, beforeMigrationInstant.toEpochMilli(), false, requester); + } + + private void waitUntilOffsetsAvailableOnAllKafkaTopics( + Topic topic, Duration offsetsAvailableTimeout) { + Instant abortAttemptsInstant = clock.instant().plus(offsetsAvailableTimeout); + + while (!multiDCAwareService.areOffsetsAvailableOnAllKafkaTopics(topic)) { + if (clock.instant().isAfter(abortAttemptsInstant)) { + throw new OffsetsNotAvailableException(topic); + } + logger.info( + "Not all offsets related to hermes topic {} are available, will retry", + topic.getQualifiedName()); + sleep(INTERVAL_BETWEEN_OFFSETS_AVAILABLE_CHECK); } + } - private Stream notSuspendedSubscriptionsForTopic(Topic topic) { - return subscriptionRepository.listSubscriptions(topic.getName()) - .stream() - .filter(sub -> Subscription.State.SUSPENDED != sub.getState()); + private void sleep(Duration sleepDuration) { + try { + Thread.sleep(sleepDuration.toMillis()); + } catch (InterruptedException e) { + throw new RuntimeException(e); } + } + + private boolean allSubscriptionsHaveConsumersAssigned(Topic topic) { + List notSuspendedSubscriptions = + notSuspendedSubscriptionsForTopic(topic).collect(Collectors.toList()); + return multiDCAwareService.allSubscriptionsHaveConsumersAssigned( + topic, notSuspendedSubscriptions); + } + + private Stream notSuspendedSubscriptionsForTopic(Topic topic) { + return subscriptionRepository.listSubscriptions(topic.getName()).stream() + .filter(sub -> Subscription.State.SUSPENDED != sub.getState()); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/TopicMetricsRepository.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/TopicMetricsRepository.java index 47a0c76a8c..a78fac1bbc 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/TopicMetricsRepository.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/TopicMetricsRepository.java @@ -5,6 +5,5 @@ public interface TopicMetricsRepository { - TopicMetrics loadMetrics(TopicName topicName); - + TopicMetrics loadMetrics(TopicName topicName); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/TopicOwnerCache.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/TopicOwnerCache.java index d84f13e1df..ce76b8d97d 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/TopicOwnerCache.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/TopicOwnerCache.java @@ -5,6 +5,10 @@ import com.google.common.collect.Multimaps; import com.google.common.util.concurrent.ThreadFactoryBuilder; import jakarta.annotation.PreDestroy; +import java.util.Collection; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -15,68 +19,66 @@ import pl.allegro.tech.hermes.domain.topic.TopicRepository; import pl.allegro.tech.hermes.management.domain.group.GroupService; -import java.util.Collection; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - @Component public class TopicOwnerCache { - private static final Logger logger = LoggerFactory.getLogger(TopicOwnerCache.class); + private static final Logger logger = LoggerFactory.getLogger(TopicOwnerCache.class); - private final TopicRepository topicRepository; - private final GroupService groupService; - private final ScheduledExecutorService scheduledExecutorService; + private final TopicRepository topicRepository; + private final GroupService groupService; + private final ScheduledExecutorService scheduledExecutorService; - private Multimap cache = Multimaps.synchronizedMultimap(ArrayListMultimap.create()); + private Multimap cache = + Multimaps.synchronizedMultimap(ArrayListMultimap.create()); - public TopicOwnerCache(TopicRepository topicRepository, GroupService groupService, - @Value("${topicOwnerCache.refreshRateInSeconds}") int refreshRateInSeconds) { - this.topicRepository = topicRepository; - this.groupService = groupService; - scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( - new ThreadFactoryBuilder() - .setNameFormat("topic-owner-cache-%d") - .build()); - scheduledExecutorService.scheduleAtFixedRate(this::refillCache, 0, refreshRateInSeconds, TimeUnit.SECONDS); - } + public TopicOwnerCache( + TopicRepository topicRepository, + GroupService groupService, + @Value("${topicOwnerCache.refreshRateInSeconds}") int refreshRateInSeconds) { + this.topicRepository = topicRepository; + this.groupService = groupService; + scheduledExecutorService = + Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder().setNameFormat("topic-owner-cache-%d").build()); + scheduledExecutorService.scheduleAtFixedRate( + this::refillCache, 0, refreshRateInSeconds, TimeUnit.SECONDS); + } - @PreDestroy - public void stop() { - scheduledExecutorService.shutdown(); - } + @PreDestroy + public void stop() { + scheduledExecutorService.shutdown(); + } - public Collection get(OwnerId ownerId) { - return cache.get(ownerId); - } + public Collection get(OwnerId ownerId) { + return cache.get(ownerId); + } - private void refillCache() { - try { - logger.info("Starting filling TopicOwnerCache cache"); - long start = System.currentTimeMillis(); - Multimap cache = ArrayListMultimap.create(); - groupService.listGroupNames().stream() - .flatMap(groupName -> topicRepository.listTopics(groupName).stream()) - .forEach(topic -> cache.put(topic.getOwner(), topic.getName())); - this.cache = Multimaps.synchronizedMultimap(cache); - long end = System.currentTimeMillis(); - logger.info("TopicOwnerCache filled. Took {}ms", end - start); - } catch (Exception e) { - logger.error("Error while filling TopicOwnerCache", e); - } + private void refillCache() { + try { + logger.info("Starting filling TopicOwnerCache cache"); + long start = System.currentTimeMillis(); + Multimap cache = ArrayListMultimap.create(); + groupService.listGroupNames().stream() + .flatMap(groupName -> topicRepository.listTopics(groupName).stream()) + .forEach(topic -> cache.put(topic.getOwner(), topic.getName())); + this.cache = Multimaps.synchronizedMultimap(cache); + long end = System.currentTimeMillis(); + logger.info("TopicOwnerCache filled. Took {}ms", end - start); + } catch (Exception e) { + logger.error("Error while filling TopicOwnerCache", e); } + } - void onRemovedTopic(Topic topic) { - cache.remove(topic.getOwner(), topic.getName()); - } + void onRemovedTopic(Topic topic) { + cache.remove(topic.getOwner(), topic.getName()); + } - void onCreatedTopic(Topic topic) { - cache.put(topic.getOwner(), topic.getName()); - } + void onCreatedTopic(Topic topic) { + cache.put(topic.getOwner(), topic.getName()); + } - void onUpdatedTopic(Topic oldTopic, Topic newTopic) { - cache.remove(oldTopic.getOwner(), oldTopic.getName()); - cache.put(newTopic.getOwner(), newTopic.getName()); - } + void onUpdatedTopic(Topic oldTopic, Topic newTopic) { + cache.remove(oldTopic.getOwner(), oldTopic.getName()); + cache.put(newTopic.getOwner(), newTopic.getName()); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/TopicRemovalDisabledException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/TopicRemovalDisabledException.java index 37c302ae57..400a4b27d2 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/TopicRemovalDisabledException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/TopicRemovalDisabledException.java @@ -6,13 +6,15 @@ public class TopicRemovalDisabledException extends ManagementException { - public TopicRemovalDisabledException(Topic topic) { - super(String.format("Could not remove topic %s, this operation is currently disabled.", topic.getQualifiedName())); - } - - @Override - public ErrorCode getCode() { - return ErrorCode.OPERATION_DISABLED; - } + public TopicRemovalDisabledException(Topic topic) { + super( + String.format( + "Could not remove topic %s, this operation is currently disabled.", + topic.getQualifiedName())); + } + @Override + public ErrorCode getCode() { + return ErrorCode.OPERATION_DISABLED; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/TopicSchemaExistsException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/TopicSchemaExistsException.java index a4bbb84b31..e5c8c4cc77 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/TopicSchemaExistsException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/TopicSchemaExistsException.java @@ -5,12 +5,12 @@ public class TopicSchemaExistsException extends ManagementException { - TopicSchemaExistsException(String topic) { - super("Schema already exists for topic " + topic + ", please remove it before creating topic."); - } + TopicSchemaExistsException(String topic) { + super("Schema already exists for topic " + topic + ", please remove it before creating topic."); + } - @Override - public ErrorCode getCode() { - return ErrorCode.SCHEMA_ALREADY_EXISTS; - } + @Override + public ErrorCode getCode() { + return ErrorCode.SCHEMA_ALREADY_EXISTS; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/TopicService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/TopicService.java index 26fe3ae834..4841fad67c 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/TopicService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/TopicService.java @@ -1,6 +1,21 @@ package pl.allegro.tech.hermes.management.domain.topic; +import static java.util.stream.Collectors.toList; +import static pl.allegro.tech.hermes.api.ContentType.AVRO; +import static pl.allegro.tech.hermes.api.TopicWithSchema.topicWithSchema; + import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.nio.charset.StandardCharsets; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -38,392 +53,402 @@ import pl.allegro.tech.hermes.management.domain.topic.validator.TopicValidator; import pl.allegro.tech.hermes.management.infrastructure.kafka.MultiDCAwareService; -import java.nio.charset.StandardCharsets; -import java.time.Clock; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import static java.util.stream.Collectors.toList; -import static pl.allegro.tech.hermes.api.ContentType.AVRO; -import static pl.allegro.tech.hermes.api.TopicWithSchema.topicWithSchema; - @Component public class TopicService { - private static final Logger logger = LoggerFactory.getLogger(TopicService.class); - - private final TopicRepository topicRepository; - private final GroupService groupService; - private final TopicProperties topicProperties; - private final SchemaService schemaService; - - private final TopicMetricsRepository metricRepository; - private final MultiDCAwareService multiDCAwareService; - private final TopicBlacklistService topicBlacklistService; - private final TopicValidator topicValidator; - private final TopicContentTypeMigrationService topicContentTypeMigrationService; - private final Clock clock; - private final Auditor auditor; - private final MultiDatacenterRepositoryCommandExecutor multiDcExecutor; - private final RepositoryManager repositoryManager; - private final TopicOwnerCache topicOwnerCache; - private final SubscriptionRemover subscriptionRemover; - private final ScheduledExecutorService scheduledTopicExecutor = Executors.newSingleThreadScheduledExecutor( - new ThreadFactoryBuilder() - .setNameFormat("scheduled-topic-executor-%d") - .build()); - - @Autowired - public TopicService(MultiDCAwareService multiDCAwareService, - TopicRepository topicRepository, - GroupService groupService, - TopicProperties topicProperties, - SchemaService schemaService, TopicMetricsRepository metricRepository, - TopicBlacklistService topicBlacklistService, TopicValidator topicValidator, - TopicContentTypeMigrationService topicContentTypeMigrationService, - Clock clock, - Auditor auditor, - MultiDatacenterRepositoryCommandExecutor multiDcExecutor, - RepositoryManager repositoryManager, - TopicOwnerCache topicOwnerCache, - SubscriptionRemover subscriptionRemover) { - this.multiDCAwareService = multiDCAwareService; - this.topicRepository = topicRepository; - this.groupService = groupService; - this.topicProperties = topicProperties; - this.schemaService = schemaService; - this.metricRepository = metricRepository; - this.topicBlacklistService = topicBlacklistService; - this.topicValidator = topicValidator; - this.topicContentTypeMigrationService = topicContentTypeMigrationService; - this.clock = clock; - this.auditor = auditor; - this.multiDcExecutor = multiDcExecutor; - this.repositoryManager = repositoryManager; - this.topicOwnerCache = topicOwnerCache; - this.subscriptionRemover = subscriptionRemover; - } - - public void createTopicWithSchema(TopicWithSchema topicWithSchema, RequestUser createdBy, CreatorRights isAllowedToManage) { - Topic topic = topicWithSchema.getTopic(); - auditor.beforeObjectCreation(createdBy.getUsername(), topic); - groupService.checkGroupExists(topic.getName().getGroupName()); - topicValidator.ensureCreatedTopicIsValid(topic, createdBy, isAllowedToManage); - ensureTopicDoesNotExist(topic); - - boolean validateAndRegisterSchema = AVRO.equals(topic.getContentType()) || (topic.isJsonToAvroDryRunEnabled() - && topicWithSchema.getSchema() != null); - - validateSchema(validateAndRegisterSchema, topicWithSchema, topic); - registerAvroSchema(validateAndRegisterSchema, topicWithSchema, createdBy); - createTopic(topic, createdBy, isAllowedToManage); - } - - public void removeTopicWithSchema(Topic topic, RequestUser removedBy) { - auditor.beforeObjectRemoval(removedBy.getUsername(), Topic.class.getSimpleName(), topic.getQualifiedName()); - subscriptionRemover.removeSubscriptionRelatedToTopic(topic, removedBy); - removeSchema(topic); - if (!topicProperties.isAllowRemoval()) { - throw new TopicRemovalDisabledException(topic); - } - if (topicBlacklistService.isBlacklisted(topic.getQualifiedName())) { - topicBlacklistService.unblacklist(topic.getQualifiedName(), removedBy); - } - removeTopic(topic, removedBy); - } - - public void updateTopicWithSchema(TopicName topicName, PatchData patch, RequestUser modifiedBy) { - Topic topic = getTopicDetails(topicName); - extractSchema(patch) - .ifPresent(schema -> { - schemaService.registerSchema(topic, schema); - scheduleTouchTopic(topicName, modifiedBy); - }); - updateTopic(topicName, patch, modifiedBy); - } - - public void updateTopic(TopicName topicName, PatchData patch, RequestUser modifiedBy) { - auditor.beforeObjectUpdate(modifiedBy.getUsername(), Topic.class.getSimpleName(), topicName, patch); - groupService.checkGroupExists(topicName.getGroupName()); - - Topic retrieved = getTopicDetails(topicName); - Topic modified = Patch.apply(retrieved, patch); - - topicValidator.ensureUpdatedTopicIsValid(modified, retrieved, modifiedBy); - - if (!retrieved.equals(modified)) { - Instant beforeMigrationInstant = clock.instant(); - // TODO: this does not work as intended, uses == instead of equals - if (retrieved.getRetentionTime() != modified.getRetentionTime()) { - multiDCAwareService.manageTopic(brokerTopicManagement -> - brokerTopicManagement.updateTopic(modified) - ); - } - multiDcExecutor.executeByUser(new UpdateTopicRepositoryCommand(modified), modifiedBy); - - if (!retrieved.wasMigratedFromJsonType() && modified.wasMigratedFromJsonType()) { - logger.info( - "Waiting until all subscriptions have consumers assigned during topic {} content type migration...", - topicName.qualifiedName()); - topicContentTypeMigrationService.waitUntilAllSubscriptionsHasConsumersAssigned(modified, - Duration.ofSeconds(topicProperties.getSubscriptionsAssignmentsCompletedTimeoutSeconds())); - logger.info("Notifying subscriptions' consumers about changes in topic {} content type...", topicName.qualifiedName()); - topicContentTypeMigrationService.notifySubscriptions(modified, beforeMigrationInstant, modifiedBy); - } - auditor.objectUpdated(modifiedBy.getUsername(), retrieved, modified); - topicOwnerCache.onUpdatedTopic(retrieved, modified); - } - } - - public void touchTopic(TopicName topicName, RequestUser touchedBy) { - logger.info("Touching topic {}", topicName.qualifiedName()); - multiDcExecutor.executeByUser(new TouchTopicRepositoryCommand(topicName), touchedBy); - } - - /** - * Topic is touched so other Hermes instances are notified to read latest topic schema from schema-registry. - * However, schema-registry can be distributed so when schema is written there then it can not be available on all nodes immediately. - * This is the reason why we delay touch of topic here, to wait until schema is distributed on schema-registry nodes. - */ - public void scheduleTouchTopic(TopicName topicName, RequestUser touchedBy) { - if (topicProperties.isTouchSchedulerEnabled()) { - logger.info("Scheduling touch of topic {}", topicName.qualifiedName()); - scheduledTopicExecutor.schedule(() -> touchTopic(topicName, touchedBy), - topicProperties.getTouchDelayInSeconds(), - TimeUnit.SECONDS - ); - } else { - touchTopic(topicName, touchedBy); - } - } - - public List listQualifiedTopicNames(String groupName) { - return topicRepository.listTopicNames(groupName).stream() - .map(topicName -> new TopicName(groupName, topicName).qualifiedName()) - .collect(toList()); - } - - public List listQualifiedTopicNames() { - return groupService.listGroupNames().stream() - .map(this::listQualifiedTopicNames) - .flatMap(List::stream) - .sorted() - .collect(toList()); - } - - public List listTopics(String groupName) { - return topicRepository.listTopics(groupName); - } - - public Topic getTopicDetails(TopicName topicName) { - return topicRepository.getTopicDetails(topicName); - } - - public TopicWithSchema getTopicWithSchema(TopicName topicName) { - Topic topic = getTopicDetails(topicName); - Optional schema = Optional.empty(); - if (AVRO.equals(topic.getContentType())) { - schema = schemaService.getSchema(topicName.qualifiedName()); - } - return schema - .map(s -> topicWithSchema(topic, s.value())) - .orElseGet(() -> topicWithSchema(topic)); - } - - public TopicMetrics getTopicMetrics(TopicName topicName) { - return topicRepository.topicExists(topicName) ? metricRepository.loadMetrics(topicName) : TopicMetrics.unavailable(); - } - - public String fetchSingleMessageFromPrimary(String brokersClusterName, TopicName topicName, Integer partition, Long offset) { - return multiDCAwareService.readMessageFromPrimary(brokersClusterName, getTopicDetails(topicName), partition, offset); - } - - public List listTrackedTopicNames() { - return groupService.listGroupNames().stream() - .map(topicRepository::listTopics) - .flatMap(List::stream) - .filter(Topic::isTrackingEnabled) - .map(Topic::getQualifiedName) - .collect(toList()); - } - - public List listTrackedTopicNames(String groupName) { - return listTopics(groupName).stream() - .filter(Topic::isTrackingEnabled) - .map(Topic::getQualifiedName) - .collect(toList()); - } - - public List listFilteredTopicNames(Query query) { - return queryTopic(query) - .stream() - .map(Topic::getQualifiedName) - .collect(toList()); - } - - public List listFilteredTopicNames(String groupName, Query query) { - return query.filter(listTopics(groupName)) - .map(Topic::getQualifiedName) - .collect(toList()); - } - - public List queryTopic(Query query) { - return query - .filter(getAllTopics()) - .collect(toList()); - } - - public List getAllTopics() { - return topicRepository.listAllTopics(); - } - - public Optional preview(TopicName topicName, int idx) { - List result = loadMessagePreviewsFromAllDc(topicName) - .stream() - .map(MessagePreview::getContent) - .collect(toList()); - - if (idx >= 0 && idx < result.size()) { - return Optional.of(result.get(idx)); - } else { - return Optional.empty(); - } - } - - public List previewText(TopicName topicName) { - return loadMessagePreviewsFromAllDc(topicName).stream() - .map(p -> new MessageTextPreview(new String(p.getContent(), StandardCharsets.UTF_8), p.isTruncated())) - .collect(toList()); - } - - public List queryTopicsMetrics(Query query) { - List filteredNames = query.filterNames(getAllTopics()) - .collect(toList()); - return query.filter(getTopicsMetrics(filteredNames)) - .collect(toList()); - } - - public TopicStats getStats() { - List topics = getAllTopics(); - long ackAllTopicCount = topics.stream().filter(t -> t.getAck() == Topic.Ack.ALL).count(); - long trackingEnabledTopicCount = topics.stream().filter(Topic::isTrackingEnabled).count(); - long avroTopicCount = topics.stream().filter(t -> t.getContentType() == AVRO).count(); - return new TopicStats(topics.size(), ackAllTopicCount, trackingEnabledTopicCount, avroTopicCount); - } - - private void ensureTopicDoesNotExist(Topic topic) { - if (topicRepository.topicExists(topic.getName())) { - throw new TopicAlreadyExistsException(topic.getName()); - } - } - - private void validateSchema(boolean shouldValidate, TopicWithSchema topicWithSchema, Topic topic) { - if (shouldValidate) { - schemaService.validateSchema(topic, topicWithSchema.getSchema()); - boolean schemaAlreadyRegistered = schemaService.getSchema(topic.getQualifiedName()).isPresent(); - if (schemaAlreadyRegistered) { - throw new TopicSchemaExistsException(topic.getQualifiedName()); - } - } - } - - private void registerAvroSchema(boolean shouldRegister, TopicWithSchema topicWithSchema, RequestUser createdBy) { - if (shouldRegister) { - try { - schemaService.registerSchema(topicWithSchema.getTopic(), topicWithSchema.getSchema()); - } catch (Exception e) { - logger.error("Rolling back topic {} creation due to schema registration error", topicWithSchema.getQualifiedName(), e); - removeTopic(topicWithSchema.getTopic(), createdBy); - throw e; - } - } - } - - private void createTopic(Topic topic, RequestUser createdBy, CreatorRights creatorRights) { - topicValidator.ensureCreatedTopicIsValid(topic, createdBy, creatorRights); - - if (!multiDCAwareService.topicExists(topic)) { - createTopicInBrokers(topic, createdBy); - auditor.objectCreated(createdBy.getUsername(), topic); - topicOwnerCache.onCreatedTopic(topic); - } else { - logger.info("Skipping creation of topic {} on brokers, topic already exists", topic.getQualifiedName()); - } - - multiDcExecutor.executeByUser(new CreateTopicRepositoryCommand(topic), createdBy); - } - - private void createTopicInBrokers(Topic topic, RequestUser createdBy) { - try { - multiDCAwareService.manageTopic(brokerTopicManagement -> - brokerTopicManagement.createTopic(topic) - ); - } catch (Exception exception) { - logger.error( - String.format("Could not create topic %s, rollback topic creation.", topic.getQualifiedName()), - exception - ); - multiDcExecutor.executeByUser(new RemoveTopicRepositoryCommand(topic.getName()), createdBy); - } - } - - private void removeSchema(Topic topic) { - if (AVRO.equals(topic.getContentType()) && topicProperties.isRemoveSchema()) { - schemaService.getSchema(topic.getQualifiedName()).ifPresent(s -> - schemaService.deleteAllSchemaVersions(topic.getQualifiedName())); - } - } - - private void removeTopic(Topic topic, RequestUser removedBy) { - logger.info("Removing topic: {} from ZK clusters", topic.getQualifiedName()); - long start = System.currentTimeMillis(); - multiDcExecutor.executeByUser(new RemoveTopicRepositoryCommand(topic.getName()), removedBy); - logger.info("Removed topic: {} from ZK clusters in: {} ms", topic.getQualifiedName(), System.currentTimeMillis() - start); - logger.info("Removing topic: {} from Kafka clusters", topic.getQualifiedName()); - start = System.currentTimeMillis(); - multiDCAwareService.manageTopic(brokerTopicManagement -> brokerTopicManagement.removeTopic(topic)); - logger.info("Removed topic: {} from Kafka clusters in: {} ms", topic.getQualifiedName(), System.currentTimeMillis() - start); - auditor.objectRemoved(removedBy.getUsername(), topic); - topicOwnerCache.onRemovedTopic(topic); - } - - private Optional extractSchema(PatchData patch) { - return Optional.ofNullable(patch.getPatch().get("schema")).map(o -> (String) o); - } - - private List loadMessagePreviewsFromAllDc(TopicName topicName) { - List> repositories = - repositoryManager.getRepositories(MessagePreviewRepository.class); - List previews = new ArrayList<>(); - for (DatacenterBoundRepositoryHolder holder : repositories) { - try { - previews.addAll(holder.getRepository().loadPreview(topicName)); - } catch (Exception e) { - logger.warn("Could not load message preview for DC: {}", holder.getDatacenterName()); - } - } - return previews; - } - - private List getTopicsMetrics(List topics) { - return topics - .stream() - .map(t -> { - TopicMetrics metrics = metricRepository.loadMetrics(t.getName()); - return TopicNameWithMetrics.from(metrics, t.getQualifiedName()); - }) - .collect(toList()); - } - - public List listForOwnerId(OwnerId ownerId) { - Collection topicNames = topicOwnerCache.get(ownerId); - return topicRepository.getTopicsDetails(topicNames); - } + private static final Logger logger = LoggerFactory.getLogger(TopicService.class); + + private final TopicRepository topicRepository; + private final GroupService groupService; + private final TopicProperties topicProperties; + private final SchemaService schemaService; + + private final TopicMetricsRepository metricRepository; + private final MultiDCAwareService multiDCAwareService; + private final TopicBlacklistService topicBlacklistService; + private final TopicValidator topicValidator; + private final TopicContentTypeMigrationService topicContentTypeMigrationService; + private final Clock clock; + private final Auditor auditor; + private final MultiDatacenterRepositoryCommandExecutor multiDcExecutor; + private final RepositoryManager repositoryManager; + private final TopicOwnerCache topicOwnerCache; + private final SubscriptionRemover subscriptionRemover; + private final ScheduledExecutorService scheduledTopicExecutor = + Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder().setNameFormat("scheduled-topic-executor-%d").build()); + + @Autowired + public TopicService( + MultiDCAwareService multiDCAwareService, + TopicRepository topicRepository, + GroupService groupService, + TopicProperties topicProperties, + SchemaService schemaService, + TopicMetricsRepository metricRepository, + TopicBlacklistService topicBlacklistService, + TopicValidator topicValidator, + TopicContentTypeMigrationService topicContentTypeMigrationService, + Clock clock, + Auditor auditor, + MultiDatacenterRepositoryCommandExecutor multiDcExecutor, + RepositoryManager repositoryManager, + TopicOwnerCache topicOwnerCache, + SubscriptionRemover subscriptionRemover) { + this.multiDCAwareService = multiDCAwareService; + this.topicRepository = topicRepository; + this.groupService = groupService; + this.topicProperties = topicProperties; + this.schemaService = schemaService; + this.metricRepository = metricRepository; + this.topicBlacklistService = topicBlacklistService; + this.topicValidator = topicValidator; + this.topicContentTypeMigrationService = topicContentTypeMigrationService; + this.clock = clock; + this.auditor = auditor; + this.multiDcExecutor = multiDcExecutor; + this.repositoryManager = repositoryManager; + this.topicOwnerCache = topicOwnerCache; + this.subscriptionRemover = subscriptionRemover; + } + + public void createTopicWithSchema( + TopicWithSchema topicWithSchema, RequestUser createdBy, CreatorRights isAllowedToManage) { + Topic topic = topicWithSchema.getTopic(); + auditor.beforeObjectCreation(createdBy.getUsername(), topic); + groupService.checkGroupExists(topic.getName().getGroupName()); + topicValidator.ensureCreatedTopicIsValid(topic, createdBy, isAllowedToManage); + ensureTopicDoesNotExist(topic); + + boolean validateAndRegisterSchema = + AVRO.equals(topic.getContentType()) + || (topic.isJsonToAvroDryRunEnabled() && topicWithSchema.getSchema() != null); + + validateSchema(validateAndRegisterSchema, topicWithSchema, topic); + registerAvroSchema(validateAndRegisterSchema, topicWithSchema, createdBy); + createTopic(topic, createdBy, isAllowedToManage); + } + + public void removeTopicWithSchema(Topic topic, RequestUser removedBy) { + auditor.beforeObjectRemoval( + removedBy.getUsername(), Topic.class.getSimpleName(), topic.getQualifiedName()); + subscriptionRemover.removeSubscriptionRelatedToTopic(topic, removedBy); + removeSchema(topic); + if (!topicProperties.isAllowRemoval()) { + throw new TopicRemovalDisabledException(topic); + } + if (topicBlacklistService.isBlacklisted(topic.getQualifiedName())) { + topicBlacklistService.unblacklist(topic.getQualifiedName(), removedBy); + } + removeTopic(topic, removedBy); + } + + public void updateTopicWithSchema(TopicName topicName, PatchData patch, RequestUser modifiedBy) { + Topic topic = getTopicDetails(topicName); + extractSchema(patch) + .ifPresent( + schema -> { + schemaService.registerSchema(topic, schema); + scheduleTouchTopic(topicName, modifiedBy); + }); + updateTopic(topicName, patch, modifiedBy); + } + + public void updateTopic(TopicName topicName, PatchData patch, RequestUser modifiedBy) { + auditor.beforeObjectUpdate( + modifiedBy.getUsername(), Topic.class.getSimpleName(), topicName, patch); + groupService.checkGroupExists(topicName.getGroupName()); + + Topic retrieved = getTopicDetails(topicName); + Topic modified = Patch.apply(retrieved, patch); + + topicValidator.ensureUpdatedTopicIsValid(modified, retrieved, modifiedBy); + + if (!retrieved.equals(modified)) { + Instant beforeMigrationInstant = clock.instant(); + // TODO: this does not work as intended, uses == instead of equals + if (retrieved.getRetentionTime() != modified.getRetentionTime()) { + multiDCAwareService.manageTopic( + brokerTopicManagement -> brokerTopicManagement.updateTopic(modified)); + } + multiDcExecutor.executeByUser(new UpdateTopicRepositoryCommand(modified), modifiedBy); + + if (!retrieved.wasMigratedFromJsonType() && modified.wasMigratedFromJsonType()) { + logger.info( + "Waiting until all subscriptions have consumers assigned during topic {} content type migration...", + topicName.qualifiedName()); + topicContentTypeMigrationService.waitUntilAllSubscriptionsHasConsumersAssigned( + modified, + Duration.ofSeconds( + topicProperties.getSubscriptionsAssignmentsCompletedTimeoutSeconds())); + logger.info( + "Notifying subscriptions' consumers about changes in topic {} content type...", + topicName.qualifiedName()); + topicContentTypeMigrationService.notifySubscriptions( + modified, beforeMigrationInstant, modifiedBy); + } + auditor.objectUpdated(modifiedBy.getUsername(), retrieved, modified); + topicOwnerCache.onUpdatedTopic(retrieved, modified); + } + } + + public void touchTopic(TopicName topicName, RequestUser touchedBy) { + logger.info("Touching topic {}", topicName.qualifiedName()); + multiDcExecutor.executeByUser(new TouchTopicRepositoryCommand(topicName), touchedBy); + } + + /** + * Topic is touched so other Hermes instances are notified to read latest topic schema from + * schema-registry. However, schema-registry can be distributed so when schema is written there + * then it can not be available on all nodes immediately. This is the reason why we delay touch of + * topic here, to wait until schema is distributed on schema-registry nodes. + */ + public void scheduleTouchTopic(TopicName topicName, RequestUser touchedBy) { + if (topicProperties.isTouchSchedulerEnabled()) { + logger.info("Scheduling touch of topic {}", topicName.qualifiedName()); + scheduledTopicExecutor.schedule( + () -> touchTopic(topicName, touchedBy), + topicProperties.getTouchDelayInSeconds(), + TimeUnit.SECONDS); + } else { + touchTopic(topicName, touchedBy); + } + } + + public List listQualifiedTopicNames(String groupName) { + return topicRepository.listTopicNames(groupName).stream() + .map(topicName -> new TopicName(groupName, topicName).qualifiedName()) + .collect(toList()); + } + + public List listQualifiedTopicNames() { + return groupService.listGroupNames().stream() + .map(this::listQualifiedTopicNames) + .flatMap(List::stream) + .sorted() + .collect(toList()); + } + + public List listTopics(String groupName) { + return topicRepository.listTopics(groupName); + } + + public Topic getTopicDetails(TopicName topicName) { + return topicRepository.getTopicDetails(topicName); + } + + public TopicWithSchema getTopicWithSchema(TopicName topicName) { + Topic topic = getTopicDetails(topicName); + Optional schema = Optional.empty(); + if (AVRO.equals(topic.getContentType())) { + schema = schemaService.getSchema(topicName.qualifiedName()); + } + return schema + .map(s -> topicWithSchema(topic, s.value())) + .orElseGet(() -> topicWithSchema(topic)); + } + + public TopicMetrics getTopicMetrics(TopicName topicName) { + return topicRepository.topicExists(topicName) + ? metricRepository.loadMetrics(topicName) + : TopicMetrics.unavailable(); + } + + public String fetchSingleMessageFromPrimary( + String brokersClusterName, TopicName topicName, Integer partition, Long offset) { + return multiDCAwareService.readMessageFromPrimary( + brokersClusterName, getTopicDetails(topicName), partition, offset); + } + + public List listTrackedTopicNames() { + return groupService.listGroupNames().stream() + .map(topicRepository::listTopics) + .flatMap(List::stream) + .filter(Topic::isTrackingEnabled) + .map(Topic::getQualifiedName) + .collect(toList()); + } + + public List listTrackedTopicNames(String groupName) { + return listTopics(groupName).stream() + .filter(Topic::isTrackingEnabled) + .map(Topic::getQualifiedName) + .collect(toList()); + } + + public List listFilteredTopicNames(Query query) { + return queryTopic(query).stream().map(Topic::getQualifiedName).collect(toList()); + } + + public List listFilteredTopicNames(String groupName, Query query) { + return query.filter(listTopics(groupName)).map(Topic::getQualifiedName).collect(toList()); + } + + public List queryTopic(Query query) { + return query.filter(getAllTopics()).collect(toList()); + } + + public List getAllTopics() { + return topicRepository.listAllTopics(); + } + + public Optional preview(TopicName topicName, int idx) { + List result = + loadMessagePreviewsFromAllDc(topicName).stream() + .map(MessagePreview::getContent) + .collect(toList()); + + if (idx >= 0 && idx < result.size()) { + return Optional.of(result.get(idx)); + } else { + return Optional.empty(); + } + } + + public List previewText(TopicName topicName) { + return loadMessagePreviewsFromAllDc(topicName).stream() + .map( + p -> + new MessageTextPreview( + new String(p.getContent(), StandardCharsets.UTF_8), p.isTruncated())) + .collect(toList()); + } + + public List queryTopicsMetrics(Query query) { + List filteredNames = query.filterNames(getAllTopics()).collect(toList()); + return query.filter(getTopicsMetrics(filteredNames)).collect(toList()); + } + + public TopicStats getStats() { + List topics = getAllTopics(); + long ackAllTopicCount = topics.stream().filter(t -> t.getAck() == Topic.Ack.ALL).count(); + long trackingEnabledTopicCount = topics.stream().filter(Topic::isTrackingEnabled).count(); + long avroTopicCount = topics.stream().filter(t -> t.getContentType() == AVRO).count(); + return new TopicStats( + topics.size(), ackAllTopicCount, trackingEnabledTopicCount, avroTopicCount); + } + + private void ensureTopicDoesNotExist(Topic topic) { + if (topicRepository.topicExists(topic.getName())) { + throw new TopicAlreadyExistsException(topic.getName()); + } + } + + private void validateSchema( + boolean shouldValidate, TopicWithSchema topicWithSchema, Topic topic) { + if (shouldValidate) { + schemaService.validateSchema(topic, topicWithSchema.getSchema()); + boolean schemaAlreadyRegistered = + schemaService.getSchema(topic.getQualifiedName()).isPresent(); + if (schemaAlreadyRegistered) { + throw new TopicSchemaExistsException(topic.getQualifiedName()); + } + } + } + + private void registerAvroSchema( + boolean shouldRegister, TopicWithSchema topicWithSchema, RequestUser createdBy) { + if (shouldRegister) { + try { + schemaService.registerSchema(topicWithSchema.getTopic(), topicWithSchema.getSchema()); + } catch (Exception e) { + logger.error( + "Rolling back topic {} creation due to schema registration error", + topicWithSchema.getQualifiedName(), + e); + removeTopic(topicWithSchema.getTopic(), createdBy); + throw e; + } + } + } + + private void createTopic(Topic topic, RequestUser createdBy, CreatorRights creatorRights) { + topicValidator.ensureCreatedTopicIsValid(topic, createdBy, creatorRights); + + if (!multiDCAwareService.topicExists(topic)) { + createTopicInBrokers(topic, createdBy); + auditor.objectCreated(createdBy.getUsername(), topic); + topicOwnerCache.onCreatedTopic(topic); + } else { + logger.info( + "Skipping creation of topic {} on brokers, topic already exists", + topic.getQualifiedName()); + } + + multiDcExecutor.executeByUser(new CreateTopicRepositoryCommand(topic), createdBy); + } + + private void createTopicInBrokers(Topic topic, RequestUser createdBy) { + try { + multiDCAwareService.manageTopic( + brokerTopicManagement -> brokerTopicManagement.createTopic(topic)); + } catch (Exception exception) { + logger.error( + String.format( + "Could not create topic %s, rollback topic creation.", topic.getQualifiedName()), + exception); + multiDcExecutor.executeByUser(new RemoveTopicRepositoryCommand(topic.getName()), createdBy); + } + } + + private void removeSchema(Topic topic) { + if (AVRO.equals(topic.getContentType()) && topicProperties.isRemoveSchema()) { + schemaService + .getSchema(topic.getQualifiedName()) + .ifPresent(s -> schemaService.deleteAllSchemaVersions(topic.getQualifiedName())); + } + } + + private void removeTopic(Topic topic, RequestUser removedBy) { + logger.info("Removing topic: {} from ZK clusters", topic.getQualifiedName()); + long start = System.currentTimeMillis(); + multiDcExecutor.executeByUser(new RemoveTopicRepositoryCommand(topic.getName()), removedBy); + logger.info( + "Removed topic: {} from ZK clusters in: {} ms", + topic.getQualifiedName(), + System.currentTimeMillis() - start); + logger.info("Removing topic: {} from Kafka clusters", topic.getQualifiedName()); + start = System.currentTimeMillis(); + multiDCAwareService.manageTopic( + brokerTopicManagement -> brokerTopicManagement.removeTopic(topic)); + logger.info( + "Removed topic: {} from Kafka clusters in: {} ms", + topic.getQualifiedName(), + System.currentTimeMillis() - start); + auditor.objectRemoved(removedBy.getUsername(), topic); + topicOwnerCache.onRemovedTopic(topic); + } + + private Optional extractSchema(PatchData patch) { + return Optional.ofNullable(patch.getPatch().get("schema")).map(o -> (String) o); + } + + private List loadMessagePreviewsFromAllDc(TopicName topicName) { + List> repositories = + repositoryManager.getRepositories(MessagePreviewRepository.class); + List previews = new ArrayList<>(); + for (DatacenterBoundRepositoryHolder holder : repositories) { + try { + previews.addAll(holder.getRepository().loadPreview(topicName)); + } catch (Exception e) { + logger.warn("Could not load message preview for DC: {}", holder.getDatacenterName()); + } + } + return previews; + } + + private List getTopicsMetrics(List topics) { + return topics.stream() + .map( + t -> { + TopicMetrics metrics = metricRepository.loadMetrics(t.getName()); + return TopicNameWithMetrics.from(metrics, t.getQualifiedName()); + }) + .collect(toList()); + } + + public List listForOwnerId(OwnerId ownerId) { + Collection topicNames = topicOwnerCache.get(ownerId); + return topicRepository.getTopicsDetails(topicNames); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/UnableToMoveOffsetsException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/UnableToMoveOffsetsException.java index f7112861a2..ad75546904 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/UnableToMoveOffsetsException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/UnableToMoveOffsetsException.java @@ -6,12 +6,17 @@ public class UnableToMoveOffsetsException extends ManagementException { - public UnableToMoveOffsetsException(Topic topic, String subscriptionName) { - super("Not all offsets related to hermes subscription" + topic.getQualifiedName() + "$" + subscriptionName + " were moved."); - } + public UnableToMoveOffsetsException(Topic topic, String subscriptionName) { + super( + "Not all offsets related to hermes subscription" + + topic.getQualifiedName() + + "$" + + subscriptionName + + " were moved."); + } - @Override - public ErrorCode getCode() { - return ErrorCode.UNABLE_TO_MOVE_OFFSETS_EXCEPTION; - } + @Override + public ErrorCode getCode() { + return ErrorCode.UNABLE_TO_MOVE_OFFSETS_EXCEPTION; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/commands/CreateTopicRepositoryCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/commands/CreateTopicRepositoryCommand.java index 2842b7e988..0ad8397046 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/commands/CreateTopicRepositoryCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/commands/CreateTopicRepositoryCommand.java @@ -7,36 +7,37 @@ public class CreateTopicRepositoryCommand extends RepositoryCommand { - private final Topic topic; - - public CreateTopicRepositoryCommand(Topic topic) { - this.topic = topic; - } - - @Override - public void backup(DatacenterBoundRepositoryHolder holder) {} - - @Override - public void execute(DatacenterBoundRepositoryHolder holder) { - holder.getRepository().createTopic(topic); - } - - @Override - public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) { - /* - We don't want to do a rollback due to possible race conditions with creating a topic on Kafka. - It increases the probability of discrepancies between Kafka and Zookeeper: topic exists in Kafka, - but not in the Zookeeper and vice versa. - */ - } - - @Override - public Class getRepositoryType() { - return TopicRepository.class; - } - - @Override - public String toString() { - return "CreateTopic(" + topic.getQualifiedName() + ")"; - } + private final Topic topic; + + public CreateTopicRepositoryCommand(Topic topic) { + this.topic = topic; + } + + @Override + public void backup(DatacenterBoundRepositoryHolder holder) {} + + @Override + public void execute(DatacenterBoundRepositoryHolder holder) { + holder.getRepository().createTopic(topic); + } + + @Override + public void rollback( + DatacenterBoundRepositoryHolder holder, Exception exception) { + /* + We don't want to do a rollback due to possible race conditions with creating a topic on Kafka. + It increases the probability of discrepancies between Kafka and Zookeeper: topic exists in Kafka, + but not in the Zookeeper and vice versa. + */ + } + + @Override + public Class getRepositoryType() { + return TopicRepository.class; + } + + @Override + public String toString() { + return "CreateTopic(" + topic.getQualifiedName() + ")"; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/commands/RemoveTopicRepositoryCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/commands/RemoveTopicRepositoryCommand.java index 397c60c295..cc182285ac 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/commands/RemoveTopicRepositoryCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/commands/RemoveTopicRepositoryCommand.java @@ -7,36 +7,37 @@ public class RemoveTopicRepositoryCommand extends RepositoryCommand { - private final TopicName topicName; - - public RemoveTopicRepositoryCommand(TopicName topicName) { - this.topicName = topicName; - } - - @Override - public void backup(DatacenterBoundRepositoryHolder holder) {} - - @Override - public void execute(DatacenterBoundRepositoryHolder holder) { - holder.getRepository().removeTopic(topicName); - } - - @Override - public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) { - /* - We don't want to do a rollback due to possible race conditions with creating a topic on Kafka. - It increases the probability of discrepancies between Kafka and Zookeeper: topic exists in Kafka, - but not in the Zookeeper and vice versa. - */ - } - - @Override - public Class getRepositoryType() { - return TopicRepository.class; - } - - @Override - public String toString() { - return "RemoveTopic(" + topicName + ")"; - } + private final TopicName topicName; + + public RemoveTopicRepositoryCommand(TopicName topicName) { + this.topicName = topicName; + } + + @Override + public void backup(DatacenterBoundRepositoryHolder holder) {} + + @Override + public void execute(DatacenterBoundRepositoryHolder holder) { + holder.getRepository().removeTopic(topicName); + } + + @Override + public void rollback( + DatacenterBoundRepositoryHolder holder, Exception exception) { + /* + We don't want to do a rollback due to possible race conditions with creating a topic on Kafka. + It increases the probability of discrepancies between Kafka and Zookeeper: topic exists in Kafka, + but not in the Zookeeper and vice versa. + */ + } + + @Override + public Class getRepositoryType() { + return TopicRepository.class; + } + + @Override + public String toString() { + return "RemoveTopic(" + topicName + ")"; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/commands/TouchTopicRepositoryCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/commands/TouchTopicRepositoryCommand.java index d4a850f681..f40e7e1c0d 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/commands/TouchTopicRepositoryCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/commands/TouchTopicRepositoryCommand.java @@ -7,30 +7,31 @@ public class TouchTopicRepositoryCommand extends RepositoryCommand { - private final TopicName topicName; + private final TopicName topicName; - public TouchTopicRepositoryCommand(TopicName topicName) { - this.topicName = topicName; - } + public TouchTopicRepositoryCommand(TopicName topicName) { + this.topicName = topicName; + } - @Override - public void backup(DatacenterBoundRepositoryHolder holder) {} + @Override + public void backup(DatacenterBoundRepositoryHolder holder) {} - @Override - public void execute(DatacenterBoundRepositoryHolder holder) { - holder.getRepository().touchTopic(topicName); - } + @Override + public void execute(DatacenterBoundRepositoryHolder holder) { + holder.getRepository().touchTopic(topicName); + } - @Override - public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) {} + @Override + public void rollback( + DatacenterBoundRepositoryHolder holder, Exception exception) {} - @Override - public Class getRepositoryType() { - return TopicRepository.class; - } + @Override + public Class getRepositoryType() { + return TopicRepository.class; + } - @Override - public String toString() { - return "TouchTopic(" + topicName + ")"; - } + @Override + public String toString() { + return "TouchTopic(" + topicName + ")"; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/commands/UpdateTopicRepositoryCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/commands/UpdateTopicRepositoryCommand.java index 0f007f5fe9..5ec65be6e5 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/commands/UpdateTopicRepositoryCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/commands/UpdateTopicRepositoryCommand.java @@ -7,36 +7,37 @@ public class UpdateTopicRepositoryCommand extends RepositoryCommand { - private final Topic topic; - - private Topic backup; - - public UpdateTopicRepositoryCommand(Topic topic) { - this.topic = topic; - } - - @Override - public void backup(DatacenterBoundRepositoryHolder holder) { - backup = holder.getRepository().getTopicDetails(topic.getName()); - } - - @Override - public void execute(DatacenterBoundRepositoryHolder holder) { - holder.getRepository().updateTopic(topic); - } - - @Override - public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) { - holder.getRepository().updateTopic(backup); - } - - @Override - public Class getRepositoryType() { - return TopicRepository.class; - } - - @Override - public String toString() { - return "UpdateTopic(" + topic.getQualifiedName() + ")"; - } + private final Topic topic; + + private Topic backup; + + public UpdateTopicRepositoryCommand(Topic topic) { + this.topic = topic; + } + + @Override + public void backup(DatacenterBoundRepositoryHolder holder) { + backup = holder.getRepository().getTopicDetails(topic.getName()); + } + + @Override + public void execute(DatacenterBoundRepositoryHolder holder) { + holder.getRepository().updateTopic(topic); + } + + @Override + public void rollback( + DatacenterBoundRepositoryHolder holder, Exception exception) { + holder.getRepository().updateTopic(backup); + } + + @Override + public Class getRepositoryType() { + return TopicRepository.class; + } + + @Override + public String toString() { + return "UpdateTopic(" + topic.getQualifiedName() + ")"; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/SchemaRemovalDisabledException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/SchemaRemovalDisabledException.java index e6f1b8aaff..95b9b1c22d 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/SchemaRemovalDisabledException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/SchemaRemovalDisabledException.java @@ -5,12 +5,12 @@ public class SchemaRemovalDisabledException extends HermesException { - public SchemaRemovalDisabledException() { - super("Removing schema is disabled."); - } + public SchemaRemovalDisabledException() { + super("Removing schema is disabled."); + } - @Override - public ErrorCode getCode() { - return ErrorCode.OPERATION_DISABLED; - } + @Override + public ErrorCode getCode() { + return ErrorCode.OPERATION_DISABLED; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/SchemaService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/SchemaService.java index c0d3bcba2c..4d17a8ef20 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/SchemaService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/schema/SchemaService.java @@ -1,5 +1,9 @@ package pl.allegro.tech.hermes.management.domain.topic.schema; +import static pl.allegro.tech.hermes.api.ContentType.AVRO; +import static pl.allegro.tech.hermes.api.TopicName.fromQualifiedName; + +import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -14,75 +18,74 @@ import pl.allegro.tech.hermes.schema.SchemaId; import pl.allegro.tech.hermes.schema.SchemaVersion; -import java.util.Optional; - -import static pl.allegro.tech.hermes.api.ContentType.AVRO; -import static pl.allegro.tech.hermes.api.TopicName.fromQualifiedName; - @Component public class SchemaService { - private final RawSchemaClient rawSchemaClient; - private final SchemaValidatorProvider validatorProvider; - private final TopicProperties topicProperties; + private final RawSchemaClient rawSchemaClient; + private final SchemaValidatorProvider validatorProvider; + private final TopicProperties topicProperties; - private static final Logger logger = LoggerFactory.getLogger(SchemaService.class); + private static final Logger logger = LoggerFactory.getLogger(SchemaService.class); - @Autowired - public SchemaService(RawSchemaClient rawSchemaClient, - SchemaValidatorProvider validatorProvider, - TopicProperties topicProperties) { - this.rawSchemaClient = rawSchemaClient; - this.validatorProvider = validatorProvider; - this.topicProperties = topicProperties; - } + @Autowired + public SchemaService( + RawSchemaClient rawSchemaClient, + SchemaValidatorProvider validatorProvider, + TopicProperties topicProperties) { + this.rawSchemaClient = rawSchemaClient; + this.validatorProvider = validatorProvider; + this.topicProperties = topicProperties; + } - public Optional getSchema(String qualifiedTopicName) { - return rawSchemaClient - .getLatestRawSchemaWithMetadata(fromQualifiedName(qualifiedTopicName)) - .map(RawSchemaWithMetadata::getSchema); - } + public Optional getSchema(String qualifiedTopicName) { + return rawSchemaClient + .getLatestRawSchemaWithMetadata(fromQualifiedName(qualifiedTopicName)) + .map(RawSchemaWithMetadata::getSchema); + } - public Optional getSchema(String qualifiedTopicName, SchemaVersion version) { - return rawSchemaClient - .getRawSchemaWithMetadata(fromQualifiedName(qualifiedTopicName), version) - .map(RawSchemaWithMetadata::getSchema); - } + public Optional getSchema(String qualifiedTopicName, SchemaVersion version) { + return rawSchemaClient + .getRawSchemaWithMetadata(fromQualifiedName(qualifiedTopicName), version) + .map(RawSchemaWithMetadata::getSchema); + } - public Optional getSchema(String qualifiedTopicName, SchemaId id) { - return rawSchemaClient - .getRawSchemaWithMetadata(fromQualifiedName(qualifiedTopicName), id) - .map(RawSchemaWithMetadata::getSchema); - } + public Optional getSchema(String qualifiedTopicName, SchemaId id) { + return rawSchemaClient + .getRawSchemaWithMetadata(fromQualifiedName(qualifiedTopicName), id) + .map(RawSchemaWithMetadata::getSchema); + } - public void registerSchema(Topic topic, String schema) { - boolean validate = AVRO.equals(topic.getContentType()); - registerSchema(topic, schema, validate); - } + public void registerSchema(Topic topic, String schema) { + boolean validate = AVRO.equals(topic.getContentType()); + registerSchema(topic, schema, validate); + } - public void registerSchema(Topic topic, String schema, boolean validate) { - if (validate) { - SchemaValidator validator = validatorProvider.provide(topic.getContentType()); - validator.check(schema); - } - rawSchemaClient.registerSchema(topic.getName(), RawSchema.valueOf(schema)); + public void registerSchema(Topic topic, String schema, boolean validate) { + if (validate) { + SchemaValidator validator = validatorProvider.provide(topic.getContentType()); + validator.check(schema); } + rawSchemaClient.registerSchema(topic.getName(), RawSchema.valueOf(schema)); + } - public void deleteAllSchemaVersions(String qualifiedTopicName) { - if (!topicProperties.isRemoveSchema()) { - throw new SchemaRemovalDisabledException(); - } - logger.info("Removing all schema versions for topic: {}", qualifiedTopicName); - long start = System.currentTimeMillis(); - rawSchemaClient.deleteAllSchemaVersions(fromQualifiedName(qualifiedTopicName)); - logger.info("Removed all schema versions for topic: {} in {} ms", qualifiedTopicName, System.currentTimeMillis() - start); + public void deleteAllSchemaVersions(String qualifiedTopicName) { + if (!topicProperties.isRemoveSchema()) { + throw new SchemaRemovalDisabledException(); } + logger.info("Removing all schema versions for topic: {}", qualifiedTopicName); + long start = System.currentTimeMillis(); + rawSchemaClient.deleteAllSchemaVersions(fromQualifiedName(qualifiedTopicName)); + logger.info( + "Removed all schema versions for topic: {} in {} ms", + qualifiedTopicName, + System.currentTimeMillis() - start); + } - public void validateSchema(Topic topic, String schema) { - if (AVRO.equals(topic.getContentType())) { - SchemaValidator validator = validatorProvider.provide(AVRO); - validator.check(schema); - } - rawSchemaClient.validateSchema(topic.getName(), RawSchema.valueOf(schema)); + public void validateSchema(Topic topic, String schema) { + if (AVRO.equals(topic.getContentType())) { + SchemaValidator validator = validatorProvider.provide(AVRO); + validator.check(schema); } + rawSchemaClient.validateSchema(topic.getName(), RawSchema.valueOf(schema)); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/validator/ContentTypeValidator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/validator/ContentTypeValidator.java index 5471281a0a..1f64b8a7f6 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/validator/ContentTypeValidator.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/validator/ContentTypeValidator.java @@ -1,26 +1,27 @@ package pl.allegro.tech.hermes.management.domain.topic.validator; +import java.util.EnumSet; +import java.util.Set; import org.springframework.stereotype.Component; import pl.allegro.tech.hermes.api.ContentType; import pl.allegro.tech.hermes.management.config.TopicProperties; -import java.util.EnumSet; -import java.util.Set; - @Component public class ContentTypeValidator { - private static final String ERROR_MESSAGE = "Content type %s is not within allowed content types %s"; + private static final String ERROR_MESSAGE = + "Content type %s is not within allowed content types %s"; - private final Set allowedContentTypes; + private final Set allowedContentTypes; - public ContentTypeValidator(TopicProperties topicProperties) { - this.allowedContentTypes = EnumSet.copyOf(topicProperties.getAllowedContentTypes()); - } + public ContentTypeValidator(TopicProperties topicProperties) { + this.allowedContentTypes = EnumSet.copyOf(topicProperties.getAllowedContentTypes()); + } - public void check(ContentType contentType) { - if (!allowedContentTypes.contains(contentType)) { - throw new TopicValidationException(String.format(ERROR_MESSAGE, contentType, allowedContentTypes)); - } + public void check(ContentType contentType) { + if (!allowedContentTypes.contains(contentType)) { + throw new TopicValidationException( + String.format(ERROR_MESSAGE, contentType, allowedContentTypes)); } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/validator/TopicLabelsValidator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/validator/TopicLabelsValidator.java index 57a13604f4..664ba9cdd6 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/validator/TopicLabelsValidator.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/validator/TopicLabelsValidator.java @@ -1,25 +1,25 @@ package pl.allegro.tech.hermes.management.domain.topic.validator; +import java.util.Set; import org.springframework.stereotype.Component; import pl.allegro.tech.hermes.api.TopicLabel; import pl.allegro.tech.hermes.management.config.TopicProperties; -import java.util.Set; - @Component public class TopicLabelsValidator { - private static final String ERROR_MESSAGE = "One of topic labels %s is not within allowed topic labels %s"; + private static final String ERROR_MESSAGE = + "One of topic labels %s is not within allowed topic labels %s"; - private final Set allowedTopicLabels; + private final Set allowedTopicLabels; - public TopicLabelsValidator(TopicProperties topicProperties) { - this.allowedTopicLabels = topicProperties.getAllowedTopicLabels(); - } + public TopicLabelsValidator(TopicProperties topicProperties) { + this.allowedTopicLabels = topicProperties.getAllowedTopicLabels(); + } - public void check(Set labels) { - if (!allowedTopicLabels.containsAll(labels)) { - throw new TopicValidationException(String.format(ERROR_MESSAGE, labels, allowedTopicLabels)); - } + public void check(Set labels) { + if (!allowedTopicLabels.containsAll(labels)) { + throw new TopicValidationException(String.format(ERROR_MESSAGE, labels, allowedTopicLabels)); } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/validator/TopicValidationException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/validator/TopicValidationException.java index 4ab80fad3d..41a6b20ae1 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/validator/TopicValidationException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/validator/TopicValidationException.java @@ -5,16 +5,16 @@ public class TopicValidationException extends ManagementException { - public TopicValidationException(String message) { - super(message); - } + public TopicValidationException(String message) { + super(message); + } - public TopicValidationException(String message, Exception cause) { - super(message, cause); - } + public TopicValidationException(String message, Exception cause) { + super(message, cause); + } - @Override - public ErrorCode getCode() { - return ErrorCode.VALIDATION_ERROR; - } + @Override + public ErrorCode getCode() { + return ErrorCode.VALIDATION_ERROR; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/validator/TopicValidator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/validator/TopicValidator.java index fd1e22d8da..1c320c4b0f 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/validator/TopicValidator.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/topic/validator/TopicValidator.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.management.domain.topic.validator; +import java.util.Arrays; +import java.util.concurrent.TimeUnit; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import pl.allegro.tech.hermes.api.ContentType; @@ -17,175 +19,205 @@ import pl.allegro.tech.hermes.schema.SchemaNotFoundException; import pl.allegro.tech.hermes.schema.SchemaRepository; -import java.util.Arrays; -import java.util.concurrent.TimeUnit; - @Component public class TopicValidator { - private final OwnerIdValidator ownerIdValidator; - private final ContentTypeValidator contentTypeValidator; - private final TopicLabelsValidator topicLabelsValidator; - private final SchemaRepository schemaRepository; - private final ApiPreconditions apiPreconditions; - private final TopicProperties topicProperties; - - @Autowired - public TopicValidator(OwnerIdValidator ownerIdValidator, - ContentTypeValidator contentTypeValidator, - TopicLabelsValidator topicLabelsValidator, - SchemaRepository schemaRepository, - ApiPreconditions apiPreconditions, - TopicProperties topicProperties) { - this.ownerIdValidator = ownerIdValidator; - this.contentTypeValidator = contentTypeValidator; - this.topicLabelsValidator = topicLabelsValidator; - this.schemaRepository = schemaRepository; - this.apiPreconditions = apiPreconditions; - this.topicProperties = topicProperties; + private final OwnerIdValidator ownerIdValidator; + private final ContentTypeValidator contentTypeValidator; + private final TopicLabelsValidator topicLabelsValidator; + private final SchemaRepository schemaRepository; + private final ApiPreconditions apiPreconditions; + private final TopicProperties topicProperties; + + @Autowired + public TopicValidator( + OwnerIdValidator ownerIdValidator, + ContentTypeValidator contentTypeValidator, + TopicLabelsValidator topicLabelsValidator, + SchemaRepository schemaRepository, + ApiPreconditions apiPreconditions, + TopicProperties topicProperties) { + this.ownerIdValidator = ownerIdValidator; + this.contentTypeValidator = contentTypeValidator; + this.topicLabelsValidator = topicLabelsValidator; + this.schemaRepository = schemaRepository; + this.apiPreconditions = apiPreconditions; + this.topicProperties = topicProperties; + } + + public void ensureCreatedTopicIsValid( + Topic created, RequestUser createdBy, CreatorRights creatorRights) { + apiPreconditions.checkConstraints(created, createdBy.isAdmin()); + checkOwner(created); + checkContentType(created); + checkTopicLabels(created); + + if ((created.isFallbackToRemoteDatacenterEnabled() + != topicProperties.isDefaultFallbackToRemoteDatacenterEnabled()) + && !createdBy.isAdmin()) { + throw new TopicValidationException( + "User is not allowed to set non-default fallback to remote datacenter for this topic"); } - public void ensureCreatedTopicIsValid(Topic created, RequestUser createdBy, CreatorRights creatorRights) { - apiPreconditions.checkConstraints(created, createdBy.isAdmin()); - checkOwner(created); - checkContentType(created); - checkTopicLabels(created); - - if ((created.isFallbackToRemoteDatacenterEnabled() != topicProperties.isDefaultFallbackToRemoteDatacenterEnabled()) && !createdBy.isAdmin()) { - throw new TopicValidationException("User is not allowed to set non-default fallback to remote datacenter for this topic"); - } - - if (created.getChaos().mode() != ChaosMode.DISABLED && !createdBy.isAdmin()) { - throw new TopicValidationException("User is not allowed to set chaos policy for this topic"); - } - validateChaosPolicy(created.getChaos()); - - if (created.wasMigratedFromJsonType()) { - throw new TopicValidationException("Newly created topic cannot have migratedFromJsonType flag set to true"); - } + if (created.getChaos().mode() != ChaosMode.DISABLED && !createdBy.isAdmin()) { + throw new TopicValidationException("User is not allowed to set chaos policy for this topic"); + } + validateChaosPolicy(created.getChaos()); - if (!creatorRights.allowedToManage(created)) { - throw new TopicValidationException("Provide an owner that includes you, you would not be able to manage this topic later"); - } + if (created.wasMigratedFromJsonType()) { + throw new TopicValidationException( + "Newly created topic cannot have migratedFromJsonType flag set to true"); + } - ensureCreatedTopicRetentionTimeValid(created, createdBy); + if (!creatorRights.allowedToManage(created)) { + throw new TopicValidationException( + "Provide an owner that includes you, you would not be able to manage this topic later"); } - public void ensureUpdatedTopicIsValid(Topic updated, Topic previous, RequestUser modifiedBy) { - apiPreconditions.checkConstraints(updated, modifiedBy.isAdmin()); - checkOwner(updated); - checkTopicLabels(updated); + ensureCreatedTopicRetentionTimeValid(created, createdBy); + } - if ((previous.isFallbackToRemoteDatacenterEnabled() != updated.isFallbackToRemoteDatacenterEnabled()) && !modifiedBy.isAdmin()) { - throw new TopicValidationException("User is not allowed to update fallback to remote datacenter for this topic"); - } + public void ensureUpdatedTopicIsValid(Topic updated, Topic previous, RequestUser modifiedBy) { + apiPreconditions.checkConstraints(updated, modifiedBy.isAdmin()); + checkOwner(updated); + checkTopicLabels(updated); - if (!previous.getChaos().equals(updated.getChaos()) && !modifiedBy.isAdmin()) { - throw new TopicValidationException("User is not allowed to update chaos policy for this topic"); - } - validateChaosPolicy(updated.getChaos()); + if ((previous.isFallbackToRemoteDatacenterEnabled() + != updated.isFallbackToRemoteDatacenterEnabled()) + && !modifiedBy.isAdmin()) { + throw new TopicValidationException( + "User is not allowed to update fallback to remote datacenter for this topic"); + } - if (migrationFromJsonTypeFlagChangedToTrue(updated, previous)) { - if (updated.getContentType() != ContentType.AVRO) { - throw new TopicValidationException("Change content type to AVRO together with setting migratedFromJsonType flag"); - } + if (!previous.getChaos().equals(updated.getChaos()) && !modifiedBy.isAdmin()) { + throw new TopicValidationException( + "User is not allowed to update chaos policy for this topic"); + } + validateChaosPolicy(updated.getChaos()); + + if (migrationFromJsonTypeFlagChangedToTrue(updated, previous)) { + if (updated.getContentType() != ContentType.AVRO) { + throw new TopicValidationException( + "Change content type to AVRO together with setting migratedFromJsonType flag"); + } + + try { + schemaRepository.getLatestAvroSchema(updated); + } catch (CouldNotLoadSchemaException | SchemaNotFoundException e) { + throw new TopicValidationException("Avro schema not available, migration not permitted", e); + } + } else if (contentTypeChanged(updated, previous)) { + throw new TopicValidationException( + "Cannot change content type, except for migration to Avro with setting migratedFromJsonType flag."); + } else if (migrationFromJsonTypeFlagChangedToFalse(updated, previous)) { + throw new TopicValidationException("Cannot migrate back to JSON!"); + } - try { - schemaRepository.getLatestAvroSchema(updated); - } catch (CouldNotLoadSchemaException | SchemaNotFoundException e) { - throw new TopicValidationException("Avro schema not available, migration not permitted", e); - } - } else if (contentTypeChanged(updated, previous)) { - throw new TopicValidationException( - "Cannot change content type, except for migration to Avro with setting migratedFromJsonType flag."); - } else if (migrationFromJsonTypeFlagChangedToFalse(updated, previous)) { - throw new TopicValidationException("Cannot migrate back to JSON!"); - } + ensureUpdatedTopicRetentionTimeValid(updated, previous, modifiedBy); + } - ensureUpdatedTopicRetentionTimeValid(updated, previous, modifiedBy); + private void ensureCreatedTopicRetentionTimeValid(Topic created, RequestUser modifiedBy) { + if (modifiedBy.isAdmin()) { + return; } - private void ensureCreatedTopicRetentionTimeValid(Topic created, RequestUser modifiedBy) { - if (modifiedBy.isAdmin()) { - return; - } + checkTopicRetentionTimeUnit(created.getRetentionTime().getRetentionUnit()); - checkTopicRetentionTimeUnit(created.getRetentionTime().getRetentionUnit()); + long seconds = + created + .getRetentionTime() + .getRetentionUnit() + .toSeconds(created.getRetentionTime().getDuration()); - long seconds = created.getRetentionTime().getRetentionUnit().toSeconds(created.getRetentionTime().getDuration()); + checkTopicRetentionLimit(seconds); + } - checkTopicRetentionLimit(seconds); + private void ensureUpdatedTopicRetentionTimeValid( + Topic updated, Topic previous, RequestUser modifiedBy) { + if (modifiedBy.isAdmin()) { + return; } - private void ensureUpdatedTopicRetentionTimeValid(Topic updated, Topic previous, RequestUser modifiedBy) { - if (modifiedBy.isAdmin()) { - return; - } - - checkTopicRetentionTimeUnit(updated.getRetentionTime().getRetentionUnit()); - - long updatedSeconds = updated.getRetentionTime().getRetentionUnit().toSeconds(updated.getRetentionTime().getDuration()); - long previousSeconds = previous.getRetentionTime().getRetentionUnit().toSeconds(previous.getRetentionTime().getDuration()); + checkTopicRetentionTimeUnit(updated.getRetentionTime().getRetentionUnit()); + + long updatedSeconds = + updated + .getRetentionTime() + .getRetentionUnit() + .toSeconds(updated.getRetentionTime().getDuration()); + long previousSeconds = + previous + .getRetentionTime() + .getRetentionUnit() + .toSeconds(previous.getRetentionTime().getDuration()); + + if (updatedSeconds == previousSeconds) { + return; + } - if (updatedSeconds == previousSeconds) { - return; - } + checkTopicRetentionLimit(updatedSeconds); + } - checkTopicRetentionLimit(updatedSeconds); + private void checkTopicRetentionTimeUnit(TimeUnit toCheck) { + if (!RetentionTime.allowedUnits.contains(toCheck)) { + throw new TopicValidationException( + "Retention time unit must be one of: " + + Arrays.toString(RetentionTime.allowedUnits.toArray())); } + } - private void checkTopicRetentionTimeUnit(TimeUnit toCheck) { - if (!RetentionTime.allowedUnits.contains(toCheck)) { - throw new TopicValidationException("Retention time unit must be one of: " + Arrays.toString(RetentionTime.allowedUnits.toArray())); - } + private void checkTopicRetentionLimit(long retentionSeconds) { + if (retentionSeconds + > RetentionTime.MAX.getRetentionUnit().toSeconds(RetentionTime.MAX.getDuration())) { + throw new TopicValidationException( + "Retention time larger than 7 days can't be configured by non admin users"); } + } - private void checkTopicRetentionLimit(long retentionSeconds) { - if (retentionSeconds > RetentionTime.MAX.getRetentionUnit().toSeconds(RetentionTime.MAX.getDuration())) { - throw new TopicValidationException("Retention time larger than 7 days can't be configured by non admin users"); - } - } + private boolean contentTypeChanged(Topic updated, Topic previous) { + return previous.getContentType() != updated.getContentType(); + } - private boolean contentTypeChanged(Topic updated, Topic previous) { - return previous.getContentType() != updated.getContentType(); - } + private boolean migrationFromJsonTypeFlagChangedToTrue(Topic updated, Topic previous) { + return !previous.wasMigratedFromJsonType() && updated.wasMigratedFromJsonType(); + } - private boolean migrationFromJsonTypeFlagChangedToTrue(Topic updated, Topic previous) { - return !previous.wasMigratedFromJsonType() && updated.wasMigratedFromJsonType(); - } + private boolean migrationFromJsonTypeFlagChangedToFalse(Topic updated, Topic previous) { + return previous.wasMigratedFromJsonType() && !updated.wasMigratedFromJsonType(); + } - private boolean migrationFromJsonTypeFlagChangedToFalse(Topic updated, Topic previous) { - return previous.wasMigratedFromJsonType() && !updated.wasMigratedFromJsonType(); - } + private void checkOwner(Topic checked) { + ownerIdValidator.check(checked.getOwner()); + } - private void checkOwner(Topic checked) { - ownerIdValidator.check(checked.getOwner()); - } + private void checkContentType(Topic checked) { + contentTypeValidator.check(checked.getContentType()); + } - private void checkContentType(Topic checked) { - contentTypeValidator.check(checked.getContentType()); - } + private void checkTopicLabels(Topic checked) { + topicLabelsValidator.check(checked.getLabels()); + } - private void checkTopicLabels(Topic checked) { - topicLabelsValidator.check(checked.getLabels()); + private void validateChaosPolicy(PublishingChaosPolicy chaosPolicy) { + for (ChaosPolicy policy : chaosPolicy.datacenterPolicies().values()) { + validate(policy); } + validate(chaosPolicy.globalPolicy()); + } - private void validateChaosPolicy(PublishingChaosPolicy chaosPolicy) { - for (ChaosPolicy policy : chaosPolicy.datacenterPolicies().values()) { - validate(policy); - } - validate(chaosPolicy.globalPolicy()); + private void validate(ChaosPolicy chaosPolicy) { + if (chaosPolicy == null) { + return; } - - private void validate(ChaosPolicy chaosPolicy) { - if (chaosPolicy == null) { - return; - } - if (chaosPolicy.delayFrom() < 0 || chaosPolicy.delayTo() < 0 || chaosPolicy.delayFrom() > chaosPolicy.delayTo()) { - throw new TopicValidationException("Invalid chaos policy: 'delayFrom' and 'delayTo' must be >= 0, and 'delayFrom' <= 'delayTo'."); - } - if (chaosPolicy.probability() < 0 || chaosPolicy.probability() > 100) { - throw new TopicValidationException("Invalid chaos policy: 'probability' must be within the range 0 to 100"); - } + if (chaosPolicy.delayFrom() < 0 + || chaosPolicy.delayTo() < 0 + || chaosPolicy.delayFrom() > chaosPolicy.delayTo()) { + throw new TopicValidationException( + "Invalid chaos policy: 'delayFrom' and 'delayTo' must be >= 0, and 'delayFrom' <= 'delayTo'."); + } + if (chaosPolicy.probability() < 0 || chaosPolicy.probability() > 100) { + throw new TopicValidationException( + "Invalid chaos policy: 'probability' must be within the range 0 to 100"); } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/WorkloadConstraintsService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/WorkloadConstraintsService.java index aeafeb0aac..0356af3b3b 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/WorkloadConstraintsService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/WorkloadConstraintsService.java @@ -18,48 +18,61 @@ @Service public class WorkloadConstraintsService { - private final WorkloadConstraintsRepository workloadConstraintsRepository; - private final MultiDatacenterRepositoryCommandExecutor commandExecutor; + private final WorkloadConstraintsRepository workloadConstraintsRepository; + private final MultiDatacenterRepositoryCommandExecutor commandExecutor; - public WorkloadConstraintsService(WorkloadConstraintsRepository workloadConstraintsRepository, - MultiDatacenterRepositoryCommandExecutor commandExecutor) { - this.workloadConstraintsRepository = workloadConstraintsRepository; - this.commandExecutor = commandExecutor; - } + public WorkloadConstraintsService( + WorkloadConstraintsRepository workloadConstraintsRepository, + MultiDatacenterRepositoryCommandExecutor commandExecutor) { + this.workloadConstraintsRepository = workloadConstraintsRepository; + this.commandExecutor = commandExecutor; + } - public ConsumersWorkloadConstraints getConsumersWorkloadConstraints() { - return workloadConstraintsRepository.getConsumersWorkloadConstraints(); - } + public ConsumersWorkloadConstraints getConsumersWorkloadConstraints() { + return workloadConstraintsRepository.getConsumersWorkloadConstraints(); + } - public void createConstraints(TopicName topicName, Constraints constraints, RequestUser requester) { - commandExecutor.executeByUser(new CreateTopicConstraintsRepositoryCommand(topicName, constraints), requester); - } + public void createConstraints( + TopicName topicName, Constraints constraints, RequestUser requester) { + commandExecutor.executeByUser( + new CreateTopicConstraintsRepositoryCommand(topicName, constraints), requester); + } - public void createConstraints(SubscriptionName subscriptionName, Constraints constraints, RequestUser requester) { - commandExecutor.executeByUser(new CreateSubscriptionConstraintsRepositoryCommand(subscriptionName, constraints), requester); - } + public void createConstraints( + SubscriptionName subscriptionName, Constraints constraints, RequestUser requester) { + commandExecutor.executeByUser( + new CreateSubscriptionConstraintsRepositoryCommand(subscriptionName, constraints), + requester); + } - public void updateConstraints(TopicName topicName, Constraints constraints, RequestUser requester) { - commandExecutor.executeByUser(new UpdateTopicConstraintsRepositoryCommand(topicName, constraints), requester); - } + public void updateConstraints( + TopicName topicName, Constraints constraints, RequestUser requester) { + commandExecutor.executeByUser( + new UpdateTopicConstraintsRepositoryCommand(topicName, constraints), requester); + } - public void updateConstraints(SubscriptionName subscriptionName, Constraints constraints, RequestUser requester) { - commandExecutor.executeByUser(new UpdateSubscriptionConstraintsRepositoryCommand(subscriptionName, constraints), requester); - } + public void updateConstraints( + SubscriptionName subscriptionName, Constraints constraints, RequestUser requester) { + commandExecutor.executeByUser( + new UpdateSubscriptionConstraintsRepositoryCommand(subscriptionName, constraints), + requester); + } - public void deleteConstraints(TopicName topicName, RequestUser requester) { - commandExecutor.executeByUser(new DeleteTopicConstraintsRepositoryCommand(topicName), requester); - } + public void deleteConstraints(TopicName topicName, RequestUser requester) { + commandExecutor.executeByUser( + new DeleteTopicConstraintsRepositoryCommand(topicName), requester); + } - public void deleteConstraints(SubscriptionName subscriptionName, RequestUser requester) { - commandExecutor.executeByUser(new DeleteSubscriptionConstraintsRepositoryCommand(subscriptionName), requester); - } + public void deleteConstraints(SubscriptionName subscriptionName, RequestUser requester) { + commandExecutor.executeByUser( + new DeleteSubscriptionConstraintsRepositoryCommand(subscriptionName), requester); + } - public boolean constraintsExist(TopicName topicName) { - return workloadConstraintsRepository.constraintsExist(topicName); - } + public boolean constraintsExist(TopicName topicName) { + return workloadConstraintsRepository.constraintsExist(topicName); + } - public boolean constraintsExist(SubscriptionName subscriptionName) { - return workloadConstraintsRepository.constraintsExist(subscriptionName); - } + public boolean constraintsExist(SubscriptionName subscriptionName) { + return workloadConstraintsRepository.constraintsExist(subscriptionName); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/command/CreateSubscriptionConstraintsRepositoryCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/command/CreateSubscriptionConstraintsRepositoryCommand.java index bf38652177..979daf340f 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/command/CreateSubscriptionConstraintsRepositoryCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/command/CreateSubscriptionConstraintsRepositoryCommand.java @@ -6,41 +6,44 @@ import pl.allegro.tech.hermes.management.domain.dc.DatacenterBoundRepositoryHolder; import pl.allegro.tech.hermes.management.domain.dc.RepositoryCommand; -public class CreateSubscriptionConstraintsRepositoryCommand extends RepositoryCommand { - - private final SubscriptionName subscriptionName; - private final Constraints constraints; - private boolean exists; - - public CreateSubscriptionConstraintsRepositoryCommand(SubscriptionName subscriptionName, Constraints constraints) { - this.subscriptionName = subscriptionName; - this.constraints = constraints; - } - - @Override - public void backup(DatacenterBoundRepositoryHolder holder) { - exists = holder.getRepository().constraintsExist(subscriptionName); - } - - @Override - public void execute(DatacenterBoundRepositoryHolder holder) { - holder.getRepository().createConstraints(subscriptionName, constraints); +public class CreateSubscriptionConstraintsRepositoryCommand + extends RepositoryCommand { + + private final SubscriptionName subscriptionName; + private final Constraints constraints; + private boolean exists; + + public CreateSubscriptionConstraintsRepositoryCommand( + SubscriptionName subscriptionName, Constraints constraints) { + this.subscriptionName = subscriptionName; + this.constraints = constraints; + } + + @Override + public void backup(DatacenterBoundRepositoryHolder holder) { + exists = holder.getRepository().constraintsExist(subscriptionName); + } + + @Override + public void execute(DatacenterBoundRepositoryHolder holder) { + holder.getRepository().createConstraints(subscriptionName, constraints); + } + + @Override + public void rollback( + DatacenterBoundRepositoryHolder holder, Exception exception) { + if (!exists) { + holder.getRepository().deleteConstraints(subscriptionName); } + } - @Override - public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) { - if (!exists) { - holder.getRepository().deleteConstraints(subscriptionName); - } - } + @Override + public Class getRepositoryType() { + return WorkloadConstraintsRepository.class; + } - @Override - public Class getRepositoryType() { - return WorkloadConstraintsRepository.class; - } - - @Override - public String toString() { - return String.format("CreateSubscriptionConstraints(%s)", subscriptionName.getQualifiedName()); - } + @Override + public String toString() { + return String.format("CreateSubscriptionConstraints(%s)", subscriptionName.getQualifiedName()); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/command/CreateTopicConstraintsRepositoryCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/command/CreateTopicConstraintsRepositoryCommand.java index e96ba4875d..0f83d13fce 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/command/CreateTopicConstraintsRepositoryCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/command/CreateTopicConstraintsRepositoryCommand.java @@ -6,41 +6,43 @@ import pl.allegro.tech.hermes.management.domain.dc.DatacenterBoundRepositoryHolder; import pl.allegro.tech.hermes.management.domain.dc.RepositoryCommand; -public class CreateTopicConstraintsRepositoryCommand extends RepositoryCommand { - - private final TopicName topicName; - private final Constraints constraints; - private boolean exist; - - public CreateTopicConstraintsRepositoryCommand(TopicName topicName, Constraints constraints) { - this.topicName = topicName; - this.constraints = constraints; - } - - @Override - public void backup(DatacenterBoundRepositoryHolder holder) { - exist = holder.getRepository().constraintsExist(topicName); - } - - @Override - public void execute(DatacenterBoundRepositoryHolder holder) { - holder.getRepository().createConstraints(topicName, constraints); +public class CreateTopicConstraintsRepositoryCommand + extends RepositoryCommand { + + private final TopicName topicName; + private final Constraints constraints; + private boolean exist; + + public CreateTopicConstraintsRepositoryCommand(TopicName topicName, Constraints constraints) { + this.topicName = topicName; + this.constraints = constraints; + } + + @Override + public void backup(DatacenterBoundRepositoryHolder holder) { + exist = holder.getRepository().constraintsExist(topicName); + } + + @Override + public void execute(DatacenterBoundRepositoryHolder holder) { + holder.getRepository().createConstraints(topicName, constraints); + } + + @Override + public void rollback( + DatacenterBoundRepositoryHolder holder, Exception exception) { + if (!exist) { + holder.getRepository().deleteConstraints(topicName); } + } - @Override - public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) { - if (!exist) { - holder.getRepository().deleteConstraints(topicName); - } - } + @Override + public Class getRepositoryType() { + return WorkloadConstraintsRepository.class; + } - @Override - public Class getRepositoryType() { - return WorkloadConstraintsRepository.class; - } - - @Override - public String toString() { - return String.format("CreateTopicConstraints(%s)", topicName.qualifiedName()); - } + @Override + public String toString() { + return String.format("CreateTopicConstraints(%s)", topicName.qualifiedName()); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/command/DeleteSubscriptionConstraintsRepositoryCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/command/DeleteSubscriptionConstraintsRepositoryCommand.java index 3f0f713367..e087ada847 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/command/DeleteSubscriptionConstraintsRepositoryCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/command/DeleteSubscriptionConstraintsRepositoryCommand.java @@ -6,39 +6,46 @@ import pl.allegro.tech.hermes.management.domain.dc.DatacenterBoundRepositoryHolder; import pl.allegro.tech.hermes.management.domain.dc.RepositoryCommand; -public class DeleteSubscriptionConstraintsRepositoryCommand extends RepositoryCommand { - - private final SubscriptionName subscriptionName; - private Constraints backup; - - public DeleteSubscriptionConstraintsRepositoryCommand(SubscriptionName subscriptionName) { - this.subscriptionName = subscriptionName; - } - - @Override - public void backup(DatacenterBoundRepositoryHolder holder) { - backup = holder.getRepository().getConsumersWorkloadConstraints().getSubscriptionConstraints().get(subscriptionName); - } - - @Override - public void execute(DatacenterBoundRepositoryHolder holder) { - holder.getRepository().deleteConstraints(subscriptionName); +public class DeleteSubscriptionConstraintsRepositoryCommand + extends RepositoryCommand { + + private final SubscriptionName subscriptionName; + private Constraints backup; + + public DeleteSubscriptionConstraintsRepositoryCommand(SubscriptionName subscriptionName) { + this.subscriptionName = subscriptionName; + } + + @Override + public void backup(DatacenterBoundRepositoryHolder holder) { + backup = + holder + .getRepository() + .getConsumersWorkloadConstraints() + .getSubscriptionConstraints() + .get(subscriptionName); + } + + @Override + public void execute(DatacenterBoundRepositoryHolder holder) { + holder.getRepository().deleteConstraints(subscriptionName); + } + + @Override + public void rollback( + DatacenterBoundRepositoryHolder holder, Exception exception) { + if (backup != null) { + holder.getRepository().createConstraints(subscriptionName, backup); } + } - @Override - public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) { - if (backup != null) { - holder.getRepository().createConstraints(subscriptionName, backup); - } - } + @Override + public Class getRepositoryType() { + return WorkloadConstraintsRepository.class; + } - @Override - public Class getRepositoryType() { - return WorkloadConstraintsRepository.class; - } - - @Override - public String toString() { - return String.format("DeleteSubscriptionConstraints(%s)", subscriptionName.getQualifiedName()); - } + @Override + public String toString() { + return String.format("DeleteSubscriptionConstraints(%s)", subscriptionName.getQualifiedName()); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/command/DeleteTopicConstraintsRepositoryCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/command/DeleteTopicConstraintsRepositoryCommand.java index 981a198968..609b8c4974 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/command/DeleteTopicConstraintsRepositoryCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/command/DeleteTopicConstraintsRepositoryCommand.java @@ -6,39 +6,46 @@ import pl.allegro.tech.hermes.management.domain.dc.DatacenterBoundRepositoryHolder; import pl.allegro.tech.hermes.management.domain.dc.RepositoryCommand; -public class DeleteTopicConstraintsRepositoryCommand extends RepositoryCommand { - - private final TopicName topicName; - private Constraints backup; - - public DeleteTopicConstraintsRepositoryCommand(TopicName topicName) { - this.topicName = topicName; - } - - @Override - public void backup(DatacenterBoundRepositoryHolder holder) { - backup = holder.getRepository().getConsumersWorkloadConstraints().getTopicConstraints().get(topicName); - } - - @Override - public void execute(DatacenterBoundRepositoryHolder holder) { - holder.getRepository().deleteConstraints(topicName); +public class DeleteTopicConstraintsRepositoryCommand + extends RepositoryCommand { + + private final TopicName topicName; + private Constraints backup; + + public DeleteTopicConstraintsRepositoryCommand(TopicName topicName) { + this.topicName = topicName; + } + + @Override + public void backup(DatacenterBoundRepositoryHolder holder) { + backup = + holder + .getRepository() + .getConsumersWorkloadConstraints() + .getTopicConstraints() + .get(topicName); + } + + @Override + public void execute(DatacenterBoundRepositoryHolder holder) { + holder.getRepository().deleteConstraints(topicName); + } + + @Override + public void rollback( + DatacenterBoundRepositoryHolder holder, Exception exception) { + if (backup != null) { + holder.getRepository().createConstraints(topicName, backup); } + } - @Override - public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) { - if (backup != null) { - holder.getRepository().createConstraints(topicName, backup); - } - } + @Override + public Class getRepositoryType() { + return WorkloadConstraintsRepository.class; + } - @Override - public Class getRepositoryType() { - return WorkloadConstraintsRepository.class; - } - - @Override - public String toString() { - return String.format("DeleteTopicConstraints(%s)", topicName.qualifiedName()); - } + @Override + public String toString() { + return String.format("DeleteTopicConstraints(%s)", topicName.qualifiedName()); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/command/UpdateSubscriptionConstraintsRepositoryCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/command/UpdateSubscriptionConstraintsRepositoryCommand.java index ade0564ee3..fc7900f050 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/command/UpdateSubscriptionConstraintsRepositoryCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/command/UpdateSubscriptionConstraintsRepositoryCommand.java @@ -6,41 +6,49 @@ import pl.allegro.tech.hermes.management.domain.dc.DatacenterBoundRepositoryHolder; import pl.allegro.tech.hermes.management.domain.dc.RepositoryCommand; -public class UpdateSubscriptionConstraintsRepositoryCommand extends RepositoryCommand { - - private final SubscriptionName subscriptionName; - private final Constraints constraints; - private Constraints backup; - - public UpdateSubscriptionConstraintsRepositoryCommand(SubscriptionName subscriptionName, Constraints constraints) { - this.subscriptionName = subscriptionName; - this.constraints = constraints; - } - - @Override - public void backup(DatacenterBoundRepositoryHolder holder) { - backup = holder.getRepository().getConsumersWorkloadConstraints().getSubscriptionConstraints().get(subscriptionName); - } - - @Override - public void execute(DatacenterBoundRepositoryHolder holder) { - holder.getRepository().updateConstraints(subscriptionName, constraints); +public class UpdateSubscriptionConstraintsRepositoryCommand + extends RepositoryCommand { + + private final SubscriptionName subscriptionName; + private final Constraints constraints; + private Constraints backup; + + public UpdateSubscriptionConstraintsRepositoryCommand( + SubscriptionName subscriptionName, Constraints constraints) { + this.subscriptionName = subscriptionName; + this.constraints = constraints; + } + + @Override + public void backup(DatacenterBoundRepositoryHolder holder) { + backup = + holder + .getRepository() + .getConsumersWorkloadConstraints() + .getSubscriptionConstraints() + .get(subscriptionName); + } + + @Override + public void execute(DatacenterBoundRepositoryHolder holder) { + holder.getRepository().updateConstraints(subscriptionName, constraints); + } + + @Override + public void rollback( + DatacenterBoundRepositoryHolder holder, Exception exception) { + if (backup != null) { + holder.getRepository().updateConstraints(subscriptionName, backup); } + } - @Override - public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) { - if (backup != null) { - holder.getRepository().updateConstraints(subscriptionName, backup); - } - } + @Override + public Class getRepositoryType() { + return WorkloadConstraintsRepository.class; + } - @Override - public Class getRepositoryType() { - return WorkloadConstraintsRepository.class; - } - - @Override - public String toString() { - return String.format("UpdateSubscriptionConstraints(%s)", subscriptionName.getQualifiedName()); - } + @Override + public String toString() { + return String.format("UpdateSubscriptionConstraints(%s)", subscriptionName.getQualifiedName()); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/command/UpdateTopicConstraintsRepositoryCommand.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/command/UpdateTopicConstraintsRepositoryCommand.java index e69d08642b..9a09e8fd7a 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/command/UpdateTopicConstraintsRepositoryCommand.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/domain/workload/constraints/command/UpdateTopicConstraintsRepositoryCommand.java @@ -6,41 +6,48 @@ import pl.allegro.tech.hermes.management.domain.dc.DatacenterBoundRepositoryHolder; import pl.allegro.tech.hermes.management.domain.dc.RepositoryCommand; -public class UpdateTopicConstraintsRepositoryCommand extends RepositoryCommand { - - private final TopicName topicName; - private final Constraints constraints; - private Constraints backup; - - public UpdateTopicConstraintsRepositoryCommand(TopicName topicName, Constraints constraints) { - this.topicName = topicName; - this.constraints = constraints; - } - - @Override - public void backup(DatacenterBoundRepositoryHolder holder) { - backup = holder.getRepository().getConsumersWorkloadConstraints().getTopicConstraints().get(topicName); - } - - @Override - public void execute(DatacenterBoundRepositoryHolder holder) { - holder.getRepository().updateConstraints(topicName, constraints); +public class UpdateTopicConstraintsRepositoryCommand + extends RepositoryCommand { + + private final TopicName topicName; + private final Constraints constraints; + private Constraints backup; + + public UpdateTopicConstraintsRepositoryCommand(TopicName topicName, Constraints constraints) { + this.topicName = topicName; + this.constraints = constraints; + } + + @Override + public void backup(DatacenterBoundRepositoryHolder holder) { + backup = + holder + .getRepository() + .getConsumersWorkloadConstraints() + .getTopicConstraints() + .get(topicName); + } + + @Override + public void execute(DatacenterBoundRepositoryHolder holder) { + holder.getRepository().updateConstraints(topicName, constraints); + } + + @Override + public void rollback( + DatacenterBoundRepositoryHolder holder, Exception exception) { + if (backup != null) { + holder.getRepository().updateConstraints(topicName, backup); } + } - @Override - public void rollback(DatacenterBoundRepositoryHolder holder, Exception exception) { - if (backup != null) { - holder.getRepository().updateConstraints(topicName, backup); - } - } + @Override + public Class getRepositoryType() { + return WorkloadConstraintsRepository.class; + } - @Override - public Class getRepositoryType() { - return WorkloadConstraintsRepository.class; - } - - @Override - public String toString() { - return String.format("UpdateTopicConstraints(%s)", topicName.qualifiedName()); - } + @Override + public String toString() { + return String.format("UpdateTopicConstraints(%s)", topicName.qualifiedName()); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/audit/AuditEvent.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/audit/AuditEvent.java index 9461f38131..070329c07b 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/audit/AuditEvent.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/audit/AuditEvent.java @@ -1,37 +1,42 @@ package pl.allegro.tech.hermes.management.infrastructure.audit; public class AuditEvent { - private final AuditEventType eventType; - private final String resourceName; - private final String payloadClass; - private final String payload; - private final String username; + private final AuditEventType eventType; + private final String resourceName; + private final String payloadClass; + private final String payload; + private final String username; - public AuditEvent(AuditEventType eventType, String payload, String payloadClass, String resourceName, String username) { - this.eventType = eventType; - this.payload = payload; - this.payloadClass = payloadClass; - this.resourceName = resourceName; - this.username = username; - } + public AuditEvent( + AuditEventType eventType, + String payload, + String payloadClass, + String resourceName, + String username) { + this.eventType = eventType; + this.payload = payload; + this.payloadClass = payloadClass; + this.resourceName = resourceName; + this.username = username; + } - public AuditEventType getEventType() { - return eventType; - } + public AuditEventType getEventType() { + return eventType; + } - public String getPayloadClass() { - return payloadClass; - } + public String getPayloadClass() { + return payloadClass; + } - public String getPayload() { - return payload; - } + public String getPayload() { + return payload; + } - public String getUsername() { - return username; - } + public String getUsername() { + return username; + } - public String getResourceName() { - return resourceName; - } + public String getResourceName() { + return resourceName; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/audit/AuditEventType.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/audit/AuditEventType.java index dc5796e9b6..689d4da0d9 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/audit/AuditEventType.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/audit/AuditEventType.java @@ -1,7 +1,10 @@ package pl.allegro.tech.hermes.management.infrastructure.audit; public enum AuditEventType { - BEFORE_CREATION, CREATED, - BEFORE_UPDATE, UPDATED, - BEFORE_REMOVAL, REMOVED + BEFORE_CREATION, + CREATED, + BEFORE_UPDATE, + UPDATED, + BEFORE_REMOVAL, + REMOVED } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/audit/CompositeAuditor.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/audit/CompositeAuditor.java index 4750b68dc6..4309ba5a43 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/audit/CompositeAuditor.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/audit/CompositeAuditor.java @@ -1,47 +1,50 @@ package pl.allegro.tech.hermes.management.infrastructure.audit; -import pl.allegro.tech.hermes.api.PatchData; -import pl.allegro.tech.hermes.management.domain.Auditor; +import static com.google.common.base.Preconditions.checkNotNull; import java.util.Collection; - -import static com.google.common.base.Preconditions.checkNotNull; +import pl.allegro.tech.hermes.api.PatchData; +import pl.allegro.tech.hermes.management.domain.Auditor; public class CompositeAuditor implements Auditor { - private final Collection auditors; - - public CompositeAuditor(Collection auditors) { - this.auditors = checkNotNull(auditors); - } - - @Override - public void beforeObjectCreation(String username, Object createdObject) { - auditors.forEach(auditor -> auditor.beforeObjectCreation(username, createdObject)); - } - - @Override - public void beforeObjectRemoval(String username, String removedObjectType, String removedObjectName) { - auditors.forEach(auditor -> auditor.beforeObjectRemoval(username, removedObjectType, removedObjectName)); - } - - @Override - public void beforeObjectUpdate(String username, String objectClassName, Object objectName, PatchData patchData) { - auditors.forEach(auditor -> auditor.beforeObjectUpdate(username, objectClassName, objectName, patchData)); - } - - @Override - public void objectCreated(String username, Object createdObject) { - auditors.forEach(auditor -> auditor.objectCreated(username, createdObject)); - } - - @Override - public void objectRemoved(String username, Object removedObject) { - auditors.forEach(auditor -> auditor.objectRemoved(username, removedObject)); - } - - @Override - public void objectUpdated(String username, Object oldObject, Object newObject) { - auditors.forEach(auditor -> auditor.objectUpdated(username, oldObject, newObject)); - } + private final Collection auditors; + + public CompositeAuditor(Collection auditors) { + this.auditors = checkNotNull(auditors); + } + + @Override + public void beforeObjectCreation(String username, Object createdObject) { + auditors.forEach(auditor -> auditor.beforeObjectCreation(username, createdObject)); + } + + @Override + public void beforeObjectRemoval( + String username, String removedObjectType, String removedObjectName) { + auditors.forEach( + auditor -> auditor.beforeObjectRemoval(username, removedObjectType, removedObjectName)); + } + + @Override + public void beforeObjectUpdate( + String username, String objectClassName, Object objectName, PatchData patchData) { + auditors.forEach( + auditor -> auditor.beforeObjectUpdate(username, objectClassName, objectName, patchData)); + } + + @Override + public void objectCreated(String username, Object createdObject) { + auditors.forEach(auditor -> auditor.objectCreated(username, createdObject)); + } + + @Override + public void objectRemoved(String username, Object removedObject) { + auditors.forEach(auditor -> auditor.objectRemoved(username, removedObject)); + } + + @Override + public void objectUpdated(String username, Object oldObject, Object newObject) { + auditors.forEach(auditor -> auditor.objectUpdated(username, oldObject, newObject)); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/audit/EventAuditor.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/audit/EventAuditor.java index 5562b69a17..a07a9a2440 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/audit/EventAuditor.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/audit/EventAuditor.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.management.infrastructure.audit; +import static com.google.common.base.Preconditions.checkNotNull; + import com.fasterxml.jackson.databind.ObjectMapper; import org.javers.core.Javers; import org.javers.core.diff.Diff; @@ -9,115 +11,136 @@ import pl.allegro.tech.hermes.api.PatchData; import pl.allegro.tech.hermes.management.domain.Auditor; -import static com.google.common.base.Preconditions.checkNotNull; - public class EventAuditor implements Auditor { - private static final Logger logger = LoggerFactory.getLogger(EventAuditor.class); - - private final Javers javers; - - private final RestTemplate restTemplate; - - private final String eventDestination; - - private final ObjectMapper objectMapper; - - public EventAuditor(Javers javers, RestTemplate restTemplate, String eventDestination, ObjectMapper objectMapper) { - this.javers = checkNotNull(javers); - this.restTemplate = checkNotNull(restTemplate); - this.eventDestination = eventDestination; - this.objectMapper = objectMapper; - } - - @Override - public void beforeObjectCreation(String username, Object createdObject) { - ignoringExceptions(() -> { - String createdObjectToString = objectMapper.writeValueAsString(createdObject); - AuditEvent event = new AuditEvent(AuditEventType.BEFORE_CREATION, - createdObjectToString, - createdObject.getClass().getSimpleName(), - createdObject.toString(), - username); - restTemplate.postForObject(eventDestination, event, Void.class); + private static final Logger logger = LoggerFactory.getLogger(EventAuditor.class); + + private final Javers javers; + + private final RestTemplate restTemplate; + + private final String eventDestination; + + private final ObjectMapper objectMapper; + + public EventAuditor( + Javers javers, + RestTemplate restTemplate, + String eventDestination, + ObjectMapper objectMapper) { + this.javers = checkNotNull(javers); + this.restTemplate = checkNotNull(restTemplate); + this.eventDestination = eventDestination; + this.objectMapper = objectMapper; + } + + @Override + public void beforeObjectCreation(String username, Object createdObject) { + ignoringExceptions( + () -> { + String createdObjectToString = objectMapper.writeValueAsString(createdObject); + AuditEvent event = + new AuditEvent( + AuditEventType.BEFORE_CREATION, + createdObjectToString, + createdObject.getClass().getSimpleName(), + createdObject.toString(), + username); + restTemplate.postForObject(eventDestination, event, Void.class); }); - } - - @Override - public void beforeObjectRemoval(String username, String removedObjectType, String removedObjectName) { - ignoringExceptions(() -> { - AuditEvent event = new AuditEvent(AuditEventType.BEFORE_REMOVAL, - removedObjectName, - removedObjectType, - removedObjectName, - username); - restTemplate.postForObject(eventDestination, event, Void.class); + } + + @Override + public void beforeObjectRemoval( + String username, String removedObjectType, String removedObjectName) { + ignoringExceptions( + () -> { + AuditEvent event = + new AuditEvent( + AuditEventType.BEFORE_REMOVAL, + removedObjectName, + removedObjectType, + removedObjectName, + username); + restTemplate.postForObject(eventDestination, event, Void.class); }); - } - - @Override - public void beforeObjectUpdate(String username, String objectClassName, Object objectName, PatchData patchData) { - ignoringExceptions(() -> { - String patchDataToString = objectMapper.writeValueAsString(patchData); - AuditEvent event = new AuditEvent(AuditEventType.BEFORE_UPDATE, - patchDataToString, - patchData.getClass().getSimpleName(), - objectName.toString(), - username); - restTemplate.postForObject(eventDestination, event, Void.class); + } + + @Override + public void beforeObjectUpdate( + String username, String objectClassName, Object objectName, PatchData patchData) { + ignoringExceptions( + () -> { + String patchDataToString = objectMapper.writeValueAsString(patchData); + AuditEvent event = + new AuditEvent( + AuditEventType.BEFORE_UPDATE, + patchDataToString, + patchData.getClass().getSimpleName(), + objectName.toString(), + username); + restTemplate.postForObject(eventDestination, event, Void.class); }); - } - - @Override - public void objectCreated(String username, Object createdObject) { - ignoringExceptions(() -> { - String createdObjectToString = objectMapper.writeValueAsString(createdObject); - AuditEvent event = new AuditEvent(AuditEventType.CREATED, - createdObjectToString, - createdObject.getClass().getSimpleName(), - createdObject.toString(), - username); - restTemplate.postForObject(eventDestination, event, Void.class); + } + + @Override + public void objectCreated(String username, Object createdObject) { + ignoringExceptions( + () -> { + String createdObjectToString = objectMapper.writeValueAsString(createdObject); + AuditEvent event = + new AuditEvent( + AuditEventType.CREATED, + createdObjectToString, + createdObject.getClass().getSimpleName(), + createdObject.toString(), + username); + restTemplate.postForObject(eventDestination, event, Void.class); }); - } - - @Override - public void objectRemoved(String username, Object removedObject) { - ignoringExceptions(() -> { - String removedObjectToString = objectMapper.writeValueAsString(removedObject); - AuditEvent event = new AuditEvent(AuditEventType.REMOVED, - removedObjectToString, - removedObject.getClass().getSimpleName(), - removedObject.toString(), - username); - restTemplate.postForObject(eventDestination, event, Void.class); + } + + @Override + public void objectRemoved(String username, Object removedObject) { + ignoringExceptions( + () -> { + String removedObjectToString = objectMapper.writeValueAsString(removedObject); + AuditEvent event = + new AuditEvent( + AuditEventType.REMOVED, + removedObjectToString, + removedObject.getClass().getSimpleName(), + removedObject.toString(), + username); + restTemplate.postForObject(eventDestination, event, Void.class); }); - } - - @Override - public void objectUpdated(String username, Object oldObject, Object newObject) { - ignoringExceptions(() -> { - Diff diff = javers.compare(oldObject, newObject); - AuditEvent event = new AuditEvent(AuditEventType.UPDATED, - diff.toString(), - oldObject.getClass().getSimpleName(), - oldObject.toString(), - username); - restTemplate.postForObject(eventDestination, event, Void.class); + } + + @Override + public void objectUpdated(String username, Object oldObject, Object newObject) { + ignoringExceptions( + () -> { + Diff diff = javers.compare(oldObject, newObject); + AuditEvent event = + new AuditEvent( + AuditEventType.UPDATED, + diff.toString(), + oldObject.getClass().getSimpleName(), + oldObject.toString(), + username); + restTemplate.postForObject(eventDestination, event, Void.class); }); - } - - private void ignoringExceptions(Wrapped wrapped) { - try { - wrapped.execute(); - } catch (Exception e) { - logger.info("Audit event emission failed.", e); - } - } + } - @FunctionalInterface - private interface Wrapped { - void execute() throws Exception; + private void ignoringExceptions(Wrapped wrapped) { + try { + wrapped.execute(); + } catch (Exception e) { + logger.info("Audit event emission failed.", e); } + } + @FunctionalInterface + private interface Wrapped { + void execute() throws Exception; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/audit/LoggingAuditor.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/audit/LoggingAuditor.java index 9c50655d68..f544b5b27e 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/audit/LoggingAuditor.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/audit/LoggingAuditor.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.management.infrastructure.audit; +import static com.google.common.base.Preconditions.checkNotNull; + import com.fasterxml.jackson.databind.ObjectMapper; import org.javers.core.Javers; import org.javers.core.diff.Diff; @@ -8,73 +10,91 @@ import pl.allegro.tech.hermes.api.PatchData; import pl.allegro.tech.hermes.management.domain.Auditor; -import static com.google.common.base.Preconditions.checkNotNull; - public class LoggingAuditor implements Auditor { - private static final Logger logger = LoggerFactory.getLogger(LoggingAuditor.class); + private static final Logger logger = LoggerFactory.getLogger(LoggingAuditor.class); - private final Javers javers; - private final ObjectMapper objectMapper; + private final Javers javers; + private final ObjectMapper objectMapper; - public LoggingAuditor(Javers javers, ObjectMapper objectMapper) { - this.javers = checkNotNull(javers); - this.objectMapper = checkNotNull(objectMapper); - } + public LoggingAuditor(Javers javers, ObjectMapper objectMapper) { + this.javers = checkNotNull(javers); + this.objectMapper = checkNotNull(objectMapper); + } - @Override - public void beforeObjectCreation(String username, Object createdObject) { - ignoringExceptions(() -> - logger.info("User {} tries creating new {} {}.", username, createdObject.getClass().getSimpleName(), - objectMapper.writeValueAsString(createdObject))); - } + @Override + public void beforeObjectCreation(String username, Object createdObject) { + ignoringExceptions( + () -> + logger.info( + "User {} tries creating new {} {}.", + username, + createdObject.getClass().getSimpleName(), + objectMapper.writeValueAsString(createdObject))); + } - @Override - public void beforeObjectRemoval(String username, String removedObjectType, String removedObjectName) { - logger.info("User {} tries removing {} {}.", username, removedObjectType, removedObjectName); - } + @Override + public void beforeObjectRemoval( + String username, String removedObjectType, String removedObjectName) { + logger.info("User {} tries removing {} {}.", username, removedObjectType, removedObjectName); + } - @Override - public void beforeObjectUpdate(String username, String objectClassName, Object objectName, PatchData patchData) { - ignoringExceptions(() -> { - logger.info("User {} tries updating {} {}. {}", username, objectClassName, - objectName, patchData); + @Override + public void beforeObjectUpdate( + String username, String objectClassName, Object objectName, PatchData patchData) { + ignoringExceptions( + () -> { + logger.info( + "User {} tries updating {} {}. {}", username, objectClassName, objectName, patchData); }); - } + } - @Override - public void objectCreated(String username, Object createdObject) { - ignoringExceptions(() -> - logger.info("User {} has created new {} {}.", username, createdObject.getClass().getSimpleName(), - objectMapper.writeValueAsString(createdObject))); - } + @Override + public void objectCreated(String username, Object createdObject) { + ignoringExceptions( + () -> + logger.info( + "User {} has created new {} {}.", + username, + createdObject.getClass().getSimpleName(), + objectMapper.writeValueAsString(createdObject))); + } - @Override - public void objectRemoved(String username, Object removedObject) { - ignoringExceptions(() -> - logger.info("User {} has removed {} {}.", username, removedObject.getClass().getSimpleName(), + @Override + public void objectRemoved(String username, Object removedObject) { + ignoringExceptions( + () -> + logger.info( + "User {} has removed {} {}.", + username, + removedObject.getClass().getSimpleName(), objectMapper.writeValueAsString(removedObject))); - } + } - @Override - public void objectUpdated(String username, Object oldObject, Object newObject) { - ignoringExceptions(() -> { - Diff diff = javers.compare(oldObject, newObject); - logger.info("User {} has updated {} {}. {}", username, oldObject.getClass().getSimpleName(), - objectMapper.writeValueAsString(oldObject), diff); + @Override + public void objectUpdated(String username, Object oldObject, Object newObject) { + ignoringExceptions( + () -> { + Diff diff = javers.compare(oldObject, newObject); + logger.info( + "User {} has updated {} {}. {}", + username, + oldObject.getClass().getSimpleName(), + objectMapper.writeValueAsString(oldObject), + diff); }); - } + } - private void ignoringExceptions(Wrapped wrapped) { - try { - wrapped.execute(); - } catch (Exception e) { - logger.info("Audit log failed {}.", e); - } + private void ignoringExceptions(Wrapped wrapped) { + try { + wrapped.execute(); + } catch (Exception e) { + logger.info("Audit log failed {}.", e); } + } - @FunctionalInterface - private interface Wrapped { - void execute() throws Exception; - } + @FunctionalInterface + private interface Wrapped { + void execute() throws Exception; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/blacklist/ZookeeperTopicBlacklistRepository.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/blacklist/ZookeeperTopicBlacklistRepository.java index 1ba3da65bd..87b99e09c9 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/blacklist/ZookeeperTopicBlacklistRepository.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/blacklist/ZookeeperTopicBlacklistRepository.java @@ -1,6 +1,7 @@ package pl.allegro.tech.hermes.management.infrastructure.blacklist; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.List; import org.apache.curator.framework.CuratorFramework; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -9,40 +10,41 @@ import pl.allegro.tech.hermes.management.domain.blacklist.NotUnblacklistedException; import pl.allegro.tech.hermes.management.domain.blacklist.TopicBlacklistRepository; -import java.util.List; - -public class ZookeeperTopicBlacklistRepository extends ZookeeperBasedRepository implements TopicBlacklistRepository { - - private static final Logger logger = LoggerFactory.getLogger(ZookeeperTopicBlacklistRepository.class); - - public ZookeeperTopicBlacklistRepository(CuratorFramework zookeeper, ObjectMapper mapper, ZookeeperPaths paths) { - super(zookeeper, mapper, paths); +public class ZookeeperTopicBlacklistRepository extends ZookeeperBasedRepository + implements TopicBlacklistRepository { + + private static final Logger logger = + LoggerFactory.getLogger(ZookeeperTopicBlacklistRepository.class); + + public ZookeeperTopicBlacklistRepository( + CuratorFramework zookeeper, ObjectMapper mapper, ZookeeperPaths paths) { + super(zookeeper, mapper, paths); + } + + @Override + public void add(String qualifiedTopicName) { + logger.info("Adding topic {} to Blacklist", qualifiedTopicName); + ensurePathExists(paths.blacklistedTopicPath(qualifiedTopicName)); + } + + @Override + public void remove(String qualifiedTopicName) { + logger.info("Removing topic {} from Blacklist", qualifiedTopicName); + try { + super.remove(paths.blacklistedTopicPath(qualifiedTopicName)); + } catch (Exception e) { + logger.warn("Removing topic {} from Blacklist caused an exception", qualifiedTopicName, e); + throw new NotUnblacklistedException(qualifiedTopicName, e); } + } - @Override - public void add(String qualifiedTopicName) { - logger.info("Adding topic {} to Blacklist", qualifiedTopicName); - ensurePathExists(paths.blacklistedTopicPath(qualifiedTopicName)); - } + @Override + public boolean isBlacklisted(String qualifiedTopicName) { + return pathExists(paths.blacklistedTopicPath(qualifiedTopicName)); + } - @Override - public void remove(String qualifiedTopicName) { - logger.info("Removing topic {} from Blacklist", qualifiedTopicName); - try { - super.remove(paths.blacklistedTopicPath(qualifiedTopicName)); - } catch (Exception e) { - logger.warn("Removing topic {} from Blacklist caused an exception", qualifiedTopicName, e); - throw new NotUnblacklistedException(qualifiedTopicName, e); - } - } - - @Override - public boolean isBlacklisted(String qualifiedTopicName) { - return pathExists(paths.blacklistedTopicPath(qualifiedTopicName)); - } - - @Override - public List list() { - return childrenOf(paths.topicsBlacklistPath()); - } + @Override + public List list() { + return childrenOf(paths.topicsBlacklistPath()); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/console/ClasspathFileConsoleConfigurationRepository.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/console/ClasspathFileConsoleConfigurationRepository.java index 9bfb0f4f11..c24075cd0e 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/console/ClasspathFileConsoleConfigurationRepository.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/console/ClasspathFileConsoleConfigurationRepository.java @@ -1,32 +1,32 @@ package pl.allegro.tech.hermes.management.infrastructure.console; +import java.nio.charset.StandardCharsets; import org.springframework.core.io.ClassPathResource; import org.springframework.util.FileCopyUtils; import pl.allegro.tech.hermes.management.config.console.ConsoleConfigProperties; import pl.allegro.tech.hermes.management.domain.console.ConsoleConfigurationRepository; -import java.nio.charset.StandardCharsets; - public class ClasspathFileConsoleConfigurationRepository implements ConsoleConfigurationRepository { - private String configuration; + private String configuration; - public ClasspathFileConsoleConfigurationRepository(ConsoleConfigProperties properties) { - configuration = loadConfiguration(properties.getLocation()); - } + public ClasspathFileConsoleConfigurationRepository(ConsoleConfigProperties properties) { + configuration = loadConfiguration(properties.getLocation()); + } - @Override - public String getConfiguration() { - return configuration; - } + @Override + public String getConfiguration() { + return configuration; + } - private String loadConfiguration(String location) { - try { - ClassPathResource resource = new ClassPathResource(location); - byte[] bytes = FileCopyUtils.copyToByteArray(resource.getInputStream()); - return new String(bytes, StandardCharsets.UTF_8); - } catch (Exception e) { - throw new IllegalArgumentException("Error reading Hermes Console configuration from " + location, e); - } + private String loadConfiguration(String location) { + try { + ClassPathResource resource = new ClassPathResource(location); + byte[] bytes = FileCopyUtils.copyToByteArray(resource.getInputStream()); + return new String(bytes, StandardCharsets.UTF_8); + } catch (Exception e) { + throw new IllegalArgumentException( + "Error reading Hermes Console configuration from " + location, e); } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/console/FrontendRoutesFilter.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/console/FrontendRoutesFilter.java index 66bf4f622a..6bee83e894 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/console/FrontendRoutesFilter.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/console/FrontendRoutesFilter.java @@ -5,23 +5,21 @@ import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; -import org.springframework.web.filter.OncePerRequestFilter; - import java.io.IOException; +import org.springframework.web.filter.OncePerRequestFilter; public class FrontendRoutesFilter extends OncePerRequestFilter { - private final String frontendEndpoint = "/"; + private final String frontendEndpoint = "/"; - @Override - protected void doFilterInternal( - HttpServletRequest request, - HttpServletResponse response, - FilterChain filterChain) throws ServletException, IOException { - if (request.getRequestURI().startsWith("/ui")) { - RequestDispatcher rd = request.getRequestDispatcher(frontendEndpoint); - rd.forward(request, response); - } else { - filterChain.doFilter(request, response); - } + @Override + protected void doFilterInternal( + HttpServletRequest request, HttpServletResponse response, FilterChain filterChain) + throws ServletException, IOException { + if (request.getRequestURI().startsWith("/ui")) { + RequestDispatcher rd = request.getRequestDispatcher(frontendEndpoint); + rd.forward(request, response); + } else { + filterChain.doFilter(request, response); } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/console/HttpConsoleConfigurationRepository.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/console/HttpConsoleConfigurationRepository.java index 8e44e0bfb4..b83b46adf7 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/console/HttpConsoleConfigurationRepository.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/console/HttpConsoleConfigurationRepository.java @@ -6,22 +6,24 @@ public class HttpConsoleConfigurationRepository implements ConsoleConfigurationRepository { - private String configuration; + private String configuration; - public HttpConsoleConfigurationRepository(ConsoleConfigProperties properties, RestTemplate restTemplate) { - configuration = loadConfiguration(properties.getLocation(), restTemplate); - } + public HttpConsoleConfigurationRepository( + ConsoleConfigProperties properties, RestTemplate restTemplate) { + configuration = loadConfiguration(properties.getLocation(), restTemplate); + } - @Override - public String getConfiguration() { - return configuration; - } + @Override + public String getConfiguration() { + return configuration; + } - private String loadConfiguration(String location, RestTemplate restTemplate) { - try { - return restTemplate.getForObject(location, String.class); - } catch (Exception e) { - throw new IllegalArgumentException("Error reading Hermes Console configuration from " + location, e); - } + private String loadConfiguration(String location, RestTemplate restTemplate) { + try { + return restTemplate.getForObject(location, String.class); + } catch (Exception e) { + throw new IllegalArgumentException( + "Error reading Hermes Console configuration from " + location, e); } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/console/SpringConfigConsoleConfigurationRepository.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/console/SpringConfigConsoleConfigurationRepository.java index 4dd1e33ba2..65daa14469 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/console/SpringConfigConsoleConfigurationRepository.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/console/SpringConfigConsoleConfigurationRepository.java @@ -7,19 +7,20 @@ public class SpringConfigConsoleConfigurationRepository implements ConsoleConfigurationRepository { - private final String jsonConfig; + private final String jsonConfig; - public SpringConfigConsoleConfigurationRepository(ObjectMapper objectMapper, ConsoleProperties consoleProperties) { - try { - jsonConfig = objectMapper.writeValueAsString(consoleProperties); - } catch (JsonProcessingException e) { - throw new IllegalArgumentException("Error during spring console config conversion to JSON", e); - } - } - - @Override - public String getConfiguration() { - return jsonConfig; + public SpringConfigConsoleConfigurationRepository( + ObjectMapper objectMapper, ConsoleProperties consoleProperties) { + try { + jsonConfig = objectMapper.writeValueAsString(consoleProperties); + } catch (JsonProcessingException e) { + throw new IllegalArgumentException( + "Error during spring console config conversion to JSON", e); } + } + @Override + public String getConfiguration() { + return jsonConfig; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/graphite/RestTemplateGraphiteClient.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/graphite/RestTemplateGraphiteClient.java index e69de29bb2..8b13789179 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/graphite/RestTemplateGraphiteClient.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/graphite/RestTemplateGraphiteClient.java @@ -0,0 +1 @@ + diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/BrokersClusterCommunicationException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/BrokersClusterCommunicationException.java index a7d48dd5c8..3e06ef8f7a 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/BrokersClusterCommunicationException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/BrokersClusterCommunicationException.java @@ -5,12 +5,12 @@ public class BrokersClusterCommunicationException extends ManagementException { - public BrokersClusterCommunicationException(Throwable t) { - super(t); - } + public BrokersClusterCommunicationException(Throwable t) { + super(t); + } - @Override - public ErrorCode getCode() { - return ErrorCode.BROKERS_CLUSTER_COMMUNICATION_EXCEPTION; - } + @Override + public ErrorCode getCode() { + return ErrorCode.BROKERS_CLUSTER_COMMUNICATION_EXCEPTION; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/BrokersClusterNotFoundException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/BrokersClusterNotFoundException.java index 4e0dad4465..62a9a7f3fa 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/BrokersClusterNotFoundException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/BrokersClusterNotFoundException.java @@ -1,19 +1,19 @@ package pl.allegro.tech.hermes.management.infrastructure.kafka; -import pl.allegro.tech.hermes.api.ErrorCode; -import pl.allegro.tech.hermes.management.domain.ManagementException; - import static java.lang.String.format; import static pl.allegro.tech.hermes.api.ErrorCode.BROKERS_CLUSTER_NOT_FOUND_EXCEPTION; +import pl.allegro.tech.hermes.api.ErrorCode; +import pl.allegro.tech.hermes.management.domain.ManagementException; + public class BrokersClusterNotFoundException extends ManagementException { - public BrokersClusterNotFoundException(String clusterName) { - super(format("Brokers cluster with name %s not found", clusterName)); - } + public BrokersClusterNotFoundException(String clusterName) { + super(format("Brokers cluster with name %s not found", clusterName)); + } - @Override - public ErrorCode getCode() { - return BROKERS_CLUSTER_NOT_FOUND_EXCEPTION; - } + @Override + public ErrorCode getCode() { + return BROKERS_CLUSTER_NOT_FOUND_EXCEPTION; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/MovingSubscriptionOffsetsValidationException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/MovingSubscriptionOffsetsValidationException.java index b338424e7d..d7afa018c4 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/MovingSubscriptionOffsetsValidationException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/MovingSubscriptionOffsetsValidationException.java @@ -4,12 +4,12 @@ import pl.allegro.tech.hermes.management.domain.ManagementException; public class MovingSubscriptionOffsetsValidationException extends ManagementException { - public MovingSubscriptionOffsetsValidationException(String msg) { - super(msg); - } + public MovingSubscriptionOffsetsValidationException(String msg) { + super(msg); + } - @Override - public ErrorCode getCode() { - return ErrorCode.MOVING_SUBSCRIPTION_OFFSETS_VALIDATION_ERROR; - } + @Override + public ErrorCode getCode() { + return ErrorCode.MOVING_SUBSCRIPTION_OFFSETS_VALIDATION_ERROR; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/MultiDCAwareService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/MultiDCAwareService.java index 10b314ca2a..17729e6d21 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/MultiDCAwareService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/MultiDCAwareService.java @@ -1,5 +1,17 @@ package pl.allegro.tech.hermes.management.infrastructure.kafka; +import static java.util.stream.Collectors.toList; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.ConsumerGroup; @@ -14,139 +26,147 @@ import pl.allegro.tech.hermes.management.domain.topic.UnableToMoveOffsetsException; import pl.allegro.tech.hermes.management.infrastructure.kafka.service.BrokersClusterService; -import java.time.Clock; -import java.time.Duration; -import java.time.Instant; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.function.Consumer; -import java.util.stream.Collectors; - -import static java.util.stream.Collectors.toList; - public class MultiDCAwareService { - private static final Logger logger = LoggerFactory.getLogger(MultiDCAwareService.class); - - private final List clusters; - private final Clock clock; - private final Duration intervalBetweenCheckingIfOffsetsMoved; - private final Duration offsetsMovedTimeout; - private final MultiDatacenterRepositoryCommandExecutor multiDcExecutor; - - public MultiDCAwareService(List clusters, Clock clock, - Duration intervalBetweenCheckingIfOffsetsMoved, Duration offsetsMovedTimeout, - MultiDatacenterRepositoryCommandExecutor multiDcExecutor) { - this.clusters = clusters; - this.clock = clock; - this.intervalBetweenCheckingIfOffsetsMoved = intervalBetweenCheckingIfOffsetsMoved; - this.offsetsMovedTimeout = offsetsMovedTimeout; - this.multiDcExecutor = multiDcExecutor; - } - - public void manageTopic(Consumer manageFunction) { - clusters.forEach(kafkaService -> kafkaService.manageTopic(manageFunction)); - } - - public String readMessageFromPrimary(String clusterName, Topic topic, Integer partition, Long offset) { - return clusters.stream() - .filter(cluster -> clusterName.equals(cluster.getClusterName())) - .findFirst() - .orElseThrow(() -> new BrokersClusterNotFoundException(clusterName)) - .readMessageFromPrimary(topic, partition, offset); - } - - public MultiDCOffsetChangeSummary retransmit(Topic topic, - String subscriptionName, - Long timestamp, - boolean dryRun, - RequestUser requester) { - MultiDCOffsetChangeSummary multiDCOffsetChangeSummary = new MultiDCOffsetChangeSummary(); - - clusters.forEach(cluster -> multiDCOffsetChangeSummary.addPartitionOffsetList( + private static final Logger logger = LoggerFactory.getLogger(MultiDCAwareService.class); + + private final List clusters; + private final Clock clock; + private final Duration intervalBetweenCheckingIfOffsetsMoved; + private final Duration offsetsMovedTimeout; + private final MultiDatacenterRepositoryCommandExecutor multiDcExecutor; + + public MultiDCAwareService( + List clusters, + Clock clock, + Duration intervalBetweenCheckingIfOffsetsMoved, + Duration offsetsMovedTimeout, + MultiDatacenterRepositoryCommandExecutor multiDcExecutor) { + this.clusters = clusters; + this.clock = clock; + this.intervalBetweenCheckingIfOffsetsMoved = intervalBetweenCheckingIfOffsetsMoved; + this.offsetsMovedTimeout = offsetsMovedTimeout; + this.multiDcExecutor = multiDcExecutor; + } + + public void manageTopic(Consumer manageFunction) { + clusters.forEach(kafkaService -> kafkaService.manageTopic(manageFunction)); + } + + public String readMessageFromPrimary( + String clusterName, Topic topic, Integer partition, Long offset) { + return clusters.stream() + .filter(cluster -> clusterName.equals(cluster.getClusterName())) + .findFirst() + .orElseThrow(() -> new BrokersClusterNotFoundException(clusterName)) + .readMessageFromPrimary(topic, partition, offset); + } + + public MultiDCOffsetChangeSummary retransmit( + Topic topic, String subscriptionName, Long timestamp, boolean dryRun, RequestUser requester) { + MultiDCOffsetChangeSummary multiDCOffsetChangeSummary = new MultiDCOffsetChangeSummary(); + + clusters.forEach( + cluster -> + multiDCOffsetChangeSummary.addPartitionOffsetList( cluster.getClusterName(), cluster.indicateOffsetChange(topic, subscriptionName, timestamp, dryRun))); - if (!dryRun) { - logger.info("Starting retransmission for subscription {}. Requested by {}. Retransmission timestamp: {}", - topic.getQualifiedName() + "$" + subscriptionName, requester.getUsername(), timestamp); - multiDcExecutor.executeByUser(new RetransmitCommand(new SubscriptionName(subscriptionName, topic.getName())), requester); - clusters.forEach(clusters -> waitUntilOffsetsAreMoved(topic, subscriptionName)); - logger.info( - "Successfully moved offsets for retransmission of subscription {}. Requested by user: {}. Retransmission timestamp: {}", - topic.getQualifiedName() + "$" + subscriptionName, requester.getUsername(), timestamp); - } - - return multiDCOffsetChangeSummary; + if (!dryRun) { + logger.info( + "Starting retransmission for subscription {}. Requested by {}. Retransmission timestamp: {}", + topic.getQualifiedName() + "$" + subscriptionName, + requester.getUsername(), + timestamp); + multiDcExecutor.executeByUser( + new RetransmitCommand(new SubscriptionName(subscriptionName, topic.getName())), + requester); + clusters.forEach(clusters -> waitUntilOffsetsAreMoved(topic, subscriptionName)); + logger.info( + "Successfully moved offsets for retransmission of subscription {}. Requested by user: {}. Retransmission timestamp: {}", + topic.getQualifiedName() + "$" + subscriptionName, + requester.getUsername(), + timestamp); } - public boolean areOffsetsAvailableOnAllKafkaTopics(Topic topic) { - return clusters.stream().allMatch(cluster -> cluster.areOffsetsAvailableOnAllKafkaTopics(topic)); + return multiDCOffsetChangeSummary; + } + + public boolean areOffsetsAvailableOnAllKafkaTopics(Topic topic) { + return clusters.stream() + .allMatch(cluster -> cluster.areOffsetsAvailableOnAllKafkaTopics(topic)); + } + + public boolean topicExists(Topic topic) { + return clusters.stream() + .allMatch(brokersClusterService -> brokersClusterService.topicExists(topic)); + } + + public Set listTopicFromAllDC() { + return clusters.stream() + .map(BrokersClusterService::listTopicsFromCluster) + .flatMap(Collection::stream) + .collect(Collectors.toCollection(HashSet::new)); + } + + public void removeTopicByName(String topicName) { + clusters.forEach(brokersClusterService -> brokersClusterService.removeTopicByName(topicName)); + } + + public void createConsumerGroups(Topic topic, Subscription subscription) { + clusters.forEach(clusterService -> clusterService.createConsumerGroup(topic, subscription)); + } + + private void waitUntilOffsetsAreMoved(Topic topic, String subscriptionName) { + Instant abortAttemptsInstant = clock.instant().plus(offsetsMovedTimeout); + + while (!areOffsetsMoved(topic, subscriptionName)) { + if (clock.instant().isAfter(abortAttemptsInstant)) { + logger.error( + "Not all offsets related to hermes subscription {}${} were moved.", + topic.getQualifiedName(), + subscriptionName); + throw new UnableToMoveOffsetsException(topic, subscriptionName); + } + logger.debug( + "Not all offsets related to hermes subscription {} were moved, will retry", + topic.getQualifiedName()); + + sleep(intervalBetweenCheckingIfOffsetsMoved); } + } - public boolean topicExists(Topic topic) { - return clusters.stream().allMatch(brokersClusterService -> brokersClusterService.topicExists(topic)); - } - - public Set listTopicFromAllDC() { - return clusters.stream() - .map(BrokersClusterService::listTopicsFromCluster) - .flatMap(Collection::stream) - .collect(Collectors.toCollection(HashSet::new)); - } + private boolean areOffsetsMoved(Topic topic, String subscriptionName) { + return clusters.stream().allMatch(cluster -> cluster.areOffsetsMoved(topic, subscriptionName)); + } - public void removeTopicByName(String topicName) { - clusters.forEach(brokersClusterService -> brokersClusterService.removeTopicByName(topicName)); + private void sleep(Duration sleepDuration) { + try { + Thread.sleep(sleepDuration.toMillis()); + } catch (InterruptedException e) { + throw new InternalProcessingException(e); } + } - public void createConsumerGroups(Topic topic, Subscription subscription) { - clusters.forEach(clusterService -> clusterService.createConsumerGroup(topic, subscription)); - } - - private void waitUntilOffsetsAreMoved(Topic topic, String subscriptionName) { - Instant abortAttemptsInstant = clock.instant().plus(offsetsMovedTimeout); - - while (!areOffsetsMoved(topic, subscriptionName)) { - if (clock.instant().isAfter(abortAttemptsInstant)) { - logger.error("Not all offsets related to hermes subscription {}${} were moved.", topic.getQualifiedName(), - subscriptionName); - throw new UnableToMoveOffsetsException(topic, subscriptionName); - } - logger.debug("Not all offsets related to hermes subscription {} were moved, will retry", topic.getQualifiedName()); - - sleep(intervalBetweenCheckingIfOffsetsMoved); - } - } - - private boolean areOffsetsMoved(Topic topic, String subscriptionName) { - return clusters.stream() - .allMatch(cluster -> cluster.areOffsetsMoved(topic, subscriptionName)); - } - - private void sleep(Duration sleepDuration) { - try { - Thread.sleep(sleepDuration.toMillis()); - } catch (InterruptedException e) { - throw new InternalProcessingException(e); - } - } - - public boolean allSubscriptionsHaveConsumersAssigned(Topic topic, List subscriptions) { - return clusters.stream().allMatch(brokersClusterService -> + public boolean allSubscriptionsHaveConsumersAssigned( + Topic topic, List subscriptions) { + return clusters.stream() + .allMatch( + brokersClusterService -> brokersClusterService.allSubscriptionsHaveConsumersAssigned(topic, subscriptions)); - } - - public List describeConsumerGroups(Topic topic, String subscriptionName) { - return clusters.stream().map(brokersClusterService -> brokersClusterService.describeConsumerGroup(topic, subscriptionName)) - .filter(Optional::isPresent) - .map(Optional::get) - .collect(toList()); - } - - public void moveOffsetsToTheEnd(Topic topic, SubscriptionName subscription) { - clusters.forEach(c -> c.moveOffsetsToTheEnd(topic, subscription)); - } + } + + public List describeConsumerGroups(Topic topic, String subscriptionName) { + return clusters.stream() + .map( + brokersClusterService -> + brokersClusterService.describeConsumerGroup(topic, subscriptionName)) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(toList()); + } + + public void moveOffsetsToTheEnd(Topic topic, SubscriptionName subscription) { + clusters.forEach(c -> c.moveOffsetsToTheEnd(topic, subscription)); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/MultiDCOffsetChangeSummary.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/MultiDCOffsetChangeSummary.java index c35449ddb1..202dd817d7 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/MultiDCOffsetChangeSummary.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/MultiDCOffsetChangeSummary.java @@ -3,27 +3,25 @@ import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.google.common.collect.ImmutableMap; -import pl.allegro.tech.hermes.common.kafka.offset.PartitionOffset; - import java.util.HashMap; import java.util.List; import java.util.Map; +import pl.allegro.tech.hermes.common.kafka.offset.PartitionOffset; public class MultiDCOffsetChangeSummary { - private Map> partitionOffsetListPerBrokerName = new HashMap<>(); - - public MultiDCOffsetChangeSummary() { - } + private Map> partitionOffsetListPerBrokerName = new HashMap<>(); - @JsonAnySetter - public void addPartitionOffsetList(String clusterName, List partitionOffsetChange) { - partitionOffsetListPerBrokerName.put(clusterName, partitionOffsetChange); - } + public MultiDCOffsetChangeSummary() {} - @JsonAnyGetter - public Map> getPartitionOffsetListPerBrokerName() { - return ImmutableMap.copyOf(partitionOffsetListPerBrokerName); - } + @JsonAnySetter + public void addPartitionOffsetList( + String clusterName, List partitionOffsetChange) { + partitionOffsetListPerBrokerName.put(clusterName, partitionOffsetChange); + } + @JsonAnyGetter + public Map> getPartitionOffsetListPerBrokerName() { + return ImmutableMap.copyOf(partitionOffsetListPerBrokerName); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/BrokersClusterService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/BrokersClusterService.java index d9ceed57be..2dcf3103f2 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/BrokersClusterService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/BrokersClusterService.java @@ -1,5 +1,20 @@ package pl.allegro.tech.hermes.management.infrastructure.kafka.service; +import static java.lang.String.format; +import static java.util.stream.Collectors.toMap; +import static java.util.stream.Collectors.toSet; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.kafka.clients.admin.AdminClient; @@ -22,182 +37,195 @@ import pl.allegro.tech.hermes.management.domain.topic.SingleMessageReader; import pl.allegro.tech.hermes.management.infrastructure.kafka.MovingSubscriptionOffsetsValidationException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.function.Consumer; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static java.lang.String.format; -import static java.util.stream.Collectors.toMap; -import static java.util.stream.Collectors.toSet; - public class BrokersClusterService { - private static final Logger logger = LoggerFactory.getLogger(BrokersClusterService.class); - - private final String clusterName; - private final SingleMessageReader singleMessageReader; - private final RetransmissionService retransmissionService; - private final BrokerTopicManagement brokerTopicManagement; - private final KafkaNamesMapper kafkaNamesMapper; - private final OffsetsAvailableChecker offsetsAvailableChecker; - private final ConsumerGroupsDescriber consumerGroupsDescriber; - private final AdminClient adminClient; - private final ConsumerGroupManager consumerGroupManager; - private final KafkaConsumerManager kafkaConsumerManager; - - public BrokersClusterService(String clusterName, SingleMessageReader singleMessageReader, - RetransmissionService retransmissionService, BrokerTopicManagement brokerTopicManagement, - KafkaNamesMapper kafkaNamesMapper, OffsetsAvailableChecker offsetsAvailableChecker, - LogEndOffsetChecker logEndOffsetChecker, AdminClient adminClient, - ConsumerGroupManager consumerGroupManager, - KafkaConsumerManager kafkaConsumerManager) { - this.clusterName = clusterName; - this.singleMessageReader = singleMessageReader; - this.retransmissionService = retransmissionService; - this.brokerTopicManagement = brokerTopicManagement; - this.kafkaNamesMapper = kafkaNamesMapper; - this.offsetsAvailableChecker = offsetsAvailableChecker; - this.consumerGroupsDescriber = new ConsumerGroupsDescriber( - kafkaNamesMapper, - adminClient, - logEndOffsetChecker, - clusterName - ); - this.adminClient = adminClient; - this.consumerGroupManager = consumerGroupManager; - this.kafkaConsumerManager = kafkaConsumerManager; - } - - public String getClusterName() { - return clusterName; - } - - public void manageTopic(Consumer manageFunction) { - manageFunction.accept(brokerTopicManagement); - } - - public String readMessageFromPrimary(Topic topic, Integer partition, Long offset) { - return singleMessageReader.readMessageAsJson(topic, kafkaNamesMapper.toKafkaTopics(topic).getPrimary(), partition, offset); - } - - public List indicateOffsetChange(Topic topic, String subscriptionName, Long timestamp, boolean dryRun) { - return retransmissionService.indicateOffsetChange(topic, subscriptionName, clusterName, timestamp, dryRun); - } - - public boolean areOffsetsAvailableOnAllKafkaTopics(Topic topic) { - return kafkaNamesMapper.toKafkaTopics(topic).allMatch(offsetsAvailableChecker::areOffsetsAvailable); - } - - public boolean topicExists(Topic topic) { - return brokerTopicManagement.topicExists(topic); - } - - public List listTopicsFromCluster() { - try { - return new ArrayList<>(adminClient.listTopics().names().get()); - } catch (ExecutionException | InterruptedException e) { - logger.error("Failed to list topics names", e); - return Collections.emptyList(); - } - } - - public void removeTopicByName(String topicName) { - adminClient.deleteTopics(Collections.singletonList(topicName)); - } - - public boolean areOffsetsMoved(Topic topic, String subscriptionName) { - return retransmissionService.areOffsetsMoved(topic, subscriptionName, clusterName); - } - - public boolean allSubscriptionsHaveConsumersAssigned(Topic topic, List subscriptions) { - List consumerGroupsForSubscriptions = subscriptions.stream() - .map(sub -> kafkaNamesMapper.toConsumerGroupId(sub.getQualifiedName()).asString()) - .collect(Collectors.toList()); - - try { - int requiredTotalNumberOfAssignments = numberOfPartitionsForTopic(topic) * subscriptions.size(); - return numberOfAssignmentsForConsumersGroups(consumerGroupsForSubscriptions) == requiredTotalNumberOfAssignments; - } catch (Exception e) { - logger.error("Failed to check assignments for topic " + topic.getQualifiedName() + " subscriptions", e); - return false; - } - } - - public void createConsumerGroup(Topic topic, Subscription subscription) { - consumerGroupManager.createConsumerGroup(topic, subscription); - } - - public Optional describeConsumerGroup(Topic topic, String subscriptionName) { - return consumerGroupsDescriber.describeConsumerGroup(topic, subscriptionName); - } - - public void moveOffsetsToTheEnd(Topic topic, SubscriptionName subscription) { - validateIfOffsetsCanBeMoved(topic, subscription); - - KafkaConsumer consumer = kafkaConsumerManager.createConsumer(subscription); - String kafkaTopicName = kafkaNamesMapper.toKafkaTopics(topic).getPrimary().name().asString(); - Set topicPartitions = getTopicPartitions(consumer, kafkaTopicName); - consumer.assign(topicPartitions); - - Map endOffsets = consumer.endOffsets(topicPartitions); - Map endOffsetsMetadata = buildOffsetsMetadata(endOffsets); - consumer.commitSync(endOffsetsMetadata); - consumer.close(); - - logger.info("Successfully moved offset to the end position for subscription {} and consumer group {}", - subscription.getQualifiedName(), kafkaNamesMapper.toConsumerGroupId(subscription)); - } - - private int numberOfAssignmentsForConsumersGroups(List consumerGroupsIds) throws ExecutionException, InterruptedException { - Collection consumerGroupsDescriptions = - adminClient.describeConsumerGroups(consumerGroupsIds).all().get().values(); - Stream memberDescriptions = consumerGroupsDescriptions.stream().flatMap(desc -> desc.members().stream()); - return memberDescriptions.flatMap(memberDescription -> memberDescription.assignment().topicPartitions().stream()) - .collect(Collectors.toList()).size(); - } - - private void validateIfOffsetsCanBeMoved(Topic topic, SubscriptionName subscription) { - describeConsumerGroup(topic, subscription.getName()) - .ifPresentOrElse( - group -> { - if (!group.getMembers().isEmpty()) { - String s = format("Consumer group %s for subscription %s has still active members.", - group.getGroupId(), subscription.getQualifiedName()); - throw new MovingSubscriptionOffsetsValidationException(s); - } - }, () -> { - String s = format("No consumer group for subscription %s exists.", subscription.getQualifiedName()); - throw new MovingSubscriptionOffsetsValidationException(s); - }); - } - - private int numberOfPartitionsForTopic(Topic topic) throws ExecutionException, InterruptedException { - List kafkaTopicsNames = kafkaNamesMapper.toKafkaTopics(topic).stream() - .map(kafkaTopic -> kafkaTopic.name().asString()) - .collect(Collectors.toList()); - - return adminClient.describeTopics(kafkaTopicsNames).all().get().values().stream() - .map(v -> v.partitions().size()) - .reduce(0, Integer::sum); - } - - private Set getTopicPartitions(KafkaConsumer consumer, String kafkaTopicName) { - return consumer.partitionsFor(kafkaTopicName).stream() - .map(info -> new TopicPartition(info.topic(), info.partition())) - .collect(toSet()); - } - - private Map buildOffsetsMetadata(Map offsets) { - return offsets.entrySet().stream() - .map(entry -> ImmutablePair.of(entry.getKey(), new OffsetAndMetadata(entry.getValue()))) - .collect(toMap(Pair::getKey, Pair::getValue)); - } + private static final Logger logger = LoggerFactory.getLogger(BrokersClusterService.class); + + private final String clusterName; + private final SingleMessageReader singleMessageReader; + private final RetransmissionService retransmissionService; + private final BrokerTopicManagement brokerTopicManagement; + private final KafkaNamesMapper kafkaNamesMapper; + private final OffsetsAvailableChecker offsetsAvailableChecker; + private final ConsumerGroupsDescriber consumerGroupsDescriber; + private final AdminClient adminClient; + private final ConsumerGroupManager consumerGroupManager; + private final KafkaConsumerManager kafkaConsumerManager; + + public BrokersClusterService( + String clusterName, + SingleMessageReader singleMessageReader, + RetransmissionService retransmissionService, + BrokerTopicManagement brokerTopicManagement, + KafkaNamesMapper kafkaNamesMapper, + OffsetsAvailableChecker offsetsAvailableChecker, + LogEndOffsetChecker logEndOffsetChecker, + AdminClient adminClient, + ConsumerGroupManager consumerGroupManager, + KafkaConsumerManager kafkaConsumerManager) { + this.clusterName = clusterName; + this.singleMessageReader = singleMessageReader; + this.retransmissionService = retransmissionService; + this.brokerTopicManagement = brokerTopicManagement; + this.kafkaNamesMapper = kafkaNamesMapper; + this.offsetsAvailableChecker = offsetsAvailableChecker; + this.consumerGroupsDescriber = + new ConsumerGroupsDescriber( + kafkaNamesMapper, adminClient, logEndOffsetChecker, clusterName); + this.adminClient = adminClient; + this.consumerGroupManager = consumerGroupManager; + this.kafkaConsumerManager = kafkaConsumerManager; + } + + public String getClusterName() { + return clusterName; + } + + public void manageTopic(Consumer manageFunction) { + manageFunction.accept(brokerTopicManagement); + } + + public String readMessageFromPrimary(Topic topic, Integer partition, Long offset) { + return singleMessageReader.readMessageAsJson( + topic, kafkaNamesMapper.toKafkaTopics(topic).getPrimary(), partition, offset); + } + + public List indicateOffsetChange( + Topic topic, String subscriptionName, Long timestamp, boolean dryRun) { + return retransmissionService.indicateOffsetChange( + topic, subscriptionName, clusterName, timestamp, dryRun); + } + + public boolean areOffsetsAvailableOnAllKafkaTopics(Topic topic) { + return kafkaNamesMapper + .toKafkaTopics(topic) + .allMatch(offsetsAvailableChecker::areOffsetsAvailable); + } + + public boolean topicExists(Topic topic) { + return brokerTopicManagement.topicExists(topic); + } + + public List listTopicsFromCluster() { + try { + return new ArrayList<>(adminClient.listTopics().names().get()); + } catch (ExecutionException | InterruptedException e) { + logger.error("Failed to list topics names", e); + return Collections.emptyList(); + } + } + + public void removeTopicByName(String topicName) { + adminClient.deleteTopics(Collections.singletonList(topicName)); + } + + public boolean areOffsetsMoved(Topic topic, String subscriptionName) { + return retransmissionService.areOffsetsMoved(topic, subscriptionName, clusterName); + } + + public boolean allSubscriptionsHaveConsumersAssigned( + Topic topic, List subscriptions) { + List consumerGroupsForSubscriptions = + subscriptions.stream() + .map(sub -> kafkaNamesMapper.toConsumerGroupId(sub.getQualifiedName()).asString()) + .collect(Collectors.toList()); + + try { + int requiredTotalNumberOfAssignments = + numberOfPartitionsForTopic(topic) * subscriptions.size(); + return numberOfAssignmentsForConsumersGroups(consumerGroupsForSubscriptions) + == requiredTotalNumberOfAssignments; + } catch (Exception e) { + logger.error( + "Failed to check assignments for topic " + topic.getQualifiedName() + " subscriptions", + e); + return false; + } + } + + public void createConsumerGroup(Topic topic, Subscription subscription) { + consumerGroupManager.createConsumerGroup(topic, subscription); + } + + public Optional describeConsumerGroup(Topic topic, String subscriptionName) { + return consumerGroupsDescriber.describeConsumerGroup(topic, subscriptionName); + } + + public void moveOffsetsToTheEnd(Topic topic, SubscriptionName subscription) { + validateIfOffsetsCanBeMoved(topic, subscription); + + KafkaConsumer consumer = kafkaConsumerManager.createConsumer(subscription); + String kafkaTopicName = kafkaNamesMapper.toKafkaTopics(topic).getPrimary().name().asString(); + Set topicPartitions = getTopicPartitions(consumer, kafkaTopicName); + consumer.assign(topicPartitions); + + Map endOffsets = consumer.endOffsets(topicPartitions); + Map endOffsetsMetadata = buildOffsetsMetadata(endOffsets); + consumer.commitSync(endOffsetsMetadata); + consumer.close(); + + logger.info( + "Successfully moved offset to the end position for subscription {} and consumer group {}", + subscription.getQualifiedName(), + kafkaNamesMapper.toConsumerGroupId(subscription)); + } + + private int numberOfAssignmentsForConsumersGroups(List consumerGroupsIds) + throws ExecutionException, InterruptedException { + Collection consumerGroupsDescriptions = + adminClient.describeConsumerGroups(consumerGroupsIds).all().get().values(); + Stream memberDescriptions = + consumerGroupsDescriptions.stream().flatMap(desc -> desc.members().stream()); + return memberDescriptions + .flatMap(memberDescription -> memberDescription.assignment().topicPartitions().stream()) + .collect(Collectors.toList()) + .size(); + } + + private void validateIfOffsetsCanBeMoved(Topic topic, SubscriptionName subscription) { + describeConsumerGroup(topic, subscription.getName()) + .ifPresentOrElse( + group -> { + if (!group.getMembers().isEmpty()) { + String s = + format( + "Consumer group %s for subscription %s has still active members.", + group.getGroupId(), subscription.getQualifiedName()); + throw new MovingSubscriptionOffsetsValidationException(s); + } + }, + () -> { + String s = + format( + "No consumer group for subscription %s exists.", + subscription.getQualifiedName()); + throw new MovingSubscriptionOffsetsValidationException(s); + }); + } + + private int numberOfPartitionsForTopic(Topic topic) + throws ExecutionException, InterruptedException { + List kafkaTopicsNames = + kafkaNamesMapper.toKafkaTopics(topic).stream() + .map(kafkaTopic -> kafkaTopic.name().asString()) + .collect(Collectors.toList()); + + return adminClient.describeTopics(kafkaTopicsNames).all().get().values().stream() + .map(v -> v.partitions().size()) + .reduce(0, Integer::sum); + } + + private Set getTopicPartitions( + KafkaConsumer consumer, String kafkaTopicName) { + return consumer.partitionsFor(kafkaTopicName).stream() + .map(info -> new TopicPartition(info.topic(), info.partition())) + .collect(toSet()); + } + + private Map buildOffsetsMetadata( + Map offsets) { + return offsets.entrySet().stream() + .map(entry -> ImmutablePair.of(entry.getKey(), new OffsetAndMetadata(entry.getValue()))) + .collect(toMap(Pair::getKey, Pair::getValue)); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/ConsumerGroupsDescriber.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/ConsumerGroupsDescriber.java index 46d87599f5..36aef4bbe8 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/ConsumerGroupsDescriber.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/ConsumerGroupsDescriber.java @@ -1,5 +1,15 @@ package pl.allegro.tech.hermes.management.infrastructure.kafka.service; +import static java.util.stream.Collectors.toMap; +import static java.util.stream.Collectors.toSet; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Collections; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutionException; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.ConsumerGroupDescription; import org.apache.kafka.clients.admin.MemberDescription; @@ -20,103 +30,109 @@ import pl.allegro.tech.hermes.common.kafka.KafkaTopics; import pl.allegro.tech.hermes.management.infrastructure.kafka.BrokersClusterCommunicationException; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.Collections; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ExecutionException; - -import static java.util.stream.Collectors.toMap; -import static java.util.stream.Collectors.toSet; - class ConsumerGroupsDescriber { - private static final Logger logger = LoggerFactory.getLogger(ConsumerGroupsDescriber.class); + private static final Logger logger = LoggerFactory.getLogger(ConsumerGroupsDescriber.class); - private final KafkaNamesMapper kafkaNamesMapper; - private final AdminClient adminClient; - private final LogEndOffsetChecker logEndOffsetChecker; - private final String clusterName; + private final KafkaNamesMapper kafkaNamesMapper; + private final AdminClient adminClient; + private final LogEndOffsetChecker logEndOffsetChecker; + private final String clusterName; - ConsumerGroupsDescriber(KafkaNamesMapper kafkaNamesMapper, AdminClient adminClient, - LogEndOffsetChecker logEndOffsetChecker, String clusterName) { - this.kafkaNamesMapper = kafkaNamesMapper; - this.adminClient = adminClient; - this.logEndOffsetChecker = logEndOffsetChecker; - this.clusterName = clusterName; - } + ConsumerGroupsDescriber( + KafkaNamesMapper kafkaNamesMapper, + AdminClient adminClient, + LogEndOffsetChecker logEndOffsetChecker, + String clusterName) { + this.kafkaNamesMapper = kafkaNamesMapper; + this.adminClient = adminClient; + this.logEndOffsetChecker = logEndOffsetChecker; + this.clusterName = clusterName; + } - Optional describeConsumerGroup(Topic topic, String subscriptionName) { - ConsumerGroupId consumerGroupId = kafkaNamesMapper.toConsumerGroupId(new SubscriptionName(subscriptionName, topic.getName())); - KafkaTopics kafkaTopics = kafkaNamesMapper.toKafkaTopics(topic); - try { - return describeConsumerGroup(consumerGroupId, kafkaTopics); - } catch (Exception e) { - logger.error("Failed to describe group with id: {}", consumerGroupId.asString(), e); - throw new BrokersClusterCommunicationException(e); - } + Optional describeConsumerGroup(Topic topic, String subscriptionName) { + ConsumerGroupId consumerGroupId = + kafkaNamesMapper.toConsumerGroupId(new SubscriptionName(subscriptionName, topic.getName())); + KafkaTopics kafkaTopics = kafkaNamesMapper.toKafkaTopics(topic); + try { + return describeConsumerGroup(consumerGroupId, kafkaTopics); + } catch (Exception e) { + logger.error("Failed to describe group with id: {}", consumerGroupId.asString(), e); + throw new BrokersClusterCommunicationException(e); } + } - private Optional describeConsumerGroup(ConsumerGroupId consumerGroupId, - KafkaTopics kafkaTopics) throws ExecutionException, InterruptedException { - Map kafkaTopicContentTypes = kafkaTopics.stream() - .collect(toMap(KafkaTopic::name, KafkaTopic::contentType)); - Map topicPartitionOffsets = adminClient - .listConsumerGroupOffsets(consumerGroupId.asString()) - .partitionsToOffsetAndMetadata() - .get(); - Optional description = adminClient - .describeConsumerGroups(Collections.singletonList(consumerGroupId.asString())) - .all() - .get() - .values() - .stream() - .findFirst(); + private Optional describeConsumerGroup( + ConsumerGroupId consumerGroupId, KafkaTopics kafkaTopics) + throws ExecutionException, InterruptedException { + Map kafkaTopicContentTypes = + kafkaTopics.stream().collect(toMap(KafkaTopic::name, KafkaTopic::contentType)); + Map topicPartitionOffsets = + adminClient + .listConsumerGroupOffsets(consumerGroupId.asString()) + .partitionsToOffsetAndMetadata() + .get(); + Optional description = + adminClient + .describeConsumerGroups(Collections.singletonList(consumerGroupId.asString())) + .all() + .get() + .values() + .stream() + .findFirst(); - return description - .map(d -> d.state() != ConsumerGroupState.DEAD ? d : null) - .map(d -> getKafkaConsumerGroup(topicPartitionOffsets, kafkaTopicContentTypes, d)); - } + return description + .map(d -> d.state() != ConsumerGroupState.DEAD ? d : null) + .map(d -> getKafkaConsumerGroup(topicPartitionOffsets, kafkaTopicContentTypes, d)); + } - private ConsumerGroup getKafkaConsumerGroup(Map topicPartitionOffsets, - Map kafkaTopicContentTypes, - ConsumerGroupDescription description) { - Set groupMembers = description.members().stream() - .map(member -> getKafkaConsumerGroupMember(topicPartitionOffsets, kafkaTopicContentTypes, member)) - .collect(toSet()); + private ConsumerGroup getKafkaConsumerGroup( + Map topicPartitionOffsets, + Map kafkaTopicContentTypes, + ConsumerGroupDescription description) { + Set groupMembers = + description.members().stream() + .map( + member -> + getKafkaConsumerGroupMember( + topicPartitionOffsets, kafkaTopicContentTypes, member)) + .collect(toSet()); - return new ConsumerGroup(clusterName, description.groupId(), description.state().toString(), groupMembers); - } + return new ConsumerGroup( + clusterName, description.groupId(), description.state().toString(), groupMembers); + } - private ConsumerGroupMember getKafkaConsumerGroupMember(Map topicPartitionOffsets, - Map kafkaTopicContentTypes, - MemberDescription member) { - Set kafkaTopicPartitions = member.assignment().topicPartitions().stream().map( + private ConsumerGroupMember getKafkaConsumerGroupMember( + Map topicPartitionOffsets, + Map kafkaTopicContentTypes, + MemberDescription member) { + Set kafkaTopicPartitions = + member.assignment().topicPartitions().stream() + .map( topicPartition -> { - Optional offset = Optional.ofNullable(topicPartitionOffsets.get(topicPartition)); - return new pl.allegro.tech.hermes.api.TopicPartition( - topicPartition.partition(), - topicPartition.topic(), - offset.map(OffsetAndMetadata::offset).orElse(0L), - logEndOffsetChecker.check(topicPartition), - offset.map(OffsetAndMetadata::metadata).orElse(""), - kafkaTopicContentTypes.get(KafkaTopicName.valueOf(topicPartition.topic())) - ); - } - ).collect(toSet()); - return new ConsumerGroupMember(member.consumerId(), member.clientId(), toHostName(member.host()), kafkaTopicPartitions); - } + Optional offset = + Optional.ofNullable(topicPartitionOffsets.get(topicPartition)); + return new pl.allegro.tech.hermes.api.TopicPartition( + topicPartition.partition(), + topicPartition.topic(), + offset.map(OffsetAndMetadata::offset).orElse(0L), + logEndOffsetChecker.check(topicPartition), + offset.map(OffsetAndMetadata::metadata).orElse(""), + kafkaTopicContentTypes.get(KafkaTopicName.valueOf(topicPartition.topic()))); + }) + .collect(toSet()); + return new ConsumerGroupMember( + member.consumerId(), member.clientId(), toHostName(member.host()), kafkaTopicPartitions); + } - private static String toHostName(String inetAddressStringRepresentation) { - String[] parts = inetAddressStringRepresentation.split("/"); - String ip = parts[parts.length - 1]; - try { - InetAddress addr = InetAddress.getByName(ip); - return addr.getHostName(); - } catch (UnknownHostException e) { - return inetAddressStringRepresentation; - } + private static String toHostName(String inetAddressStringRepresentation) { + String[] parts = inetAddressStringRepresentation.split("/"); + String ip = parts[parts.length - 1]; + try { + InetAddress addr = InetAddress.getByName(ip); + return addr.getHostName(); + } catch (UnknownHostException e) { + return inetAddressStringRepresentation; } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/KafkaBrokerTopicManagement.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/KafkaBrokerTopicManagement.java index f799c0b03c..537821f91f 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/KafkaBrokerTopicManagement.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/KafkaBrokerTopicManagement.java @@ -1,5 +1,11 @@ package pl.allegro.tech.hermes.management.infrastructure.kafka.service; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.Config; import org.apache.kafka.clients.admin.ConfigEntry; @@ -19,133 +25,143 @@ import pl.allegro.tech.hermes.management.domain.topic.BrokerTopicManagement; import pl.allegro.tech.hermes.management.infrastructure.kafka.BrokersClusterCommunicationException; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; - public class KafkaBrokerTopicManagement implements BrokerTopicManagement { - private final TopicProperties topicProperties; - - private final AdminClient kafkaAdminClient; - - private final KafkaNamesMapper kafkaNamesMapper; - - private final String datacenterName; - - private static final Logger logger = LoggerFactory.getLogger(KafkaBrokerTopicManagement.class); - - - public KafkaBrokerTopicManagement(TopicProperties topicProperties, AdminClient kafkaAdminClient, KafkaNamesMapper kafkaNamesMapper, String datacenterName) { - this.topicProperties = topicProperties; - this.kafkaAdminClient = kafkaAdminClient; - this.kafkaNamesMapper = kafkaNamesMapper; - this.datacenterName = datacenterName; - } - - @Override - public void createTopic(Topic topic) { - Map config = createTopicConfig(topic.getRetentionTime().getDurationInMillis(), topicProperties); - - kafkaNamesMapper.toKafkaTopics(topic).stream().map(k -> - kafkaAdminClient.createTopics(Collections.singletonList( - new NewTopic( - k.name().asString(), - topicProperties.getPartitions(), - (short) topicProperties.getReplicationFactor() - ).configs(config) - )) - ).map(CreateTopicsResult::all) - .forEach(this::waitForKafkaFuture); - } - - @Override - public void removeTopic(Topic topic) { - kafkaNamesMapper.toKafkaTopics(topic).stream() - .map(k -> kafkaAdminClient.deleteTopics(Collections.singletonList(k.name().asString()))) - .map(DeleteTopicsResult::all) - .forEach(future -> { - logger.info("Removing topic: {} from Kafka dc: {}", topic, datacenterName); - long start = System.currentTimeMillis(); - waitForKafkaFuture(future); - logger.info("Removed topic: {} from Kafka dc: {} in {} ms", topic, datacenterName, System.currentTimeMillis() - start); - }); - } - - @Override - public void updateTopic(Topic topic) { - Map config = createTopicConfig(topic.getRetentionTime().getDurationInMillis(), topicProperties); - KafkaTopics kafkaTopics = kafkaNamesMapper.toKafkaTopics(topic); - - if (isMigrationToNewKafkaTopic(kafkaTopics)) { - KafkaFuture createTopicsFuture = kafkaAdminClient.createTopics(Collections.singletonList( - new NewTopic( - kafkaTopics.getPrimary().name().asString(), - topicProperties.getPartitions(), - (short) topicProperties.getReplicationFactor() - ).configs(config) - )).all(); - waitForKafkaFuture(createTopicsFuture); - } else { - doUpdateTopic(kafkaTopics.getPrimary(), config); - } - - kafkaTopics.getSecondary().ifPresent(secondary -> - doUpdateTopic(secondary, config) - ); - } - - @Override - public boolean topicExists(Topic topic) { - return kafkaNamesMapper.toKafkaTopics(topic) - .allMatch(this::doesTopicExist); - } - - private boolean isMigrationToNewKafkaTopic(KafkaTopics kafkaTopics) { - return kafkaTopics.getSecondary().isPresent() - && !doesTopicExist(kafkaTopics.getPrimary()); - } - - private boolean doesTopicExist(KafkaTopic topic) { - KafkaFuture topicExistsFuture = - kafkaAdminClient.listTopics().names().thenApply(names -> names.contains(topic.name().asString())); - return waitForKafkaFuture(topicExistsFuture); - } - - private void doUpdateTopic(KafkaTopic topic, Map configMap) { - ConfigResource topicConfigResource = new ConfigResource( - ConfigResource.Type.TOPIC, - topic.name().asString() - ); - - Collection configEntries = configMap.entrySet().stream().map(entry -> - new ConfigEntry(entry.getKey(), entry.getValue()) - ).collect(Collectors.toList()); - - Map configUpdates = new HashMap<>(); - configUpdates.put(topicConfigResource, new Config(configEntries)); - - KafkaFuture updateTopicFuture = kafkaAdminClient.alterConfigs(configUpdates).all(); - waitForKafkaFuture(updateTopicFuture); - } - - private Map createTopicConfig(long retentionPolicy, TopicProperties topicProperties) { - Map props = new HashMap<>(); - props.put(TopicConfig.RETENTION_MS_CONFIG, String.valueOf(retentionPolicy)); - props.put(TopicConfig.UNCLEAN_LEADER_ELECTION_ENABLE_CONFIG, Boolean.toString(topicProperties.isUncleanLeaderElectionEnabled())); - props.put(TopicConfig.MAX_MESSAGE_BYTES_CONFIG, String.valueOf(topicProperties.getMaxMessageSize())); - - return props; + private final TopicProperties topicProperties; + + private final AdminClient kafkaAdminClient; + + private final KafkaNamesMapper kafkaNamesMapper; + + private final String datacenterName; + + private static final Logger logger = LoggerFactory.getLogger(KafkaBrokerTopicManagement.class); + + public KafkaBrokerTopicManagement( + TopicProperties topicProperties, + AdminClient kafkaAdminClient, + KafkaNamesMapper kafkaNamesMapper, + String datacenterName) { + this.topicProperties = topicProperties; + this.kafkaAdminClient = kafkaAdminClient; + this.kafkaNamesMapper = kafkaNamesMapper; + this.datacenterName = datacenterName; + } + + @Override + public void createTopic(Topic topic) { + Map config = + createTopicConfig(topic.getRetentionTime().getDurationInMillis(), topicProperties); + + kafkaNamesMapper.toKafkaTopics(topic).stream() + .map( + k -> + kafkaAdminClient.createTopics( + Collections.singletonList( + new NewTopic( + k.name().asString(), + topicProperties.getPartitions(), + (short) topicProperties.getReplicationFactor()) + .configs(config)))) + .map(CreateTopicsResult::all) + .forEach(this::waitForKafkaFuture); + } + + @Override + public void removeTopic(Topic topic) { + kafkaNamesMapper.toKafkaTopics(topic).stream() + .map(k -> kafkaAdminClient.deleteTopics(Collections.singletonList(k.name().asString()))) + .map(DeleteTopicsResult::all) + .forEach( + future -> { + logger.info("Removing topic: {} from Kafka dc: {}", topic, datacenterName); + long start = System.currentTimeMillis(); + waitForKafkaFuture(future); + logger.info( + "Removed topic: {} from Kafka dc: {} in {} ms", + topic, + datacenterName, + System.currentTimeMillis() - start); + }); + } + + @Override + public void updateTopic(Topic topic) { + Map config = + createTopicConfig(topic.getRetentionTime().getDurationInMillis(), topicProperties); + KafkaTopics kafkaTopics = kafkaNamesMapper.toKafkaTopics(topic); + + if (isMigrationToNewKafkaTopic(kafkaTopics)) { + KafkaFuture createTopicsFuture = + kafkaAdminClient + .createTopics( + Collections.singletonList( + new NewTopic( + kafkaTopics.getPrimary().name().asString(), + topicProperties.getPartitions(), + (short) topicProperties.getReplicationFactor()) + .configs(config))) + .all(); + waitForKafkaFuture(createTopicsFuture); + } else { + doUpdateTopic(kafkaTopics.getPrimary(), config); } - private T waitForKafkaFuture(KafkaFuture future) { - try { - return future.get(); - } catch (InterruptedException | ExecutionException e) { - throw new BrokersClusterCommunicationException(e); - } + kafkaTopics.getSecondary().ifPresent(secondary -> doUpdateTopic(secondary, config)); + } + + @Override + public boolean topicExists(Topic topic) { + return kafkaNamesMapper.toKafkaTopics(topic).allMatch(this::doesTopicExist); + } + + private boolean isMigrationToNewKafkaTopic(KafkaTopics kafkaTopics) { + return kafkaTopics.getSecondary().isPresent() && !doesTopicExist(kafkaTopics.getPrimary()); + } + + private boolean doesTopicExist(KafkaTopic topic) { + KafkaFuture topicExistsFuture = + kafkaAdminClient + .listTopics() + .names() + .thenApply(names -> names.contains(topic.name().asString())); + return waitForKafkaFuture(topicExistsFuture); + } + + private void doUpdateTopic(KafkaTopic topic, Map configMap) { + ConfigResource topicConfigResource = + new ConfigResource(ConfigResource.Type.TOPIC, topic.name().asString()); + + Collection configEntries = + configMap.entrySet().stream() + .map(entry -> new ConfigEntry(entry.getKey(), entry.getValue())) + .collect(Collectors.toList()); + + Map configUpdates = new HashMap<>(); + configUpdates.put(topicConfigResource, new Config(configEntries)); + + KafkaFuture updateTopicFuture = kafkaAdminClient.alterConfigs(configUpdates).all(); + waitForKafkaFuture(updateTopicFuture); + } + + private Map createTopicConfig( + long retentionPolicy, TopicProperties topicProperties) { + Map props = new HashMap<>(); + props.put(TopicConfig.RETENTION_MS_CONFIG, String.valueOf(retentionPolicy)); + props.put( + TopicConfig.UNCLEAN_LEADER_ELECTION_ENABLE_CONFIG, + Boolean.toString(topicProperties.isUncleanLeaderElectionEnabled())); + props.put( + TopicConfig.MAX_MESSAGE_BYTES_CONFIG, String.valueOf(topicProperties.getMaxMessageSize())); + + return props; + } + + private T waitForKafkaFuture(KafkaFuture future) { + try { + return future.get(); + } catch (InterruptedException | ExecutionException e) { + throw new BrokersClusterCommunicationException(e); } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/KafkaConsumerGroupManager.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/KafkaConsumerGroupManager.java index 09d19d761c..8957d84e08 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/KafkaConsumerGroupManager.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/KafkaConsumerGroupManager.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.management.infrastructure.kafka.service; +import static java.util.stream.Collectors.toMap; +import static java.util.stream.Collectors.toSet; + +import java.util.Map; +import java.util.Set; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.kafka.clients.consumer.KafkaConsumer; @@ -13,59 +18,66 @@ import pl.allegro.tech.hermes.management.config.kafka.KafkaProperties; import pl.allegro.tech.hermes.management.domain.subscription.ConsumerGroupManager; -import java.util.Map; -import java.util.Set; - -import static java.util.stream.Collectors.toMap; -import static java.util.stream.Collectors.toSet; - public class KafkaConsumerGroupManager implements ConsumerGroupManager { - private final Logger logger = LoggerFactory.getLogger(KafkaConsumerGroupManager.class); + private final Logger logger = LoggerFactory.getLogger(KafkaConsumerGroupManager.class); - private final KafkaNamesMapper kafkaNamesMapper; - private final String clusterName; - private final KafkaConsumerManager consumerManager; + private final KafkaNamesMapper kafkaNamesMapper; + private final String clusterName; + private final KafkaConsumerManager consumerManager; - public KafkaConsumerGroupManager(KafkaNamesMapper kafkaNamesMapper, - String clusterName, - String brokerList, - KafkaProperties kafkaProperties) { - this.kafkaNamesMapper = kafkaNamesMapper; - this.clusterName = clusterName; - this.consumerManager = new KafkaConsumerManager(kafkaProperties, kafkaNamesMapper, brokerList); - } + public KafkaConsumerGroupManager( + KafkaNamesMapper kafkaNamesMapper, + String clusterName, + String brokerList, + KafkaProperties kafkaProperties) { + this.kafkaNamesMapper = kafkaNamesMapper; + this.clusterName = clusterName; + this.consumerManager = new KafkaConsumerManager(kafkaProperties, kafkaNamesMapper, brokerList); + } - @Override - public void createConsumerGroup(Topic topic, Subscription subscription) { - logger.info("Creating consumer group for subscription {}, cluster: {}", subscription.getQualifiedName(), clusterName); + @Override + public void createConsumerGroup(Topic topic, Subscription subscription) { + logger.info( + "Creating consumer group for subscription {}, cluster: {}", + subscription.getQualifiedName(), + clusterName); - KafkaConsumer kafkaConsumer = consumerManager.createConsumer(subscription.getQualifiedName()); - try { - String kafkaTopicName = kafkaNamesMapper.toKafkaTopics(topic).getPrimary().name().asString(); - Set topicPartitions = kafkaConsumer.partitionsFor(kafkaTopicName).stream() - .map(info -> new TopicPartition(info.topic(), info.partition())) - .collect(toSet()); + KafkaConsumer kafkaConsumer = + consumerManager.createConsumer(subscription.getQualifiedName()); + try { + String kafkaTopicName = kafkaNamesMapper.toKafkaTopics(topic).getPrimary().name().asString(); + Set topicPartitions = + kafkaConsumer.partitionsFor(kafkaTopicName).stream() + .map(info -> new TopicPartition(info.topic(), info.partition())) + .collect(toSet()); - logger.info("Received partitions: {}, cluster: {}", topicPartitions, clusterName); + logger.info("Received partitions: {}, cluster: {}", topicPartitions, clusterName); - kafkaConsumer.assign(topicPartitions); + kafkaConsumer.assign(topicPartitions); - Map topicPartitionByOffset = topicPartitions.stream() - .map(topicPartition -> { - long offset = kafkaConsumer.position(topicPartition); - return ImmutablePair.of(topicPartition, new OffsetAndMetadata(offset)); - }) - .collect(toMap(Pair::getKey, Pair::getValue)); + Map topicPartitionByOffset = + topicPartitions.stream() + .map( + topicPartition -> { + long offset = kafkaConsumer.position(topicPartition); + return ImmutablePair.of(topicPartition, new OffsetAndMetadata(offset)); + }) + .collect(toMap(Pair::getKey, Pair::getValue)); - kafkaConsumer.commitSync(topicPartitionByOffset); - kafkaConsumer.close(); + kafkaConsumer.commitSync(topicPartitionByOffset); + kafkaConsumer.close(); - logger.info("Successfully created consumer group for subscription {}, cluster: {}", - subscription.getQualifiedName(), clusterName); - } catch (Exception e) { - logger.error("Failed to create consumer group for subscription {}, cluster: {}", - subscription.getQualifiedName(), clusterName, e); - } + logger.info( + "Successfully created consumer group for subscription {}, cluster: {}", + subscription.getQualifiedName(), + clusterName); + } catch (Exception e) { + logger.error( + "Failed to create consumer group for subscription {}, cluster: {}", + subscription.getQualifiedName(), + clusterName, + e); } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/KafkaConsumerManager.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/KafkaConsumerManager.java index 068b47ebdb..061fecf4b0 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/KafkaConsumerManager.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/KafkaConsumerManager.java @@ -1,13 +1,5 @@ package pl.allegro.tech.hermes.management.infrastructure.kafka.service; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import pl.allegro.tech.hermes.api.SubscriptionName; -import pl.allegro.tech.hermes.common.kafka.ConsumerGroupId; -import pl.allegro.tech.hermes.common.kafka.KafkaNamesMapper; -import pl.allegro.tech.hermes.management.config.kafka.KafkaProperties; - -import java.util.Properties; - import static org.apache.kafka.clients.CommonClientConfigs.SECURITY_PROTOCOL_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG; @@ -19,37 +11,49 @@ import static org.apache.kafka.common.config.SaslConfigs.SASL_JAAS_CONFIG; import static org.apache.kafka.common.config.SaslConfigs.SASL_MECHANISM; -public class KafkaConsumerManager { - - private final KafkaNamesMapper kafkaNamesMapper; - private final String brokerList; - private final KafkaProperties kafkaProperties; - - public KafkaConsumerManager(KafkaProperties kafkaProperties, KafkaNamesMapper kafkaNamesMapper, String brokerList) { - this.kafkaNamesMapper = kafkaNamesMapper; - this.brokerList = brokerList; - this.kafkaProperties = kafkaProperties; - } +import java.util.Properties; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import pl.allegro.tech.hermes.api.SubscriptionName; +import pl.allegro.tech.hermes.common.kafka.ConsumerGroupId; +import pl.allegro.tech.hermes.common.kafka.KafkaNamesMapper; +import pl.allegro.tech.hermes.management.config.kafka.KafkaProperties; - public KafkaConsumer createConsumer(SubscriptionName subscription) { - ConsumerGroupId groupId = kafkaNamesMapper.toConsumerGroupId(subscription); - return new KafkaConsumer<>(properties(groupId)); - } +public class KafkaConsumerManager { - private Properties properties(ConsumerGroupId groupId) { - Properties props = new Properties(); - props.put(BOOTSTRAP_SERVERS_CONFIG, brokerList); - props.put(GROUP_ID_CONFIG, groupId.asString()); - props.put(ENABLE_AUTO_COMMIT_CONFIG, false); - props.put(REQUEST_TIMEOUT_MS_CONFIG, 5000); - props.put(DEFAULT_API_TIMEOUT_MS_CONFIG, 5000); - props.put(KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArrayDeserializer"); - props.put(VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArrayDeserializer"); - if (kafkaProperties.getAuthentication().isEnabled()) { - props.put(SASL_MECHANISM, kafkaProperties.getAuthentication().getMechanism()); - props.put(SECURITY_PROTOCOL_CONFIG, kafkaProperties.getAuthentication().getProtocol()); - props.put(SASL_JAAS_CONFIG, kafkaProperties.getAuthentication().getJaasConfig()); - } - return props; + private final KafkaNamesMapper kafkaNamesMapper; + private final String brokerList; + private final KafkaProperties kafkaProperties; + + public KafkaConsumerManager( + KafkaProperties kafkaProperties, KafkaNamesMapper kafkaNamesMapper, String brokerList) { + this.kafkaNamesMapper = kafkaNamesMapper; + this.brokerList = brokerList; + this.kafkaProperties = kafkaProperties; + } + + public KafkaConsumer createConsumer(SubscriptionName subscription) { + ConsumerGroupId groupId = kafkaNamesMapper.toConsumerGroupId(subscription); + return new KafkaConsumer<>(properties(groupId)); + } + + private Properties properties(ConsumerGroupId groupId) { + Properties props = new Properties(); + props.put(BOOTSTRAP_SERVERS_CONFIG, brokerList); + props.put(GROUP_ID_CONFIG, groupId.asString()); + props.put(ENABLE_AUTO_COMMIT_CONFIG, false); + props.put(REQUEST_TIMEOUT_MS_CONFIG, 5000); + props.put(DEFAULT_API_TIMEOUT_MS_CONFIG, 5000); + props.put( + KEY_DESERIALIZER_CLASS_CONFIG, + "org.apache.kafka.common.serialization.ByteArrayDeserializer"); + props.put( + VALUE_DESERIALIZER_CLASS_CONFIG, + "org.apache.kafka.common.serialization.ByteArrayDeserializer"); + if (kafkaProperties.getAuthentication().isEnabled()) { + props.put(SASL_MECHANISM, kafkaProperties.getAuthentication().getMechanism()); + props.put(SECURITY_PROTOCOL_CONFIG, kafkaProperties.getAuthentication().getProtocol()); + props.put(SASL_JAAS_CONFIG, kafkaProperties.getAuthentication().getJaasConfig()); } + return props; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/KafkaRawMessageReader.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/KafkaRawMessageReader.java index 199b540693..936a5ddf3c 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/KafkaRawMessageReader.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/KafkaRawMessageReader.java @@ -1,5 +1,9 @@ package pl.allegro.tech.hermes.management.infrastructure.kafka.service; +import static java.lang.String.format; + +import java.time.Duration; +import java.util.Collections; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; @@ -10,59 +14,59 @@ import pl.allegro.tech.hermes.common.kafka.KafkaTopic; import pl.allegro.tech.hermes.management.domain.topic.SingleMessageReaderException; -import java.time.Duration; -import java.util.Collections; - -import static java.lang.String.format; - public class KafkaRawMessageReader { - private static final Logger logger = LoggerFactory.getLogger(KafkaRawMessageReader.class); + private static final Logger logger = LoggerFactory.getLogger(KafkaRawMessageReader.class); - private final KafkaConsumerPool consumerPool; - private final int pollTimeoutMillis; + private final KafkaConsumerPool consumerPool; + private final int pollTimeoutMillis; - public KafkaRawMessageReader(KafkaConsumerPool consumerPool, - int pollTimeoutMillis) { - this.consumerPool = consumerPool; - this.pollTimeoutMillis = pollTimeoutMillis; - } + public KafkaRawMessageReader(KafkaConsumerPool consumerPool, int pollTimeoutMillis) { + this.consumerPool = consumerPool; + this.pollTimeoutMillis = pollTimeoutMillis; + } - private static SingleMessageReaderException messageNotFoundException(KafkaTopic topic, int partition, long offset) { - String cause = buildErrorMessage(topic, partition, offset, "Cannot find message"); - logger.error(cause); - return new SingleMessageReaderException(cause); - } + private static SingleMessageReaderException messageNotFoundException( + KafkaTopic topic, int partition, long offset) { + String cause = buildErrorMessage(topic, partition, offset, "Cannot find message"); + logger.error(cause); + return new SingleMessageReaderException(cause); + } - private static SingleMessageReaderException pollingException(KafkaTopic topic, int partition, long offset, Throwable throwable) { - String cause = buildErrorMessage(topic, partition, offset, "Error during polling kafka message"); - logger.error(cause, throwable); - return new SingleMessageReaderException(cause, throwable); - } + private static SingleMessageReaderException pollingException( + KafkaTopic topic, int partition, long offset, Throwable throwable) { + String cause = + buildErrorMessage(topic, partition, offset, "Error during polling kafka message"); + logger.error(cause, throwable); + return new SingleMessageReaderException(cause, throwable); + } - private static String buildErrorMessage(KafkaTopic topic, int partition, long offset, String message) { - return format("%s [offset %d, kafka_topic %s, partition %d]", message, offset, - topic.name().asString(), partition); - } + private static String buildErrorMessage( + KafkaTopic topic, int partition, long offset, String message) { + return format( + "%s [offset %d, kafka_topic %s, partition %d]", + message, offset, topic.name().asString(), partition); + } - byte[] readMessage(KafkaTopic topic, int partition, long offset) { - KafkaConsumer kafkaConsumer = consumerPool.get(topic, partition); - TopicPartition topicPartition = new TopicPartition(topic.name().asString(), partition); + byte[] readMessage(KafkaTopic topic, int partition, long offset) { + KafkaConsumer kafkaConsumer = consumerPool.get(topic, partition); + TopicPartition topicPartition = new TopicPartition(topic.name().asString(), partition); - try { - kafkaConsumer.assign(Collections.singleton(topicPartition)); - kafkaConsumer.poll(Duration.ZERO); - kafkaConsumer.seek(topicPartition, offset); - ConsumerRecords records = kafkaConsumer.poll(Duration.ofMillis(pollTimeoutMillis)); - for (ConsumerRecord record : records.records(topicPartition)) { - if (record.offset() == offset) { - return record.value(); - } - logger.info("Found an old offset: {} Expecting: {}", record.offset(), offset); - } - throw messageNotFoundException(topic, partition, offset); - } catch (Exception e) { - throw pollingException(topic, partition, offset, e); + try { + kafkaConsumer.assign(Collections.singleton(topicPartition)); + kafkaConsumer.poll(Duration.ZERO); + kafkaConsumer.seek(topicPartition, offset); + ConsumerRecords records = + kafkaConsumer.poll(Duration.ofMillis(pollTimeoutMillis)); + for (ConsumerRecord record : records.records(topicPartition)) { + if (record.offset() == offset) { + return record.value(); } + logger.info("Found an old offset: {} Expecting: {}", record.offset(), offset); + } + throw messageNotFoundException(topic, partition, offset); + } catch (Exception e) { + throw pollingException(topic, partition, offset, e); } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/KafkaSingleMessageReader.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/KafkaSingleMessageReader.java index 34abac0c5b..508c3b16c9 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/KafkaSingleMessageReader.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/KafkaSingleMessageReader.java @@ -1,5 +1,6 @@ package pl.allegro.tech.hermes.management.infrastructure.kafka.service; +import java.nio.charset.Charset; import pl.allegro.tech.hermes.api.ContentType; import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.common.kafka.KafkaTopic; @@ -9,37 +10,37 @@ import pl.allegro.tech.hermes.schema.SchemaRepository; import tech.allegro.schema.json2avro.converter.JsonAvroConverter; -import java.nio.charset.Charset; - public class KafkaSingleMessageReader implements SingleMessageReader { - private final KafkaRawMessageReader kafkaRawMessageReader; - private final SchemaRepository schemaRepository; - private final JsonAvroConverter converter; - - public KafkaSingleMessageReader(KafkaRawMessageReader kafkaRawMessageReader, - SchemaRepository schemaRepository, - JsonAvroConverter converter) { - this.kafkaRawMessageReader = kafkaRawMessageReader; - this.schemaRepository = schemaRepository; - this.converter = converter; + private final KafkaRawMessageReader kafkaRawMessageReader; + private final SchemaRepository schemaRepository; + private final JsonAvroConverter converter; + + public KafkaSingleMessageReader( + KafkaRawMessageReader kafkaRawMessageReader, + SchemaRepository schemaRepository, + JsonAvroConverter converter) { + this.kafkaRawMessageReader = kafkaRawMessageReader; + this.schemaRepository = schemaRepository; + this.converter = converter; + } + + @Override + public String readMessageAsJson(Topic topic, KafkaTopic kafkaTopic, int partition, long offset) { + byte[] bytes = kafkaRawMessageReader.readMessage(kafkaTopic, partition, offset); + if (topic.getContentType() == ContentType.AVRO) { + bytes = convertAvroToJson(topic, bytes); } - - @Override - public String readMessageAsJson(Topic topic, KafkaTopic kafkaTopic, int partition, long offset) { - byte[] bytes = kafkaRawMessageReader.readMessage(kafkaTopic, partition, offset); - if (topic.getContentType() == ContentType.AVRO) { - bytes = convertAvroToJson(topic, bytes); - } - return new String(bytes, Charset.forName("UTF-8")); - } - - private byte[] convertAvroToJson(Topic topic, byte[] bytes) { - if (topic.isSchemaIdAwareSerializationEnabled()) { - SchemaAwarePayload payload = SchemaAwareSerDe.deserialize(bytes); - return converter.convertToJson(payload.getPayload(), schemaRepository.getAvroSchema(topic, payload.getSchemaId()).getSchema()); - } - - return converter.convertToJson(bytes, schemaRepository.getLatestAvroSchema(topic).getSchema()); + return new String(bytes, Charset.forName("UTF-8")); + } + + private byte[] convertAvroToJson(Topic topic, byte[] bytes) { + if (topic.isSchemaIdAwareSerializationEnabled()) { + SchemaAwarePayload payload = SchemaAwareSerDe.deserialize(bytes); + return converter.convertToJson( + payload.getPayload(), + schemaRepository.getAvroSchema(topic, payload.getSchemaId()).getSchema()); } + return converter.convertToJson(bytes, schemaRepository.getLatestAvroSchema(topic).getSchema()); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/LogEndOffsetChecker.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/LogEndOffsetChecker.java index 5eefcdd181..dd91f3aea5 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/LogEndOffsetChecker.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/LogEndOffsetChecker.java @@ -1,22 +1,21 @@ package pl.allegro.tech.hermes.management.infrastructure.kafka.service; +import java.util.Collections; import org.apache.kafka.common.TopicPartition; import pl.allegro.tech.hermes.common.kafka.KafkaConsumerPool; -import java.util.Collections; - public class LogEndOffsetChecker { - private final KafkaConsumerPool kafkaConsumerPool; + private final KafkaConsumerPool kafkaConsumerPool; - public LogEndOffsetChecker(KafkaConsumerPool kafkaConsumerPool) { - this.kafkaConsumerPool = kafkaConsumerPool; - } + public LogEndOffsetChecker(KafkaConsumerPool kafkaConsumerPool) { + this.kafkaConsumerPool = kafkaConsumerPool; + } - public long check(TopicPartition topicPartition) { - return kafkaConsumerPool - .get(topicPartition.topic(), topicPartition.partition()) - .endOffsets(Collections.singletonList(topicPartition)) - .get(topicPartition); - } + public long check(TopicPartition topicPartition) { + return kafkaConsumerPool + .get(topicPartition.topic(), topicPartition.partition()) + .endOffsets(Collections.singletonList(topicPartition)) + .get(topicPartition); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/NoOpConsumerGroupManager.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/NoOpConsumerGroupManager.java index f89f37e91f..9de89e9c81 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/NoOpConsumerGroupManager.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/NoOpConsumerGroupManager.java @@ -5,8 +5,8 @@ import pl.allegro.tech.hermes.management.domain.subscription.ConsumerGroupManager; public class NoOpConsumerGroupManager implements ConsumerGroupManager { - @Override - public void createConsumerGroup(Topic topic, Subscription subscription) { - // no operation - } + @Override + public void createConsumerGroup(Topic topic, Subscription subscription) { + // no operation + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/OffsetsAvailableChecker.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/OffsetsAvailableChecker.java index 9e21f200f3..4c40f16483 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/OffsetsAvailableChecker.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/OffsetsAvailableChecker.java @@ -1,5 +1,6 @@ package pl.allegro.tech.hermes.management.infrastructure.kafka.service; +import java.util.Collections; import org.apache.kafka.common.TopicPartition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -9,31 +10,35 @@ import pl.allegro.tech.hermes.common.kafka.KafkaConsumerPool; import pl.allegro.tech.hermes.common.kafka.KafkaTopic; -import java.util.Collections; - public class OffsetsAvailableChecker { - private static final Logger logger = LoggerFactory.getLogger(OffsetsAvailableChecker.class); + private static final Logger logger = LoggerFactory.getLogger(OffsetsAvailableChecker.class); - private final KafkaConsumerPool consumerPool; - private final BrokerStorage storage; + private final KafkaConsumerPool consumerPool; + private final BrokerStorage storage; - public OffsetsAvailableChecker(KafkaConsumerPool consumerPool, BrokerStorage storage) { - this.consumerPool = consumerPool; - this.storage = storage; - } + public OffsetsAvailableChecker(KafkaConsumerPool consumerPool, BrokerStorage storage) { + this.consumerPool = consumerPool; + this.storage = storage; + } - boolean areOffsetsAvailable(KafkaTopic topic) { - try { - return storage.readPartitionsIds(topic.name().asString()).stream().allMatch(partition -> { - TopicPartition topicPartition = new TopicPartition(topic.name().asString(), partition); - consumerPool.get(topic, partition).beginningOffsets(Collections.singleton(topicPartition)); + boolean areOffsetsAvailable(KafkaTopic topic) { + try { + return storage.readPartitionsIds(topic.name().asString()).stream() + .allMatch( + partition -> { + TopicPartition topicPartition = + new TopicPartition(topic.name().asString(), partition); + consumerPool + .get(topic, partition) + .beginningOffsets(Collections.singleton(topicPartition)); return true; - }); - } catch (PartitionsNotFoundForGivenTopicException | BrokerNotFoundForPartitionException - | org.apache.kafka.common.errors.TimeoutException e) { - logger.debug("Offsets reported as not available due to failure", e); - return false; - } + }); + } catch (PartitionsNotFoundForGivenTopicException + | BrokerNotFoundForPartitionException + | org.apache.kafka.common.errors.TimeoutException e) { + logger.debug("Offsets reported as not available due to failure", e); + return false; } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/retransmit/KafkaRetransmissionService.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/retransmit/KafkaRetransmissionService.java index 6b640d4753..d805d639c7 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/retransmit/KafkaRetransmissionService.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/retransmit/KafkaRetransmissionService.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.management.infrastructure.kafka.service.retransmit; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.consumer.OffsetAndTimestamp; import org.apache.kafka.common.TopicPartition; @@ -12,80 +17,94 @@ import pl.allegro.tech.hermes.common.kafka.offset.SubscriptionOffsetChangeIndicator; import pl.allegro.tech.hermes.management.domain.message.RetransmissionService; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; - public class KafkaRetransmissionService implements RetransmissionService { - private final BrokerStorage brokerStorage; - private final SubscriptionOffsetChangeIndicator subscriptionOffsetChange; - private final KafkaConsumerPool consumerPool; - private final KafkaNamesMapper kafkaNamesMapper; + private final BrokerStorage brokerStorage; + private final SubscriptionOffsetChangeIndicator subscriptionOffsetChange; + private final KafkaConsumerPool consumerPool; + private final KafkaNamesMapper kafkaNamesMapper; - public KafkaRetransmissionService( - BrokerStorage brokerStorage, - SubscriptionOffsetChangeIndicator subscriptionOffsetChange, - KafkaConsumerPool consumerPool, - KafkaNamesMapper kafkaNamesMapper) { + public KafkaRetransmissionService( + BrokerStorage brokerStorage, + SubscriptionOffsetChangeIndicator subscriptionOffsetChange, + KafkaConsumerPool consumerPool, + KafkaNamesMapper kafkaNamesMapper) { - this.brokerStorage = brokerStorage; - this.subscriptionOffsetChange = subscriptionOffsetChange; - this.consumerPool = consumerPool; - this.kafkaNamesMapper = kafkaNamesMapper; - } + this.brokerStorage = brokerStorage; + this.subscriptionOffsetChange = subscriptionOffsetChange; + this.consumerPool = consumerPool; + this.kafkaNamesMapper = kafkaNamesMapper; + } - @Override - public List indicateOffsetChange(Topic topic, String subscription, String brokersClusterName, - long timestamp, boolean dryRun) { + @Override + public List indicateOffsetChange( + Topic topic, String subscription, String brokersClusterName, long timestamp, boolean dryRun) { - List partitionOffsetList = new ArrayList<>(); - kafkaNamesMapper.toKafkaTopics(topic).forEach(k -> { - List partitionsIds = brokerStorage.readPartitionsIds(k.name().asString()); + List partitionOffsetList = new ArrayList<>(); + kafkaNamesMapper + .toKafkaTopics(topic) + .forEach( + k -> { + List partitionsIds = brokerStorage.readPartitionsIds(k.name().asString()); - for (Integer partitionId : partitionsIds) { + for (Integer partitionId : partitionsIds) { KafkaConsumer consumer = createKafkaConsumer(k, partitionId); - long offset = findClosestOffsetJustBeforeTimestamp(consumer, k, partitionId, timestamp); - PartitionOffset partitionOffset = new PartitionOffset(k.name(), offset, partitionId); + long offset = + findClosestOffsetJustBeforeTimestamp(consumer, k, partitionId, timestamp); + PartitionOffset partitionOffset = + new PartitionOffset(k.name(), offset, partitionId); partitionOffsetList.add(partitionOffset); if (!dryRun) { - subscriptionOffsetChange.setSubscriptionOffset(topic.getName(), subscription, brokersClusterName, partitionOffset); + subscriptionOffsetChange.setSubscriptionOffset( + topic.getName(), subscription, brokersClusterName, partitionOffset); } - } - }); + } + }); - return partitionOffsetList; - } + return partitionOffsetList; + } - @Override - public boolean areOffsetsMoved(Topic topic, String subscriptionName, String brokersClusterName) { - return kafkaNamesMapper.toKafkaTopics(topic).allMatch(kafkaTopic -> { - List partitionIds = brokerStorage.readPartitionsIds(kafkaTopic.name().asString()); - return subscriptionOffsetChange.areOffsetsMoved( - topic.getName(), subscriptionName, brokersClusterName, kafkaTopic, partitionIds); - }); - } + @Override + public boolean areOffsetsMoved(Topic topic, String subscriptionName, String brokersClusterName) { + return kafkaNamesMapper + .toKafkaTopics(topic) + .allMatch( + kafkaTopic -> { + List partitionIds = + brokerStorage.readPartitionsIds(kafkaTopic.name().asString()); + return subscriptionOffsetChange.areOffsetsMoved( + topic.getName(), subscriptionName, brokersClusterName, kafkaTopic, partitionIds); + }); + } - private KafkaConsumer createKafkaConsumer(KafkaTopic kafkaTopic, int partition) { - return consumerPool.get(kafkaTopic, partition); - } + private KafkaConsumer createKafkaConsumer(KafkaTopic kafkaTopic, int partition) { + return consumerPool.get(kafkaTopic, partition); + } - private long findClosestOffsetJustBeforeTimestamp(KafkaConsumer consumer, - KafkaTopic kafkaTopic, - int partition, - long timestamp) { - long endOffset = getEndingOffset(consumer, kafkaTopic, partition); - TopicPartition topicPartition = new TopicPartition(kafkaTopic.name().asString(), partition); - return Optional.ofNullable(consumer.offsetsForTimes(Collections.singletonMap(topicPartition, timestamp)).get(topicPartition)) - .orElse(new OffsetAndTimestamp(endOffset, timestamp)).offset(); - } + private long findClosestOffsetJustBeforeTimestamp( + KafkaConsumer consumer, + KafkaTopic kafkaTopic, + int partition, + long timestamp) { + long endOffset = getEndingOffset(consumer, kafkaTopic, partition); + TopicPartition topicPartition = new TopicPartition(kafkaTopic.name().asString(), partition); + return Optional.ofNullable( + consumer + .offsetsForTimes(Collections.singletonMap(topicPartition, timestamp)) + .get(topicPartition)) + .orElse(new OffsetAndTimestamp(endOffset, timestamp)) + .offset(); + } - private long getEndingOffset(KafkaConsumer kafkaConsumer, KafkaTopic topicName, int partition) { - TopicPartition topicPartition = new TopicPartition(topicName.name().asString(), partition); - Map offsets = kafkaConsumer.endOffsets(Collections.singleton(topicPartition)); - return Optional.ofNullable(offsets.get(topicPartition)) - .orElseThrow(() -> new OffsetNotFoundException(String.format("Ending offset for partition %s not found", topicPartition))); - } + private long getEndingOffset( + KafkaConsumer kafkaConsumer, KafkaTopic topicName, int partition) { + TopicPartition topicPartition = new TopicPartition(topicName.name().asString(), partition); + Map offsets = + kafkaConsumer.endOffsets(Collections.singleton(topicPartition)); + return Optional.ofNullable(offsets.get(topicPartition)) + .orElseThrow( + () -> + new OffsetNotFoundException( + String.format("Ending offset for partition %s not found", topicPartition))); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/retransmit/OffsetNotFoundException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/retransmit/OffsetNotFoundException.java index 6c43c00ee1..6220c31f3b 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/retransmit/OffsetNotFoundException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/retransmit/OffsetNotFoundException.java @@ -1,18 +1,18 @@ package pl.allegro.tech.hermes.management.infrastructure.kafka.service.retransmit; +import static pl.allegro.tech.hermes.api.ErrorCode.OFFSET_NOT_FOUND_EXCEPTION; + import pl.allegro.tech.hermes.api.ErrorCode; import pl.allegro.tech.hermes.management.domain.ManagementException; -import static pl.allegro.tech.hermes.api.ErrorCode.OFFSET_NOT_FOUND_EXCEPTION; - class OffsetNotFoundException extends ManagementException { - OffsetNotFoundException(String message) { - super(message); - } + OffsetNotFoundException(String message) { + super(message); + } - @Override - public ErrorCode getCode() { - return OFFSET_NOT_FOUND_EXCEPTION; - } + @Override + public ErrorCode getCode() { + return OFFSET_NOT_FOUND_EXCEPTION; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/HybridSubscriptionMetricsRepository.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/HybridSubscriptionMetricsRepository.java index 712a5bba17..e343a54c55 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/HybridSubscriptionMetricsRepository.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/HybridSubscriptionMetricsRepository.java @@ -1,5 +1,8 @@ package pl.allegro.tech.hermes.management.infrastructure.metrics; +import static org.apache.commons.lang3.exception.ExceptionUtils.getRootCauseMessage; + +import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -12,81 +15,92 @@ import pl.allegro.tech.hermes.management.domain.subscription.SubscriptionMetricsRepository; import pl.allegro.tech.hermes.management.infrastructure.metrics.MonitoringSubscriptionMetricsProvider.MonitoringSubscriptionMetrics; -import java.util.function.Supplier; - -import static org.apache.commons.lang3.exception.ExceptionUtils.getRootCauseMessage; - - @Component public class HybridSubscriptionMetricsRepository implements SubscriptionMetricsRepository { - private static final Logger logger = LoggerFactory.getLogger(HybridSubscriptionMetricsRepository.class); - - private final MonitoringSubscriptionMetricsProvider monitoringSubscriptionMetricsProvider; - private final SummedSharedCounter summedSharedCounter; - private final ZookeeperPaths zookeeperPaths; - private final SubscriptionLagSource lagSource; - - public HybridSubscriptionMetricsRepository(MonitoringSubscriptionMetricsProvider monitoringSubscriptionMetricsProvider, - SummedSharedCounter summedSharedCounter, - ZookeeperPaths zookeeperPaths, SubscriptionLagSource lagSource) { - this.monitoringSubscriptionMetricsProvider = monitoringSubscriptionMetricsProvider; - this.summedSharedCounter = summedSharedCounter; - this.zookeeperPaths = zookeeperPaths; - this.lagSource = lagSource; - } - - @Override - public SubscriptionMetrics loadMetrics(TopicName topicName, String subscriptionName) { - SubscriptionName name = new SubscriptionName(subscriptionName, topicName); - - MonitoringSubscriptionMetrics monitoringMetrics = monitoringSubscriptionMetricsProvider.subscriptionMetrics(name); - ZookeeperMetrics zookeeperMetrics = readZookeeperMetrics(name); - - return SubscriptionMetrics.Builder.subscriptionMetrics() - .withRate(monitoringMetrics.rate()) - .withCodes2xx(monitoringMetrics.codes2xx()) - .withCodes4xx(monitoringMetrics.code4xx()) - .withCodes5xx(monitoringMetrics.code5xx()) - .withRetries(monitoringMetrics.retries()) - .withTimeouts(monitoringMetrics.timeouts()) - .withOtherErrors(monitoringMetrics.otherErrors()) - .withThroughput(monitoringMetrics.throughput()) - .withBatchRate(monitoringMetrics.metricPathBatchRate()) - .withDiscarded(zookeeperMetrics.discarded) - .withDelivered(zookeeperMetrics.delivered) - .withVolume(zookeeperMetrics.volume) - .withLag(lagSource.getLag(topicName, subscriptionName)) - .build(); - } - - @Override - public PersistentSubscriptionMetrics loadZookeeperMetrics(TopicName topicName, String subscriptionName) { - SubscriptionName name = new SubscriptionName(subscriptionName, topicName); - ZookeeperMetrics zookeeperMetrics = readZookeeperMetrics(name); - - return new PersistentSubscriptionMetrics(zookeeperMetrics.delivered, zookeeperMetrics.discarded, zookeeperMetrics.volume); - } - - private ZookeeperMetrics readZookeeperMetrics(SubscriptionName name) { - return new ZookeeperMetrics( - readZookeeperMetric(() -> summedSharedCounter.getValue(zookeeperPaths.subscriptionMetricPath(name, "delivered")), name), - readZookeeperMetric(() -> summedSharedCounter.getValue(zookeeperPaths.subscriptionMetricPath(name, "discarded")), name), - readZookeeperMetric(() -> summedSharedCounter.getValue(zookeeperPaths.subscriptionMetricPath(name, "volume")), name) - ); - } - - private long readZookeeperMetric(Supplier supplier, SubscriptionName name) { - try { - return supplier.get(); - } catch (Exception exception) { - logger.warn( - "Failed to read Zookeeper metrics for subscription: {}; root cause: {}", - name.getQualifiedName(), getRootCauseMessage(exception) - ); - return -1; - } + private static final Logger logger = + LoggerFactory.getLogger(HybridSubscriptionMetricsRepository.class); + + private final MonitoringSubscriptionMetricsProvider monitoringSubscriptionMetricsProvider; + private final SummedSharedCounter summedSharedCounter; + private final ZookeeperPaths zookeeperPaths; + private final SubscriptionLagSource lagSource; + + public HybridSubscriptionMetricsRepository( + MonitoringSubscriptionMetricsProvider monitoringSubscriptionMetricsProvider, + SummedSharedCounter summedSharedCounter, + ZookeeperPaths zookeeperPaths, + SubscriptionLagSource lagSource) { + this.monitoringSubscriptionMetricsProvider = monitoringSubscriptionMetricsProvider; + this.summedSharedCounter = summedSharedCounter; + this.zookeeperPaths = zookeeperPaths; + this.lagSource = lagSource; + } + + @Override + public SubscriptionMetrics loadMetrics(TopicName topicName, String subscriptionName) { + SubscriptionName name = new SubscriptionName(subscriptionName, topicName); + + MonitoringSubscriptionMetrics monitoringMetrics = + monitoringSubscriptionMetricsProvider.subscriptionMetrics(name); + ZookeeperMetrics zookeeperMetrics = readZookeeperMetrics(name); + + return SubscriptionMetrics.Builder.subscriptionMetrics() + .withRate(monitoringMetrics.rate()) + .withCodes2xx(monitoringMetrics.codes2xx()) + .withCodes4xx(monitoringMetrics.code4xx()) + .withCodes5xx(monitoringMetrics.code5xx()) + .withRetries(monitoringMetrics.retries()) + .withTimeouts(monitoringMetrics.timeouts()) + .withOtherErrors(monitoringMetrics.otherErrors()) + .withThroughput(monitoringMetrics.throughput()) + .withBatchRate(monitoringMetrics.metricPathBatchRate()) + .withDiscarded(zookeeperMetrics.discarded) + .withDelivered(zookeeperMetrics.delivered) + .withVolume(zookeeperMetrics.volume) + .withLag(lagSource.getLag(topicName, subscriptionName)) + .build(); + } + + @Override + public PersistentSubscriptionMetrics loadZookeeperMetrics( + TopicName topicName, String subscriptionName) { + SubscriptionName name = new SubscriptionName(subscriptionName, topicName); + ZookeeperMetrics zookeeperMetrics = readZookeeperMetrics(name); + + return new PersistentSubscriptionMetrics( + zookeeperMetrics.delivered, zookeeperMetrics.discarded, zookeeperMetrics.volume); + } + + private ZookeeperMetrics readZookeeperMetrics(SubscriptionName name) { + return new ZookeeperMetrics( + readZookeeperMetric( + () -> + summedSharedCounter.getValue( + zookeeperPaths.subscriptionMetricPath(name, "delivered")), + name), + readZookeeperMetric( + () -> + summedSharedCounter.getValue( + zookeeperPaths.subscriptionMetricPath(name, "discarded")), + name), + readZookeeperMetric( + () -> + summedSharedCounter.getValue(zookeeperPaths.subscriptionMetricPath(name, "volume")), + name)); + } + + private long readZookeeperMetric(Supplier supplier, SubscriptionName name) { + try { + return supplier.get(); + } catch (Exception exception) { + logger.warn( + "Failed to read Zookeeper metrics for subscription: {}; root cause: {}", + name.getQualifiedName(), + getRootCauseMessage(exception)); + return -1; } + } - private record ZookeeperMetrics(long delivered, long discarded, long volume) { } + private record ZookeeperMetrics(long delivered, long discarded, long volume) {} } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/HybridTopicMetricsRepository.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/HybridTopicMetricsRepository.java index 90d6255994..a63a6260b8 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/HybridTopicMetricsRepository.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/HybridTopicMetricsRepository.java @@ -10,34 +10,39 @@ @Component public class HybridTopicMetricsRepository implements TopicMetricsRepository { - private final MonitoringTopicMetricsProvider monitoringTopicMetricsProvider; - - private final SummedSharedCounter summedSharedCounter; - - private final ZookeeperPaths zookeeperPaths; - - private final SubscriptionRepository subscriptionRepository; - - public HybridTopicMetricsRepository(MonitoringTopicMetricsProvider monitoringTopicMetricsProvider, - SummedSharedCounter summedSharedCounter, ZookeeperPaths zookeeperPaths, - SubscriptionRepository subscriptionRepository) { - this.monitoringTopicMetricsProvider = monitoringTopicMetricsProvider; - this.summedSharedCounter = summedSharedCounter; - this.zookeeperPaths = zookeeperPaths; - this.subscriptionRepository = subscriptionRepository; - } - - @Override - public TopicMetrics loadMetrics(TopicName topicName) { - MonitoringTopicMetricsProvider.MonitoringTopicMetrics metrics = monitoringTopicMetricsProvider.topicMetrics(topicName); - - return TopicMetrics.Builder.topicMetrics() - .withRate(metrics.rate()) - .withDeliveryRate(metrics.deliveryRate()) - .withThroughput(metrics.throughput()) - .withPublished(summedSharedCounter.getValue(zookeeperPaths.topicMetricPath(topicName, "published"))) - .withVolume(summedSharedCounter.getValue(zookeeperPaths.topicMetricPath(topicName, "volume"))) - .withSubscriptions(subscriptionRepository.listSubscriptionNames(topicName).size()) - .build(); - } + private final MonitoringTopicMetricsProvider monitoringTopicMetricsProvider; + + private final SummedSharedCounter summedSharedCounter; + + private final ZookeeperPaths zookeeperPaths; + + private final SubscriptionRepository subscriptionRepository; + + public HybridTopicMetricsRepository( + MonitoringTopicMetricsProvider monitoringTopicMetricsProvider, + SummedSharedCounter summedSharedCounter, + ZookeeperPaths zookeeperPaths, + SubscriptionRepository subscriptionRepository) { + this.monitoringTopicMetricsProvider = monitoringTopicMetricsProvider; + this.summedSharedCounter = summedSharedCounter; + this.zookeeperPaths = zookeeperPaths; + this.subscriptionRepository = subscriptionRepository; + } + + @Override + public TopicMetrics loadMetrics(TopicName topicName) { + MonitoringTopicMetricsProvider.MonitoringTopicMetrics metrics = + monitoringTopicMetricsProvider.topicMetrics(topicName); + + return TopicMetrics.Builder.topicMetrics() + .withRate(metrics.rate()) + .withDeliveryRate(metrics.deliveryRate()) + .withThroughput(metrics.throughput()) + .withPublished( + summedSharedCounter.getValue(zookeeperPaths.topicMetricPath(topicName, "published"))) + .withVolume( + summedSharedCounter.getValue(zookeeperPaths.topicMetricPath(topicName, "volume"))) + .withSubscriptions(subscriptionRepository.listSubscriptionNames(topicName).size()) + .build(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/MonitoringMetricsContainer.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/MonitoringMetricsContainer.java index cb6ebdcdcf..ba6958369d 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/MonitoringMetricsContainer.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/MonitoringMetricsContainer.java @@ -1,50 +1,49 @@ package pl.allegro.tech.hermes.management.infrastructure.metrics; -import pl.allegro.tech.hermes.api.MetricDecimalValue; - import java.util.HashMap; import java.util.Map; +import pl.allegro.tech.hermes.api.MetricDecimalValue; public class MonitoringMetricsContainer { - private static final MetricDecimalValue DEFAULT_VALUE = MetricDecimalValue.of("0.0"); + private static final MetricDecimalValue DEFAULT_VALUE = MetricDecimalValue.of("0.0"); - private final Map metrics; - private final boolean isAvailable; + private final Map metrics; + private final boolean isAvailable; - private MonitoringMetricsContainer(boolean isAvailable, Map metrics) { - this.metrics = metrics; - this.isAvailable = isAvailable; - } + private MonitoringMetricsContainer(boolean isAvailable, Map metrics) { + this.metrics = metrics; + this.isAvailable = isAvailable; + } - public static MonitoringMetricsContainer createEmpty() { - return new MonitoringMetricsContainer(true, new HashMap<>()); - } + public static MonitoringMetricsContainer createEmpty() { + return new MonitoringMetricsContainer(true, new HashMap<>()); + } - public static MonitoringMetricsContainer initialized(Map metrics) { - return new MonitoringMetricsContainer(true, metrics); - } + public static MonitoringMetricsContainer initialized(Map metrics) { + return new MonitoringMetricsContainer(true, metrics); + } - public static MonitoringMetricsContainer unavailable() { - return new MonitoringMetricsContainer(false, new HashMap<>()); - } + public static MonitoringMetricsContainer unavailable() { + return new MonitoringMetricsContainer(false, new HashMap<>()); + } - public MonitoringMetricsContainer addMetricValue(String query, MetricDecimalValue value) { - if (!isAvailable) { - throw new IllegalStateException("Adding value to unavailable metrics container"); - } - this.metrics.put(query, value); - return this; + public MonitoringMetricsContainer addMetricValue(String query, MetricDecimalValue value) { + if (!isAvailable) { + throw new IllegalStateException("Adding value to unavailable metrics container"); } + this.metrics.put(query, value); + return this; + } - public MetricDecimalValue metricValue(String query) { - if (!isAvailable) { - return MetricDecimalValue.unavailable(); - } - return metrics.getOrDefault(query, DEFAULT_VALUE); + public MetricDecimalValue metricValue(String query) { + if (!isAvailable) { + return MetricDecimalValue.unavailable(); } + return metrics.getOrDefault(query, DEFAULT_VALUE); + } - public boolean hasUnavailableMetrics() { - return !isAvailable || metrics.entrySet().stream().anyMatch(e -> !e.getValue().isAvailable()); - } + public boolean hasUnavailableMetrics() { + return !isAvailable || metrics.entrySet().stream().anyMatch(e -> !e.getValue().isAvailable()); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/MonitoringSubscriptionMetricsProvider.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/MonitoringSubscriptionMetricsProvider.java index 8745aa1d1c..67dd4fb16e 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/MonitoringSubscriptionMetricsProvider.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/MonitoringSubscriptionMetricsProvider.java @@ -4,82 +4,90 @@ import pl.allegro.tech.hermes.api.SubscriptionName; public interface MonitoringSubscriptionMetricsProvider { - MonitoringSubscriptionMetrics subscriptionMetrics(SubscriptionName subscriptionName); + MonitoringSubscriptionMetrics subscriptionMetrics(SubscriptionName subscriptionName); - record MonitoringSubscriptionMetrics(MetricDecimalValue rate, - MetricDecimalValue timeouts, - MetricDecimalValue throughput, - MetricDecimalValue otherErrors, - MetricDecimalValue codes2xx, - MetricDecimalValue code4xx, - MetricDecimalValue code5xx, - MetricDecimalValue retries, - MetricDecimalValue metricPathBatchRate) { - } + record MonitoringSubscriptionMetrics( + MetricDecimalValue rate, + MetricDecimalValue timeouts, + MetricDecimalValue throughput, + MetricDecimalValue otherErrors, + MetricDecimalValue codes2xx, + MetricDecimalValue code4xx, + MetricDecimalValue code5xx, + MetricDecimalValue retries, + MetricDecimalValue metricPathBatchRate) {} - static MetricsBuilder metricsBuilder() { - return new MetricsBuilder(); - } + static MetricsBuilder metricsBuilder() { + return new MetricsBuilder(); + } - class MetricsBuilder { - private MetricDecimalValue rate; - private MetricDecimalValue timeouts; - private MetricDecimalValue throughput; - private MetricDecimalValue otherErrors; - private MetricDecimalValue codes2xx; - private MetricDecimalValue code4xx; - private MetricDecimalValue code5xx; - private MetricDecimalValue retries; - private MetricDecimalValue metricPathBatchRate; + class MetricsBuilder { + private MetricDecimalValue rate; + private MetricDecimalValue timeouts; + private MetricDecimalValue throughput; + private MetricDecimalValue otherErrors; + private MetricDecimalValue codes2xx; + private MetricDecimalValue code4xx; + private MetricDecimalValue code5xx; + private MetricDecimalValue retries; + private MetricDecimalValue metricPathBatchRate; - public MetricsBuilder withRate(MetricDecimalValue rate) { - this.rate = rate; - return this; - } + public MetricsBuilder withRate(MetricDecimalValue rate) { + this.rate = rate; + return this; + } - public MetricsBuilder withTimeouts(MetricDecimalValue timeouts) { - this.timeouts = timeouts; - return this; - } + public MetricsBuilder withTimeouts(MetricDecimalValue timeouts) { + this.timeouts = timeouts; + return this; + } - public MetricsBuilder withThroughput(MetricDecimalValue throughput) { - this.throughput = throughput; - return this; - } + public MetricsBuilder withThroughput(MetricDecimalValue throughput) { + this.throughput = throughput; + return this; + } - public MetricsBuilder withOtherErrors(MetricDecimalValue otherErrors) { - this.otherErrors = otherErrors; - return this; - } + public MetricsBuilder withOtherErrors(MetricDecimalValue otherErrors) { + this.otherErrors = otherErrors; + return this; + } - public MetricsBuilder withCodes2xx(MetricDecimalValue codes2xx) { - this.codes2xx = codes2xx; - return this; - } + public MetricsBuilder withCodes2xx(MetricDecimalValue codes2xx) { + this.codes2xx = codes2xx; + return this; + } - public MetricsBuilder withCode4xx(MetricDecimalValue code4xx) { - this.code4xx = code4xx; - return this; - } + public MetricsBuilder withCode4xx(MetricDecimalValue code4xx) { + this.code4xx = code4xx; + return this; + } - public MetricsBuilder withCode5xx(MetricDecimalValue code5xx) { - this.code5xx = code5xx; - return this; - } + public MetricsBuilder withCode5xx(MetricDecimalValue code5xx) { + this.code5xx = code5xx; + return this; + } - public MetricsBuilder withRetries(MetricDecimalValue retries) { - this.retries = retries; - return this; - } + public MetricsBuilder withRetries(MetricDecimalValue retries) { + this.retries = retries; + return this; + } - public MetricsBuilder withMetricPathBatchRate(MetricDecimalValue metricPathBatchRate) { - this.metricPathBatchRate = metricPathBatchRate; - return this; - } + public MetricsBuilder withMetricPathBatchRate(MetricDecimalValue metricPathBatchRate) { + this.metricPathBatchRate = metricPathBatchRate; + return this; + } - public MonitoringSubscriptionMetrics build() { - return new MonitoringSubscriptionMetrics(rate, timeouts, throughput, otherErrors, codes2xx, - code4xx, code5xx, retries, metricPathBatchRate); - } + public MonitoringSubscriptionMetrics build() { + return new MonitoringSubscriptionMetrics( + rate, + timeouts, + throughput, + otherErrors, + codes2xx, + code4xx, + code5xx, + retries, + metricPathBatchRate); } -} \ No newline at end of file + } +} diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/MonitoringTopicMetricsProvider.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/MonitoringTopicMetricsProvider.java index 176f61f358..916666de71 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/MonitoringTopicMetricsProvider.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/MonitoringTopicMetricsProvider.java @@ -4,39 +4,37 @@ import pl.allegro.tech.hermes.api.TopicName; public interface MonitoringTopicMetricsProvider { - MonitoringTopicMetrics topicMetrics(TopicName topicName); + MonitoringTopicMetrics topicMetrics(TopicName topicName); - record MonitoringTopicMetrics(MetricDecimalValue rate, - MetricDecimalValue deliveryRate, - MetricDecimalValue throughput) { + record MonitoringTopicMetrics( + MetricDecimalValue rate, MetricDecimalValue deliveryRate, MetricDecimalValue throughput) {} + + static MetricsBuilder metricsBuilder() { + return new MetricsBuilder(); + } + + class MetricsBuilder { + private MetricDecimalValue rate; + private MetricDecimalValue deliveryRate; + private MetricDecimalValue throughput; + + public MetricsBuilder withRate(MetricDecimalValue rate) { + this.rate = rate; + return this; + } + + public MetricsBuilder withDeliveryRate(MetricDecimalValue deliveryRate) { + this.deliveryRate = deliveryRate; + return this; } - static MetricsBuilder metricsBuilder() { - return new MetricsBuilder(); + public MetricsBuilder withThroughput(MetricDecimalValue throughput) { + this.throughput = throughput; + return this; } - class MetricsBuilder { - private MetricDecimalValue rate; - private MetricDecimalValue deliveryRate; - private MetricDecimalValue throughput; - - public MetricsBuilder withRate(MetricDecimalValue rate) { - this.rate = rate; - return this; - } - - public MetricsBuilder withDeliveryRate(MetricDecimalValue deliveryRate) { - this.deliveryRate = deliveryRate; - return this; - } - - public MetricsBuilder withThroughput(MetricDecimalValue throughput) { - this.throughput = throughput; - return this; - } - - public MonitoringTopicMetrics build() { - return new MonitoringTopicMetrics(rate, deliveryRate, throughput); - } + public MonitoringTopicMetrics build() { + return new MonitoringTopicMetrics(rate, deliveryRate, throughput); } -} \ No newline at end of file + } +} diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/NoOpSubscriptionLagSource.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/NoOpSubscriptionLagSource.java index 60da162174..ec185b5bf9 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/NoOpSubscriptionLagSource.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/NoOpSubscriptionLagSource.java @@ -5,8 +5,8 @@ import pl.allegro.tech.hermes.management.domain.subscription.SubscriptionLagSource; public class NoOpSubscriptionLagSource implements SubscriptionLagSource { - @Override - public MetricLongValue getLag(TopicName topicName, String subscriptionName) { - return MetricLongValue.of(-1); - } + @Override + public MetricLongValue getLag(TopicName topicName, String subscriptionName) { + return MetricLongValue.of(-1); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/SummedSharedCounter.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/SummedSharedCounter.java index a5395f9786..cbd8f4a317 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/SummedSharedCounter.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/metrics/SummedSharedCounter.java @@ -3,99 +3,99 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong; import org.apache.curator.retry.ExponentialBackoffRetry; import pl.allegro.tech.hermes.infrastructure.zookeeper.counter.ZookeeperCounterException; import pl.allegro.tech.hermes.management.infrastructure.zookeeper.ZookeeperClient; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - public class SummedSharedCounter { - private final LoadingCache counterAggregators; + private final LoadingCache counterAggregators; - public SummedSharedCounter(List zookeeperClients, - int expireAfter, - int distributedLoaderBackoff, - int distributedLoaderRetries) { - this.counterAggregators = buildLoadingCache(zookeeperClients, expireAfter, distributedLoaderBackoff, distributedLoaderRetries); - } + public SummedSharedCounter( + List zookeeperClients, + int expireAfter, + int distributedLoaderBackoff, + int distributedLoaderRetries) { + this.counterAggregators = + buildLoadingCache( + zookeeperClients, expireAfter, distributedLoaderBackoff, distributedLoaderRetries); + } - public long getValue(String path) { - try { - return counterAggregators.get(path).aggregate(); - } catch (ZookeeperCounterException e) { - throw e; - } catch (Exception e) { - throw new ZookeeperCounterException(path, e); - } + public long getValue(String path) { + try { + return counterAggregators.get(path).aggregate(); + } catch (ZookeeperCounterException e) { + throw e; + } catch (Exception e) { + throw new ZookeeperCounterException(path, e); } + } - private LoadingCache buildLoadingCache(List zookeeperClients, - int expireAfter, - int distributedLoaderBackoff, - int distributedLoaderRetries) { - return CacheBuilder.newBuilder() - .expireAfterAccess(expireAfter, TimeUnit.HOURS) - .build(new CacheLoader<>() { - @Override - public CounterAggregator load(String key) { - return new CounterAggregator( - key, - zookeeperClients, - distributedLoaderBackoff, - distributedLoaderRetries - ); - } - } - ); - } + private LoadingCache buildLoadingCache( + List zookeeperClients, + int expireAfter, + int distributedLoaderBackoff, + int distributedLoaderRetries) { + return CacheBuilder.newBuilder() + .expireAfterAccess(expireAfter, TimeUnit.HOURS) + .build( + new CacheLoader<>() { + @Override + public CounterAggregator load(String key) { + return new CounterAggregator( + key, zookeeperClients, distributedLoaderBackoff, distributedLoaderRetries); + } + }); + } - private static class CounterAggregator { + private static class CounterAggregator { - private final String counterName; - private final Map curatorPerDatacenter = new HashMap<>(); - private final Map counterPerDatacenter = new HashMap<>(); + private final String counterName; + private final Map curatorPerDatacenter = new HashMap<>(); + private final Map counterPerDatacenter = new HashMap<>(); - CounterAggregator(String counterName, - List zookeeperClients, - int distributedLoaderBackoff, - int distributedLoaderRetries) { - this.counterName = counterName; - for (ZookeeperClient zookeeperClient : zookeeperClients) { - CuratorFramework curatorFramework = zookeeperClient.getCuratorFramework(); - curatorPerDatacenter.put(zookeeperClient.getDatacenterName(), curatorFramework); - DistributedAtomicLong distributedAtomicLong = new DistributedAtomicLong( - curatorFramework, - counterName, - new ExponentialBackoffRetry(distributedLoaderBackoff, distributedLoaderRetries) - ); - counterPerDatacenter.put(zookeeperClient.getDatacenterName(), distributedAtomicLong); - } - } + CounterAggregator( + String counterName, + List zookeeperClients, + int distributedLoaderBackoff, + int distributedLoaderRetries) { + this.counterName = counterName; + for (ZookeeperClient zookeeperClient : zookeeperClients) { + CuratorFramework curatorFramework = zookeeperClient.getCuratorFramework(); + curatorPerDatacenter.put(zookeeperClient.getDatacenterName(), curatorFramework); + DistributedAtomicLong distributedAtomicLong = + new DistributedAtomicLong( + curatorFramework, + counterName, + new ExponentialBackoffRetry(distributedLoaderBackoff, distributedLoaderRetries)); + counterPerDatacenter.put(zookeeperClient.getDatacenterName(), distributedAtomicLong); + } + } - long aggregate() throws Exception { - long sum = 0; - for (Map.Entry counterEntry : counterPerDatacenter.entrySet()) { - ensureConnected(counterEntry.getKey()); - DistributedAtomicLong counter = counterEntry.getValue(); - sum += counter.get().preValue(); - } - return sum; - } + long aggregate() throws Exception { + long sum = 0; + for (Map.Entry counterEntry : + counterPerDatacenter.entrySet()) { + ensureConnected(counterEntry.getKey()); + DistributedAtomicLong counter = counterEntry.getValue(); + sum += counter.get().preValue(); + } + return sum; + } - private void ensureConnected(String datacenter) { - CuratorFramework curator = curatorPerDatacenter.get(datacenter); - if (!curator.getZookeeperClient().isConnected()) { - throw new ZookeeperCounterException( - counterName, - "Could not establish connection to a Zookeeper instance in " + datacenter + "." - ); - } - } + private void ensureConnected(String datacenter) { + CuratorFramework curator = curatorPerDatacenter.get(datacenter); + if (!curator.getZookeeperClient().isConnected()) { + throw new ZookeeperCounterException( + counterName, + "Could not establish connection to a Zookeeper instance in " + datacenter + "."); + } } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/CachingPrometheusClient.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/CachingPrometheusClient.java index 36675b8b6f..25c37519ee 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/CachingPrometheusClient.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/CachingPrometheusClient.java @@ -1,60 +1,64 @@ package pl.allegro.tech.hermes.management.infrastructure.prometheus; +import static java.util.concurrent.TimeUnit.SECONDS; + import com.google.common.base.Ticker; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import pl.allegro.tech.hermes.management.infrastructure.metrics.MonitoringMetricsContainer; - import java.util.List; import java.util.concurrent.ExecutionException; +import pl.allegro.tech.hermes.management.infrastructure.metrics.MonitoringMetricsContainer; -import static java.util.concurrent.TimeUnit.SECONDS; +public class CachingPrometheusClient implements PrometheusClient { + private final PrometheusClient underlyingPrometheusClient; + /* + Metrics will always be requested in the context of a single subscription/topic. The single sub/topic will + always result in the same list of metrics queries. There is no overlapping between metrics used in the context of + topic or subscriptions. That's why it is safe to use a list of queries as a caching key. -public class CachingPrometheusClient implements PrometheusClient { + Maybe it will be worth to cache it per query except of queries when there will be too much overhead + of refreshing all sub/topic metrics if the single fetch fails (currently we invalidate whole metrics container + when one of the sub metric is unavailable) + */ + private final LoadingCache, MonitoringMetricsContainer> prometheusMetricsCache; - private final PrometheusClient underlyingPrometheusClient; - /* - Metrics will always be requested in the context of a single subscription/topic. The single sub/topic will - always result in the same list of metrics queries. There is no overlapping between metrics used in the context of - topic or subscriptions. That's why it is safe to use a list of queries as a caching key. - - Maybe it will be worth to cache it per query except of queries when there will be too much overhead - of refreshing all sub/topic metrics if the single fetch fails (currently we invalidate whole metrics container - when one of the sub metric is unavailable) - */ - private final LoadingCache, MonitoringMetricsContainer> prometheusMetricsCache; - - public CachingPrometheusClient(PrometheusClient underlyingPrometheusClient, Ticker ticker, - long cacheTtlInSeconds, long cacheSize) { - this.underlyingPrometheusClient = underlyingPrometheusClient; - this.prometheusMetricsCache = CacheBuilder.newBuilder() - .ticker(ticker) - .expireAfterWrite(cacheTtlInSeconds, SECONDS) - .maximumSize(cacheSize) - .build(new PrometheusMetricsCacheLoader()); - } + public CachingPrometheusClient( + PrometheusClient underlyingPrometheusClient, + Ticker ticker, + long cacheTtlInSeconds, + long cacheSize) { + this.underlyingPrometheusClient = underlyingPrometheusClient; + this.prometheusMetricsCache = + CacheBuilder.newBuilder() + .ticker(ticker) + .expireAfterWrite(cacheTtlInSeconds, SECONDS) + .maximumSize(cacheSize) + .build(new PrometheusMetricsCacheLoader()); + } - @Override - public MonitoringMetricsContainer readMetrics(List queries) { - try { - MonitoringMetricsContainer monitoringMetricsContainer = prometheusMetricsCache.get(List.copyOf(queries)); - if (monitoringMetricsContainer.hasUnavailableMetrics()) { - // try to reload the on the next fetch - prometheusMetricsCache.invalidate(queries); - } - return monitoringMetricsContainer; - } catch (ExecutionException e) { - // should never happen because the loader does not throw any exceptions - throw new RuntimeException(e); - } + @Override + public MonitoringMetricsContainer readMetrics(List queries) { + try { + MonitoringMetricsContainer monitoringMetricsContainer = + prometheusMetricsCache.get(List.copyOf(queries)); + if (monitoringMetricsContainer.hasUnavailableMetrics()) { + // try to reload the on the next fetch + prometheusMetricsCache.invalidate(queries); + } + return monitoringMetricsContainer; + } catch (ExecutionException e) { + // should never happen because the loader does not throw any exceptions + throw new RuntimeException(e); } + } - private class PrometheusMetricsCacheLoader extends CacheLoader, MonitoringMetricsContainer> { - @Override - public MonitoringMetricsContainer load(List queries) { - return underlyingPrometheusClient.readMetrics(queries); - } + private class PrometheusMetricsCacheLoader + extends CacheLoader, MonitoringMetricsContainer> { + @Override + public MonitoringMetricsContainer load(List queries) { + return underlyingPrometheusClient.readMetrics(queries); } -} \ No newline at end of file + } +} diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/PrometheusClient.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/PrometheusClient.java index 33170fb9ee..fe0142483d 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/PrometheusClient.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/PrometheusClient.java @@ -1,44 +1,53 @@ package pl.allegro.tech.hermes.management.infrastructure.prometheus; +import java.util.List; import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.api.TopicName; import pl.allegro.tech.hermes.management.infrastructure.metrics.MonitoringMetricsContainer; -import java.util.List; - - public interface PrometheusClient { - String SUBSCRIPTION_QUERY_FORMAT = "sum by (group, topic, subscription)" - + " (irate({__name__='%s', group='%s', topic='%s', subscription='%s', %s}[1m]))"; - - String SUBSCRIPTION_QUERY_FORMAT_STATUS_CODE = "sum by (group, topic, subscription)" - + " (irate({__name__='%s', group='%s', topic='%s', subscription='%s', status_code=~'%s', %s}[1m]))"; - - String TOPIC_QUERY_FORMAT = "sum by (group, topic) (irate({__name__='%s', group='%s', " - + "topic='%s', %s}[1m]))"; - - default MonitoringMetricsContainer readMetrics(String... query) { - return readMetrics(List.of(query)); - } - - MonitoringMetricsContainer readMetrics(List queries); - - static String forSubscription(String name, SubscriptionName subscriptionName, String additionalFilters) { - return String.format(SUBSCRIPTION_QUERY_FORMAT, name, - subscriptionName.getTopicName().getGroupName(), subscriptionName.getTopicName().getName(), - subscriptionName.getName(), additionalFilters); - } - - static String forSubscriptionStatusCode(String name, SubscriptionName subscriptionName, - String regex, String additionalFilters) { - return String.format(SUBSCRIPTION_QUERY_FORMAT_STATUS_CODE, name, - subscriptionName.getTopicName().getGroupName(), subscriptionName.getTopicName().getName(), - subscriptionName.getName(), regex, additionalFilters); - } - - - static String forTopic(String name, TopicName topicName, String additionalFilters) { - return String.format(TOPIC_QUERY_FORMAT, name, - topicName.getGroupName(), topicName.getName(), additionalFilters); - } + String SUBSCRIPTION_QUERY_FORMAT = + "sum by (group, topic, subscription)" + + " (irate({__name__='%s', group='%s', topic='%s', subscription='%s', %s}[1m]))"; + + String SUBSCRIPTION_QUERY_FORMAT_STATUS_CODE = + "sum by (group, topic, subscription)" + + " (irate({__name__='%s', group='%s', topic='%s', subscription='%s', status_code=~'%s', %s}[1m]))"; + + String TOPIC_QUERY_FORMAT = + "sum by (group, topic) (irate({__name__='%s', group='%s', " + "topic='%s', %s}[1m]))"; + + default MonitoringMetricsContainer readMetrics(String... query) { + return readMetrics(List.of(query)); + } + + MonitoringMetricsContainer readMetrics(List queries); + + static String forSubscription( + String name, SubscriptionName subscriptionName, String additionalFilters) { + return String.format( + SUBSCRIPTION_QUERY_FORMAT, + name, + subscriptionName.getTopicName().getGroupName(), + subscriptionName.getTopicName().getName(), + subscriptionName.getName(), + additionalFilters); + } + + static String forSubscriptionStatusCode( + String name, SubscriptionName subscriptionName, String regex, String additionalFilters) { + return String.format( + SUBSCRIPTION_QUERY_FORMAT_STATUS_CODE, + name, + subscriptionName.getTopicName().getGroupName(), + subscriptionName.getTopicName().getName(), + subscriptionName.getName(), + regex, + additionalFilters); + } + + static String forTopic(String name, TopicName topicName, String additionalFilters) { + return String.format( + TOPIC_QUERY_FORMAT, name, topicName.getGroupName(), topicName.getName(), additionalFilters); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/PrometheusMetricsProvider.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/PrometheusMetricsProvider.java index 7121281b5a..ab404e6c30 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/PrometheusMetricsProvider.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/PrometheusMetricsProvider.java @@ -1,94 +1,132 @@ package pl.allegro.tech.hermes.management.infrastructure.prometheus; +import static pl.allegro.tech.hermes.management.infrastructure.prometheus.PrometheusClient.forSubscription; +import static pl.allegro.tech.hermes.management.infrastructure.prometheus.PrometheusClient.forSubscriptionStatusCode; +import static pl.allegro.tech.hermes.management.infrastructure.prometheus.PrometheusClient.forTopic; + import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.api.TopicName; import pl.allegro.tech.hermes.management.infrastructure.metrics.MonitoringMetricsContainer; import pl.allegro.tech.hermes.management.infrastructure.metrics.MonitoringSubscriptionMetricsProvider; import pl.allegro.tech.hermes.management.infrastructure.metrics.MonitoringTopicMetricsProvider; -import static pl.allegro.tech.hermes.management.infrastructure.prometheus.PrometheusClient.forSubscription; -import static pl.allegro.tech.hermes.management.infrastructure.prometheus.PrometheusClient.forSubscriptionStatusCode; -import static pl.allegro.tech.hermes.management.infrastructure.prometheus.PrometheusClient.forTopic; - - -public class PrometheusMetricsProvider implements MonitoringSubscriptionMetricsProvider, MonitoringTopicMetricsProvider { +public class PrometheusMetricsProvider + implements MonitoringSubscriptionMetricsProvider, MonitoringTopicMetricsProvider { - private static final String SUBSCRIPTION_DELIVERED = "subscription_delivered_total"; - private static final String SUBSCRIPTION_TIMEOUTS = "subscription_timeouts_total"; - private static final String SUBSCRIPTION_THROUGHPUT = "subscription_throughput_bytes_total"; - private static final String SUBSCRIPTION_OTHER_ERRORS = "subscription_other_errors_total"; - private static final String SUBSCRIPTION_BATCHES = "subscription_batches_total"; - private static final String SUBSCRIPTION_STATUS_CODES = "subscription_http_status_codes_total"; - private static final String SUBSCRIPTION_RETRIES = "subscription_retries_total"; + private static final String SUBSCRIPTION_DELIVERED = "subscription_delivered_total"; + private static final String SUBSCRIPTION_TIMEOUTS = "subscription_timeouts_total"; + private static final String SUBSCRIPTION_THROUGHPUT = "subscription_throughput_bytes_total"; + private static final String SUBSCRIPTION_OTHER_ERRORS = "subscription_other_errors_total"; + private static final String SUBSCRIPTION_BATCHES = "subscription_batches_total"; + private static final String SUBSCRIPTION_STATUS_CODES = "subscription_http_status_codes_total"; + private static final String SUBSCRIPTION_RETRIES = "subscription_retries_total"; - private static final String TOPIC_RATE = "topic_requests_total"; - private static final String TOPIC_DELIVERY_RATE = "subscription_delivered_total"; - private static final String TOPIC_THROUGHPUT_RATE = "topic_throughput_bytes_total"; + private static final String TOPIC_RATE = "topic_requests_total"; + private static final String TOPIC_DELIVERY_RATE = "subscription_delivered_total"; + private static final String TOPIC_THROUGHPUT_RATE = "topic_throughput_bytes_total"; - private final String consumersMetricsPrefix; - private final String frontendMetricsPrefix; - private final String additionalFilters; - private final PrometheusClient prometheusClient; + private final String consumersMetricsPrefix; + private final String frontendMetricsPrefix; + private final String additionalFilters; + private final PrometheusClient prometheusClient; - public PrometheusMetricsProvider(PrometheusClient prometheusClient, String consumersMetricsPrefix, - String frontendMetricsPrefix, String additionalFilters) { - this.prometheusClient = prometheusClient; - this.consumersMetricsPrefix = consumersMetricsPrefix.isEmpty() ? "" : consumersMetricsPrefix + "_"; - this.frontendMetricsPrefix = frontendMetricsPrefix.isEmpty() ? "" : frontendMetricsPrefix + "_"; - this.additionalFilters = additionalFilters; - } + public PrometheusMetricsProvider( + PrometheusClient prometheusClient, + String consumersMetricsPrefix, + String frontendMetricsPrefix, + String additionalFilters) { + this.prometheusClient = prometheusClient; + this.consumersMetricsPrefix = + consumersMetricsPrefix.isEmpty() ? "" : consumersMetricsPrefix + "_"; + this.frontendMetricsPrefix = frontendMetricsPrefix.isEmpty() ? "" : frontendMetricsPrefix + "_"; + this.additionalFilters = additionalFilters; + } - @Override - public MonitoringSubscriptionMetrics subscriptionMetrics(SubscriptionName subscriptionName) { - String subscriptionDeliveredQuery = forSubscription(consumerMetricName(SUBSCRIPTION_DELIVERED), subscriptionName, additionalFilters); - String subscriptionTimeoutsQuery = forSubscription(consumerMetricName(SUBSCRIPTION_TIMEOUTS), subscriptionName, additionalFilters); - String subscriptionThroughputQuery = forSubscription(consumerMetricName(SUBSCRIPTION_THROUGHPUT), subscriptionName, additionalFilters); - String subscriptionOtherErrorsQuery = forSubscription(consumerMetricName(SUBSCRIPTION_OTHER_ERRORS), subscriptionName, additionalFilters); - String subscriptionBatchesQuery = forSubscription(consumerMetricName(SUBSCRIPTION_BATCHES), subscriptionName, additionalFilters); - String subscriptionRetriesQuery = forSubscription(consumerMetricName(SUBSCRIPTION_RETRIES), subscriptionName, additionalFilters); - String subscription2xx = forSubscriptionStatusCode(consumerMetricName(SUBSCRIPTION_STATUS_CODES), subscriptionName, "2.*", additionalFilters); - String subscription4xx = forSubscriptionStatusCode(consumerMetricName(SUBSCRIPTION_STATUS_CODES), subscriptionName, "4.*", additionalFilters); - String subscription5xx = forSubscriptionStatusCode(consumerMetricName(SUBSCRIPTION_STATUS_CODES), subscriptionName, "5.*", additionalFilters); + @Override + public MonitoringSubscriptionMetrics subscriptionMetrics(SubscriptionName subscriptionName) { + String subscriptionDeliveredQuery = + forSubscription( + consumerMetricName(SUBSCRIPTION_DELIVERED), subscriptionName, additionalFilters); + String subscriptionTimeoutsQuery = + forSubscription( + consumerMetricName(SUBSCRIPTION_TIMEOUTS), subscriptionName, additionalFilters); + String subscriptionThroughputQuery = + forSubscription( + consumerMetricName(SUBSCRIPTION_THROUGHPUT), subscriptionName, additionalFilters); + String subscriptionOtherErrorsQuery = + forSubscription( + consumerMetricName(SUBSCRIPTION_OTHER_ERRORS), subscriptionName, additionalFilters); + String subscriptionBatchesQuery = + forSubscription( + consumerMetricName(SUBSCRIPTION_BATCHES), subscriptionName, additionalFilters); + String subscriptionRetriesQuery = + forSubscription( + consumerMetricName(SUBSCRIPTION_RETRIES), subscriptionName, additionalFilters); + String subscription2xx = + forSubscriptionStatusCode( + consumerMetricName(SUBSCRIPTION_STATUS_CODES), + subscriptionName, + "2.*", + additionalFilters); + String subscription4xx = + forSubscriptionStatusCode( + consumerMetricName(SUBSCRIPTION_STATUS_CODES), + subscriptionName, + "4.*", + additionalFilters); + String subscription5xx = + forSubscriptionStatusCode( + consumerMetricName(SUBSCRIPTION_STATUS_CODES), + subscriptionName, + "5.*", + additionalFilters); - MonitoringMetricsContainer prometheusMetricsContainer = prometheusClient.readMetrics( - subscriptionDeliveredQuery, subscriptionTimeoutsQuery, subscriptionRetriesQuery, subscriptionThroughputQuery, - subscriptionOtherErrorsQuery, subscriptionBatchesQuery, subscription2xx, subscription4xx, subscription5xx - ); - return MonitoringSubscriptionMetricsProvider - .metricsBuilder() - .withRate(prometheusMetricsContainer.metricValue(subscriptionDeliveredQuery)) - .withTimeouts(prometheusMetricsContainer.metricValue(subscriptionTimeoutsQuery)) - .withThroughput(prometheusMetricsContainer.metricValue(subscriptionThroughputQuery)) - .withOtherErrors(prometheusMetricsContainer.metricValue(subscriptionOtherErrorsQuery)) - .withMetricPathBatchRate(prometheusMetricsContainer.metricValue(subscriptionBatchesQuery)) - .withCodes2xx(prometheusMetricsContainer.metricValue(subscription2xx)) - .withCode4xx(prometheusMetricsContainer.metricValue(subscription4xx)) - .withCode5xx(prometheusMetricsContainer.metricValue(subscription5xx)) - .withRetries(prometheusMetricsContainer.metricValue(subscriptionRetriesQuery)) - .build(); - } + MonitoringMetricsContainer prometheusMetricsContainer = + prometheusClient.readMetrics( + subscriptionDeliveredQuery, + subscriptionTimeoutsQuery, + subscriptionRetriesQuery, + subscriptionThroughputQuery, + subscriptionOtherErrorsQuery, + subscriptionBatchesQuery, + subscription2xx, + subscription4xx, + subscription5xx); + return MonitoringSubscriptionMetricsProvider.metricsBuilder() + .withRate(prometheusMetricsContainer.metricValue(subscriptionDeliveredQuery)) + .withTimeouts(prometheusMetricsContainer.metricValue(subscriptionTimeoutsQuery)) + .withThroughput(prometheusMetricsContainer.metricValue(subscriptionThroughputQuery)) + .withOtherErrors(prometheusMetricsContainer.metricValue(subscriptionOtherErrorsQuery)) + .withMetricPathBatchRate(prometheusMetricsContainer.metricValue(subscriptionBatchesQuery)) + .withCodes2xx(prometheusMetricsContainer.metricValue(subscription2xx)) + .withCode4xx(prometheusMetricsContainer.metricValue(subscription4xx)) + .withCode5xx(prometheusMetricsContainer.metricValue(subscription5xx)) + .withRetries(prometheusMetricsContainer.metricValue(subscriptionRetriesQuery)) + .build(); + } - @Override - public MonitoringTopicMetrics topicMetrics(TopicName topicName) { - String topicRateQuery = forTopic(frontendMetricName(TOPIC_RATE), topicName, additionalFilters); - String topicDeliveryRateQuery = forTopic(consumerMetricName(TOPIC_DELIVERY_RATE), topicName, additionalFilters); - String topicThroughputQuery = forTopic(frontendMetricName(TOPIC_THROUGHPUT_RATE), topicName, additionalFilters); + @Override + public MonitoringTopicMetrics topicMetrics(TopicName topicName) { + String topicRateQuery = forTopic(frontendMetricName(TOPIC_RATE), topicName, additionalFilters); + String topicDeliveryRateQuery = + forTopic(consumerMetricName(TOPIC_DELIVERY_RATE), topicName, additionalFilters); + String topicThroughputQuery = + forTopic(frontendMetricName(TOPIC_THROUGHPUT_RATE), topicName, additionalFilters); - MonitoringMetricsContainer prometheusMetricsContainer = prometheusClient.readMetrics( - topicRateQuery, topicDeliveryRateQuery, topicThroughputQuery); - return MonitoringTopicMetricsProvider - .metricsBuilder() - .withRate(prometheusMetricsContainer.metricValue(topicRateQuery)) - .withDeliveryRate(prometheusMetricsContainer.metricValue(topicDeliveryRateQuery)) - .withThroughput(prometheusMetricsContainer.metricValue(topicThroughputQuery)) - .build(); - } + MonitoringMetricsContainer prometheusMetricsContainer = + prometheusClient.readMetrics(topicRateQuery, topicDeliveryRateQuery, topicThroughputQuery); + return MonitoringTopicMetricsProvider.metricsBuilder() + .withRate(prometheusMetricsContainer.metricValue(topicRateQuery)) + .withDeliveryRate(prometheusMetricsContainer.metricValue(topicDeliveryRateQuery)) + .withThroughput(prometheusMetricsContainer.metricValue(topicThroughputQuery)) + .build(); + } - private String consumerMetricName(String name) { - return consumersMetricsPrefix + name; - } + private String consumerMetricName(String name) { + return consumersMetricsPrefix + name; + } - private String frontendMetricName(String name) { - return frontendMetricsPrefix + name; - } -} \ No newline at end of file + private String frontendMetricName(String name) { + return frontendMetricsPrefix + name; + } +} diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/PrometheusResponse.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/PrometheusResponse.java index 74c1c29c9e..69d005148d 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/PrometheusResponse.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/PrometheusResponse.java @@ -2,37 +2,34 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.List; import java.util.Optional; -record PrometheusResponse(@JsonProperty("status") String status, - @JsonProperty("data") Data data) { +record PrometheusResponse(@JsonProperty("status") String status, @JsonProperty("data") Data data) { - boolean isSuccess() { - return status.equals("success") && data.isVector(); - } + boolean isSuccess() { + return status.equals("success") && data.isVector(); + } - record Data(@JsonProperty("resultType") String resultType, - @JsonProperty("result") List results) { - boolean isVector() { - return resultType.equals("vector"); - } + record Data( + @JsonProperty("resultType") String resultType, + @JsonProperty("result") List results) { + boolean isVector() { + return resultType.equals("vector"); } + } - @JsonIgnoreProperties(ignoreUnknown = true) - record VectorResult( - @JsonProperty("value") List vector) { + @JsonIgnoreProperties(ignoreUnknown = true) + record VectorResult(@JsonProperty("value") List vector) { - private static final int VALID_VECTOR_LENGTH = 2; - private static final int SCALAR_INDEX_VALUE = 1; + private static final int VALID_VECTOR_LENGTH = 2; + private static final int SCALAR_INDEX_VALUE = 1; - Optional getValue() { - if (vector.size() != VALID_VECTOR_LENGTH) { - return Optional.empty(); - } - return Optional.of(Double.parseDouble(vector.get(SCALAR_INDEX_VALUE))); - } + Optional getValue() { + if (vector.size() != VALID_VECTOR_LENGTH) { + return Optional.empty(); + } + return Optional.of(Double.parseDouble(vector.get(SCALAR_INDEX_VALUE))); } - + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/RestTemplatePrometheusClient.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/RestTemplatePrometheusClient.java index d10e09e425..6b7d2e62ee 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/RestTemplatePrometheusClient.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/prometheus/RestTemplatePrometheusClient.java @@ -1,7 +1,18 @@ package pl.allegro.tech.hermes.management.infrastructure.prometheus; +import static java.net.URLEncoder.encode; +import static java.nio.charset.StandardCharsets.UTF_8; + import com.google.common.base.Preconditions; import io.micrometer.core.instrument.MeterRegistry; +import java.net.URI; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -12,109 +23,107 @@ import pl.allegro.tech.hermes.api.MetricDecimalValue; import pl.allegro.tech.hermes.management.infrastructure.metrics.MonitoringMetricsContainer; -import java.net.URI; -import java.time.Duration; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import static java.net.URLEncoder.encode; -import static java.nio.charset.StandardCharsets.UTF_8; - - public class RestTemplatePrometheusClient implements PrometheusClient { - private static final Logger logger = LoggerFactory.getLogger(RestTemplatePrometheusClient.class); - - private final URI prometheusUri; - private final RestTemplate restTemplate; - private final ExecutorService executorService; - private final Duration fetchingTimeout; - private final MeterRegistry meterRegistry; - - - public RestTemplatePrometheusClient(RestTemplate restTemplate, - URI prometheusUri, - ExecutorService executorService, - Duration fetchingTimeoutMillis, - MeterRegistry meterRegistry) { - this.restTemplate = restTemplate; - this.prometheusUri = prometheusUri; - this.executorService = executorService; - this.fetchingTimeout = fetchingTimeoutMillis; - this.meterRegistry = meterRegistry; - } - - @Override - public MonitoringMetricsContainer readMetrics(List queries) { - return fetchInParallelFromPrometheus(queries); + private static final Logger logger = LoggerFactory.getLogger(RestTemplatePrometheusClient.class); + + private final URI prometheusUri; + private final RestTemplate restTemplate; + private final ExecutorService executorService; + private final Duration fetchingTimeout; + private final MeterRegistry meterRegistry; + + public RestTemplatePrometheusClient( + RestTemplate restTemplate, + URI prometheusUri, + ExecutorService executorService, + Duration fetchingTimeoutMillis, + MeterRegistry meterRegistry) { + this.restTemplate = restTemplate; + this.prometheusUri = prometheusUri; + this.executorService = executorService; + this.fetchingTimeout = fetchingTimeoutMillis; + this.meterRegistry = meterRegistry; + } + + @Override + public MonitoringMetricsContainer readMetrics(List queries) { + return fetchInParallelFromPrometheus(queries); + } + + private MonitoringMetricsContainer fetchInParallelFromPrometheus(List queries) { + CompletableFuture> aggregatedFuture = + getAggregatedCompletableFuture(queries); + + try { + Map metrics = + aggregatedFuture.get(fetchingTimeout.toMillis(), TimeUnit.MILLISECONDS); + return MonitoringMetricsContainer.initialized(metrics); + } catch (InterruptedException e) { + // possibly let know the caller that the thread was interrupted + Thread.currentThread().interrupt(); + logger.warn("Prometheus fetching thread was interrupted...", e); + return MonitoringMetricsContainer.unavailable(); + } catch (Exception ex) { + logger.warn("Unexpected exception during fetching metrics from prometheus...", ex); + return MonitoringMetricsContainer.unavailable(); } - - private MonitoringMetricsContainer fetchInParallelFromPrometheus(List queries) { - CompletableFuture> aggregatedFuture = getAggregatedCompletableFuture(queries); - - try { - Map metrics = aggregatedFuture.get(fetchingTimeout.toMillis(), TimeUnit.MILLISECONDS); - return MonitoringMetricsContainer.initialized(metrics); - } catch (InterruptedException e) { - // possibly let know the caller that the thread was interrupted - Thread.currentThread().interrupt(); - logger.warn("Prometheus fetching thread was interrupted...", e); - return MonitoringMetricsContainer.unavailable(); - } catch (Exception ex) { - logger.warn("Unexpected exception during fetching metrics from prometheus...", ex); - return MonitoringMetricsContainer.unavailable(); - } - } - - private CompletableFuture> getAggregatedCompletableFuture(List queries) { - // has to be collected to run in parallel - List>> futures = queries.stream() - .map(this::readSingleMetric) - .toList(); - - return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) - .thenApply( - v -> futures.stream().map(CompletableFuture::join) - .collect(Collectors.toMap(Pair::getKey, Pair::getValue)) - ); - } - - private CompletableFuture> readSingleMetric(String query) { - return CompletableFuture.supplyAsync(() -> queryPrometheus(query), executorService); - } - - private Pair queryPrometheus(String query) { - try { - URI queryUri = URI.create(prometheusUri.toString() + "/api/v1/query?query=" + encode(query, UTF_8)); - PrometheusResponse response = restTemplate.exchange(queryUri, - HttpMethod.GET, HttpEntity.EMPTY, PrometheusResponse.class).getBody(); - - Preconditions.checkNotNull(response, "Prometheus response is null"); - Preconditions.checkState(response.isSuccess(), "Prometheus response does not contain valid data"); - - MetricDecimalValue result = parseResponse(response); - meterRegistry.counter("read-metric-from-prometheus.success").increment(); - return Pair.of(query, result); - } catch (HttpStatusCodeException ex) { - logger.warn("Unable to read from Prometheus. Query: {}, Status code: {}. Response body: {}", - query, ex.getStatusCode(), ex.getResponseBodyAsString(), ex); - return Pair.of(query, MetricDecimalValue.unavailable()); - } catch (Exception ex) { - logger.warn("Unable to read from Prometheus. Query: {}", query, ex); - meterRegistry.counter("read-metric-from-prometheus.error").increment(); - return Pair.of(query, MetricDecimalValue.unavailable()); - } - } - - private MetricDecimalValue parseResponse(PrometheusResponse response) { - return response.data().results().stream() - .findFirst() - .flatMap(PrometheusResponse.VectorResult::getValue) - .map(value -> MetricDecimalValue.of(value.toString())) - .orElse(MetricDecimalValue.defaultValue()); + } + + private CompletableFuture> getAggregatedCompletableFuture( + List queries) { + // has to be collected to run in parallel + List>> futures = + queries.stream().map(this::readSingleMetric).toList(); + + return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) + .thenApply( + v -> + futures.stream() + .map(CompletableFuture::join) + .collect(Collectors.toMap(Pair::getKey, Pair::getValue))); + } + + private CompletableFuture> readSingleMetric(String query) { + return CompletableFuture.supplyAsync(() -> queryPrometheus(query), executorService); + } + + private Pair queryPrometheus(String query) { + try { + URI queryUri = + URI.create(prometheusUri.toString() + "/api/v1/query?query=" + encode(query, UTF_8)); + PrometheusResponse response = + restTemplate + .exchange(queryUri, HttpMethod.GET, HttpEntity.EMPTY, PrometheusResponse.class) + .getBody(); + + Preconditions.checkNotNull(response, "Prometheus response is null"); + Preconditions.checkState( + response.isSuccess(), "Prometheus response does not contain valid data"); + + MetricDecimalValue result = parseResponse(response); + meterRegistry.counter("read-metric-from-prometheus.success").increment(); + return Pair.of(query, result); + } catch (HttpStatusCodeException ex) { + logger.warn( + "Unable to read from Prometheus. Query: {}, Status code: {}. Response body: {}", + query, + ex.getStatusCode(), + ex.getResponseBodyAsString(), + ex); + return Pair.of(query, MetricDecimalValue.unavailable()); + } catch (Exception ex) { + logger.warn("Unable to read from Prometheus. Query: {}", query, ex); + meterRegistry.counter("read-metric-from-prometheus.error").increment(); + return Pair.of(query, MetricDecimalValue.unavailable()); } + } + + private MetricDecimalValue parseResponse(PrometheusResponse response) { + return response.data().results().stream() + .findFirst() + .flatMap(PrometheusResponse.VectorResult::getValue) + .map(value -> MetricDecimalValue.of(value.toString())) + .orElse(MetricDecimalValue.defaultValue()); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/MatcherQuery.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/MatcherQuery.java index 2363343342..75a3f62756 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/MatcherQuery.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/MatcherQuery.java @@ -1,6 +1,9 @@ package pl.allegro.tech.hermes.management.infrastructure.query; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Map; +import java.util.function.Predicate; +import java.util.stream.Stream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.Query; @@ -8,67 +11,63 @@ import pl.allegro.tech.hermes.management.infrastructure.query.matcher.MatcherException; import pl.allegro.tech.hermes.management.infrastructure.query.matcher.MatcherInputException; -import java.util.Map; -import java.util.function.Predicate; -import java.util.stream.Stream; - public class MatcherQuery implements Query { - private static final Logger logger = LoggerFactory.getLogger(MatcherQuery.class); + private static final Logger logger = LoggerFactory.getLogger(MatcherQuery.class); - private final Matcher matcher; - private final ObjectMapper objectMapper; + private final Matcher matcher; + private final ObjectMapper objectMapper; - private MatcherQuery(Matcher matcher, ObjectMapper objectMapper) { - this.matcher = matcher; - this.objectMapper = objectMapper; - } + private MatcherQuery(Matcher matcher, ObjectMapper objectMapper) { + this.matcher = matcher; + this.objectMapper = objectMapper; + } - @Override - public Stream filter(Stream input) { - return input.filter(getPredicate()); - } + @Override + public Stream filter(Stream input) { + return input.filter(getPredicate()); + } - @Override - public Stream filterNames(Stream input) { - return input.filter(getSoftPredicate()); - } + @Override + public Stream filterNames(Stream input) { + return input.filter(getSoftPredicate()); + } - public Predicate getPredicate() { - return (value) -> { - try { - return matcher.match(convertToMap(value)); - } catch (MatcherException e) { - logger.info("Failed to match {}, skipping", value, e); - return false; - } catch (MatcherInputException e) { - logger.error("Not existing query property", e); - throw new IllegalArgumentException(e); - } - }; - } + public Predicate getPredicate() { + return (value) -> { + try { + return matcher.match(convertToMap(value)); + } catch (MatcherException e) { + logger.info("Failed to match {}, skipping", value, e); + return false; + } catch (MatcherInputException e) { + logger.error("Not existing query property", e); + throw new IllegalArgumentException(e); + } + }; + } - public Predicate getSoftPredicate() { - return (value) -> { - try { - return matcher.match(convertToMap(value)); - } catch (MatcherException e) { - logger.info("Failed to match {}, skipping", value, e); - return false; - } catch (MatcherInputException e) { - //Non-existing property is normal when querying objects with non-default class - return true; - } - }; - } + public Predicate getSoftPredicate() { + return (value) -> { + try { + return matcher.match(convertToMap(value)); + } catch (MatcherException e) { + logger.info("Failed to match {}, skipping", value, e); + return false; + } catch (MatcherInputException e) { + // Non-existing property is normal when querying objects with non-default class + return true; + } + }; + } - @SuppressWarnings("unchecked") - //workaround for type which is not java bean - private Map convertToMap(K value) { - return objectMapper.convertValue(value, Map.class); - } + @SuppressWarnings("unchecked") + // workaround for type which is not java bean + private Map convertToMap(K value) { + return objectMapper.convertValue(value, Map.class); + } - public static Query fromMatcher(Matcher matcher, ObjectMapper objectMapper) { - return new MatcherQuery<>(matcher, objectMapper); - } + public static Query fromMatcher(Matcher matcher, ObjectMapper objectMapper) { + return new MatcherQuery<>(matcher, objectMapper); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/graph/JXPathAttribute.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/graph/JXPathAttribute.java index a584a94dcb..832d364c9c 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/graph/JXPathAttribute.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/graph/JXPathAttribute.java @@ -5,19 +5,19 @@ public class JXPathAttribute implements ObjectAttribute { - private final Object target; + private final Object target; - private final String path; + private final String path; - public JXPathAttribute(Object target, String path) { - this.target = target; - this.path = path.replace('.', '/'); - } + public JXPathAttribute(Object target, String path) { + this.target = target; + this.path = path.replace('.', '/'); + } - @Override - public Object value() { - JXPathContext context = JXPathContext.newContext(target); - context.setFunctions(new FunctionLibrary()); - return context.getValue(path); - } + @Override + public Object value() { + JXPathContext context = JXPathContext.newContext(target); + context.setFunctions(new FunctionLibrary()); + return context.getValue(path); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/graph/ObjectAttribute.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/graph/ObjectAttribute.java index 7536e1f8e7..0585ab4690 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/graph/ObjectAttribute.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/graph/ObjectAttribute.java @@ -2,5 +2,5 @@ public interface ObjectAttribute { - Object value(); + Object value(); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/graph/ObjectGraph.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/graph/ObjectGraph.java index c0cf2ed01d..68a5b8e6b9 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/graph/ObjectGraph.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/graph/ObjectGraph.java @@ -2,17 +2,17 @@ public class ObjectGraph { - private final Object target; + private final Object target; - private ObjectGraph(Object target) { - this.target = target; - } + private ObjectGraph(Object target) { + this.target = target; + } - public ObjectAttribute navigate(String path) { - return new JXPathAttribute(target, path); - } + public ObjectAttribute navigate(String path) { + return new JXPathAttribute(target, path); + } - public static ObjectGraph from(Object target) { - return new ObjectGraph(target); - } + public static ObjectGraph from(Object target) { + return new ObjectGraph(target); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/AndMatcher.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/AndMatcher.java index 18057af836..79e8b42f98 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/AndMatcher.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/AndMatcher.java @@ -4,18 +4,18 @@ public class AndMatcher implements Matcher { - private Collection matchers; + private Collection matchers; - public AndMatcher(Collection matchers) { - this.matchers = matchers; - } + public AndMatcher(Collection matchers) { + this.matchers = matchers; + } - @Override - public boolean match(Object value) { - return matchers.stream().reduce( - true, - (match, matcher) -> match && matcher.match(value), - (first, second) -> first && second - ); - } + @Override + public boolean match(Object value) { + return matchers.stream() + .reduce( + true, + (match, matcher) -> match && matcher.match(value), + (first, second) -> first && second); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/ComparisonMatcher.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/ComparisonMatcher.java index 244043e9bc..bcaae73d31 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/ComparisonMatcher.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/ComparisonMatcher.java @@ -1,68 +1,69 @@ package pl.allegro.tech.hermes.management.infrastructure.query.matcher; -import org.apache.commons.jxpath.JXPathException; -import pl.allegro.tech.hermes.management.infrastructure.query.graph.ObjectGraph; +import static java.lang.Double.parseDouble; import java.util.Optional; - -import static java.lang.Double.parseDouble; +import org.apache.commons.jxpath.JXPathException; +import pl.allegro.tech.hermes.management.infrastructure.query.graph.ObjectGraph; public class ComparisonMatcher implements Matcher { - private final String attribute; + private final String attribute; - private final Object rightSideValue; + private final Object rightSideValue; - private final ComparisonOperator operator; + private final ComparisonOperator operator; - public ComparisonMatcher(String attribute, Object rightSideValue, ComparisonOperator operator) { - this.attribute = attribute; - this.rightSideValue = rightSideValue; - this.operator = operator; - } + public ComparisonMatcher(String attribute, Object rightSideValue, ComparisonOperator operator) { + this.attribute = attribute; + this.rightSideValue = rightSideValue; + this.operator = operator; + } - @Override - public boolean match(Object object) { - Object leftSideValue = extractAttributeValue(object); + @Override + public boolean match(Object object) { + Object leftSideValue = extractAttributeValue(object); - Optional leftSideValueAsNumber = tryParseNumber(leftSideValue); - Optional rightSideValueAsNumber = tryParseNumber(rightSideValue); + Optional leftSideValueAsNumber = tryParseNumber(leftSideValue); + Optional rightSideValueAsNumber = tryParseNumber(rightSideValue); - if (!leftSideValueAsNumber.isPresent()) { - return true; - } - if (!rightSideValueAsNumber.isPresent()) { - throw new MatcherInputException("Comparison operator requires numerical data"); - } - return makeComparison(leftSideValueAsNumber.get(), rightSideValueAsNumber.get()); + if (!leftSideValueAsNumber.isPresent()) { + return true; } - - private Object extractAttributeValue(Object object) { - try { - Object value = ObjectGraph.from(object).navigate(attribute).value(); - - if (value == null) { - throw new MatcherInputException(String.format("Cannot find '%s' attribute", this.attribute)); - } - return value; - } catch (JXPathException e) { - throw new MatcherException(String.format("Could not navigate to specific path: '%s'", attribute), e); - } + if (!rightSideValueAsNumber.isPresent()) { + throw new MatcherInputException("Comparison operator requires numerical data"); } - - private Optional tryParseNumber(Object value) { - try { - return Optional.of(parseDouble(asString(value))); - } catch (NumberFormatException e) { - return Optional.empty(); - } + return makeComparison(leftSideValueAsNumber.get(), rightSideValueAsNumber.get()); + } + + private Object extractAttributeValue(Object object) { + try { + Object value = ObjectGraph.from(object).navigate(attribute).value(); + + if (value == null) { + throw new MatcherInputException( + String.format("Cannot find '%s' attribute", this.attribute)); + } + return value; + } catch (JXPathException e) { + throw new MatcherException( + String.format("Could not navigate to specific path: '%s'", attribute), e); } + } - private boolean makeComparison(Double leftSideNumber, Double rightSideNumber) { - return operator.compare(leftSideNumber, rightSideNumber); + private Optional tryParseNumber(Object value) { + try { + return Optional.of(parseDouble(asString(value))); + } catch (NumberFormatException e) { + return Optional.empty(); } + } - private String asString(Object value) { - return String.valueOf(value); - } + private boolean makeComparison(Double leftSideNumber, Double rightSideNumber) { + return operator.compare(leftSideNumber, rightSideNumber); + } + + private String asString(Object value) { + return String.valueOf(value); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/ComparisonOperator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/ComparisonOperator.java index 5d7709f607..d111e3de38 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/ComparisonOperator.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/ComparisonOperator.java @@ -1,6 +1,5 @@ package pl.allegro.tech.hermes.management.infrastructure.query.matcher; - public interface ComparisonOperator { - boolean compare(double a, double b); + boolean compare(double a, double b); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/EqualityMatcher.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/EqualityMatcher.java index 0b264bc7f5..555dfac31d 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/EqualityMatcher.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/EqualityMatcher.java @@ -5,30 +5,30 @@ public class EqualityMatcher implements Matcher { - private final String attribute; + private final String attribute; - private final Object expected; + private final Object expected; - public EqualityMatcher(String attribute, Object expected) { - this.attribute = attribute; - this.expected = expected; - } + public EqualityMatcher(String attribute, Object expected) { + this.attribute = attribute; + this.expected = expected; + } - @Override - public boolean match(Object value) { - try { - if (expected == null) { - return false; - } - Object actual = ObjectGraph.from(value).navigate(attribute).value(); - return expected.equals(actual) - || asString(expected).equals(asString(actual)); - } catch (JXPathException e) { - throw new MatcherException(String.format("Could not navigate to specific path: '%s'", attribute), e); - } + @Override + public boolean match(Object value) { + try { + if (expected == null) { + return false; + } + Object actual = ObjectGraph.from(value).navigate(attribute).value(); + return expected.equals(actual) || asString(expected).equals(asString(actual)); + } catch (JXPathException e) { + throw new MatcherException( + String.format("Could not navigate to specific path: '%s'", attribute), e); } + } - private static String asString(Object value) { - return String.valueOf(value); - } + private static String asString(Object value) { + return String.valueOf(value); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/InMatcher.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/InMatcher.java index 296bb76e8f..836f827312 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/InMatcher.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/InMatcher.java @@ -1,41 +1,40 @@ package pl.allegro.tech.hermes.management.infrastructure.query.matcher; +import java.util.Arrays; import org.apache.commons.jxpath.JXPathException; import pl.allegro.tech.hermes.management.infrastructure.query.graph.ObjectGraph; -import java.util.Arrays; - public class InMatcher implements Matcher { - private final String attribute; + private final String attribute; - private final Object[] values; + private final Object[] values; - public InMatcher(String attribute, Object[] values) { - this.attribute = attribute; - this.values = values; - } + public InMatcher(String attribute, Object[] values) { + this.attribute = attribute; + this.values = values; + } - @Override - public boolean match(Object value) { - - try { - if (values == null || values.length == 0) { - return false; - } - Object actual = ObjectGraph.from(value).navigate(attribute).value(); - return actual != null && (contains(actual) || contains(asString(actual))); - } catch (JXPathException e) { - throw new MatcherException(String.format("Could not navigate to specific path: '%s'", attribute), e); - } - } + @Override + public boolean match(Object value) { - private boolean contains(Object actual) { - return Arrays.stream(values) - .anyMatch(actual::equals); + try { + if (values == null || values.length == 0) { + return false; + } + Object actual = ObjectGraph.from(value).navigate(attribute).value(); + return actual != null && (contains(actual) || contains(asString(actual))); + } catch (JXPathException e) { + throw new MatcherException( + String.format("Could not navigate to specific path: '%s'", attribute), e); } + } - private String asString(Object actual) { - return String.valueOf(actual); - } + private boolean contains(Object actual) { + return Arrays.stream(values).anyMatch(actual::equals); + } + + private String asString(Object actual) { + return String.valueOf(actual); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/LikeMatcher.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/LikeMatcher.java index 4662ab6140..20cdd22d23 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/LikeMatcher.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/LikeMatcher.java @@ -1,37 +1,38 @@ package pl.allegro.tech.hermes.management.infrastructure.query.matcher; -import org.apache.commons.jxpath.JXPathException; -import pl.allegro.tech.hermes.management.infrastructure.query.graph.ObjectGraph; - import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; +import org.apache.commons.jxpath.JXPathException; +import pl.allegro.tech.hermes.management.infrastructure.query.graph.ObjectGraph; public class LikeMatcher implements Matcher { - private final String attribute; + private final String attribute; - private final Pattern expected; + private final Pattern expected; - public LikeMatcher(String attribute, Object expected) { - this.attribute = attribute; - try { - this.expected = Pattern.compile(asString(expected)); - } catch (PatternSyntaxException e) { - throw new MatcherException(String.format("Could not parse regexp pattern: '%s'", expected), e); - } + public LikeMatcher(String attribute, Object expected) { + this.attribute = attribute; + try { + this.expected = Pattern.compile(asString(expected)); + } catch (PatternSyntaxException e) { + throw new MatcherException( + String.format("Could not parse regexp pattern: '%s'", expected), e); } - - @Override - public boolean match(Object value) { - try { - Object actual = ObjectGraph.from(value).navigate(attribute).value(); - return expected.matcher(asString(actual)).matches(); - } catch (JXPathException e) { - throw new MatcherException(String.format("Could not navigate to specific path: '%s'", attribute), e); - } + } + + @Override + public boolean match(Object value) { + try { + Object actual = ObjectGraph.from(value).navigate(attribute).value(); + return expected.matcher(asString(actual)).matches(); + } catch (JXPathException e) { + throw new MatcherException( + String.format("Could not navigate to specific path: '%s'", attribute), e); } + } - private static String asString(Object value) { - return String.valueOf(value); - } + private static String asString(Object value) { + return String.valueOf(value); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/Matcher.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/Matcher.java index 987bba8065..734554f95e 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/Matcher.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/Matcher.java @@ -2,5 +2,5 @@ public interface Matcher { - boolean match(Object value); + boolean match(Object value); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/MatcherException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/MatcherException.java index 05b49d4f75..60e3f05ada 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/MatcherException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/MatcherException.java @@ -2,11 +2,11 @@ public class MatcherException extends RuntimeException { - public MatcherException(String message) { - super(message); - } + public MatcherException(String message) { + super(message); + } - public MatcherException(String message, Throwable cause) { - super(message, cause); - } + public MatcherException(String message, Throwable cause) { + super(message, cause); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/MatcherFactories.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/MatcherFactories.java index 4c21696b30..f26c043153 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/MatcherFactories.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/MatcherFactories.java @@ -1,10 +1,5 @@ package pl.allegro.tech.hermes.management.infrastructure.query.matcher; -import pl.allegro.tech.hermes.management.infrastructure.query.parser.Operator; - -import java.util.EnumMap; -import java.util.Map; - import static pl.allegro.tech.hermes.management.infrastructure.query.parser.Operator.AND; import static pl.allegro.tech.hermes.management.infrastructure.query.parser.Operator.EQ; import static pl.allegro.tech.hermes.management.infrastructure.query.parser.Operator.GREATER_THAN; @@ -15,41 +10,51 @@ import static pl.allegro.tech.hermes.management.infrastructure.query.parser.Operator.NOT; import static pl.allegro.tech.hermes.management.infrastructure.query.parser.Operator.OR; -public class MatcherFactories { - - private static final Map FACTORIES = new EnumMap<>(Operator.class); - - static { - registerFactories(); - } +import java.util.EnumMap; +import java.util.Map; +import pl.allegro.tech.hermes.management.infrastructure.query.parser.Operator; - public static MatcherFactory getMatcherFactory(String operator) { - try { - return getMatcherFactory(Operator.from(operator)); - } catch (IllegalArgumentException e) { - throw new MatcherNotFoundException( - String.format("Unrecognized operator: '%s'", operator) - ); - } - } +public class MatcherFactories { - private static MatcherFactory getMatcherFactory(Operator operator) { - return FACTORIES.get(operator); - } + private static final Map FACTORIES = new EnumMap<>(Operator.class); - public static MatcherFactory defaultMatcher() { - return getMatcherFactory(EQ); - } + static { + registerFactories(); + } - private static void registerFactories() { - FACTORIES.put(EQ, (path, node, parser) -> new EqualityMatcher(path, parser.parseValue(node))); - FACTORIES.put(NE, (path, node, parser) -> new NotMatcher(new EqualityMatcher(path, parser.parseValue(node)))); - FACTORIES.put(LIKE, (path, node, parser) -> new LikeMatcher(path, parser.parseValue(node))); - FACTORIES.put(IN, (path, node, parser) -> new InMatcher(path, parser.parseArrayValue(node))); - FACTORIES.put(NOT, (path, node, parser) -> new NotMatcher(parser.parseNode(node))); - FACTORIES.put(AND, (path, node, parser) -> new AndMatcher(parser.parseArrayNodes(node))); - FACTORIES.put(OR, (path, node, parser) -> new OrMatcher(parser.parseArrayNodes(node))); - FACTORIES.put(GREATER_THAN, (path, node, parser) -> new ComparisonMatcher(path, parser.parseValue(node), (a, b) -> a > b)); - FACTORIES.put(LOWER_THAN, (path, node, parser) -> new ComparisonMatcher(path, parser.parseValue(node), (a, b) -> a < b)); + public static MatcherFactory getMatcherFactory(String operator) { + try { + return getMatcherFactory(Operator.from(operator)); + } catch (IllegalArgumentException e) { + throw new MatcherNotFoundException(String.format("Unrecognized operator: '%s'", operator)); } + } + + private static MatcherFactory getMatcherFactory(Operator operator) { + return FACTORIES.get(operator); + } + + public static MatcherFactory defaultMatcher() { + return getMatcherFactory(EQ); + } + + private static void registerFactories() { + FACTORIES.put(EQ, (path, node, parser) -> new EqualityMatcher(path, parser.parseValue(node))); + FACTORIES.put( + NE, + (path, node, parser) -> new NotMatcher(new EqualityMatcher(path, parser.parseValue(node)))); + FACTORIES.put(LIKE, (path, node, parser) -> new LikeMatcher(path, parser.parseValue(node))); + FACTORIES.put(IN, (path, node, parser) -> new InMatcher(path, parser.parseArrayValue(node))); + FACTORIES.put(NOT, (path, node, parser) -> new NotMatcher(parser.parseNode(node))); + FACTORIES.put(AND, (path, node, parser) -> new AndMatcher(parser.parseArrayNodes(node))); + FACTORIES.put(OR, (path, node, parser) -> new OrMatcher(parser.parseArrayNodes(node))); + FACTORIES.put( + GREATER_THAN, + (path, node, parser) -> + new ComparisonMatcher(path, parser.parseValue(node), (a, b) -> a > b)); + FACTORIES.put( + LOWER_THAN, + (path, node, parser) -> + new ComparisonMatcher(path, parser.parseValue(node), (a, b) -> a < b)); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/MatcherFactory.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/MatcherFactory.java index 7916c38307..ba50732293 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/MatcherFactory.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/MatcherFactory.java @@ -5,5 +5,5 @@ public interface MatcherFactory { - Matcher createMatcher(String path, JsonNode value, QueryParserContext context); + Matcher createMatcher(String path, JsonNode value, QueryParserContext context); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/MatcherInputException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/MatcherInputException.java index 2805dbdf62..62f32a29a9 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/MatcherInputException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/MatcherInputException.java @@ -5,12 +5,12 @@ public class MatcherInputException extends ManagementException { - public MatcherInputException(String message) { - super(message); - } + public MatcherInputException(String message) { + super(message); + } - @Override - public ErrorCode getCode() { - return ErrorCode.INVALID_QUERY; - } + @Override + public ErrorCode getCode() { + return ErrorCode.INVALID_QUERY; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/MatcherNotFoundException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/MatcherNotFoundException.java index e356892b46..5941c00480 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/MatcherNotFoundException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/MatcherNotFoundException.java @@ -2,11 +2,11 @@ public class MatcherNotFoundException extends RuntimeException { - public MatcherNotFoundException(String message) { - super(message); - } + public MatcherNotFoundException(String message) { + super(message); + } - public MatcherNotFoundException(String message, Throwable cause) { - super(message, cause); - } + public MatcherNotFoundException(String message, Throwable cause) { + super(message, cause); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/NotMatcher.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/NotMatcher.java index 118a942284..1915f8ef54 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/NotMatcher.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/NotMatcher.java @@ -2,14 +2,14 @@ public class NotMatcher implements Matcher { - private final Matcher matcher; + private final Matcher matcher; - public NotMatcher(Matcher matcher) { - this.matcher = matcher; - } + public NotMatcher(Matcher matcher) { + this.matcher = matcher; + } - @Override - public boolean match(Object value) { - return !matcher.match(value); - } + @Override + public boolean match(Object value) { + return !matcher.match(value); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/OrMatcher.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/OrMatcher.java index 95c7485a6c..9fab841bac 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/OrMatcher.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/matcher/OrMatcher.java @@ -4,18 +4,18 @@ public class OrMatcher implements Matcher { - private final Collection matchers; + private final Collection matchers; - public OrMatcher(Collection matchers) { - this.matchers = matchers; - } + public OrMatcher(Collection matchers) { + this.matchers = matchers; + } - @Override - public boolean match(Object value) { - return matchers.stream().reduce( - false, - (match, matcher) -> match || matcher.match(value), - (first, second) -> first || second - ); - } + @Override + public boolean match(Object value) { + return matchers.stream() + .reduce( + false, + (match, matcher) -> match || matcher.match(value), + (first, second) -> first || second); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/parser/Operator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/parser/Operator.java index 189cedc1be..126f8f4f00 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/parser/Operator.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/parser/Operator.java @@ -4,38 +4,35 @@ import java.util.Optional; public enum Operator { - - EQ("eq"), - NE("ne"), - LIKE("like"), - IN("in"), - NOT("not"), - AND("and"), - OR("or"), - GREATER_THAN("gt"), - LOWER_THAN("lt"); - - - private String name; - - Operator(String name) { - this.name = name; - } - - public static Operator from(String name) { - return fromOptional(name) - .orElseThrow( - () -> new IllegalArgumentException(String.format("No operator matching '%s' could be found", name)) - ); - } - - public static boolean isValid(String name) { - return fromOptional(name).isPresent(); - } - - private static Optional fromOptional(String name) { - return Arrays.stream(values()) - .filter(value -> value.name.equalsIgnoreCase(name)) - .findFirst(); - } + EQ("eq"), + NE("ne"), + LIKE("like"), + IN("in"), + NOT("not"), + AND("and"), + OR("or"), + GREATER_THAN("gt"), + LOWER_THAN("lt"); + + private String name; + + Operator(String name) { + this.name = name; + } + + public static Operator from(String name) { + return fromOptional(name) + .orElseThrow( + () -> + new IllegalArgumentException( + String.format("No operator matching '%s' could be found", name))); + } + + public static boolean isValid(String name) { + return fromOptional(name).isPresent(); + } + + private static Optional fromOptional(String name) { + return Arrays.stream(values()).filter(value -> value.name.equalsIgnoreCase(name)).findFirst(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/parser/ParseException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/parser/ParseException.java index 7449f40976..81daefb32d 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/parser/ParseException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/parser/ParseException.java @@ -2,11 +2,11 @@ public class ParseException extends RuntimeException { - public ParseException(String message) { - super(message); - } + public ParseException(String message) { + super(message); + } - public ParseException(String message, Throwable cause) { - super(message, cause); - } + public ParseException(String message, Throwable cause) { + super(message, cause); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/parser/QueryParser.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/parser/QueryParser.java index ac3a27de87..49153983dd 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/parser/QueryParser.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/parser/QueryParser.java @@ -1,12 +1,11 @@ package pl.allegro.tech.hermes.management.infrastructure.query.parser; -import pl.allegro.tech.hermes.api.Query; - import java.io.InputStream; +import pl.allegro.tech.hermes.api.Query; public interface QueryParser { - Query parse(InputStream input, Class type); + Query parse(InputStream input, Class type); - Query parse(String query, Class type); + Query parse(String query, Class type); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/parser/QueryParserContext.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/parser/QueryParserContext.java index 257e2fe8e7..b34143598e 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/parser/QueryParserContext.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/parser/QueryParserContext.java @@ -1,17 +1,16 @@ package pl.allegro.tech.hermes.management.infrastructure.query.parser; import com.fasterxml.jackson.databind.JsonNode; -import pl.allegro.tech.hermes.management.infrastructure.query.matcher.Matcher; - import java.util.List; +import pl.allegro.tech.hermes.management.infrastructure.query.matcher.Matcher; public interface QueryParserContext { - Matcher parseNode(JsonNode node); + Matcher parseNode(JsonNode node); - List parseArrayNodes(JsonNode node); + List parseArrayNodes(JsonNode node); - Object parseValue(JsonNode node); + Object parseValue(JsonNode node); - Object[] parseArrayValue(JsonNode node); + Object[] parseArrayValue(JsonNode node); } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/parser/json/JsonQueryParser.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/parser/json/JsonQueryParser.java index 009e345b5f..303158975d 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/parser/json/JsonQueryParser.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/query/parser/json/JsonQueryParser.java @@ -1,8 +1,18 @@ package pl.allegro.tech.hermes.management.infrastructure.query.parser.json; +import static java.util.stream.StreamSupport.stream; +import static pl.allegro.tech.hermes.management.infrastructure.query.MatcherQuery.fromMatcher; +import static pl.allegro.tech.hermes.management.infrastructure.utils.Iterators.stream; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import pl.allegro.tech.hermes.api.Query; import pl.allegro.tech.hermes.management.infrastructure.query.matcher.AndMatcher; import pl.allegro.tech.hermes.management.infrastructure.query.matcher.Matcher; @@ -13,152 +23,134 @@ import pl.allegro.tech.hermes.management.infrastructure.query.parser.QueryParser; import pl.allegro.tech.hermes.management.infrastructure.query.parser.QueryParserContext; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static java.util.stream.StreamSupport.stream; -import static pl.allegro.tech.hermes.management.infrastructure.query.MatcherQuery.fromMatcher; -import static pl.allegro.tech.hermes.management.infrastructure.utils.Iterators.stream; - public class JsonQueryParser implements QueryParser, QueryParserContext { - private static final String QUERY = "query"; - - private final ObjectMapper objectMapper; - - public JsonQueryParser(ObjectMapper objectMapper) { - this.objectMapper = objectMapper; - } - - @Override - public Query parse(InputStream input, Class type) { - try { - return parseDocument( - objectMapper.readTree(input) - ); - } catch (IOException | MatcherNotFoundException e) { - throw new ParseException("Query could not be parsed", e); - } - } - - @Override - public Query parse(String query, Class type) { - return parse(new ByteArrayInputStream(query.getBytes()), type); - } - - @Override - public Matcher parseNode(JsonNode node) { - Map.Entry entry = singleNode(node); - return parseSingleAttribute(entry.getKey(), entry.getValue()); - } - - @Override - public List parseArrayNodes(JsonNode node) { - if (!node.isArray()) { - throw new ParseException("Element value was expected to be an array"); - } - return parseObjectArray(node); - } - - @Override - public Object parseValue(JsonNode node) { - if (!node.isValueNode()) { - throw new ParseException("The node value wasn't present"); - } - if (node.isTextual()) { - return node.asText(); - } else if (node.isBoolean()) { - return node.asBoolean(); - } else if (node.isInt()) { - return node.asInt(); - } else if (node.isDouble()) { - return node.asDouble(); - } - return null; - } - - @Override - public Object[] parseArrayValue(JsonNode node) { - if (!node.isArray()) { - throw new ParseException("Element value was expected to be an array"); - } - return stream(node.spliterator(), false) - .map(this::parseValue) - .toArray(); - } - - private Query parseDocument(JsonNode document) { - validateDocument(document); - return parseQuery(document.get(QUERY)); - } - - private Query parseQuery(JsonNode node) { - return fromMatcher(parseCompoundObject(node), objectMapper); - } - - private Matcher parseCompoundObject(JsonNode node) { - return new AndMatcher(parseAllAttributes(node)); - } - - private Matcher parseSingleAttribute(String key, JsonNode node) { - if (isOperator(key)) { - return parseOperator(key, node); - } else if (node.isObject()) { - return parseObject(key, node); - } else { - return parseAttribute(key, node); - } - } - - private List parseAllAttributes(JsonNode node) { - return stream(node.fields()) - .map(entry -> parseSingleAttribute(entry.getKey(), entry.getValue())) - .collect(Collectors.toList()); - } - - private Matcher parseOperator(String key, JsonNode node) { - return MatcherFactories.getMatcherFactory(key).createMatcher(key, node, this); - } - - private Matcher parseObject(String key, JsonNode node) { - Map.Entry entry = singleNode(node); - return MatcherFactories.getMatcherFactory(entry.getKey()).createMatcher(key, entry.getValue(), this); - } - - private List parseObjectArray(JsonNode node) { - return stream(node.iterator()) - .map(this::parseCompoundObject) - .collect(Collectors.toList()); - } - - private Matcher parseAttribute(String key, JsonNode node) { - return MatcherFactories.defaultMatcher().createMatcher(key, node, this); - } - - private boolean isOperator(String key) { - return Operator.isValid(key); - } - - private Map.Entry singleNode(JsonNode node) { - List> attributes = Lists.newArrayList(node.fields()); - if (attributes.size() != 1) { - throw new MatcherNotFoundException( - String.format( - "The object must define exactly one member, but defines %s", - Lists.newArrayList(node.fieldNames()).toString() - ) - ); - } - return attributes.get(0); - } - - private void validateDocument(JsonNode document) { - if (!document.isObject() || !document.has(QUERY)) { - throw new ParseException("JSON object must contain 'query' attribute"); - } - } + private static final String QUERY = "query"; + + private final ObjectMapper objectMapper; + + public JsonQueryParser(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + @Override + public Query parse(InputStream input, Class type) { + try { + return parseDocument(objectMapper.readTree(input)); + } catch (IOException | MatcherNotFoundException e) { + throw new ParseException("Query could not be parsed", e); + } + } + + @Override + public Query parse(String query, Class type) { + return parse(new ByteArrayInputStream(query.getBytes()), type); + } + + @Override + public Matcher parseNode(JsonNode node) { + Map.Entry entry = singleNode(node); + return parseSingleAttribute(entry.getKey(), entry.getValue()); + } + + @Override + public List parseArrayNodes(JsonNode node) { + if (!node.isArray()) { + throw new ParseException("Element value was expected to be an array"); + } + return parseObjectArray(node); + } + + @Override + public Object parseValue(JsonNode node) { + if (!node.isValueNode()) { + throw new ParseException("The node value wasn't present"); + } + if (node.isTextual()) { + return node.asText(); + } else if (node.isBoolean()) { + return node.asBoolean(); + } else if (node.isInt()) { + return node.asInt(); + } else if (node.isDouble()) { + return node.asDouble(); + } + return null; + } + + @Override + public Object[] parseArrayValue(JsonNode node) { + if (!node.isArray()) { + throw new ParseException("Element value was expected to be an array"); + } + return stream(node.spliterator(), false).map(this::parseValue).toArray(); + } + + private Query parseDocument(JsonNode document) { + validateDocument(document); + return parseQuery(document.get(QUERY)); + } + + private Query parseQuery(JsonNode node) { + return fromMatcher(parseCompoundObject(node), objectMapper); + } + + private Matcher parseCompoundObject(JsonNode node) { + return new AndMatcher(parseAllAttributes(node)); + } + + private Matcher parseSingleAttribute(String key, JsonNode node) { + if (isOperator(key)) { + return parseOperator(key, node); + } else if (node.isObject()) { + return parseObject(key, node); + } else { + return parseAttribute(key, node); + } + } + + private List parseAllAttributes(JsonNode node) { + return stream(node.fields()) + .map(entry -> parseSingleAttribute(entry.getKey(), entry.getValue())) + .collect(Collectors.toList()); + } + + private Matcher parseOperator(String key, JsonNode node) { + return MatcherFactories.getMatcherFactory(key).createMatcher(key, node, this); + } + + private Matcher parseObject(String key, JsonNode node) { + Map.Entry entry = singleNode(node); + return MatcherFactories.getMatcherFactory(entry.getKey()) + .createMatcher(key, entry.getValue(), this); + } + + private List parseObjectArray(JsonNode node) { + return stream(node.iterator()).map(this::parseCompoundObject).collect(Collectors.toList()); + } + + private Matcher parseAttribute(String key, JsonNode node) { + return MatcherFactories.defaultMatcher().createMatcher(key, node, this); + } + + private boolean isOperator(String key) { + return Operator.isValid(key); + } + + private Map.Entry singleNode(JsonNode node) { + List> attributes = Lists.newArrayList(node.fields()); + if (attributes.size() != 1) { + throw new MatcherNotFoundException( + String.format( + "The object must define exactly one member, but defines %s", + Lists.newArrayList(node.fieldNames()).toString())); + } + return attributes.get(0); + } + + private void validateDocument(JsonNode document) { + if (!document.isObject() || !document.has(QUERY)) { + throw new ParseException("JSON object must contain 'query' attribute"); + } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/readiness/ZookeeperDatacenterReadinessRepository.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/readiness/ZookeeperDatacenterReadinessRepository.java index b928dcd461..70ae608271 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/readiness/ZookeeperDatacenterReadinessRepository.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/readiness/ZookeeperDatacenterReadinessRepository.java @@ -1,6 +1,8 @@ package pl.allegro.tech.hermes.management.infrastructure.readiness; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Collections; +import java.util.List; import org.apache.curator.framework.CuratorFramework; import org.apache.zookeeper.KeeperException; import pl.allegro.tech.hermes.api.DatacenterReadiness; @@ -11,41 +13,40 @@ import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; import pl.allegro.tech.hermes.management.domain.readiness.DatacenterReadinessRepository; -import java.util.Collections; -import java.util.List; - -public class ZookeeperDatacenterReadinessRepository extends ZookeeperBasedRepository implements DatacenterReadinessRepository { +public class ZookeeperDatacenterReadinessRepository extends ZookeeperBasedRepository + implements DatacenterReadinessRepository { - public ZookeeperDatacenterReadinessRepository(CuratorFramework curator, ObjectMapper mapper, ZookeeperPaths paths) { - super(curator, mapper, paths); - } + public ZookeeperDatacenterReadinessRepository( + CuratorFramework curator, ObjectMapper mapper, ZookeeperPaths paths) { + super(curator, mapper, paths); + } - @Override - public List getReadiness() { - try { - String path = paths.datacenterReadinessPath(); - return readFrom(path, DatacenterReadinessList.class).datacenters(); - } catch (InternalProcessingException e) { - if (e.getCause() instanceof KeeperException.NoNodeException) { - return Collections.emptyList(); - } - throw e; - } catch (MalformedDataException e) { - return Collections.emptyList(); - } + @Override + public List getReadiness() { + try { + String path = paths.datacenterReadinessPath(); + return readFrom(path, DatacenterReadinessList.class).datacenters(); + } catch (InternalProcessingException e) { + if (e.getCause() instanceof KeeperException.NoNodeException) { + return Collections.emptyList(); + } + throw e; + } catch (MalformedDataException e) { + return Collections.emptyList(); } + } - @Override - public void setReadiness(List datacenterReadiness) { - try { - String path = paths.datacenterReadinessPath(); - if (!pathExists(path)) { - createRecursively(path, new DatacenterReadinessList(datacenterReadiness)); - } else { - overwrite(path, new DatacenterReadinessList(datacenterReadiness)); - } - } catch (Exception ex) { - throw new InternalProcessingException(ex); - } + @Override + public void setReadiness(List datacenterReadiness) { + try { + String path = paths.datacenterReadinessPath(); + if (!pathExists(path)) { + createRecursively(path, new DatacenterReadinessList(datacenterReadiness)); + } else { + overwrite(path, new DatacenterReadinessList(datacenterReadiness)); + } + } catch (Exception ex) { + throw new InternalProcessingException(ex); } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/retransmit/ZookeeperOfflineRetransmissionRepository.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/retransmit/ZookeeperOfflineRetransmissionRepository.java index 715f9db2af..3b1d4ee66e 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/retransmit/ZookeeperOfflineRetransmissionRepository.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/retransmit/ZookeeperOfflineRetransmissionRepository.java @@ -1,6 +1,11 @@ package pl.allegro.tech.hermes.management.infrastructure.retransmit; +import static java.lang.String.format; + import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; import org.apache.curator.framework.CuratorFramework; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -11,67 +16,65 @@ import pl.allegro.tech.hermes.management.domain.retransmit.OfflineRetransmissionRepository; import pl.allegro.tech.hermes.management.domain.retransmit.OfflineRetransmissionValidationException; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - -import static java.lang.String.format; - -public class ZookeeperOfflineRetransmissionRepository extends ZookeeperBasedRepository implements OfflineRetransmissionRepository { +public class ZookeeperOfflineRetransmissionRepository extends ZookeeperBasedRepository + implements OfflineRetransmissionRepository { - private static final Logger logger = LoggerFactory.getLogger(ZookeeperOfflineRetransmissionRepository.class); + private static final Logger logger = + LoggerFactory.getLogger(ZookeeperOfflineRetransmissionRepository.class); - public ZookeeperOfflineRetransmissionRepository(CuratorFramework zookeeper, ObjectMapper mapper, ZookeeperPaths paths) { - super(zookeeper, mapper, paths); - } + public ZookeeperOfflineRetransmissionRepository( + CuratorFramework zookeeper, ObjectMapper mapper, ZookeeperPaths paths) { + super(zookeeper, mapper, paths); + } - @Override - public void saveTask(OfflineRetransmissionTask task) { - logger.info("Saving retransmission task {}", task); - try { - createRecursively(paths.offlineRetransmissionPath(task.getTaskId()), task); - logger.info("Successfully saved retransmission task {}", task); - } catch (Exception ex) { - String msg = format("Error while saving retransmission task %s", task.toString()); - throw new InternalProcessingException(msg, ex); - } + @Override + public void saveTask(OfflineRetransmissionTask task) { + logger.info("Saving retransmission task {}", task); + try { + createRecursively(paths.offlineRetransmissionPath(task.getTaskId()), task); + logger.info("Successfully saved retransmission task {}", task); + } catch (Exception ex) { + String msg = format("Error while saving retransmission task %s", task.toString()); + throw new InternalProcessingException(msg, ex); } + } - @Override - public List getAllTasks() { - try { - if (pathExists(paths.offlineRetransmissionPath())) { - return childrenOf(paths.offlineRetransmissionPath()) - .stream() - .map(id -> readFrom(paths.offlineRetransmissionPath(id), OfflineRetransmissionTask.class)) - .collect(Collectors.toList()); - } - return Collections.emptyList(); - } catch (Exception ex) { - String msg = "Error while fetching offline retransmission tasks"; - throw new InternalProcessingException(msg, ex); - } + @Override + public List getAllTasks() { + try { + if (pathExists(paths.offlineRetransmissionPath())) { + return childrenOf(paths.offlineRetransmissionPath()).stream() + .map( + id -> + readFrom(paths.offlineRetransmissionPath(id), OfflineRetransmissionTask.class)) + .collect(Collectors.toList()); + } + return Collections.emptyList(); + } catch (Exception ex) { + String msg = "Error while fetching offline retransmission tasks"; + throw new InternalProcessingException(msg, ex); } + } - @Override - public void deleteTask(String taskId) { - logger.info("Trying to delete retransmission task with id={}", taskId); - try { - ensureTaskExists(taskId); - remove(paths.offlineRetransmissionPath(taskId)); - logger.info("Successfully deleted retransmission task with id={}", taskId); - } catch (OfflineRetransmissionValidationException ex) { - throw ex; - } catch (Exception ex) { - String msg = format("Error while deleting retransmission task with id=%s", taskId); - throw new InternalProcessingException(msg, ex); - } + @Override + public void deleteTask(String taskId) { + logger.info("Trying to delete retransmission task with id={}", taskId); + try { + ensureTaskExists(taskId); + remove(paths.offlineRetransmissionPath(taskId)); + logger.info("Successfully deleted retransmission task with id={}", taskId); + } catch (OfflineRetransmissionValidationException ex) { + throw ex; + } catch (Exception ex) { + String msg = format("Error while deleting retransmission task with id=%s", taskId); + throw new InternalProcessingException(msg, ex); } + } - private void ensureTaskExists(String taskId) { - if (!pathExists(paths.offlineRetransmissionPath(taskId))) { - String msg = String.format("Retransmission task with id %s does not exist.", taskId); - throw new OfflineRetransmissionValidationException(msg); - } + private void ensureTaskExists(String taskId) { + if (!pathExists(paths.offlineRetransmissionPath(taskId))) { + String msg = String.format("Retransmission task with id %s does not exist.", taskId); + throw new OfflineRetransmissionValidationException(msg); } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/AvroSchemaValidator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/AvroSchemaValidator.java index 950ce07116..48ab6256d7 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/AvroSchemaValidator.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/AvroSchemaValidator.java @@ -1,5 +1,9 @@ package pl.allegro.tech.hermes.management.infrastructure.schema.validator; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Strings.isNullOrEmpty; + +import java.io.IOException; import org.apache.avro.Schema; import org.apache.avro.SchemaParseException; import org.apache.commons.io.IOUtils; @@ -7,70 +11,68 @@ import org.springframework.stereotype.Component; import pl.allegro.tech.hermes.management.config.TopicProperties; -import java.io.IOException; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Strings.isNullOrEmpty; - @Component public class AvroSchemaValidator implements SchemaValidator { - private static final Schema HERMES_METADATA_SCHEMA = - metadataFieldSchema(readAndParseResourceSchema("/avro-schema-metadata-field.avsc")); - private final boolean metadataFieldIsRequired; + private static final Schema HERMES_METADATA_SCHEMA = + metadataFieldSchema(readAndParseResourceSchema("/avro-schema-metadata-field.avsc")); + private final boolean metadataFieldIsRequired; - public AvroSchemaValidator(boolean metadataFieldIsRequired) { - this.metadataFieldIsRequired = metadataFieldIsRequired; - } + public AvroSchemaValidator(boolean metadataFieldIsRequired) { + this.metadataFieldIsRequired = metadataFieldIsRequired; + } - @Autowired - public AvroSchemaValidator(TopicProperties topicProperties) { - this(topicProperties.isAvroContentTypeMetadataRequired()); - } + @Autowired + public AvroSchemaValidator(TopicProperties topicProperties) { + this(topicProperties.isAvroContentTypeMetadataRequired()); + } - private static Schema metadataFieldSchema(Schema schema) { - Schema.Field metadata = schema.getField("__metadata"); - if (metadata == null) { - throw new InvalidSchemaException("Missing Hermes __metadata field"); - } - return metadata.schema(); + private static Schema metadataFieldSchema(Schema schema) { + Schema.Field metadata = schema.getField("__metadata"); + if (metadata == null) { + throw new InvalidSchemaException("Missing Hermes __metadata field"); } + return metadata.schema(); + } - private static Schema readAndParseResourceSchema(String resourceFilePath) { - try { - String schema = IOUtils.toString(AvroSchemaValidator.class.getResourceAsStream(resourceFilePath), "UTF-8"); - return parseSchema(schema); - } catch (IOException e) { - throw new RuntimeException("Could not load schema with metadata"); - } + private static Schema readAndParseResourceSchema(String resourceFilePath) { + try { + String schema = + IOUtils.toString( + AvroSchemaValidator.class.getResourceAsStream(resourceFilePath), "UTF-8"); + return parseSchema(schema); + } catch (IOException e) { + throw new RuntimeException("Could not load schema with metadata"); } + } - private static Schema parseSchema(String schema) { - try { - return new Schema.Parser().parse(schema); - } catch (SchemaParseException e) { - throw new InvalidSchemaException(e); - } + private static Schema parseSchema(String schema) { + try { + return new Schema.Parser().parse(schema); + } catch (SchemaParseException e) { + throw new InvalidSchemaException(e); } + } - @Override - public void check(String schema) throws InvalidSchemaException { - checkArgument(!isNullOrEmpty(schema), "Message schema cannot be empty"); - Schema parsedSchema = parseSchema(schema); - if (metadataFieldIsRequired) { - checkHermesMetadataField(parsedSchema); - } + @Override + public void check(String schema) throws InvalidSchemaException { + checkArgument(!isNullOrEmpty(schema), "Message schema cannot be empty"); + Schema parsedSchema = parseSchema(schema); + if (metadataFieldIsRequired) { + checkHermesMetadataField(parsedSchema); } + } - private void checkHermesMetadataField(Schema parsedSchema) { - Schema metadata = metadataFieldSchema(parsedSchema); + private void checkHermesMetadataField(Schema parsedSchema) { + Schema metadata = metadataFieldSchema(parsedSchema); - boolean metadataTypeMatches = HERMES_METADATA_SCHEMA.getType().equals(metadata.getType()); - boolean metadataNestedTypesMatch = HERMES_METADATA_SCHEMA.getTypes().equals(metadata.getTypes()); - boolean valid = metadataTypeMatches && metadataNestedTypesMatch; + boolean metadataTypeMatches = HERMES_METADATA_SCHEMA.getType().equals(metadata.getType()); + boolean metadataNestedTypesMatch = + HERMES_METADATA_SCHEMA.getTypes().equals(metadata.getTypes()); + boolean valid = metadataTypeMatches && metadataNestedTypesMatch; - if (!valid) { - throw new InvalidSchemaException("Invalid types used in field __metadata"); - } + if (!valid) { + throw new InvalidSchemaException("Invalid types used in field __metadata"); } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/InvalidSchemaException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/InvalidSchemaException.java index 1425acf169..aae90a8818 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/InvalidSchemaException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/InvalidSchemaException.java @@ -1,23 +1,23 @@ package pl.allegro.tech.hermes.management.infrastructure.schema.validator; +import static java.lang.String.format; + import org.apache.avro.SchemaParseException; import pl.allegro.tech.hermes.api.ErrorCode; import pl.allegro.tech.hermes.common.exception.HermesException; -import static java.lang.String.format; - public class InvalidSchemaException extends HermesException { - InvalidSchemaException(String cause) { - super(format("Error while trying to validate schema: %s", cause)); - } + InvalidSchemaException(String cause) { + super(format("Error while trying to validate schema: %s", cause)); + } - InvalidSchemaException(SchemaParseException cause) { - super(format("Error while trying to validate schema: %s", cause.getMessage()), cause); - } + InvalidSchemaException(SchemaParseException cause) { + super(format("Error while trying to validate schema: %s", cause.getMessage()), cause); + } - @Override - public ErrorCode getCode() { - return ErrorCode.FORMAT_ERROR; - } + @Override + public ErrorCode getCode() { + return ErrorCode.FORMAT_ERROR; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/SchemaValidator.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/SchemaValidator.java index 0bec062cb1..71914129c8 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/SchemaValidator.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/SchemaValidator.java @@ -1,5 +1,5 @@ package pl.allegro.tech.hermes.management.infrastructure.schema.validator; public interface SchemaValidator { - void check(String schema) throws InvalidSchemaException; + void check(String schema) throws InvalidSchemaException; } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/SchemaValidatorProvider.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/SchemaValidatorProvider.java index 026dccb792..10e387d968 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/SchemaValidatorProvider.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/schema/validator/SchemaValidatorProvider.java @@ -1,41 +1,40 @@ package pl.allegro.tech.hermes.management.infrastructure.schema.validator; +import java.util.EnumMap; +import java.util.Map; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import pl.allegro.tech.hermes.api.ContentType; import pl.allegro.tech.hermes.api.ErrorCode; import pl.allegro.tech.hermes.management.domain.ManagementException; -import java.util.EnumMap; -import java.util.Map; - @Component public class SchemaValidatorProvider { - private final Map validators = new EnumMap<>(ContentType.class); - - @Autowired - public SchemaValidatorProvider(AvroSchemaValidator avroSchemaValidator) { - validators.put(ContentType.AVRO, avroSchemaValidator); - } + private final Map validators = new EnumMap<>(ContentType.class); - public SchemaValidator provide(ContentType contentType) { - if (!validators.containsKey(contentType)) { - throw new SchemaValidatorNotAvailable("No schema validator for content-type: " + contentType.name()); - } + @Autowired + public SchemaValidatorProvider(AvroSchemaValidator avroSchemaValidator) { + validators.put(ContentType.AVRO, avroSchemaValidator); + } - return validators.get(contentType); + public SchemaValidator provide(ContentType contentType) { + if (!validators.containsKey(contentType)) { + throw new SchemaValidatorNotAvailable( + "No schema validator for content-type: " + contentType.name()); } - class SchemaValidatorNotAvailable extends ManagementException { - public SchemaValidatorNotAvailable(String message) { - super(message); - } + return validators.get(contentType); + } - @Override - public ErrorCode getCode() { - return ErrorCode.VALIDATION_ERROR; - } + class SchemaValidatorNotAvailable extends ManagementException { + public SchemaValidatorNotAvailable(String message) { + super(message); } + @Override + public ErrorCode getCode() { + return ErrorCode.VALIDATION_ERROR; + } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/tracker/NoOperationLogRepository.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/tracker/NoOperationLogRepository.java index a4caa9c204..c12ed3aafe 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/tracker/NoOperationLogRepository.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/tracker/NoOperationLogRepository.java @@ -1,28 +1,30 @@ package pl.allegro.tech.hermes.management.infrastructure.tracker; +import java.util.Collections; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.MessageTrace; import pl.allegro.tech.hermes.api.SentMessageTrace; import pl.allegro.tech.hermes.tracker.management.LogRepository; -import java.util.Collections; -import java.util.List; - public class NoOperationLogRepository implements LogRepository { - private static final String NO_LOG_MESSAGE = "No LogRepository implementation found, using default no-operation implementation"; - private static final Logger logger = LoggerFactory.getLogger(NoOperationLogRepository.class); + private static final String NO_LOG_MESSAGE = + "No LogRepository implementation found, using default no-operation implementation"; + private static final Logger logger = LoggerFactory.getLogger(NoOperationLogRepository.class); - @Override - public List getLastUndeliveredMessages(String topicName, String subscriptionName, int limit) { - logger.info(NO_LOG_MESSAGE); - return Collections.emptyList(); - } + @Override + public List getLastUndeliveredMessages( + String topicName, String subscriptionName, int limit) { + logger.info(NO_LOG_MESSAGE); + return Collections.emptyList(); + } - @Override - public List getMessageStatus(String qualifiedTopicName, String subscriptionName, String messageId) { - logger.info(NO_LOG_MESSAGE); - return Collections.emptyList(); - } + @Override + public List getMessageStatus( + String qualifiedTopicName, String subscriptionName, String messageId) { + logger.info(NO_LOG_MESSAGE); + return Collections.emptyList(); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/utils/Iterators.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/utils/Iterators.java index 53ae478def..ddebdb35ab 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/utils/Iterators.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/utils/Iterators.java @@ -8,10 +8,10 @@ public final class Iterators { - private Iterators() { - } + private Iterators() {} - public static Stream stream(Iterator iterator) { - return StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.NONNULL), false); - } + public static Stream stream(Iterator iterator) { + return StreamSupport.stream( + Spliterators.spliteratorUnknownSize(iterator, Spliterator.NONNULL), false); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/zookeeper/ZookeeperClient.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/zookeeper/ZookeeperClient.java index 7487d72eb6..5e740a737d 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/zookeeper/ZookeeperClient.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/zookeeper/ZookeeperClient.java @@ -1,41 +1,42 @@ package pl.allegro.tech.hermes.management.infrastructure.zookeeper; +import static org.slf4j.LoggerFactory.getLogger; + import org.apache.curator.framework.CuratorFramework; import org.apache.zookeeper.CreateMode; import org.slf4j.Logger; import pl.allegro.tech.hermes.common.exception.InternalProcessingException; -import static org.slf4j.LoggerFactory.getLogger; - public class ZookeeperClient { - private static final Logger logger = getLogger(ZookeeperClient.class); - private final CuratorFramework curatorFramework; - private final String datacenterName; - - ZookeeperClient(CuratorFramework curatorFramework, String datacenterName) { - this.curatorFramework = curatorFramework; - this.datacenterName = datacenterName; - } - - public CuratorFramework getCuratorFramework() { - return curatorFramework; - } - - public String getDatacenterName() { - return datacenterName; - } - - public void ensureEphemeralNodeExists(String path) { - try { - if (curatorFramework.checkExists().forPath(path) == null) { - curatorFramework.create() - .creatingParentsIfNeeded() - .withMode(CreateMode.EPHEMERAL) - .forPath(path); - } - } catch (Exception e) { - throw new InternalProcessingException("Could not ensure existence of path: " + path); - } + private static final Logger logger = getLogger(ZookeeperClient.class); + private final CuratorFramework curatorFramework; + private final String datacenterName; + + ZookeeperClient(CuratorFramework curatorFramework, String datacenterName) { + this.curatorFramework = curatorFramework; + this.datacenterName = datacenterName; + } + + public CuratorFramework getCuratorFramework() { + return curatorFramework; + } + + public String getDatacenterName() { + return datacenterName; + } + + public void ensureEphemeralNodeExists(String path) { + try { + if (curatorFramework.checkExists().forPath(path) == null) { + curatorFramework + .create() + .creatingParentsIfNeeded() + .withMode(CreateMode.EPHEMERAL) + .forPath(path); + } + } catch (Exception e) { + throw new InternalProcessingException("Could not ensure existence of path: " + path); } + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/zookeeper/ZookeeperClientManager.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/zookeeper/ZookeeperClientManager.java index 022b629dda..932b2b0b68 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/zookeeper/ZookeeperClientManager.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/zookeeper/ZookeeperClientManager.java @@ -1,5 +1,9 @@ package pl.allegro.tech.hermes.management.infrastructure.zookeeper; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.framework.api.ACLProvider; @@ -14,137 +18,136 @@ import pl.allegro.tech.hermes.management.config.storage.StorageClustersProperties; import pl.allegro.tech.hermes.management.config.storage.StorageProperties; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; - public class ZookeeperClientManager { - private static final Logger logger = LoggerFactory.getLogger(ZookeeperClientManager.class); - - private final StorageClustersProperties properties; - private final DatacenterNameProvider datacenterNameProvider; - private List clients; - private ZookeeperClient localClient; - - public ZookeeperClientManager(StorageClustersProperties properties, DatacenterNameProvider datacenterNameProvider) { - this.properties = properties; - this.datacenterNameProvider = datacenterNameProvider; - } - - public void start() { - createClients(); - selectLocalClient(); - waitForConnection(localClient.getCuratorFramework()); - } - - private void createClients() { - clients = getClusterProperties() - .stream() - .map(clusterProperties -> buildZookeeperClient(clusterProperties, properties)) - .collect(Collectors.toList()); - } - - private List getClusterProperties() { - if (properties.getClusters().isEmpty()) { - return Collections.singletonList(createPropertiesForSingleCluster()); - } else { - return properties.getClusters(); - } - } - - private StorageProperties createPropertiesForSingleCluster() { - StorageProperties clusterProperties = new StorageProperties(); - clusterProperties.setConnectionString(properties.getConnectionString()); - clusterProperties.setConnectTimeout(properties.getConnectTimeout()); - clusterProperties.setSessionTimeout(properties.getSessionTimeout()); - clusterProperties.setDatacenter(DefaultDatacenterNameProvider.DEFAULT_DC_NAME); - return clusterProperties; - } - - private void selectLocalClient() { - if (clients.size() == 1) { - localClient = clients.get(0); - } else { - String localDcName = datacenterNameProvider.getDatacenterName(); - localClient = clients - .stream() - .filter(client -> client.getDatacenterName().equals(localDcName)) - .findFirst() - .orElseThrow(() -> new ZookeeperClientNotFoundException(localDcName)); - } + private static final Logger logger = LoggerFactory.getLogger(ZookeeperClientManager.class); + + private final StorageClustersProperties properties; + private final DatacenterNameProvider datacenterNameProvider; + private List clients; + private ZookeeperClient localClient; + + public ZookeeperClientManager( + StorageClustersProperties properties, DatacenterNameProvider datacenterNameProvider) { + this.properties = properties; + this.datacenterNameProvider = datacenterNameProvider; + } + + public void start() { + createClients(); + selectLocalClient(); + waitForConnection(localClient.getCuratorFramework()); + } + + private void createClients() { + clients = + getClusterProperties().stream() + .map(clusterProperties -> buildZookeeperClient(clusterProperties, properties)) + .collect(Collectors.toList()); + } + + private List getClusterProperties() { + if (properties.getClusters().isEmpty()) { + return Collections.singletonList(createPropertiesForSingleCluster()); + } else { + return properties.getClusters(); } - - private ZookeeperClient buildZookeeperClient(StorageProperties clusterProperties, - StorageClustersProperties commonProperties) { - return new ZookeeperClient( - buildCuratorFramework(clusterProperties, commonProperties), - clusterProperties.getDatacenter() - ); + } + + private StorageProperties createPropertiesForSingleCluster() { + StorageProperties clusterProperties = new StorageProperties(); + clusterProperties.setConnectionString(properties.getConnectionString()); + clusterProperties.setConnectTimeout(properties.getConnectTimeout()); + clusterProperties.setSessionTimeout(properties.getSessionTimeout()); + clusterProperties.setDatacenter(DefaultDatacenterNameProvider.DEFAULT_DC_NAME); + return clusterProperties; + } + + private void selectLocalClient() { + if (clients.size() == 1) { + localClient = clients.get(0); + } else { + String localDcName = datacenterNameProvider.getDatacenterName(); + localClient = + clients.stream() + .filter(client -> client.getDatacenterName().equals(localDcName)) + .findFirst() + .orElseThrow(() -> new ZookeeperClientNotFoundException(localDcName)); } - - private CuratorFramework buildCuratorFramework(StorageProperties clusterProperties, - StorageClustersProperties commonProperties) { - ExponentialBackoffRetry retryPolicy = new ExponentialBackoffRetry(commonProperties.getRetrySleep(), - commonProperties.getRetryTimes()); - - CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder() - .connectString(clusterProperties.getConnectionString()) - .sessionTimeoutMs(clusterProperties.getSessionTimeout()) - .connectionTimeoutMs(clusterProperties.getConnectTimeout()) - .retryPolicy(retryPolicy); - - Optional.ofNullable(commonProperties.getAuthorization()).ifPresent(it -> { - builder.authorization(it.getScheme(), (it.getUser() + ":" + it.getPassword()).getBytes()); - builder.aclProvider( - new ACLProvider() { - @Override - public List getDefaultAcl() { - return ZooDefs.Ids.CREATOR_ALL_ACL; - } - - @Override - public List getAclForPath(String path) { - return ZooDefs.Ids.CREATOR_ALL_ACL; - } - } - ); - } - ); - - CuratorFramework curator = builder.build(); - curator.start(); - - return curator; + } + + private ZookeeperClient buildZookeeperClient( + StorageProperties clusterProperties, StorageClustersProperties commonProperties) { + return new ZookeeperClient( + buildCuratorFramework(clusterProperties, commonProperties), + clusterProperties.getDatacenter()); + } + + private CuratorFramework buildCuratorFramework( + StorageProperties clusterProperties, StorageClustersProperties commonProperties) { + ExponentialBackoffRetry retryPolicy = + new ExponentialBackoffRetry( + commonProperties.getRetrySleep(), commonProperties.getRetryTimes()); + + CuratorFrameworkFactory.Builder builder = + CuratorFrameworkFactory.builder() + .connectString(clusterProperties.getConnectionString()) + .sessionTimeoutMs(clusterProperties.getSessionTimeout()) + .connectionTimeoutMs(clusterProperties.getConnectTimeout()) + .retryPolicy(retryPolicy); + + Optional.ofNullable(commonProperties.getAuthorization()) + .ifPresent( + it -> { + builder.authorization( + it.getScheme(), (it.getUser() + ":" + it.getPassword()).getBytes()); + builder.aclProvider( + new ACLProvider() { + @Override + public List getDefaultAcl() { + return ZooDefs.Ids.CREATOR_ALL_ACL; + } + + @Override + public List getAclForPath(String path) { + return ZooDefs.Ids.CREATOR_ALL_ACL; + } + }); + }); + + CuratorFramework curator = builder.build(); + curator.start(); + + return curator; + } + + private void waitForConnection(CuratorFramework curator) { + try { + curator.blockUntilConnected(); + } catch (InterruptedException interruptedException) { + RuntimeException exception = + new InternalProcessingException( + "Could not start curator for storage", interruptedException); + logger.error(exception.getMessage(), interruptedException); + throw exception; } - - private void waitForConnection(CuratorFramework curator) { - try { - curator.blockUntilConnected(); - } catch (InterruptedException interruptedException) { - RuntimeException exception = new InternalProcessingException("Could not start curator for storage", - interruptedException); - logger.error(exception.getMessage(), interruptedException); - throw exception; - } + } + + public void stop() { + for (ZookeeperClient client : clients) { + try { + client.getCuratorFramework().close(); + } catch (Exception e) { + logger.warn("Failed to close Zookeeper client on DC: " + client.getDatacenterName()); + } } + } - public void stop() { - for (ZookeeperClient client : clients) { - try { - client.getCuratorFramework().close(); - } catch (Exception e) { - logger.warn("Failed to close Zookeeper client on DC: " + client.getDatacenterName()); - } - } - } + public ZookeeperClient getLocalClient() { + return localClient; + } - public ZookeeperClient getLocalClient() { - return localClient; - } - - public List getClients() { - return clients; - } + public List getClients() { + return clients; + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/zookeeper/ZookeeperClientNotFoundException.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/zookeeper/ZookeeperClientNotFoundException.java index 6f0cfe6f28..830c0605b2 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/zookeeper/ZookeeperClientNotFoundException.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/zookeeper/ZookeeperClientNotFoundException.java @@ -1,7 +1,10 @@ package pl.allegro.tech.hermes.management.infrastructure.zookeeper; public class ZookeeperClientNotFoundException extends RuntimeException { - public ZookeeperClientNotFoundException(String localDcName) { - super("No Zookeeper client is configured to connect to cluster on DC (name: " + localDcName + ")."); - } + public ZookeeperClientNotFoundException(String localDcName) { + super( + "No Zookeeper client is configured to connect to cluster on DC (name: " + + localDcName + + ")."); + } } diff --git a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/zookeeper/ZookeeperRepositoryManager.java b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/zookeeper/ZookeeperRepositoryManager.java index 8d72b11395..3082d76bb9 100644 --- a/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/zookeeper/ZookeeperRepositoryManager.java +++ b/hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/zookeeper/ZookeeperRepositoryManager.java @@ -1,6 +1,11 @@ package pl.allegro.tech.hermes.management.infrastructure.zookeeper; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import org.apache.curator.framework.CuratorFramework; import pl.allegro.tech.hermes.common.admin.AdminTool; import pl.allegro.tech.hermes.common.admin.zookeeper.ZookeeperAdminTool; @@ -34,139 +39,155 @@ import pl.allegro.tech.hermes.management.infrastructure.readiness.ZookeeperDatacenterReadinessRepository; import pl.allegro.tech.hermes.management.infrastructure.retransmit.ZookeeperOfflineRetransmissionRepository; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - public class ZookeeperRepositoryManager implements RepositoryManager { - private final DatacenterNameProvider datacenterNameProvider; - private final ObjectMapper mapper; - private final ZookeeperPaths paths; - private final ZookeeperClientManager clientManager; - private final Map, Object> repositoryByType = new HashMap<>(); - private final Map groupRepositoriesByDc = new HashMap<>(); - private final Map credentialsRepositoriesByDc = new HashMap<>(); - private final Map topicRepositoriesByDc = new HashMap<>(); - private final Map subscriptionRepositoriesByDc = new HashMap<>(); - private final Map oAuthProviderRepositoriesByDc = new HashMap<>(); - private final Map offsetChangeIndicatorsByDc = new HashMap<>(); - private final Map messagePreviewRepositoriesByDc = new HashMap<>(); - private final Map topicBlacklistRepositoriesByDc = new HashMap<>(); - private final Map workloadConstraintsRepositoriesByDc = new HashMap<>(); - private final Map lastUndeliveredMessageReaderByDc = new HashMap<>(); - private final Map adminToolByDc = new HashMap<>(); - private final Map readinessRepositoriesByDc = new HashMap<>(); - private final Map offlineRetransmissionRepositoriesByDc = new HashMap<>(); - private final ZookeeperGroupRepositoryFactory zookeeperGroupRepositoryFactory; - - public ZookeeperRepositoryManager(ZookeeperClientManager clientManager, - DatacenterNameProvider datacenterNameProvider, - ObjectMapper mapper, - ZookeeperPaths paths, - ZookeeperGroupRepositoryFactory zookeeperGroupRepositoryFactory) { - this.datacenterNameProvider = datacenterNameProvider; - this.mapper = mapper; - this.paths = paths; - this.clientManager = clientManager; - this.zookeeperGroupRepositoryFactory = zookeeperGroupRepositoryFactory; - initRepositoryTypeMap(); + private final DatacenterNameProvider datacenterNameProvider; + private final ObjectMapper mapper; + private final ZookeeperPaths paths; + private final ZookeeperClientManager clientManager; + private final Map, Object> repositoryByType = new HashMap<>(); + private final Map groupRepositoriesByDc = new HashMap<>(); + private final Map credentialsRepositoriesByDc = new HashMap<>(); + private final Map topicRepositoriesByDc = new HashMap<>(); + private final Map subscriptionRepositoriesByDc = new HashMap<>(); + private final Map oAuthProviderRepositoriesByDc = + new HashMap<>(); + private final Map offsetChangeIndicatorsByDc = + new HashMap<>(); + private final Map messagePreviewRepositoriesByDc = + new HashMap<>(); + private final Map topicBlacklistRepositoriesByDc = + new HashMap<>(); + private final Map workloadConstraintsRepositoriesByDc = + new HashMap<>(); + private final Map lastUndeliveredMessageReaderByDc = + new HashMap<>(); + private final Map adminToolByDc = new HashMap<>(); + private final Map readinessRepositoriesByDc = + new HashMap<>(); + private final Map offlineRetransmissionRepositoriesByDc = + new HashMap<>(); + private final ZookeeperGroupRepositoryFactory zookeeperGroupRepositoryFactory; + + public ZookeeperRepositoryManager( + ZookeeperClientManager clientManager, + DatacenterNameProvider datacenterNameProvider, + ObjectMapper mapper, + ZookeeperPaths paths, + ZookeeperGroupRepositoryFactory zookeeperGroupRepositoryFactory) { + this.datacenterNameProvider = datacenterNameProvider; + this.mapper = mapper; + this.paths = paths; + this.clientManager = clientManager; + this.zookeeperGroupRepositoryFactory = zookeeperGroupRepositoryFactory; + initRepositoryTypeMap(); + } + + public void start() { + for (ZookeeperClient client : clientManager.getClients()) { + String dcName = client.getDatacenterName(); + CuratorFramework zookeeper = client.getCuratorFramework(); + + GroupRepository groupRepository = + zookeeperGroupRepositoryFactory.create(zookeeper, mapper, paths); + groupRepositoriesByDc.put(dcName, groupRepository); + + CredentialsRepository credentialsRepository = + new ZookeeperCredentialsRepository(zookeeper, mapper, paths); + credentialsRepositoriesByDc.put(dcName, credentialsRepository); + + TopicRepository topicRepository = + new ZookeeperTopicRepository(zookeeper, mapper, paths, groupRepository); + topicRepositoriesByDc.put(dcName, topicRepository); + + SubscriptionRepository subscriptionRepository = + new ZookeeperSubscriptionRepository(zookeeper, mapper, paths, topicRepository); + subscriptionRepositoriesByDc.put(dcName, subscriptionRepository); + + OAuthProviderRepository oAuthProviderRepository = + new ZookeeperOAuthProviderRepository(zookeeper, mapper, paths); + oAuthProviderRepositoriesByDc.put(dcName, oAuthProviderRepository); + + SubscriptionOffsetChangeIndicator offsetChangeIndicator = + new ZookeeperSubscriptionOffsetChangeIndicator(zookeeper, paths, subscriptionRepository); + offsetChangeIndicatorsByDc.put(dcName, offsetChangeIndicator); + + MessagePreviewRepository messagePreviewRepository = + new ZookeeperMessagePreviewRepository(zookeeper, mapper, paths); + messagePreviewRepositoriesByDc.put(dcName, messagePreviewRepository); + + TopicBlacklistRepository topicBlacklistRepository = + new ZookeeperTopicBlacklistRepository(zookeeper, mapper, paths); + topicBlacklistRepositoriesByDc.put(dcName, topicBlacklistRepository); + + WorkloadConstraintsRepository workloadConstraintsRepository = + new ZookeeperWorkloadConstraintsRepository(zookeeper, mapper, paths); + workloadConstraintsRepositoriesByDc.put(dcName, workloadConstraintsRepository); + + LastUndeliveredMessageReader lastUndeliveredMessageReader = + new ZookeeperLastUndeliveredMessageReader(zookeeper, paths, mapper); + lastUndeliveredMessageReaderByDc.put(dcName, lastUndeliveredMessageReader); + + AdminTool adminTool = new ZookeeperAdminTool(paths, client.getCuratorFramework(), mapper); + adminToolByDc.put(dcName, adminTool); + + DatacenterReadinessRepository readinessRepository = + new ZookeeperDatacenterReadinessRepository(zookeeper, mapper, paths); + readinessRepositoriesByDc.put(dcName, readinessRepository); + + ZookeeperOfflineRetransmissionRepository offlineRetransmissionRepository = + new ZookeeperOfflineRetransmissionRepository(zookeeper, mapper, paths); + offlineRetransmissionRepositoriesByDc.put(dcName, offlineRetransmissionRepository); } - - public void start() { - for (ZookeeperClient client : clientManager.getClients()) { - String dcName = client.getDatacenterName(); - CuratorFramework zookeeper = client.getCuratorFramework(); - - GroupRepository groupRepository = zookeeperGroupRepositoryFactory.create(zookeeper, mapper, paths); - groupRepositoriesByDc.put(dcName, groupRepository); - - CredentialsRepository credentialsRepository = new ZookeeperCredentialsRepository(zookeeper, mapper, paths); - credentialsRepositoriesByDc.put(dcName, credentialsRepository); - - TopicRepository topicRepository = new ZookeeperTopicRepository(zookeeper, mapper, paths, groupRepository); - topicRepositoriesByDc.put(dcName, topicRepository); - - SubscriptionRepository subscriptionRepository = new ZookeeperSubscriptionRepository(zookeeper, mapper, paths, topicRepository); - subscriptionRepositoriesByDc.put(dcName, subscriptionRepository); - - OAuthProviderRepository oAuthProviderRepository = new ZookeeperOAuthProviderRepository(zookeeper, mapper, paths); - oAuthProviderRepositoriesByDc.put(dcName, oAuthProviderRepository); - - SubscriptionOffsetChangeIndicator offsetChangeIndicator = - new ZookeeperSubscriptionOffsetChangeIndicator(zookeeper, paths, subscriptionRepository); - offsetChangeIndicatorsByDc.put(dcName, offsetChangeIndicator); - - MessagePreviewRepository messagePreviewRepository = new ZookeeperMessagePreviewRepository(zookeeper, mapper, paths); - messagePreviewRepositoriesByDc.put(dcName, messagePreviewRepository); - - TopicBlacklistRepository topicBlacklistRepository = new ZookeeperTopicBlacklistRepository(zookeeper, mapper, paths); - topicBlacklistRepositoriesByDc.put(dcName, topicBlacklistRepository); - - WorkloadConstraintsRepository workloadConstraintsRepository = - new ZookeeperWorkloadConstraintsRepository(zookeeper, mapper, paths); - workloadConstraintsRepositoriesByDc.put(dcName, workloadConstraintsRepository); - - LastUndeliveredMessageReader lastUndeliveredMessageReader = new ZookeeperLastUndeliveredMessageReader(zookeeper, paths, mapper); - lastUndeliveredMessageReaderByDc.put(dcName, lastUndeliveredMessageReader); - - AdminTool adminTool = new ZookeeperAdminTool(paths, client.getCuratorFramework(), mapper); - adminToolByDc.put(dcName, adminTool); - - DatacenterReadinessRepository readinessRepository = new ZookeeperDatacenterReadinessRepository(zookeeper, mapper, paths); - readinessRepositoriesByDc.put(dcName, readinessRepository); - - ZookeeperOfflineRetransmissionRepository offlineRetransmissionRepository = - new ZookeeperOfflineRetransmissionRepository(zookeeper, mapper, paths); - offlineRetransmissionRepositoriesByDc.put(dcName, offlineRetransmissionRepository); - } - } - - public DatacenterBoundRepositoryHolder getLocalRepository(Class repositoryType) { - String dcName = datacenterNameProvider.getDatacenterName(); - T repository = getRepositoriesByType(repositoryType).get(dcName); - - if (repository == null) { - throw new InternalProcessingException("Failed to find '" + repositoryType.getSimpleName() - + "' bound with DC '" + dcName + "'."); - } - - return new DatacenterBoundRepositoryHolder<>(repository, dcName); - } - - public List> getRepositories(Class repositoryType) { - return getRepositoriesByType(repositoryType) - .entrySet() - .stream() - .sorted(Comparator.comparing(Map.Entry::getKey)) - .map(entry -> new DatacenterBoundRepositoryHolder<>(entry.getValue(), entry.getKey())) - .collect(Collectors.toList()); - } - - @SuppressWarnings("unchecked") - private Map getRepositoriesByType(Class type) { - Object repository = repositoryByType.get(type); - if (repository == null) { - throw new InternalProcessingException("Could not provide repository of type: " + type.getName()); - } - return (Map) repository; + } + + public DatacenterBoundRepositoryHolder getLocalRepository(Class repositoryType) { + String dcName = datacenterNameProvider.getDatacenterName(); + T repository = getRepositoriesByType(repositoryType).get(dcName); + + if (repository == null) { + throw new InternalProcessingException( + "Failed to find '" + + repositoryType.getSimpleName() + + "' bound with DC '" + + dcName + + "'."); } - private void initRepositoryTypeMap() { - repositoryByType.put(GroupRepository.class, groupRepositoriesByDc); - repositoryByType.put(CredentialsRepository.class, credentialsRepositoriesByDc); - repositoryByType.put(TopicRepository.class, topicRepositoriesByDc); - repositoryByType.put(SubscriptionRepository.class, subscriptionRepositoriesByDc); - repositoryByType.put(OAuthProviderRepository.class, oAuthProviderRepositoriesByDc); - repositoryByType.put(SubscriptionOffsetChangeIndicator.class, offsetChangeIndicatorsByDc); - repositoryByType.put(MessagePreviewRepository.class, messagePreviewRepositoriesByDc); - repositoryByType.put(TopicBlacklistRepository.class, topicBlacklistRepositoriesByDc); - repositoryByType.put(WorkloadConstraintsRepository.class, workloadConstraintsRepositoriesByDc); - repositoryByType.put(LastUndeliveredMessageReader.class, lastUndeliveredMessageReaderByDc); - repositoryByType.put(AdminTool.class, adminToolByDc); - repositoryByType.put(DatacenterReadinessRepository.class, readinessRepositoriesByDc); - repositoryByType.put(OfflineRetransmissionRepository.class, offlineRetransmissionRepositoriesByDc); + return new DatacenterBoundRepositoryHolder<>(repository, dcName); + } + + public List> getRepositories(Class repositoryType) { + return getRepositoriesByType(repositoryType).entrySet().stream() + .sorted(Comparator.comparing(Map.Entry::getKey)) + .map(entry -> new DatacenterBoundRepositoryHolder<>(entry.getValue(), entry.getKey())) + .collect(Collectors.toList()); + } + + @SuppressWarnings("unchecked") + private Map getRepositoriesByType(Class type) { + Object repository = repositoryByType.get(type); + if (repository == null) { + throw new InternalProcessingException( + "Could not provide repository of type: " + type.getName()); } + return (Map) repository; + } + + private void initRepositoryTypeMap() { + repositoryByType.put(GroupRepository.class, groupRepositoriesByDc); + repositoryByType.put(CredentialsRepository.class, credentialsRepositoriesByDc); + repositoryByType.put(TopicRepository.class, topicRepositoriesByDc); + repositoryByType.put(SubscriptionRepository.class, subscriptionRepositoriesByDc); + repositoryByType.put(OAuthProviderRepository.class, oAuthProviderRepositoriesByDc); + repositoryByType.put(SubscriptionOffsetChangeIndicator.class, offsetChangeIndicatorsByDc); + repositoryByType.put(MessagePreviewRepository.class, messagePreviewRepositoriesByDc); + repositoryByType.put(TopicBlacklistRepository.class, topicBlacklistRepositoriesByDc); + repositoryByType.put(WorkloadConstraintsRepository.class, workloadConstraintsRepositoriesByDc); + repositoryByType.put(LastUndeliveredMessageReader.class, lastUndeliveredMessageReaderByDc); + repositoryByType.put(AdminTool.class, adminToolByDc); + repositoryByType.put(DatacenterReadinessRepository.class, readinessRepositoriesByDc); + repositoryByType.put( + OfflineRetransmissionRepository.class, offlineRetransmissionRepositoriesByDc); + } } diff --git a/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/DefaultHermesHistogram.java b/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/DefaultHermesHistogram.java index ffc1573d94..1140b439c6 100644 --- a/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/DefaultHermesHistogram.java +++ b/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/DefaultHermesHistogram.java @@ -3,18 +3,20 @@ import io.micrometer.core.instrument.DistributionSummary; public class DefaultHermesHistogram implements HermesHistogram { - private final DistributionSummary micrometerHistogram; + private final DistributionSummary micrometerHistogram; - private DefaultHermesHistogram(io.micrometer.core.instrument.DistributionSummary micrometerHistogram) { - this.micrometerHistogram = micrometerHistogram; - } + private DefaultHermesHistogram( + io.micrometer.core.instrument.DistributionSummary micrometerHistogram) { + this.micrometerHistogram = micrometerHistogram; + } - public static DefaultHermesHistogram of(io.micrometer.core.instrument.DistributionSummary micrometerHistogram) { - return new DefaultHermesHistogram(micrometerHistogram); - } + public static DefaultHermesHistogram of( + io.micrometer.core.instrument.DistributionSummary micrometerHistogram) { + return new DefaultHermesHistogram(micrometerHistogram); + } - @Override - public void record(long value) { - micrometerHistogram.record(value); - } + @Override + public void record(long value) { + micrometerHistogram.record(value); + } } diff --git a/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/HermesCounter.java b/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/HermesCounter.java index 641cd72a5f..db786f2f3b 100644 --- a/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/HermesCounter.java +++ b/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/HermesCounter.java @@ -1,10 +1,9 @@ package pl.allegro.tech.hermes.metrics; - public interface HermesCounter { - void increment(long size); + void increment(long size); - default void increment() { - increment(1L); - } + default void increment() { + increment(1L); + } } diff --git a/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/HermesHistogram.java b/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/HermesHistogram.java index 2410154e13..21ca2a1bef 100644 --- a/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/HermesHistogram.java +++ b/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/HermesHistogram.java @@ -2,5 +2,5 @@ public interface HermesHistogram { - void record(long value); + void record(long value); } diff --git a/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/HermesRateMeter.java b/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/HermesRateMeter.java index c31ae5fa46..b4d3189f19 100644 --- a/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/HermesRateMeter.java +++ b/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/HermesRateMeter.java @@ -1,5 +1,5 @@ package pl.allegro.tech.hermes.metrics; public interface HermesRateMeter { - double getOneMinuteRate(); + double getOneMinuteRate(); } diff --git a/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/HermesTimer.java b/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/HermesTimer.java index 8102c31c94..a994b56594 100644 --- a/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/HermesTimer.java +++ b/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/HermesTimer.java @@ -3,17 +3,17 @@ import io.micrometer.core.instrument.Timer; public class HermesTimer { - private final Timer micrometerTimer; + private final Timer micrometerTimer; - private HermesTimer(Timer micrometerTimer) { - this.micrometerTimer = micrometerTimer; - } + private HermesTimer(Timer micrometerTimer) { + this.micrometerTimer = micrometerTimer; + } - public static HermesTimer from(io.micrometer.core.instrument.Timer micrometerTimer) { - return new HermesTimer(micrometerTimer); - } + public static HermesTimer from(io.micrometer.core.instrument.Timer micrometerTimer) { + return new HermesTimer(micrometerTimer); + } - public HermesTimerContext time() { - return HermesTimerContext.from(micrometerTimer); - } -} \ No newline at end of file + public HermesTimerContext time() { + return HermesTimerContext.from(micrometerTimer); + } +} diff --git a/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/HermesTimerContext.java b/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/HermesTimerContext.java index 65bb85d157..867ca43a3f 100644 --- a/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/HermesTimerContext.java +++ b/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/HermesTimerContext.java @@ -2,39 +2,38 @@ import io.micrometer.core.instrument.Clock; import io.micrometer.core.instrument.Timer; - import java.io.Closeable; import java.time.Duration; import java.util.concurrent.TimeUnit; public class HermesTimerContext implements Closeable { - private final Timer micrometerTimer; - private final Clock clock; - private final long startNanos; - - private HermesTimerContext(Timer micrometerTimer, Clock clock) { - this.micrometerTimer = micrometerTimer; - this.clock = clock; - this.startNanos = clock.monotonicTime(); - } - - public static HermesTimerContext from(Timer micrometerTimer) { - return new HermesTimerContext(micrometerTimer, Clock.SYSTEM); - } - - @Override - public void close() { - reportTimer(); - } - - public Duration closeAndGet() { - return Duration.ofNanos(reportTimer()); - } - - private long reportTimer() { - long amount = clock.monotonicTime() - startNanos; - micrometerTimer.record(amount, TimeUnit.NANOSECONDS); - return amount; - } + private final Timer micrometerTimer; + private final Clock clock; + private final long startNanos; + + private HermesTimerContext(Timer micrometerTimer, Clock clock) { + this.micrometerTimer = micrometerTimer; + this.clock = clock; + this.startNanos = clock.monotonicTime(); + } + + public static HermesTimerContext from(Timer micrometerTimer) { + return new HermesTimerContext(micrometerTimer, Clock.SYSTEM); + } + + @Override + public void close() { + reportTimer(); + } + + public Duration closeAndGet() { + return Duration.ofNanos(reportTimer()); + } + + private long reportTimer() { + long amount = clock.monotonicTime() - startNanos; + micrometerTimer.record(amount, TimeUnit.NANOSECONDS); + return amount; + } } diff --git a/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/PathContext.java b/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/PathContext.java index d96b017d9a..667cd44930 100644 --- a/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/PathContext.java +++ b/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/PathContext.java @@ -4,58 +4,56 @@ public class PathContext { - private final Optional group; - private final Optional topic; - private final Optional subscription; - - private PathContext(Optional group, - Optional topic, - Optional subscription) { - this.group = group; - this.topic = topic; - this.subscription = subscription; + private final Optional group; + private final Optional topic; + private final Optional subscription; + + private PathContext( + Optional group, Optional topic, Optional subscription) { + this.group = group; + this.topic = topic; + this.subscription = subscription; + } + + public Optional getGroup() { + return group; + } + + public Optional getTopic() { + return topic; + } + + public Optional getSubscription() { + return subscription; + } + + public static Builder pathContext() { + return new Builder(); + } + + public static class Builder { + + private Optional group = Optional.empty(); + private Optional topic = Optional.empty(); + private Optional subscription = Optional.empty(); + + public Builder withGroup(String group) { + this.group = Optional.of(group); + return this; } - public Optional getGroup() { - return group; + public Builder withTopic(String topic) { + this.topic = Optional.of(topic); + return this; } - public Optional getTopic() { - return topic; + public Builder withSubscription(String subscription) { + this.subscription = Optional.of(subscription); + return this; } - public Optional getSubscription() { - return subscription; - } - - - public static Builder pathContext() { - return new Builder(); - } - - public static class Builder { - - private Optional group = Optional.empty(); - private Optional topic = Optional.empty(); - private Optional subscription = Optional.empty(); - - public Builder withGroup(String group) { - this.group = Optional.of(group); - return this; - } - - public Builder withTopic(String topic) { - this.topic = Optional.of(topic); - return this; - } - - public Builder withSubscription(String subscription) { - this.subscription = Optional.of(subscription); - return this; - } - - public PathContext build() { - return new PathContext(group, topic, subscription); - } + public PathContext build() { + return new PathContext(group, topic, subscription); } + } } diff --git a/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/PathsCompiler.java b/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/PathsCompiler.java index c69e333740..7ac7fca8b3 100644 --- a/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/PathsCompiler.java +++ b/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/PathsCompiler.java @@ -1,40 +1,39 @@ package pl.allegro.tech.hermes.metrics; - import org.apache.commons.text.TextStringBuilder; public class PathsCompiler { - public static final String REPLACEMENT_CHAR = "_"; + public static final String REPLACEMENT_CHAR = "_"; - public static final String HOSTNAME = "$hostname"; - public static final String GROUP = "$group"; - public static final String TOPIC = "$topic"; - public static final String SUBSCRIPTION = "$subscription"; + public static final String HOSTNAME = "$hostname"; + public static final String GROUP = "$group"; + public static final String TOPIC = "$topic"; + public static final String SUBSCRIPTION = "$subscription"; - private final String hostname; + private final String hostname; - public PathsCompiler(String hostname) { - this.hostname = escapeDots(hostname); - } + public PathsCompiler(String hostname) { + this.hostname = escapeDots(hostname); + } - public String compile(String path) { - return path.replace(HOSTNAME, hostname); - } + public String compile(String path) { + return path.replace(HOSTNAME, hostname); + } - public String compile(String path, PathContext context) { - TextStringBuilder pathBuilder = new TextStringBuilder(path); + public String compile(String path, PathContext context) { + TextStringBuilder pathBuilder = new TextStringBuilder(path); - context.getGroup().ifPresent(g -> pathBuilder.replaceAll(GROUP, g)); - context.getTopic().ifPresent(t -> pathBuilder.replaceAll(TOPIC, t)); - context.getSubscription().ifPresent(s -> pathBuilder.replaceAll(SUBSCRIPTION, s)); + context.getGroup().ifPresent(g -> pathBuilder.replaceAll(GROUP, g)); + context.getTopic().ifPresent(t -> pathBuilder.replaceAll(TOPIC, t)); + context.getSubscription().ifPresent(s -> pathBuilder.replaceAll(SUBSCRIPTION, s)); - pathBuilder.replaceAll(HOSTNAME, hostname); + pathBuilder.replaceAll(HOSTNAME, hostname); - return pathBuilder.toString(); - } + return pathBuilder.toString(); + } - private String escapeDots(String value) { - return value.replaceAll("\\.", REPLACEMENT_CHAR); - } + private String escapeDots(String value) { + return value.replaceAll("\\.", REPLACEMENT_CHAR); + } } diff --git a/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/counters/DefaultHermesCounter.java b/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/counters/DefaultHermesCounter.java index c87629c534..609c4cce9d 100644 --- a/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/counters/DefaultHermesCounter.java +++ b/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/counters/DefaultHermesCounter.java @@ -1,16 +1,15 @@ package pl.allegro.tech.hermes.metrics.counters; - import pl.allegro.tech.hermes.metrics.HermesCounter; public class DefaultHermesCounter implements HermesCounter { - protected final io.micrometer.core.instrument.Counter micrometerCounter; + protected final io.micrometer.core.instrument.Counter micrometerCounter; - protected DefaultHermesCounter(io.micrometer.core.instrument.Counter micrometerCounter) { - this.micrometerCounter = micrometerCounter; - } + protected DefaultHermesCounter(io.micrometer.core.instrument.Counter micrometerCounter) { + this.micrometerCounter = micrometerCounter; + } - public void increment(long size) { - this.micrometerCounter.increment((double) size); - } + public void increment(long size) { + this.micrometerCounter.increment((double) size); + } } diff --git a/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/counters/HermesCounters.java b/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/counters/HermesCounters.java index 67b4e2f054..098b87f9ae 100644 --- a/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/counters/HermesCounters.java +++ b/hermes-metrics/src/main/java/pl/allegro/tech/hermes/metrics/counters/HermesCounters.java @@ -1,8 +1,7 @@ package pl.allegro.tech.hermes.metrics.counters; - public class HermesCounters { - public static DefaultHermesCounter from(io.micrometer.core.instrument.Counter micrometerCounter) { - return new DefaultHermesCounter(micrometerCounter); - } + public static DefaultHermesCounter from(io.micrometer.core.instrument.Counter micrometerCounter) { + return new DefaultHermesCounter(micrometerCounter); + } } diff --git a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMock.java b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMock.java index 52e690365e..ffcb675d21 100644 --- a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMock.java +++ b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMock.java @@ -1,122 +1,126 @@ package pl.allegro.tech.hermes.mock; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; +import static com.github.tomakehurst.wiremock.http.RequestMethod.POST; +import static pl.allegro.tech.hermes.mock.HermesMockHelper.startsWith; + import com.fasterxml.jackson.databind.ObjectMapper; import com.github.tomakehurst.wiremock.WireMockServer; import com.github.tomakehurst.wiremock.http.Request; import com.github.tomakehurst.wiremock.matching.RequestPattern; import com.github.tomakehurst.wiremock.matching.RequestPatternBuilder; import com.github.tomakehurst.wiremock.matching.ValueMatcher; +import java.util.function.Predicate; import org.apache.avro.Schema; import pl.allegro.tech.hermes.mock.matching.ContentMatchers; -import java.util.function.Predicate; - -import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; -import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; -import static com.github.tomakehurst.wiremock.http.RequestMethod.POST; -import static pl.allegro.tech.hermes.mock.HermesMockHelper.startsWith; - public class HermesMock { - private static final String APPLICATION_JSON = "application/json"; - private static final String AVRO_BINARY = "avro/binary"; - - private final WireMockServer wireMockServer; - - private final HermesMockDefine hermesMockDefine; - private final HermesMockExpect hermesMockExpect; - private final HermesMockQuery hermesMockQuery; - private final HermesMockHelper hermesMockHelper; - - private HermesMock(int port, int awaitSeconds, ObjectMapper objectMapper) { - wireMockServer = new WireMockServer(port); - - hermesMockHelper = new HermesMockHelper(wireMockServer, objectMapper); - hermesMockDefine = new HermesMockDefine(hermesMockHelper); - hermesMockExpect = new HermesMockExpect(hermesMockHelper, awaitSeconds); - hermesMockQuery = new HermesMockQuery(hermesMockHelper); - } - - public HermesMockDefine define() { - return hermesMockDefine; - } - - public HermesMockExpect expect() { - return hermesMockExpect; - } - - public HermesMockQuery query() { - return hermesMockQuery; - } - - public void resetReceivedRequest() { - wireMockServer.resetRequests(); - } - - public void resetReceivedAvroRequests(String topicName, Schema schema, Class clazz, Predicate predicate) { - ValueMatcher valueMatcher = ContentMatchers.matchAvro(hermesMockHelper, predicate, schema, clazz); - resetReceivedRequests(topicName, AVRO_BINARY, valueMatcher); - } - - public void resetReceivedJsonRequests(String topicName, Class clazz, Predicate predicate) { - ValueMatcher valueMatcher = - ContentMatchers.matchJson(hermesMockHelper, predicate, clazz); - resetReceivedRequests(topicName, APPLICATION_JSON, valueMatcher); - } - - private void resetReceivedRequests(String topicName, String contentType, - ValueMatcher valueMatcher) { - RequestPattern requestPattern = RequestPatternBuilder - .newRequestPattern(POST, urlEqualTo("/topics/" + topicName)) - .withHeader("Content-Type", startsWith(contentType)) - .andMatching(valueMatcher) - .build(); - wireMockServer.removeServeEventsMatching(requestPattern); + private static final String APPLICATION_JSON = "application/json"; + private static final String AVRO_BINARY = "avro/binary"; + + private final WireMockServer wireMockServer; + + private final HermesMockDefine hermesMockDefine; + private final HermesMockExpect hermesMockExpect; + private final HermesMockQuery hermesMockQuery; + private final HermesMockHelper hermesMockHelper; + + private HermesMock(int port, int awaitSeconds, ObjectMapper objectMapper) { + wireMockServer = new WireMockServer(port); + + hermesMockHelper = new HermesMockHelper(wireMockServer, objectMapper); + hermesMockDefine = new HermesMockDefine(hermesMockHelper); + hermesMockExpect = new HermesMockExpect(hermesMockHelper, awaitSeconds); + hermesMockQuery = new HermesMockQuery(hermesMockHelper); + } + + public HermesMockDefine define() { + return hermesMockDefine; + } + + public HermesMockExpect expect() { + return hermesMockExpect; + } + + public HermesMockQuery query() { + return hermesMockQuery; + } + + public void resetReceivedRequest() { + wireMockServer.resetRequests(); + } + + public void resetReceivedAvroRequests( + String topicName, Schema schema, Class clazz, Predicate predicate) { + ValueMatcher valueMatcher = + ContentMatchers.matchAvro(hermesMockHelper, predicate, schema, clazz); + resetReceivedRequests(topicName, AVRO_BINARY, valueMatcher); + } + + public void resetReceivedJsonRequests( + String topicName, Class clazz, Predicate predicate) { + ValueMatcher valueMatcher = + ContentMatchers.matchJson(hermesMockHelper, predicate, clazz); + resetReceivedRequests(topicName, APPLICATION_JSON, valueMatcher); + } + + private void resetReceivedRequests( + String topicName, + String contentType, + ValueMatcher valueMatcher) { + RequestPattern requestPattern = + RequestPatternBuilder.newRequestPattern(POST, urlEqualTo("/topics/" + topicName)) + .withHeader("Content-Type", startsWith(contentType)) + .andMatching(valueMatcher) + .build(); + wireMockServer.removeServeEventsMatching(requestPattern); + } + + public void resetMappings() { + wireMockServer.resetMappings(); + } + + public void start() { + wireMockServer.start(); + } + + public void stop() { + wireMockServer.stop(); + } + + public static class Builder { + private int port; + private int awaitSeconds; + private ObjectMapper objectMapper; + + public Builder() { + port = wireMockConfig().portNumber(); + awaitSeconds = 5; + objectMapper = new ObjectMapper().findAndRegisterModules(); } - public void resetMappings() { - wireMockServer.resetMappings(); + public HermesMock.Builder withPort(int port) { + this.port = port; + return this; } - public void start() { - wireMockServer.start(); + public HermesMock.Builder withObjectMapper(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + return this; } - public void stop() { - wireMockServer.stop(); + public HermesMock.Builder withAwaitSeconds(int awaitSeconds) { + this.awaitSeconds = awaitSeconds; + return this; } - public static class Builder { - private int port; - private int awaitSeconds; - private ObjectMapper objectMapper; - - public Builder() { - port = wireMockConfig().portNumber(); - awaitSeconds = 5; - objectMapper = new ObjectMapper().findAndRegisterModules(); - } - - public HermesMock.Builder withPort(int port) { - this.port = port; - return this; - } - - public HermesMock.Builder withObjectMapper(ObjectMapper objectMapper) { - this.objectMapper = objectMapper; - return this; - } - - public HermesMock.Builder withAwaitSeconds(int awaitSeconds) { - this.awaitSeconds = awaitSeconds; - return this; - } - - public HermesMock build() { - if (port == 0) { - port = wireMockConfig().dynamicPort().portNumber(); - } - return new HermesMock(port, awaitSeconds, objectMapper); - } + public HermesMock build() { + if (port == 0) { + port = wireMockConfig().dynamicPort().portNumber(); + } + return new HermesMock(port, awaitSeconds, objectMapper); } + } } diff --git a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockDefine.java b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockDefine.java index 8fadfc3378..3a62a48d1c 100644 --- a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockDefine.java +++ b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockDefine.java @@ -1,69 +1,73 @@ package pl.allegro.tech.hermes.mock; +import static pl.allegro.tech.hermes.mock.exchange.Response.Builder.aResponse; + import com.github.tomakehurst.wiremock.http.Request; import com.github.tomakehurst.wiremock.matching.ValueMatcher; import com.github.tomakehurst.wiremock.stubbing.StubMapping; +import java.util.function.Predicate; import org.apache.avro.Schema; import pl.allegro.tech.hermes.mock.exchange.Response; import pl.allegro.tech.hermes.mock.matching.ContentMatchers; import wiremock.org.apache.hc.core5.http.HttpStatus; -import java.util.function.Predicate; - -import static pl.allegro.tech.hermes.mock.exchange.Response.Builder.aResponse; - public class HermesMockDefine { - private static final String APPLICATION_JSON = "application/json"; - private static final String AVRO_BINARY = "avro/binary"; - private final HermesMockHelper hermesMockHelper; - - public HermesMockDefine(HermesMockHelper hermesMockHelper) { - this.hermesMockHelper = hermesMockHelper; - } - - public StubMapping jsonTopic(String topicName) { - return jsonTopic(topicName, HttpStatus.SC_CREATED); - } - - public StubMapping jsonTopic(String topicName, int statusCode) { - return addTopic(topicName, aResponse().withStatusCode(statusCode).build(), APPLICATION_JSON); - } - - public StubMapping jsonTopic(String topicName, Response response) { - return addTopic(topicName, response, APPLICATION_JSON); - } - - public StubMapping jsonTopic(String topicName, Response response, Class clazz, Predicate predicate) { - ValueMatcher jsonMatchesPattern = ContentMatchers.matchJson(hermesMockHelper, predicate, clazz); - return addTopic(topicName, response, APPLICATION_JSON, jsonMatchesPattern); - } - - public StubMapping avroTopic(String topicName) { - return avroTopic(topicName, HttpStatus.SC_CREATED); - } - - public StubMapping avroTopic(String topicName, int statusCode) { - return addTopic(topicName, aResponse().withStatusCode(statusCode).build(), AVRO_BINARY); - } - - public StubMapping avroTopic(String topicName, Response response) { - return addTopic(topicName, response, AVRO_BINARY); - } - - public StubMapping avroTopic(String topicName, Response response, Schema schema, Class clazz, Predicate predicate) { - ValueMatcher avroMatchesPattern = ContentMatchers.matchAvro(hermesMockHelper, predicate, schema, clazz); - return addTopic(topicName, response, AVRO_BINARY, avroMatchesPattern); - } - - public void removeStubMapping(StubMapping stubMapping) { - hermesMockHelper.removeStubMapping(stubMapping); - } - - private StubMapping addTopic(String topicName, Response response, String contentType) { - return hermesMockHelper.addStub(topicName, response, contentType); - } - - private StubMapping addTopic(String topicName, Response response, String contentType, ValueMatcher valueMatcher) { - return hermesMockHelper.addStub(topicName, response, contentType, valueMatcher); - } + private static final String APPLICATION_JSON = "application/json"; + private static final String AVRO_BINARY = "avro/binary"; + private final HermesMockHelper hermesMockHelper; + + public HermesMockDefine(HermesMockHelper hermesMockHelper) { + this.hermesMockHelper = hermesMockHelper; + } + + public StubMapping jsonTopic(String topicName) { + return jsonTopic(topicName, HttpStatus.SC_CREATED); + } + + public StubMapping jsonTopic(String topicName, int statusCode) { + return addTopic(topicName, aResponse().withStatusCode(statusCode).build(), APPLICATION_JSON); + } + + public StubMapping jsonTopic(String topicName, Response response) { + return addTopic(topicName, response, APPLICATION_JSON); + } + + public StubMapping jsonTopic( + String topicName, Response response, Class clazz, Predicate predicate) { + ValueMatcher jsonMatchesPattern = + ContentMatchers.matchJson(hermesMockHelper, predicate, clazz); + return addTopic(topicName, response, APPLICATION_JSON, jsonMatchesPattern); + } + + public StubMapping avroTopic(String topicName) { + return avroTopic(topicName, HttpStatus.SC_CREATED); + } + + public StubMapping avroTopic(String topicName, int statusCode) { + return addTopic(topicName, aResponse().withStatusCode(statusCode).build(), AVRO_BINARY); + } + + public StubMapping avroTopic(String topicName, Response response) { + return addTopic(topicName, response, AVRO_BINARY); + } + + public StubMapping avroTopic( + String topicName, Response response, Schema schema, Class clazz, Predicate predicate) { + ValueMatcher avroMatchesPattern = + ContentMatchers.matchAvro(hermesMockHelper, predicate, schema, clazz); + return addTopic(topicName, response, AVRO_BINARY, avroMatchesPattern); + } + + public void removeStubMapping(StubMapping stubMapping) { + hermesMockHelper.removeStubMapping(stubMapping); + } + + private StubMapping addTopic(String topicName, Response response, String contentType) { + return hermesMockHelper.addStub(topicName, response, contentType); + } + + private StubMapping addTopic( + String topicName, Response response, String contentType, ValueMatcher valueMatcher) { + return hermesMockHelper.addStub(topicName, response, contentType, valueMatcher); + } } diff --git a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockException.java b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockException.java index 539b17dd2e..f0214b0be8 100644 --- a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockException.java +++ b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockException.java @@ -1,11 +1,11 @@ package pl.allegro.tech.hermes.mock; public class HermesMockException extends RuntimeException { - public HermesMockException(String message) { - super(message); - } + public HermesMockException(String message) { + super(message); + } - public HermesMockException(String message, Throwable cause) { - super(message, cause); - } + public HermesMockException(String message, Throwable cause) { + super(message, cause); + } } diff --git a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockExpect.java b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockExpect.java index 7f503235a3..180e4d0202 100644 --- a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockExpect.java +++ b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockExpect.java @@ -1,85 +1,93 @@ package pl.allegro.tech.hermes.mock; -import org.apache.avro.Schema; -import org.awaitility.core.ConditionTimeoutException; +import static java.util.concurrent.TimeUnit.SECONDS; +import static java.util.stream.Collectors.toList; +import static org.awaitility.Awaitility.await; import java.util.List; import java.util.function.Predicate; import java.util.function.Supplier; - -import static java.util.concurrent.TimeUnit.SECONDS; -import static java.util.stream.Collectors.toList; -import static org.awaitility.Awaitility.await; +import org.apache.avro.Schema; +import org.awaitility.core.ConditionTimeoutException; public class HermesMockExpect { - private final HermesMockHelper hermesMockHelper; - private final HermesMockQuery hermesMockQuery; - private final int awaitSeconds; - - public HermesMockExpect(HermesMockHelper hermesMockHelper, int awaitSeconds) { - this.hermesMockHelper = hermesMockHelper; - this.hermesMockQuery = new HermesMockQuery(hermesMockHelper); - this.awaitSeconds = awaitSeconds; - } - - public void singleMessageOnTopic(String topicName) { - messagesOnTopic(topicName, 1); - } - - public void singleJsonMessageOnTopicAs(String topicName, Class clazz) { - jsonMessagesOnTopicAs(topicName, 1, clazz); - } - - public void singleAvroMessageOnTopic(String topicName, Schema schema) { - avroMessagesOnTopic(topicName, 1, schema); + private final HermesMockHelper hermesMockHelper; + private final HermesMockQuery hermesMockQuery; + private final int awaitSeconds; + + public HermesMockExpect(HermesMockHelper hermesMockHelper, int awaitSeconds) { + this.hermesMockHelper = hermesMockHelper; + this.hermesMockQuery = new HermesMockQuery(hermesMockHelper); + this.awaitSeconds = awaitSeconds; + } + + public void singleMessageOnTopic(String topicName) { + messagesOnTopic(topicName, 1); + } + + public void singleJsonMessageOnTopicAs(String topicName, Class clazz) { + jsonMessagesOnTopicAs(topicName, 1, clazz); + } + + public void singleAvroMessageOnTopic(String topicName, Schema schema) { + avroMessagesOnTopic(topicName, 1, schema); + } + + public void messagesOnTopic(String topicName, int count) { + expectMessages(topicName, count); + } + + public void jsonMessagesOnTopicAs(String topicName, int count, Class clazz) { + assertMessages(count, () -> hermesMockQuery.allJsonMessagesAs(topicName, clazz)); + } + + public void jsonMessagesOnTopicAs( + String topicName, int count, Class clazz, Predicate predicate) { + assertMessages( + count, () -> hermesMockQuery.matchingJsonMessagesAs(topicName, clazz, predicate)); + } + + public void avroMessagesOnTopic(String topicName, int count, Schema schema) { + assertMessages(count, () -> validateAvroMessages(topicName, schema)); + } + + public void avroMessagesOnTopic( + String topicName, int count, Schema schema, Class clazz, Predicate predicate) { + assertMessages(count, () -> validateAvroMessages(topicName, schema, clazz, predicate)); + } + + private void assertMessages(int count, Supplier> messages) { + try { + await() + .atMost(awaitSeconds, SECONDS) + .until(() -> (messages != null && messages.get().size() == count)); + } catch (ConditionTimeoutException ex) { + throw new HermesMockException( + "Hermes mock did not receive " + count + " messages, got " + messages.get().size()); } - - public void messagesOnTopic(String topicName, int count) { - expectMessages(topicName, count); - } - - public void jsonMessagesOnTopicAs(String topicName, int count, Class clazz) { - assertMessages(count, () -> hermesMockQuery.allJsonMessagesAs(topicName, clazz)); - } - - public void jsonMessagesOnTopicAs(String topicName, int count, Class clazz, Predicate predicate) { - assertMessages(count, () -> hermesMockQuery.matchingJsonMessagesAs(topicName, clazz, predicate)); - } - - public void avroMessagesOnTopic(String topicName, int count, Schema schema) { - assertMessages(count, () -> validateAvroMessages(topicName, schema)); - } - - public void avroMessagesOnTopic(String topicName, int count, Schema schema, Class clazz, Predicate predicate) { - assertMessages(count, () -> validateAvroMessages(topicName, schema, clazz, predicate)); - } - - private void assertMessages(int count, Supplier> messages) { - try { - await().atMost(awaitSeconds, SECONDS).until(() -> (messages != null && messages.get().size() == count)); - } catch (ConditionTimeoutException ex) { - throw new HermesMockException("Hermes mock did not receive " + count + " messages, got " + messages.get().size()); - } - } - - private void expectMessages(String topicName, int count) { - try { - await().atMost(awaitSeconds, SECONDS).untilAsserted(() -> hermesMockHelper.verifyRequest(count, topicName)); - } catch (ConditionTimeoutException ex) { - throw new HermesMockException("Hermes mock did not receive " + count + " messages.", ex); - } - } - - private List validateAvroMessages(String topicName, Schema schema) { - return hermesMockQuery.allAvroRawMessages(topicName).stream() - .peek(raw -> hermesMockHelper.validateAvroSchema(raw, schema)) - .collect(toList()); - } - - private List validateAvroMessages(String topicName, Schema schema, Class clazz, Predicate predicate) { - return hermesMockQuery.allAvroRawMessages(topicName).stream() - .map(raw -> hermesMockHelper.deserializeAvro(raw, schema, clazz)) - .filter(predicate) - .collect(toList()); + } + + private void expectMessages(String topicName, int count) { + try { + await() + .atMost(awaitSeconds, SECONDS) + .untilAsserted(() -> hermesMockHelper.verifyRequest(count, topicName)); + } catch (ConditionTimeoutException ex) { + throw new HermesMockException("Hermes mock did not receive " + count + " messages.", ex); } + } + + private List validateAvroMessages(String topicName, Schema schema) { + return hermesMockQuery.allAvroRawMessages(topicName).stream() + .peek(raw -> hermesMockHelper.validateAvroSchema(raw, schema)) + .collect(toList()); + } + + private List validateAvroMessages( + String topicName, Schema schema, Class clazz, Predicate predicate) { + return hermesMockQuery.allAvroRawMessages(topicName).stream() + .map(raw -> hermesMockHelper.deserializeAvro(raw, schema, clazz)) + .filter(predicate) + .collect(toList()); + } } diff --git a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockExtension.java b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockExtension.java index abcf916fda..48c77ed955 100644 --- a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockExtension.java +++ b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockExtension.java @@ -7,54 +7,57 @@ import org.junit.jupiter.api.extension.ParameterResolutionException; import org.junit.jupiter.api.extension.ParameterResolver; -public class HermesMockExtension implements ParameterResolver, BeforeEachCallback, AfterEachCallback { - - private final HermesMock hermesMock; - - public HermesMockExtension(int port) { - this(new HermesMock.Builder().withPort(port).build()); - } - - public HermesMockExtension(HermesMock hermesMock) { - this.hermesMock = hermesMock; - } - - public HermesMockDefine define() { - return hermesMock.define(); - } - - public HermesMockExpect expect() { - return hermesMock.expect(); - } - - public HermesMockQuery query() { - return hermesMock.query(); - } - - public void resetReceivedRequest() { - hermesMock.resetReceivedRequest(); - } - - @Override - public void beforeEach(ExtensionContext extensionContext) throws Exception { - hermesMock.start(); - } - - @Override - public void afterEach(ExtensionContext extensionContext) throws Exception { - hermesMock.resetReceivedRequest(); - hermesMock.stop(); - } - - @Override - public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) - throws ParameterResolutionException { - return parameterContext.getParameter().getType().isAssignableFrom(HermesMock.class); - } - - @Override - public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) - throws ParameterResolutionException { - return hermesMock; - } +public class HermesMockExtension + implements ParameterResolver, BeforeEachCallback, AfterEachCallback { + + private final HermesMock hermesMock; + + public HermesMockExtension(int port) { + this(new HermesMock.Builder().withPort(port).build()); + } + + public HermesMockExtension(HermesMock hermesMock) { + this.hermesMock = hermesMock; + } + + public HermesMockDefine define() { + return hermesMock.define(); + } + + public HermesMockExpect expect() { + return hermesMock.expect(); + } + + public HermesMockQuery query() { + return hermesMock.query(); + } + + public void resetReceivedRequest() { + hermesMock.resetReceivedRequest(); + } + + @Override + public void beforeEach(ExtensionContext extensionContext) throws Exception { + hermesMock.start(); + } + + @Override + public void afterEach(ExtensionContext extensionContext) throws Exception { + hermesMock.resetReceivedRequest(); + hermesMock.stop(); + } + + @Override + public boolean supportsParameter( + ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + return parameterContext.getParameter().getType().isAssignableFrom(HermesMock.class); + } + + @Override + public Object resolveParameter( + ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + return hermesMock; + } } diff --git a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockHelper.java b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockHelper.java index 1b94d35a6f..fd962ba103 100644 --- a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockHelper.java +++ b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockHelper.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.mock; +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; + import com.fasterxml.jackson.databind.ObjectMapper; import com.github.tomakehurst.wiremock.WireMockServer; import com.github.tomakehurst.wiremock.matching.RequestPatternBuilder; @@ -7,6 +12,11 @@ import com.github.tomakehurst.wiremock.matching.ValueMatcher; import com.github.tomakehurst.wiremock.stubbing.StubMapping; import com.github.tomakehurst.wiremock.verification.LoggedRequest; +import java.io.IOException; +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.UUID; import org.apache.avro.Schema; import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.io.BinaryDecoder; @@ -16,100 +26,91 @@ import pl.allegro.tech.hermes.mock.matching.StartsWithPattern; import tech.allegro.schema.json2avro.converter.JsonAvroConverter; -import java.io.IOException; -import java.time.Duration; -import java.util.List; -import java.util.Optional; -import java.util.UUID; - -import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; -import static com.github.tomakehurst.wiremock.client.WireMock.post; -import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; -import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; - public class HermesMockHelper { - private final WireMockServer wireMockServer; - private final ObjectMapper objectMapper; - - public HermesMockHelper(WireMockServer wireMockServer, ObjectMapper objectMapper) { - this.wireMockServer = wireMockServer; - this.objectMapper = objectMapper; + private final WireMockServer wireMockServer; + private final ObjectMapper objectMapper; + + public HermesMockHelper(WireMockServer wireMockServer, ObjectMapper objectMapper) { + this.wireMockServer = wireMockServer; + this.objectMapper = objectMapper; + } + + private static Integer toIntMilliseconds(Duration duration) { + return Optional.ofNullable(duration).map(Duration::toMillis).map(Math::toIntExact).orElse(null); + } + + public static StringValuePattern startsWith(String value) { + return new StartsWithPattern(value); + } + + public T deserializeJson(byte[] content, Class clazz) { + try { + return objectMapper.readValue(content, clazz); + } catch (IOException ex) { + throw new HermesMockException( + "Cannot read body " + content.toString() + " as " + clazz.getSimpleName(), ex); } - - private static Integer toIntMilliseconds(Duration duration) { - return Optional.ofNullable(duration) - .map(Duration::toMillis) - .map(Math::toIntExact) - .orElse(null); - } - - public static StringValuePattern startsWith(String value) { - return new StartsWithPattern(value); + } + + public T deserializeAvro(Request request, Schema schema, Class clazz) { + return deserializeAvro(request.getBody(), schema, clazz); + } + + public T deserializeAvro(byte[] raw, Schema schema, Class clazz) { + try { + byte[] json = new JsonAvroConverter().convertToJson(raw, schema); + return deserializeJson(json, clazz); + } catch (RuntimeException ex) { + throw new HermesMockException( + "Cannot decode body " + raw + " to " + clazz.getSimpleName(), ex); } - - public T deserializeJson(byte[] content, Class clazz) { - try { - return objectMapper.readValue(content, clazz); - } catch (IOException ex) { - throw new HermesMockException("Cannot read body " + content.toString() + " as " + clazz.getSimpleName(), ex); - } - } - - public T deserializeAvro(Request request, Schema schema, Class clazz) { - return deserializeAvro(request.getBody(), schema, clazz); + } + + public void validateAvroSchema(byte[] raw, Schema schema) { + try { + BinaryDecoder binaryDecoder = DecoderFactory.get().binaryDecoder(raw, null); + new GenericDatumReader<>(schema).read(null, binaryDecoder); + } catch (IOException e) { + throw new HermesMockException("Cannot convert bytes as " + schema.getName()); } - - public T deserializeAvro(byte[] raw, Schema schema, Class clazz) { - try { - byte[] json = new JsonAvroConverter().convertToJson(raw, schema); - return deserializeJson(json, clazz); - } catch (RuntimeException ex) { - throw new HermesMockException("Cannot decode body " + raw + " to " + clazz.getSimpleName(), ex); - } - } - - public void validateAvroSchema(byte[] raw, Schema schema) { - try { - BinaryDecoder binaryDecoder = DecoderFactory.get().binaryDecoder(raw, null); - new GenericDatumReader<>(schema).read(null, binaryDecoder); - } catch (IOException e) { - throw new HermesMockException("Cannot convert bytes as " + schema.getName()); - } - } - - public List findAll(RequestPatternBuilder requestPatternBuilder) { - return wireMockServer.findAll(requestPatternBuilder); - } - - public void verifyRequest(int count, String topicName) { - wireMockServer.verify(count, postRequestedFor(urlEqualTo("/topics/" + topicName))); - } - - public StubMapping addStub(String topicName, Response response, String contentType) { - return wireMockServer.stubFor(post(urlEqualTo("/topics/" + topicName)) - .withHeader("Content-Type", startsWith(contentType)) - .willReturn(aResponse() - .withStatus(response.getStatusCode()) - .withHeader("Hermes-Message-Id", UUID.randomUUID().toString()) - .withFixedDelay(toIntMilliseconds(response.getFixedDelay()))) - ); - } - - public StubMapping addStub(String topicName, Response response, String contentType, - ValueMatcher valueMatcher) { - return wireMockServer.stubFor(post(urlEqualTo("/topics/" + topicName)) - .andMatching(valueMatcher) - .withHeader("Content-Type", startsWith(contentType)) - .willReturn(aResponse() - .withStatus(response.getStatusCode()) - .withHeader("Hermes-Message-Id", UUID.randomUUID().toString()) - .withFixedDelay(toIntMilliseconds(response.getFixedDelay())) - ) - ); - } - - public void removeStubMapping(StubMapping stubMapping) { - wireMockServer.removeStubMapping(stubMapping); - } - + } + + public List findAll(RequestPatternBuilder requestPatternBuilder) { + return wireMockServer.findAll(requestPatternBuilder); + } + + public void verifyRequest(int count, String topicName) { + wireMockServer.verify(count, postRequestedFor(urlEqualTo("/topics/" + topicName))); + } + + public StubMapping addStub(String topicName, Response response, String contentType) { + return wireMockServer.stubFor( + post(urlEqualTo("/topics/" + topicName)) + .withHeader("Content-Type", startsWith(contentType)) + .willReturn( + aResponse() + .withStatus(response.getStatusCode()) + .withHeader("Hermes-Message-Id", UUID.randomUUID().toString()) + .withFixedDelay(toIntMilliseconds(response.getFixedDelay())))); + } + + public StubMapping addStub( + String topicName, + Response response, + String contentType, + ValueMatcher valueMatcher) { + return wireMockServer.stubFor( + post(urlEqualTo("/topics/" + topicName)) + .andMatching(valueMatcher) + .withHeader("Content-Type", startsWith(contentType)) + .willReturn( + aResponse() + .withStatus(response.getStatusCode()) + .withHeader("Hermes-Message-Id", UUID.randomUUID().toString()) + .withFixedDelay(toIntMilliseconds(response.getFixedDelay())))); + } + + public void removeStubMapping(StubMapping stubMapping) { + wireMockServer.removeStubMapping(stubMapping); + } } diff --git a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockQuery.java b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockQuery.java index 9b7af13069..51a1d49646 100644 --- a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockQuery.java +++ b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockQuery.java @@ -1,126 +1,120 @@ package pl.allegro.tech.hermes.mock; -import com.github.tomakehurst.wiremock.matching.RequestPatternBuilder; -import org.apache.avro.Schema; -import pl.allegro.tech.hermes.mock.exchange.Request; -import wiremock.com.google.common.collect.Lists; - -import java.util.List; -import java.util.Optional; -import java.util.function.Predicate; - import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; import static com.github.tomakehurst.wiremock.client.WireMock.urlPathMatching; import static java.util.stream.Collectors.toList; -public class HermesMockQuery { +import com.github.tomakehurst.wiremock.matching.RequestPatternBuilder; +import java.util.List; +import java.util.Optional; +import java.util.function.Predicate; +import org.apache.avro.Schema; +import pl.allegro.tech.hermes.mock.exchange.Request; +import wiremock.com.google.common.collect.Lists; - private final HermesMockHelper hermesMockHelper; - - public HermesMockQuery(HermesMockHelper hermesMockHelper) { - this.hermesMockHelper = hermesMockHelper; - } - - public List allRequests() { - return hermesMockHelper.findAll(postRequestedFor(urlPathMatching(("/topics/.*")))).stream() - .map(Request::new) - .collect(toList()); - } - - public List allRequestsOnTopic(String topicName) { - RequestPatternBuilder matcher = postRequestedFor(urlEqualTo("/topics/" + topicName)); - return hermesMockHelper.findAll(matcher).stream() - .map(Request::new) - .collect(toList()); - } - - public List matchingJsonMessagesAs(String topicName, Class clazz, Predicate predicate) { - return allRequestsOnTopic(topicName).stream() - .map(req -> hermesMockHelper.deserializeJson(req.getBody(), clazz)) - .filter(predicate) - .collect(toList()); - } - - public List allJsonMessagesAs(String topicName, Class clazz) { - return matchingJsonMessagesAs(topicName, clazz, t -> true); - } - - public long countMatchingJsonMessages(String topicName, Class clazz, Predicate predicate) { - return allRequestsOnTopic(topicName).stream() - .map(req -> hermesMockHelper.deserializeJson(req.getBody(), clazz)) - .filter(predicate) - .count(); - } - - public long countJsonMessages(String topicName, Class clazz) { - return countMatchingJsonMessages(topicName, clazz, t -> true); - } - - public List allAvroRawMessages(String topicName) { - return allRequestsOnTopic(topicName).stream() - .map(Request::getBody) - .collect(toList()); - } - - public List matchingAvroMessagesAs(String topicName, Schema schema, Class clazz, Predicate predicate) { - return allRequestsOnTopic(topicName).stream() - .map(req -> hermesMockHelper.deserializeAvro(req, schema, clazz)) - .filter(predicate) - .collect(toList()); - } - - public List allAvroMessagesAs(String topicName, Schema schema, Class clazz) { - return matchingAvroMessagesAs(topicName, schema, clazz, t -> true); - } - - public long countMatchingAvroMessages(String topicName, Schema schema, Class clazz, Predicate predicate) { - return allRequestsOnTopic(topicName).stream() - .map(req -> hermesMockHelper.deserializeAvro(req, schema, clazz)) - .filter(predicate) - .count(); - } - - public long countAvroMessages(String topicName) { - return allRequestsOnTopic(topicName).stream() - .map(Request::getBody) - .count(); - } - - public Optional lastRequest(String topicName) { - List matchingRequests = allRequestsOnTopic(topicName); - return matchingRequests.isEmpty() - ? Optional.empty() - : Optional.of(matchingRequests.get(matchingRequests.size() - 1)); - } - - public Optional lastMatchingJsonMessageAs(String topicName, Class clazz, Predicate predicate) { - return Lists.reverse(allJsonMessagesAs(topicName, clazz)) - .stream() - .filter(predicate) - .findFirst(); - } - - public Optional lastJsonMessageAs(String topicName, Class clazz) { - return lastRequest(topicName) - .map(req -> hermesMockHelper.deserializeJson(req.getBody(), clazz)); - } - - public Optional lastMatchingAvroMessageAs(String topicName, Schema schema, Class clazz, Predicate predicate) { - return Lists.reverse(allAvroMessagesAs(topicName, schema, clazz)) - .stream() - .filter(predicate) - .findFirst(); - } - - public Optional lastAvroRawMessage(String topicName) { - return lastRequest(topicName) - .map(Request::getBody); - } - - public Optional lastAvroMessageAs(String topicName, Schema schema, Class clazz) { - return lastRequest(topicName) - .map(req -> hermesMockHelper.deserializeAvro(req, schema, clazz)); - } +public class HermesMockQuery { + private final HermesMockHelper hermesMockHelper; + + public HermesMockQuery(HermesMockHelper hermesMockHelper) { + this.hermesMockHelper = hermesMockHelper; + } + + public List allRequests() { + return hermesMockHelper.findAll(postRequestedFor(urlPathMatching(("/topics/.*")))).stream() + .map(Request::new) + .collect(toList()); + } + + public List allRequestsOnTopic(String topicName) { + RequestPatternBuilder matcher = postRequestedFor(urlEqualTo("/topics/" + topicName)); + return hermesMockHelper.findAll(matcher).stream().map(Request::new).collect(toList()); + } + + public List matchingJsonMessagesAs( + String topicName, Class clazz, Predicate predicate) { + return allRequestsOnTopic(topicName).stream() + .map(req -> hermesMockHelper.deserializeJson(req.getBody(), clazz)) + .filter(predicate) + .collect(toList()); + } + + public List allJsonMessagesAs(String topicName, Class clazz) { + return matchingJsonMessagesAs(topicName, clazz, t -> true); + } + + public long countMatchingJsonMessages( + String topicName, Class clazz, Predicate predicate) { + return allRequestsOnTopic(topicName).stream() + .map(req -> hermesMockHelper.deserializeJson(req.getBody(), clazz)) + .filter(predicate) + .count(); + } + + public long countJsonMessages(String topicName, Class clazz) { + return countMatchingJsonMessages(topicName, clazz, t -> true); + } + + public List allAvroRawMessages(String topicName) { + return allRequestsOnTopic(topicName).stream().map(Request::getBody).collect(toList()); + } + + public List matchingAvroMessagesAs( + String topicName, Schema schema, Class clazz, Predicate predicate) { + return allRequestsOnTopic(topicName).stream() + .map(req -> hermesMockHelper.deserializeAvro(req, schema, clazz)) + .filter(predicate) + .collect(toList()); + } + + public List allAvroMessagesAs(String topicName, Schema schema, Class clazz) { + return matchingAvroMessagesAs(topicName, schema, clazz, t -> true); + } + + public long countMatchingAvroMessages( + String topicName, Schema schema, Class clazz, Predicate predicate) { + return allRequestsOnTopic(topicName).stream() + .map(req -> hermesMockHelper.deserializeAvro(req, schema, clazz)) + .filter(predicate) + .count(); + } + + public long countAvroMessages(String topicName) { + return allRequestsOnTopic(topicName).stream().map(Request::getBody).count(); + } + + public Optional lastRequest(String topicName) { + List matchingRequests = allRequestsOnTopic(topicName); + return matchingRequests.isEmpty() + ? Optional.empty() + : Optional.of(matchingRequests.get(matchingRequests.size() - 1)); + } + + public Optional lastMatchingJsonMessageAs( + String topicName, Class clazz, Predicate predicate) { + return Lists.reverse(allJsonMessagesAs(topicName, clazz)).stream() + .filter(predicate) + .findFirst(); + } + + public Optional lastJsonMessageAs(String topicName, Class clazz) { + return lastRequest(topicName) + .map(req -> hermesMockHelper.deserializeJson(req.getBody(), clazz)); + } + + public Optional lastMatchingAvroMessageAs( + String topicName, Schema schema, Class clazz, Predicate predicate) { + return Lists.reverse(allAvroMessagesAs(topicName, schema, clazz)).stream() + .filter(predicate) + .findFirst(); + } + + public Optional lastAvroRawMessage(String topicName) { + return lastRequest(topicName).map(Request::getBody); + } + + public Optional lastAvroMessageAs(String topicName, Schema schema, Class clazz) { + return lastRequest(topicName).map(req -> hermesMockHelper.deserializeAvro(req, schema, clazz)); + } } diff --git a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockRule.java b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockRule.java index c3e090ab01..e557a399a8 100644 --- a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockRule.java +++ b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/HermesMockRule.java @@ -1,5 +1,6 @@ package pl.allegro.tech.hermes.mock; +import java.util.function.Predicate; import org.apache.avro.Schema; import org.junit.rules.MethodRule; import org.junit.rules.TestRule; @@ -7,78 +8,78 @@ import org.junit.runners.model.FrameworkMethod; import org.junit.runners.model.Statement; -import java.util.function.Predicate; - public class HermesMockRule implements MethodRule, TestRule { - private final HermesMock hermesMock; - - public HermesMockRule(int port) { - this.hermesMock = new HermesMock.Builder().withPort(port).build(); - } - - public HermesMockRule(HermesMock hermesMock) { - this.hermesMock = hermesMock; - } - - public HermesMockDefine define() { - return hermesMock.define(); - } - - public HermesMockExpect expect() { - return hermesMock.expect(); - } - - public HermesMockQuery query() { - return hermesMock.query(); - } - - public void resetReceivedRequest() { - hermesMock.resetReceivedRequest(); - } - - public void resetReceivedAvroRequests(String topicName, Schema schema, Class clazz, Predicate predicate) { - hermesMock.resetReceivedAvroRequests(topicName, schema, clazz, predicate); - } - - public void resetReceivedJsonRequests(String topicName, Class clazz, Predicate predicate) { - hermesMock.resetReceivedJsonRequests(topicName, clazz, predicate); - } - - public void resetMappings() { - hermesMock.resetMappings(); - } - - @Override - public Statement apply(Statement base, FrameworkMethod method, Object target) { - return runHermesMock(base); - } - - @Override - public Statement apply(Statement base, Description description) { - return runHermesMock(base); - } - - private Statement runHermesMock(Statement base) { - return new Statement() { - @Override - public void evaluate() throws Throwable { - try { - createInitialContext(); - base.evaluate(); - } finally { - destroyInitialContext(); - } - } - }; - } - - private void createInitialContext() { - hermesMock.start(); - hermesMock.resetReceivedRequest(); - } - - private void destroyInitialContext() { - hermesMock.stop(); - } + private final HermesMock hermesMock; + + public HermesMockRule(int port) { + this.hermesMock = new HermesMock.Builder().withPort(port).build(); + } + + public HermesMockRule(HermesMock hermesMock) { + this.hermesMock = hermesMock; + } + + public HermesMockDefine define() { + return hermesMock.define(); + } + + public HermesMockExpect expect() { + return hermesMock.expect(); + } + + public HermesMockQuery query() { + return hermesMock.query(); + } + + public void resetReceivedRequest() { + hermesMock.resetReceivedRequest(); + } + + public void resetReceivedAvroRequests( + String topicName, Schema schema, Class clazz, Predicate predicate) { + hermesMock.resetReceivedAvroRequests(topicName, schema, clazz, predicate); + } + + public void resetReceivedJsonRequests( + String topicName, Class clazz, Predicate predicate) { + hermesMock.resetReceivedJsonRequests(topicName, clazz, predicate); + } + + public void resetMappings() { + hermesMock.resetMappings(); + } + + @Override + public Statement apply(Statement base, FrameworkMethod method, Object target) { + return runHermesMock(base); + } + + @Override + public Statement apply(Statement base, Description description) { + return runHermesMock(base); + } + + private Statement runHermesMock(Statement base) { + return new Statement() { + @Override + public void evaluate() throws Throwable { + try { + createInitialContext(); + base.evaluate(); + } finally { + destroyInitialContext(); + } + } + }; + } + + private void createInitialContext() { + hermesMock.start(); + hermesMock.resetReceivedRequest(); + } + + private void destroyInitialContext() { + hermesMock.stop(); + } } diff --git a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/exchange/Request.java b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/exchange/Request.java index 9eea0f3b65..70723a086a 100644 --- a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/exchange/Request.java +++ b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/exchange/Request.java @@ -1,64 +1,64 @@ package pl.allegro.tech.hermes.mock.exchange; +import static java.util.stream.Collectors.toMap; + import com.github.tomakehurst.wiremock.http.RequestMethod; import com.github.tomakehurst.wiremock.verification.LoggedRequest; - import java.util.HashMap; import java.util.Map; -import static java.util.stream.Collectors.toMap; - public class Request { - private final String url; - private final Method method; - private final byte[] body; - private final Map headers; + private final String url; + private final Method method; + private final byte[] body; + private final Map headers; - public Request(LoggedRequest loggedRequest) { - this.url = loggedRequest.getUrl(); - this.method = getRequestMethod(loggedRequest.getMethod()); - this.body = loggedRequest.getBody(); - this.headers = loggedRequest.getAllHeaderKeys().stream() - .collect(toMap(key -> key, loggedRequest::getHeader)); - } + public Request(LoggedRequest loggedRequest) { + this.url = loggedRequest.getUrl(); + this.method = getRequestMethod(loggedRequest.getMethod()); + this.body = loggedRequest.getBody(); + this.headers = + loggedRequest.getAllHeaderKeys().stream() + .collect(toMap(key -> key, loggedRequest::getHeader)); + } - public String getUrl() { - return url; - } + public String getUrl() { + return url; + } - public Method getMethod() { - return method; - } + public Method getMethod() { + return method; + } - public byte[] getBody() { - return body; - } + public byte[] getBody() { + return body; + } - public Map getHeaders() { - return headers; - } + public Map getHeaders() { + return headers; + } - private Method getRequestMethod(RequestMethod req) { - Map map = new HashMap<>(); - map.put(RequestMethod.GET, Method.GET); - map.put(RequestMethod.POST, Method.POST); - map.put(RequestMethod.PUT, Method.PUT); - map.put(RequestMethod.DELETE, Method.DELETE); - map.put(RequestMethod.PATCH, Method.PATCH); - map.put(RequestMethod.OPTIONS, Method.OPTIONS); - map.put(RequestMethod.HEAD, Method.HEAD); - map.put(RequestMethod.TRACE, Method.TRACE); - return map.get(req); - } + private Method getRequestMethod(RequestMethod req) { + Map map = new HashMap<>(); + map.put(RequestMethod.GET, Method.GET); + map.put(RequestMethod.POST, Method.POST); + map.put(RequestMethod.PUT, Method.PUT); + map.put(RequestMethod.DELETE, Method.DELETE); + map.put(RequestMethod.PATCH, Method.PATCH); + map.put(RequestMethod.OPTIONS, Method.OPTIONS); + map.put(RequestMethod.HEAD, Method.HEAD); + map.put(RequestMethod.TRACE, Method.TRACE); + return map.get(req); + } - enum Method { - GET, - POST, - PUT, - DELETE, - PATCH, - OPTIONS, - HEAD, - TRACE - } + enum Method { + GET, + POST, + PUT, + DELETE, + PATCH, + OPTIONS, + HEAD, + TRACE + } } diff --git a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/exchange/Response.java b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/exchange/Response.java index 725006af6e..8d13f32cef 100644 --- a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/exchange/Response.java +++ b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/exchange/Response.java @@ -1,49 +1,47 @@ package pl.allegro.tech.hermes.mock.exchange; -import wiremock.org.apache.hc.core5.http.HttpStatus; - import java.time.Duration; +import wiremock.org.apache.hc.core5.http.HttpStatus; public class Response { - private final int statusCode; - private final Duration fixedDelay; + private final int statusCode; + private final Duration fixedDelay; - public Response(int statusCode, Duration fixedDelay) { - this.statusCode = statusCode; - this.fixedDelay = fixedDelay; - } + public Response(int statusCode, Duration fixedDelay) { + this.statusCode = statusCode; + this.fixedDelay = fixedDelay; + } - public int getStatusCode() { - return statusCode; - } + public int getStatusCode() { + return statusCode; + } - public Duration getFixedDelay() { - return fixedDelay; - } + public Duration getFixedDelay() { + return fixedDelay; + } - public static final class Builder { - private int statusCode = HttpStatus.SC_CREATED; - private Duration fixedDelay; + public static final class Builder { + private int statusCode = HttpStatus.SC_CREATED; + private Duration fixedDelay; - private Builder() { - } + private Builder() {} - public static Builder aResponse() { - return new Builder(); - } + public static Builder aResponse() { + return new Builder(); + } - public Builder withStatusCode(int statusCode) { - this.statusCode = statusCode; - return this; - } + public Builder withStatusCode(int statusCode) { + this.statusCode = statusCode; + return this; + } - public Builder withFixedDelay(Duration fixedDelay) { - this.fixedDelay = fixedDelay; - return this; - } + public Builder withFixedDelay(Duration fixedDelay) { + this.fixedDelay = fixedDelay; + return this; + } - public Response build() { - return new Response(statusCode, fixedDelay); - } + public Response build() { + return new Response(statusCode, fixedDelay); } + } } diff --git a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/matching/AvroContentMatcher.java b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/matching/AvroContentMatcher.java index 88d2b5acc6..7024ddfd48 100644 --- a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/matching/AvroContentMatcher.java +++ b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/matching/AvroContentMatcher.java @@ -3,33 +3,33 @@ import com.github.tomakehurst.wiremock.http.Request; import com.github.tomakehurst.wiremock.matching.MatchResult; import com.github.tomakehurst.wiremock.matching.ValueMatcher; +import java.util.function.Predicate; import org.apache.avro.Schema; import pl.allegro.tech.hermes.mock.HermesMockException; import pl.allegro.tech.hermes.mock.HermesMockHelper; -import java.util.function.Predicate; - class AvroContentMatcher implements ValueMatcher { - private final Predicate predicate; - private final Schema schema; - private final Class clazz; - private final HermesMockHelper hermesMockHelper; + private final Predicate predicate; + private final Schema schema; + private final Class clazz; + private final HermesMockHelper hermesMockHelper; - AvroContentMatcher(HermesMockHelper hermesMockHelper, Predicate predicate, Schema schema, Class clazz) { - this.hermesMockHelper = hermesMockHelper; - this.predicate = predicate; - this.schema = schema; - this.clazz = clazz; - } + AvroContentMatcher( + HermesMockHelper hermesMockHelper, Predicate predicate, Schema schema, Class clazz) { + this.hermesMockHelper = hermesMockHelper; + this.predicate = predicate; + this.schema = schema; + this.clazz = clazz; + } - @Override - public MatchResult match(Request actual) { - try { - T body = this.hermesMockHelper.deserializeAvro(actual.getBody(), schema, clazz); - return MatchResult.of(predicate.test(body)); - } catch (HermesMockException exception) { - return MatchResult.noMatch(); - } + @Override + public MatchResult match(Request actual) { + try { + T body = this.hermesMockHelper.deserializeAvro(actual.getBody(), schema, clazz); + return MatchResult.of(predicate.test(body)); + } catch (HermesMockException exception) { + return MatchResult.noMatch(); } -} \ No newline at end of file + } +} diff --git a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/matching/ContentMatchers.java b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/matching/ContentMatchers.java index 9999f49dac..afb4b04c37 100644 --- a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/matching/ContentMatchers.java +++ b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/matching/ContentMatchers.java @@ -1,23 +1,18 @@ package pl.allegro.tech.hermes.mock.matching; +import java.util.function.Predicate; import org.apache.avro.Schema; import pl.allegro.tech.hermes.mock.HermesMockHelper; -import java.util.function.Predicate; - public class ContentMatchers { - public static AvroContentMatcher matchAvro(HermesMockHelper hermesMockHelper, - Predicate predicate, - Schema schema, - Class clazz) { - return new AvroContentMatcher(hermesMockHelper, predicate, schema, clazz); - } - - public static JsonContentMatcher matchJson(HermesMockHelper hermesMockHelper, - Predicate predicate, - Class clazz) { - return new JsonContentMatcher(hermesMockHelper, predicate, clazz); - } + public static AvroContentMatcher matchAvro( + HermesMockHelper hermesMockHelper, Predicate predicate, Schema schema, Class clazz) { + return new AvroContentMatcher(hermesMockHelper, predicate, schema, clazz); + } + public static JsonContentMatcher matchJson( + HermesMockHelper hermesMockHelper, Predicate predicate, Class clazz) { + return new JsonContentMatcher(hermesMockHelper, predicate, clazz); + } } diff --git a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/matching/JsonContentMatcher.java b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/matching/JsonContentMatcher.java index c77b953e76..2deb80111d 100644 --- a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/matching/JsonContentMatcher.java +++ b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/matching/JsonContentMatcher.java @@ -3,26 +3,25 @@ import com.github.tomakehurst.wiremock.http.Request; import com.github.tomakehurst.wiremock.matching.MatchResult; import com.github.tomakehurst.wiremock.matching.ValueMatcher; -import pl.allegro.tech.hermes.mock.HermesMockHelper; - import java.util.function.Predicate; +import pl.allegro.tech.hermes.mock.HermesMockHelper; class JsonContentMatcher implements ValueMatcher { - private final Predicate predicate; - private final Class clazz; - private final HermesMockHelper hermesMockHelper; + private final Predicate predicate; + private final Class clazz; + private final HermesMockHelper hermesMockHelper; - JsonContentMatcher(HermesMockHelper hermesMockHelper, Predicate predicate, Class clazz) { - this.hermesMockHelper = hermesMockHelper; - this.predicate = predicate; - this.clazz = clazz; - } + JsonContentMatcher(HermesMockHelper hermesMockHelper, Predicate predicate, Class clazz) { + this.hermesMockHelper = hermesMockHelper; + this.predicate = predicate; + this.clazz = clazz; + } - @Override - public MatchResult match(Request actual) { - T body = this.hermesMockHelper.deserializeJson(actual.getBody(), clazz); + @Override + public MatchResult match(Request actual) { + T body = this.hermesMockHelper.deserializeJson(actual.getBody(), clazz); - return MatchResult.of(predicate.test(body)); - } + return MatchResult.of(predicate.test(body)); + } } diff --git a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/matching/StartsWithPattern.java b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/matching/StartsWithPattern.java index 2783d91c69..5c8af93bef 100644 --- a/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/matching/StartsWithPattern.java +++ b/hermes-mock/src/main/java/pl/allegro/tech/hermes/mock/matching/StartsWithPattern.java @@ -6,16 +6,16 @@ public class StartsWithPattern extends StringValuePattern { - public StartsWithPattern(@JsonProperty("startsWith") String expectedValue) { - super(expectedValue); - } + public StartsWithPattern(@JsonProperty("startsWith") String expectedValue) { + super(expectedValue); + } - public String getStartsWith() { - return expectedValue; - } + public String getStartsWith() { + return expectedValue; + } - @Override - public MatchResult match(String value) { - return MatchResult.of(value != null && value.startsWith(expectedValue)); - } + @Override + public MatchResult match(String value) { + return MatchResult.of(value != null && value.startsWith(expectedValue)); + } } diff --git a/hermes-mock/src/test/java/pl/allegro/tech/hermes/mock/HermesMockExtensionTest.java b/hermes-mock/src/test/java/pl/allegro/tech/hermes/mock/HermesMockExtensionTest.java index a1d7e044f7..ef1ff22f3f 100644 --- a/hermes-mock/src/test/java/pl/allegro/tech/hermes/mock/HermesMockExtensionTest.java +++ b/hermes-mock/src/test/java/pl/allegro/tech/hermes/mock/HermesMockExtensionTest.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.mock; +import static org.assertj.core.api.Assertions.assertThat; + import groovy.json.JsonOutput; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -8,51 +10,48 @@ import pl.allegro.tech.hermes.test.helper.message.TestMessage; import pl.allegro.tech.hermes.test.helper.util.Ports; -import static org.assertj.core.api.Assertions.assertThat; - class HermesMockExtensionTest { - private static final int port = Ports.nextAvailable(); + private static final int port = Ports.nextAvailable(); - @RegisterExtension - static final HermesMockExtension hermes = new HermesMockExtension(port); + @RegisterExtension static final HermesMockExtension hermes = new HermesMockExtension(port); - private FrontendTestClient publisher; + private FrontendTestClient publisher; - @BeforeEach - void setup() { - publisher = new FrontendTestClient(port); - } + @BeforeEach + void setup() { + publisher = new FrontendTestClient(port); + } - @Test - void shouldPublishMessage() { - // given - String topic = "first-sample-topic"; - hermes.define().jsonTopic(topic); + @Test + void shouldPublishMessage() { + // given + String topic = "first-sample-topic"; + hermes.define().jsonTopic(topic); - // when - publishMessage(topic); + // when + publishMessage(topic); - // then - hermes.expect().singleMessageOnTopic(topic); - } + // then + hermes.expect().singleMessageOnTopic(topic); + } - @Test - void shouldInjectHermesMock(HermesMock hermesMock) { - // given - String topic = "second-sample-topic"; - hermes.define().jsonTopic(topic); + @Test + void shouldInjectHermesMock(HermesMock hermesMock) { + // given + String topic = "second-sample-topic"; + hermes.define().jsonTopic(topic); - // when - publishMessage(topic); + // when + publishMessage(topic); - // then - assertThat(hermesMock.query().allRequests()).hasSize(1); - assertThat(hermesMock.query().allRequestsOnTopic(topic)).hasSize(1); - } + // then + assertThat(hermesMock.query().allRequests()).hasSize(1); + assertThat(hermesMock.query().allRequestsOnTopic(topic)).hasSize(1); + } - private void publishMessage(String topic) { - String body = JsonOutput.toJson(TestMessage.random()); - publisher.publish(topic, body); - } + private void publishMessage(String topic) { + String body = JsonOutput.toJson(TestMessage.random()); + publisher.publish(topic, body); + } } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/BadSchemaRequestException.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/BadSchemaRequestException.java index c95e331b6c..ce3d29fda1 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/BadSchemaRequestException.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/BadSchemaRequestException.java @@ -5,17 +5,19 @@ public class BadSchemaRequestException extends SchemaException { - public BadSchemaRequestException(String subject, Response response) { - this(subject, response.getStatus(), response.readEntity(String.class)); - } + public BadSchemaRequestException(String subject, Response response) { + this(subject, response.getStatus(), response.readEntity(String.class)); + } - public BadSchemaRequestException(String subject, int statusCode, String responseBody) { - super(String.format("Bad schema request for subject %s, server response: %d %s", - subject, statusCode, responseBody)); - } + public BadSchemaRequestException(String subject, int statusCode, String responseBody) { + super( + String.format( + "Bad schema request for subject %s, server response: %d %s", + subject, statusCode, responseBody)); + } - @Override - public ErrorCode getCode() { - return ErrorCode.SCHEMA_BAD_REQUEST; - } + @Override + public ErrorCode getCode() { + return ErrorCode.SCHEMA_BAD_REQUEST; + } } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CachedCompiledSchemaRepository.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CachedCompiledSchemaRepository.java index ad134628ce..9491103b75 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CachedCompiledSchemaRepository.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CachedCompiledSchemaRepository.java @@ -3,156 +3,158 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import pl.allegro.tech.hermes.api.Topic; - import java.time.Duration; import java.util.Objects; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import pl.allegro.tech.hermes.api.Topic; public class CachedCompiledSchemaRepository implements CompiledSchemaRepository { - private final LoadingCache> topicVersionCache; - private final LoadingCache> topicIdCache; - private final CompiledSchemaRepository compiledSchemaRepository; + private final LoadingCache> topicVersionCache; + private final LoadingCache> topicIdCache; + private final CompiledSchemaRepository compiledSchemaRepository; - public CachedCompiledSchemaRepository(CompiledSchemaRepository delegate, long maximumCacheSize, Duration expireAfterAccess) { - this.topicVersionCache = CacheBuilder - .newBuilder() - .maximumSize(maximumCacheSize) - .expireAfterAccess(expireAfterAccess.toMinutes(), TimeUnit.MINUTES) - .build(new CompiledSchemaByVersionLoader<>(delegate)); + public CachedCompiledSchemaRepository( + CompiledSchemaRepository delegate, long maximumCacheSize, Duration expireAfterAccess) { + this.topicVersionCache = + CacheBuilder.newBuilder() + .maximumSize(maximumCacheSize) + .expireAfterAccess(expireAfterAccess.toMinutes(), TimeUnit.MINUTES) + .build(new CompiledSchemaByVersionLoader<>(delegate)); - this.topicIdCache = CacheBuilder - .newBuilder() + this.topicIdCache = + CacheBuilder.newBuilder() .maximumSize(maximumCacheSize) .expireAfterAccess(expireAfterAccess.toMinutes(), TimeUnit.MINUTES) .build(new CompiledSchemaByIdLoader<>(delegate)); - this.compiledSchemaRepository = delegate; + this.compiledSchemaRepository = delegate; + } + + @Override + public CompiledSchema getSchema(Topic topic, SchemaVersion version, boolean online) { + try { + if (online) { + CompiledSchema compiledSchema = compiledSchemaRepository.getSchema(topic, version); + topicVersionCache.put(new TopicAndSchemaVersion(topic, version), compiledSchema); + topicIdCache.put(new TopicAndSchemaId(topic, compiledSchema.getId()), compiledSchema); + return compiledSchema; + } + return topicVersionCache.get(new TopicAndSchemaVersion(topic, version)); + } catch (Exception e) { + throw new CouldNotLoadSchemaException(e); + } + } + + @Override + public CompiledSchema getSchema(Topic topic, SchemaId id) { + try { + return topicIdCache.get(new TopicAndSchemaId(topic, id)); + } catch (Exception e) { + throw new CouldNotLoadSchemaException(e); } + } - @Override - public CompiledSchema getSchema(Topic topic, SchemaVersion version, boolean online) { - try { - if (online) { - CompiledSchema compiledSchema = compiledSchemaRepository.getSchema(topic, version); - topicVersionCache.put(new TopicAndSchemaVersion(topic, version), compiledSchema); - topicIdCache.put(new TopicAndSchemaId(topic, compiledSchema.getId()), compiledSchema); - return compiledSchema; - } - return topicVersionCache.get(new TopicAndSchemaVersion(topic, version)); - } catch (Exception e) { - throw new CouldNotLoadSchemaException(e); - } + public void removeFromCache(Topic topic) { + Set topicWithSchemas = + topicVersionCache.asMap().keySet().stream() + .filter(topicWithSchema -> topicWithSchema.topic.equals(topic)) + .collect(Collectors.toSet()); + + Set topicAndSchemaIds = + topicIdCache.asMap().keySet().stream() + .filter(topicAndSchemaId -> topicAndSchemaId.topic.equals(topic)) + .collect(Collectors.toSet()); + + topicVersionCache.invalidateAll(topicWithSchemas); + topicIdCache.invalidateAll(topicAndSchemaIds); + } + + private static class CompiledSchemaByVersionLoader + extends CacheLoader> { + + private final CompiledSchemaRepository delegate; + + public CompiledSchemaByVersionLoader(CompiledSchemaRepository delegate) { + this.delegate = delegate; } @Override - public CompiledSchema getSchema(Topic topic, SchemaId id) { - try { - return topicIdCache.get(new TopicAndSchemaId(topic, id)); - } catch (Exception e) { - throw new CouldNotLoadSchemaException(e); - } + public CompiledSchema load(TopicAndSchemaVersion key) { + return delegate.getSchema(key.topic, key.schemaVersion); } + } - public void removeFromCache(Topic topic) { - Set topicWithSchemas = topicVersionCache.asMap().keySet().stream() - .filter(topicWithSchema -> topicWithSchema.topic.equals(topic)) - .collect(Collectors.toSet()); + private static class CompiledSchemaByIdLoader + extends CacheLoader> { - Set topicAndSchemaIds = topicIdCache.asMap().keySet().stream() - .filter(topicAndSchemaId -> topicAndSchemaId.topic.equals(topic)) - .collect(Collectors.toSet()); + private final CompiledSchemaRepository delegate; - topicVersionCache.invalidateAll(topicWithSchemas); - topicIdCache.invalidateAll(topicAndSchemaIds); + public CompiledSchemaByIdLoader(CompiledSchemaRepository delegate) { + this.delegate = delegate; } - private static class CompiledSchemaByVersionLoader extends CacheLoader> { + @Override + public CompiledSchema load(TopicAndSchemaId key) { + return delegate.getSchema(key.topic, key.schemaId); + } + } - private final CompiledSchemaRepository delegate; + private static class TopicAndSchemaVersion { - public CompiledSchemaByVersionLoader(CompiledSchemaRepository delegate) { - this.delegate = delegate; - } + private final Topic topic; + private final SchemaVersion schemaVersion; - @Override - public CompiledSchema load(TopicAndSchemaVersion key) { - return delegate.getSchema(key.topic, key.schemaVersion); - } + TopicAndSchemaVersion(Topic topic, SchemaVersion schemaVersion) { + this.topic = topic; + this.schemaVersion = schemaVersion; } - private static class CompiledSchemaByIdLoader extends CacheLoader> { + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TopicAndSchemaVersion that = (TopicAndSchemaVersion) o; + return Objects.equals(schemaVersion, that.schemaVersion) && Objects.equals(topic, that.topic); + } + + @Override + public int hashCode() { + return Objects.hash(topic, schemaVersion); + } + } - private final CompiledSchemaRepository delegate; + private static class TopicAndSchemaId { - public CompiledSchemaByIdLoader(CompiledSchemaRepository delegate) { - this.delegate = delegate; - } + private final Topic topic; + private final SchemaId schemaId; - @Override - public CompiledSchema load(TopicAndSchemaId key) { - return delegate.getSchema(key.topic, key.schemaId); - } + TopicAndSchemaId(Topic topic, SchemaId schemaId) { + this.topic = topic; + this.schemaId = schemaId; } - private static class TopicAndSchemaVersion { - - private final Topic topic; - private final SchemaVersion schemaVersion; - - TopicAndSchemaVersion(Topic topic, SchemaVersion schemaVersion) { - this.topic = topic; - this.schemaVersion = schemaVersion; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - TopicAndSchemaVersion that = (TopicAndSchemaVersion) o; - return Objects.equals(schemaVersion, that.schemaVersion) - && Objects.equals(topic, that.topic); - } - - @Override - public int hashCode() { - return Objects.hash(topic, schemaVersion); - } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TopicAndSchemaId that = (TopicAndSchemaId) o; + return Objects.equals(schemaId, that.schemaId) && Objects.equals(topic, that.topic); } - private static class TopicAndSchemaId { - - private final Topic topic; - private final SchemaId schemaId; - - TopicAndSchemaId(Topic topic, SchemaId schemaId) { - this.topic = topic; - this.schemaId = schemaId; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - TopicAndSchemaId that = (TopicAndSchemaId) o; - return Objects.equals(schemaId, that.schemaId) - && Objects.equals(topic, that.topic); - } - - @Override - public int hashCode() { - return Objects.hash(topic, schemaId); - } + @Override + public int hashCode() { + return Objects.hash(topic, schemaId); } + } } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CachedSchemaVersionsRepository.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CachedSchemaVersionsRepository.java index 444698a915..7c3b0ed8c8 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CachedSchemaVersionsRepository.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CachedSchemaVersionsRepository.java @@ -6,105 +6,125 @@ import com.google.common.cache.LoadingCache; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFutureTask; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import pl.allegro.tech.hermes.api.Topic; - import java.time.Duration; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import pl.allegro.tech.hermes.api.Topic; public class CachedSchemaVersionsRepository implements SchemaVersionsRepository { - private static final Logger logger = LoggerFactory.getLogger(CachedSchemaVersionsRepository.class); + private static final Logger logger = + LoggerFactory.getLogger(CachedSchemaVersionsRepository.class); + + private final RawSchemaClient rawSchemaClient; + private final ExecutorService versionsReloader; + private final LoadingCache> versionsCache; + + public CachedSchemaVersionsRepository( + RawSchemaClient rawSchemaClient, + ExecutorService versionsReloader, + Duration refreshAfterWrite, + Duration expireAfterWrite) { + this( + rawSchemaClient, + versionsReloader, + refreshAfterWrite, + expireAfterWrite, + Ticker.systemTicker()); + } + + CachedSchemaVersionsRepository( + RawSchemaClient rawSchemaClient, + ExecutorService versionsReloader, + Duration refreshAfterWrite, + Duration expireAfterWrite, + Ticker ticker) { + this.rawSchemaClient = rawSchemaClient; + this.versionsReloader = versionsReloader; + this.versionsCache = + CacheBuilder.newBuilder() + .ticker(ticker) + .refreshAfterWrite(refreshAfterWrite.toMinutes(), TimeUnit.MINUTES) + .expireAfterWrite(expireAfterWrite.toMinutes(), TimeUnit.MINUTES) + .build(new SchemaVersionsLoader(rawSchemaClient, versionsReloader)); + } + + @Override + public SchemaVersionsResult versions(Topic topic, boolean online) { + try { + if (online) { + List versions = rawSchemaClient.getVersions(topic.getName()); + versionsCache.put(topic, versions); + return SchemaVersionsResult.succeeded(versions); + } else { + List versions = versionsCache.get(topic); + return SchemaVersionsResult.succeeded(versions); + } + } catch (Exception e) { + logger.error("Error while loading schema versions for topic {}", topic.getQualifiedName(), e); + return SchemaVersionsResult.failed(); + } + } + + @Override + public void close() { + if (!versionsReloader.isShutdown()) { + logger.info("Shutdown of schema-source-reloader executor"); + versionsReloader.shutdownNow(); + } + } + + public void removeFromCache(Topic topic) { + versionsCache.invalidate(topic); + } + + private static class SchemaVersionsLoader extends CacheLoader> { private final RawSchemaClient rawSchemaClient; private final ExecutorService versionsReloader; - private final LoadingCache> versionsCache; - public CachedSchemaVersionsRepository(RawSchemaClient rawSchemaClient, ExecutorService versionsReloader, - Duration refreshAfterWrite, Duration expireAfterWrite) { - this(rawSchemaClient, versionsReloader, refreshAfterWrite, expireAfterWrite, Ticker.systemTicker()); - } - - CachedSchemaVersionsRepository(RawSchemaClient rawSchemaClient, ExecutorService versionsReloader, - Duration refreshAfterWrite, Duration expireAfterWrite, Ticker ticker) { - this.rawSchemaClient = rawSchemaClient; - this.versionsReloader = versionsReloader; - this.versionsCache = CacheBuilder - .newBuilder() - .ticker(ticker) - .refreshAfterWrite(refreshAfterWrite.toMinutes(), TimeUnit.MINUTES) - .expireAfterWrite(expireAfterWrite.toMinutes(), TimeUnit.MINUTES) - .build(new SchemaVersionsLoader(rawSchemaClient, versionsReloader)); + public SchemaVersionsLoader(RawSchemaClient rawSchemaClient, ExecutorService versionsReloader) { + this.rawSchemaClient = rawSchemaClient; + this.versionsReloader = versionsReloader; } @Override - public SchemaVersionsResult versions(Topic topic, boolean online) { - try { - if (online) { - List versions = rawSchemaClient.getVersions(topic.getName()); - versionsCache.put(topic, versions); - return SchemaVersionsResult.succeeded(versions); - } else { - List versions = versionsCache.get(topic); - return SchemaVersionsResult.succeeded(versions); - } - } catch (Exception e) { - logger.error("Error while loading schema versions for topic {}", topic.getQualifiedName(), e); - return SchemaVersionsResult.failed(); - } + public List load(Topic topic) throws Exception { + logger.debug("Loading schema versions for topic {}", topic.getQualifiedName()); + return rawSchemaClient.getVersions(topic.getName()); } @Override - public void close() { - if (!versionsReloader.isShutdown()) { - logger.info("Shutdown of schema-source-reloader executor"); - versionsReloader.shutdownNow(); - } - } - - public void removeFromCache(Topic topic) { - versionsCache.invalidate(topic); - } - - private static class SchemaVersionsLoader extends CacheLoader> { - - private final RawSchemaClient rawSchemaClient; - private final ExecutorService versionsReloader; - - public SchemaVersionsLoader(RawSchemaClient rawSchemaClient, ExecutorService versionsReloader) { - this.rawSchemaClient = rawSchemaClient; - this.versionsReloader = versionsReloader; - } - - @Override - public List load(Topic topic) throws Exception { - logger.debug("Loading schema versions for topic {}", topic.getQualifiedName()); - return rawSchemaClient.getVersions(topic.getName()); - } - - @Override - public ListenableFuture> reload(Topic topic, List oldVersions) { - ListenableFutureTask> task = ListenableFutureTask.create(() -> { + public ListenableFuture> reload( + Topic topic, List oldVersions) { + ListenableFutureTask> task = + ListenableFutureTask.create( + () -> { logger.debug("Reloading schema versions for topic {}", topic.getQualifiedName()); try { - return checkSchemaVersionsAreAvailable(topic, rawSchemaClient.getVersions(topic.getName())); + return checkSchemaVersionsAreAvailable( + topic, rawSchemaClient.getVersions(topic.getName())); } catch (Exception e) { - logger.error("Could not reload schema versions for topic {}, will use stale data", topic.getQualifiedName(), e); - return oldVersions; + logger.error( + "Could not reload schema versions for topic {}, will use stale data", + topic.getQualifiedName(), + e); + return oldVersions; } - }); - versionsReloader.execute(task); - return task; - } + }); + versionsReloader.execute(task); + return task; + } - private List checkSchemaVersionsAreAvailable(Topic topic, List versions) { - if (versions.isEmpty()) { - throw new NoSchemaVersionsFoundException(topic); - } - return versions; - } + private List checkSchemaVersionsAreAvailable( + Topic topic, List versions) { + if (versions.isEmpty()) { + throw new NoSchemaVersionsFoundException(topic); + } + return versions; } + } } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CompiledSchema.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CompiledSchema.java index ffe38436d4..7e658e25fe 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CompiledSchema.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CompiledSchema.java @@ -1,72 +1,71 @@ package pl.allegro.tech.hermes.schema; import com.google.common.base.MoreObjects; -import pl.allegro.tech.hermes.api.RawSchemaWithMetadata; - import java.util.Objects; +import pl.allegro.tech.hermes.api.RawSchemaWithMetadata; public class CompiledSchema { - private final T schema; - private final SchemaId id; - private final SchemaVersion version; + private final T schema; + private final SchemaId id; + private final SchemaVersion version; - public CompiledSchema(T schema, SchemaId id, SchemaVersion version) { - this.schema = schema; - this.id = id; - this.version = version; - } + public CompiledSchema(T schema, SchemaId id, SchemaVersion version) { + this.schema = schema; + this.id = id; + this.version = version; + } - public static CompiledSchema of(T schema, int id, int version) { - return new CompiledSchema<>(schema, SchemaId.valueOf(id), SchemaVersion.valueOf(version)); - } - - public static CompiledSchema of(SchemaCompiler schemaCompiler, RawSchemaWithMetadata rawSchemaWithMetadata) { - return CompiledSchema.of( - schemaCompiler.compile(rawSchemaWithMetadata.getSchema()), - rawSchemaWithMetadata.getId(), - rawSchemaWithMetadata.getVersion() - ); - } + public static CompiledSchema of(T schema, int id, int version) { + return new CompiledSchema<>(schema, SchemaId.valueOf(id), SchemaVersion.valueOf(version)); + } - public T getSchema() { - return schema; - } + public static CompiledSchema of( + SchemaCompiler schemaCompiler, RawSchemaWithMetadata rawSchemaWithMetadata) { + return CompiledSchema.of( + schemaCompiler.compile(rawSchemaWithMetadata.getSchema()), + rawSchemaWithMetadata.getId(), + rawSchemaWithMetadata.getVersion()); + } - public SchemaVersion getVersion() { - return version; - } + public T getSchema() { + return schema; + } - public SchemaId getId() { - return id; - } + public SchemaVersion getVersion() { + return version; + } - @Override - public int hashCode() { - return Objects.hash(schema, id, version); - } + public SchemaId getId() { + return id; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } + @Override + public int hashCode() { + return Objects.hash(schema, id, version); + } - CompiledSchema that = (CompiledSchema) o; - return Objects.equals(id, that.id) - && Objects.equals(version, that.version) - && Objects.equals(schema, that.schema); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("schema", schema) - .add("id", id) - .add("version", version) - .toString(); + if (o == null || getClass() != o.getClass()) { + return false; } + + CompiledSchema that = (CompiledSchema) o; + return Objects.equals(id, that.id) + && Objects.equals(version, that.version) + && Objects.equals(schema, that.schema); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("schema", schema) + .add("id", id) + .add("version", version) + .toString(); + } } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CompiledSchemaRepository.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CompiledSchemaRepository.java index 04fd962e87..ce3ea35f20 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CompiledSchemaRepository.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CompiledSchemaRepository.java @@ -4,11 +4,11 @@ public interface CompiledSchemaRepository { - default CompiledSchema getSchema(Topic topic, SchemaVersion version) { - return getSchema(topic, version, false); - } + default CompiledSchema getSchema(Topic topic, SchemaVersion version) { + return getSchema(topic, version, false); + } - CompiledSchema getSchema(Topic topic, SchemaVersion version, boolean online); + CompiledSchema getSchema(Topic topic, SchemaVersion version, boolean online); - CompiledSchema getSchema(Topic topic, SchemaId id); + CompiledSchema getSchema(Topic topic, SchemaId id); } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CouldNotLoadSchemaException.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CouldNotLoadSchemaException.java index 644ad1bc4f..7978919713 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CouldNotLoadSchemaException.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/CouldNotLoadSchemaException.java @@ -5,16 +5,20 @@ public class CouldNotLoadSchemaException extends SchemaException { - CouldNotLoadSchemaException(Throwable cause) { - super(cause); - } + CouldNotLoadSchemaException(Throwable cause) { + super(cause); + } - CouldNotLoadSchemaException(Topic topic, SchemaVersion version, Throwable cause) { - super(String.format("Schema for topic %s at version %d could not be loaded", topic.getQualifiedName(), version.value()), cause); - } + CouldNotLoadSchemaException(Topic topic, SchemaVersion version, Throwable cause) { + super( + String.format( + "Schema for topic %s at version %d could not be loaded", + topic.getQualifiedName(), version.value()), + cause); + } - @Override - public ErrorCode getCode() { - return ErrorCode.SCHEMA_COULD_NOT_BE_LOADED; - } + @Override + public ErrorCode getCode() { + return ErrorCode.SCHEMA_COULD_NOT_BE_LOADED; + } } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/DirectCompiledSchemaRepository.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/DirectCompiledSchemaRepository.java index abd9b0833d..a6f6ccf599 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/DirectCompiledSchemaRepository.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/DirectCompiledSchemaRepository.java @@ -4,26 +4,28 @@ public class DirectCompiledSchemaRepository implements CompiledSchemaRepository { - private final RawSchemaClient rawSchemaClient; - private final SchemaCompiler schemaCompiler; + private final RawSchemaClient rawSchemaClient; + private final SchemaCompiler schemaCompiler; - public DirectCompiledSchemaRepository(RawSchemaClient rawSchemaClient, - SchemaCompiler schemaCompiler) { - this.rawSchemaClient = rawSchemaClient; - this.schemaCompiler = schemaCompiler; - } + public DirectCompiledSchemaRepository( + RawSchemaClient rawSchemaClient, SchemaCompiler schemaCompiler) { + this.rawSchemaClient = rawSchemaClient; + this.schemaCompiler = schemaCompiler; + } - @Override - public CompiledSchema getSchema(Topic topic, SchemaVersion version, boolean online) { - return rawSchemaClient.getRawSchemaWithMetadata(topic.getName(), version) - .map(rawSchemaWithMetadata -> CompiledSchema.of(schemaCompiler, rawSchemaWithMetadata)) - .orElseThrow(() -> new SchemaNotFoundException(topic, version)); - } + @Override + public CompiledSchema getSchema(Topic topic, SchemaVersion version, boolean online) { + return rawSchemaClient + .getRawSchemaWithMetadata(topic.getName(), version) + .map(rawSchemaWithMetadata -> CompiledSchema.of(schemaCompiler, rawSchemaWithMetadata)) + .orElseThrow(() -> new SchemaNotFoundException(topic, version)); + } - @Override - public CompiledSchema getSchema(Topic topic, SchemaId id) { - return rawSchemaClient.getRawSchemaWithMetadata(topic.getName(), id) - .map(rawSchemaWithMetadata -> CompiledSchema.of(schemaCompiler, rawSchemaWithMetadata)) - .orElseThrow(() -> new SchemaNotFoundException(id)); - } + @Override + public CompiledSchema getSchema(Topic topic, SchemaId id) { + return rawSchemaClient + .getRawSchemaWithMetadata(topic.getName(), id) + .map(rawSchemaWithMetadata -> CompiledSchema.of(schemaCompiler, rawSchemaWithMetadata)) + .orElseThrow(() -> new SchemaNotFoundException(id)); + } } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/DirectSchemaVersionsRepository.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/DirectSchemaVersionsRepository.java index 19cfe21309..3cf3fff76f 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/DirectSchemaVersionsRepository.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/DirectSchemaVersionsRepository.java @@ -1,34 +1,34 @@ package pl.allegro.tech.hermes.schema; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.Topic; -import java.util.List; - public class DirectSchemaVersionsRepository implements SchemaVersionsRepository { - private static final Logger logger = LoggerFactory.getLogger(DirectSchemaVersionsRepository.class); + private static final Logger logger = + LoggerFactory.getLogger(DirectSchemaVersionsRepository.class); - private final RawSchemaClient rawSchemaClient; + private final RawSchemaClient rawSchemaClient; - public DirectSchemaVersionsRepository(RawSchemaClient rawSchemaClient) { - this.rawSchemaClient = rawSchemaClient; - } + public DirectSchemaVersionsRepository(RawSchemaClient rawSchemaClient) { + this.rawSchemaClient = rawSchemaClient; + } - @Override - public SchemaVersionsResult versions(Topic topic, boolean online) { - try { - List versions = rawSchemaClient.getVersions(topic.getName()); - return SchemaVersionsResult.succeeded(versions); - } catch (Exception e) { - logger.error("Error while loading schema versions for topic {}", topic.getQualifiedName(), e); - return SchemaVersionsResult.failed(); - } + @Override + public SchemaVersionsResult versions(Topic topic, boolean online) { + try { + List versions = rawSchemaClient.getVersions(topic.getName()); + return SchemaVersionsResult.succeeded(versions); + } catch (Exception e) { + logger.error("Error while loading schema versions for topic {}", topic.getQualifiedName(), e); + return SchemaVersionsResult.failed(); } + } - @Override - public void close() { - // nothing to close - } + @Override + public void close() { + // nothing to close + } } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/InternalSchemaRepositoryException.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/InternalSchemaRepositoryException.java index c7f2e399d7..30108600eb 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/InternalSchemaRepositoryException.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/InternalSchemaRepositoryException.java @@ -5,22 +5,26 @@ public class InternalSchemaRepositoryException extends SchemaException { - public InternalSchemaRepositoryException(String subject, Response response) { - this(subject, response.getStatus(), response.readEntity(String.class)); - } + public InternalSchemaRepositoryException(String subject, Response response) { + this(subject, response.getStatus(), response.readEntity(String.class)); + } - public InternalSchemaRepositoryException(String subject, int statusCode, String responseBody) { - super(String.format("Internal schema repository error for subject %s request, server response: %d %s", - subject, statusCode, responseBody)); - } + public InternalSchemaRepositoryException(String subject, int statusCode, String responseBody) { + super( + String.format( + "Internal schema repository error for subject %s request, server response: %d %s", + subject, statusCode, responseBody)); + } - public InternalSchemaRepositoryException(SchemaId schemaId, Response response) { - super(String.format("Internal schema repository error for id %s request, server response: %d %s", + public InternalSchemaRepositoryException(SchemaId schemaId, Response response) { + super( + String.format( + "Internal schema repository error for id %s request, server response: %d %s", schemaId.value(), response.getStatus(), response.readEntity(String.class))); - } + } - @Override - public ErrorCode getCode() { - return ErrorCode.SCHEMA_REPOSITORY_INTERNAL_ERROR; - } + @Override + public ErrorCode getCode() { + return ErrorCode.SCHEMA_REPOSITORY_INTERNAL_ERROR; + } } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/NoSchemaVersionsFoundException.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/NoSchemaVersionsFoundException.java index cf652ddafd..01e6bffd89 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/NoSchemaVersionsFoundException.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/NoSchemaVersionsFoundException.java @@ -1,22 +1,20 @@ package pl.allegro.tech.hermes.schema; - import pl.allegro.tech.hermes.api.ErrorCode; import pl.allegro.tech.hermes.api.Topic; public class NoSchemaVersionsFoundException extends SchemaException { - NoSchemaVersionsFoundException(String message) { - super(message); - } - - NoSchemaVersionsFoundException(Topic topic) { - this(String.format("No schema version found for topic %s", topic.getQualifiedName())); - } + NoSchemaVersionsFoundException(String message) { + super(message); + } + NoSchemaVersionsFoundException(Topic topic) { + this(String.format("No schema version found for topic %s", topic.getQualifiedName())); + } - @Override - public ErrorCode getCode() { - return ErrorCode.SCHEMA_COULD_NOT_BE_LOADED; - } + @Override + public ErrorCode getCode() { + return ErrorCode.SCHEMA_COULD_NOT_BE_LOADED; + } } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/RawSchemaClient.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/RawSchemaClient.java index 4d012e0c1f..1b5d9da36e 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/RawSchemaClient.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/RawSchemaClient.java @@ -1,25 +1,24 @@ package pl.allegro.tech.hermes.schema; +import java.util.List; +import java.util.Optional; import pl.allegro.tech.hermes.api.RawSchema; import pl.allegro.tech.hermes.api.RawSchemaWithMetadata; import pl.allegro.tech.hermes.api.TopicName; -import java.util.List; -import java.util.Optional; - public interface RawSchemaClient { - Optional getRawSchemaWithMetadata(TopicName topic, SchemaVersion version); + Optional getRawSchemaWithMetadata(TopicName topic, SchemaVersion version); - Optional getRawSchemaWithMetadata(TopicName topic, SchemaId schemaId); + Optional getRawSchemaWithMetadata(TopicName topic, SchemaId schemaId); - Optional getLatestRawSchemaWithMetadata(TopicName topic); + Optional getLatestRawSchemaWithMetadata(TopicName topic); - List getVersions(TopicName topic); + List getVersions(TopicName topic); - void registerSchema(TopicName topic, RawSchema rawSchema); + void registerSchema(TopicName topic, RawSchema rawSchema); - void deleteAllSchemaVersions(TopicName topic); + void deleteAllSchemaVersions(TopicName topic); - void validateSchema(TopicName topic, RawSchema rawSchema); + void validateSchema(TopicName topic, RawSchema rawSchema); } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaCompiler.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaCompiler.java index 18ef082650..0abe72611a 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaCompiler.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaCompiler.java @@ -5,6 +5,5 @@ @FunctionalInterface public interface SchemaCompiler { - T compile(RawSchema rawSchema); - + T compile(RawSchema rawSchema); } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaCompilersFactory.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaCompilersFactory.java index 888ce6d423..5a91b59985 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaCompilersFactory.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaCompilersFactory.java @@ -4,7 +4,7 @@ public interface SchemaCompilersFactory { - static SchemaCompiler avroSchemaCompiler() { - return source -> new Schema.Parser().parse(source.value()); - } + static SchemaCompiler avroSchemaCompiler() { + return source -> new Schema.Parser().parse(source.value()); + } } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaException.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaException.java index 1ef1f117aa..57c5eb687d 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaException.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaException.java @@ -4,17 +4,17 @@ public abstract class SchemaException extends RuntimeException { - SchemaException(String message) { - super(message); - } + SchemaException(String message) { + super(message); + } - SchemaException(Throwable cause) { - super(cause); - } + SchemaException(Throwable cause) { + super(cause); + } - SchemaException(String message, Throwable cause) { - super(message, cause); - } + SchemaException(String message, Throwable cause) { + super(message, cause); + } - public abstract ErrorCode getCode(); -} \ No newline at end of file + public abstract ErrorCode getCode(); +} diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaExistenceEnsurer.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaExistenceEnsurer.java index 23a3500ced..c882876e42 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaExistenceEnsurer.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaExistenceEnsurer.java @@ -1,55 +1,62 @@ package pl.allegro.tech.hermes.schema; -import pl.allegro.tech.hermes.api.Topic; - import static java.lang.String.format; +import pl.allegro.tech.hermes.api.Topic; public class SchemaExistenceEnsurer { - private final SchemaRepository schemaRepository; - - public SchemaExistenceEnsurer(SchemaRepository schemaRepository) { - this.schemaRepository = schemaRepository; + private final SchemaRepository schemaRepository; + + public SchemaExistenceEnsurer(SchemaRepository schemaRepository) { + this.schemaRepository = schemaRepository; + } + + public void ensureSchemaExists(Topic topic, SchemaVersion version) { + pullSchemaIfNeeded(topic, version); + } + + public void ensureSchemaExists(Topic topic, SchemaId id) { + pullSchemaIfNeeded(topic, id); + } + + private void pullSchemaIfNeeded(Topic topic, SchemaVersion version) { + try { + schemaRepository.getAvroSchema(topic, version); + } catch (SchemaException ex) { + pullVersionsOnline(topic); + throw new SchemaNotLoaded( + format( + "Could not find schema version [%s] provided in header for topic [%s]." + + " Trying pulling online...", + version, topic), + ex); } - - public void ensureSchemaExists(Topic topic, SchemaVersion version) { - pullSchemaIfNeeded(topic, version); + } + + private void pullSchemaIfNeeded(Topic topic, SchemaId id) { + try { + schemaRepository.getAvroSchema(topic, id); + } catch (SchemaException ex) { + throw new SchemaNotLoaded( + format( + "Could not find schema id [%s] provided in header for topic [%s]." + + " Trying pulling online...", + id, topic), + ex); } + } - public void ensureSchemaExists(Topic topic, SchemaId id) { - pullSchemaIfNeeded(topic, id); - } + private void pullVersionsOnline(Topic topic) { + schemaRepository.refreshVersions(topic); + } - private void pullSchemaIfNeeded(Topic topic, SchemaVersion version) { - try { - schemaRepository.getAvroSchema(topic, version); - } catch (SchemaException ex) { - pullVersionsOnline(topic); - throw new SchemaNotLoaded(format("Could not find schema version [%s] provided in header for topic [%s]." - + " Trying pulling online...", version, topic), ex); - } + public static class SchemaNotLoaded extends RuntimeException { + SchemaNotLoaded(String msg, Throwable th) { + super(msg, th); } - private void pullSchemaIfNeeded(Topic topic, SchemaId id) { - try { - schemaRepository.getAvroSchema(topic, id); - } catch (SchemaException ex) { - throw new SchemaNotLoaded(format("Could not find schema id [%s] provided in header for topic [%s]." - + " Trying pulling online...", id, topic), ex); - } - } - - private void pullVersionsOnline(Topic topic) { - schemaRepository.refreshVersions(topic); - } - - public static class SchemaNotLoaded extends RuntimeException { - SchemaNotLoaded(String msg, Throwable th) { - super(msg, th); - } - - SchemaNotLoaded(String msg) { - super(msg); - } + SchemaNotLoaded(String msg) { + super(msg); } + } } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaId.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaId.java index b355a0c891..1eeb9fa20e 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaId.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaId.java @@ -4,39 +4,39 @@ public final class SchemaId { - private final int value; + private final int value; - private SchemaId(int value) { - this.value = value; - } - - public static SchemaId valueOf(int id) { - return new SchemaId(id); - } + private SchemaId(int value) { + this.value = value; + } - public int value() { - return value; - } + public static SchemaId valueOf(int id) { + return new SchemaId(id); + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - SchemaId that = (SchemaId) o; - return value == that.value; - } + public int value() { + return value; + } - @Override - public int hashCode() { - return Objects.hash(value); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public String toString() { - return "SchemaId(" + value + ")"; + if (o == null || getClass() != o.getClass()) { + return false; } + SchemaId that = (SchemaId) o; + return value == that.value; + } + + @Override + public int hashCode() { + return Objects.hash(value); + } + + @Override + public String toString() { + return "SchemaId(" + value + ")"; + } } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaNotFoundException.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaNotFoundException.java index b285cd2d21..e273c1e275 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaNotFoundException.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaNotFoundException.java @@ -5,20 +5,24 @@ public class SchemaNotFoundException extends SchemaException { - SchemaNotFoundException(Topic topic, SchemaVersion schemaVersion) { - super("No schema source for topic " + topic.getQualifiedName() + " at version " + schemaVersion.value()); - } + SchemaNotFoundException(Topic topic, SchemaVersion schemaVersion) { + super( + "No schema source for topic " + + topic.getQualifiedName() + + " at version " + + schemaVersion.value()); + } - SchemaNotFoundException(Topic topic) { - super("No schema source for topic " + topic.getQualifiedName()); - } + SchemaNotFoundException(Topic topic) { + super("No schema source for topic " + topic.getQualifiedName()); + } - SchemaNotFoundException(SchemaId id) { - super("No schema source for id " + id.value()); - } + SchemaNotFoundException(SchemaId id) { + super("No schema source for id " + id.value()); + } - @Override - public ErrorCode getCode() { - return ErrorCode.OTHER; - } + @Override + public ErrorCode getCode() { + return ErrorCode.OTHER; + } } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaRepository.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaRepository.java index 03acf7b644..9c06a8b22b 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaRepository.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaRepository.java @@ -3,71 +3,74 @@ import org.apache.avro.Schema; import pl.allegro.tech.hermes.api.Topic; - public class SchemaRepository { - private static final boolean ONLINE = true; + private static final boolean ONLINE = true; - private final SchemaVersionsRepository schemaVersionsRepository; - private final CompiledSchemaRepository compiledAvroSchemaRepository; + private final SchemaVersionsRepository schemaVersionsRepository; + private final CompiledSchemaRepository compiledAvroSchemaRepository; - public SchemaRepository(SchemaVersionsRepository schemaVersionsRepository, - CompiledSchemaRepository compiledAvroSchemaRepository) { - this.schemaVersionsRepository = schemaVersionsRepository; - this.compiledAvroSchemaRepository = compiledAvroSchemaRepository; - } + public SchemaRepository( + SchemaVersionsRepository schemaVersionsRepository, + CompiledSchemaRepository compiledAvroSchemaRepository) { + this.schemaVersionsRepository = schemaVersionsRepository; + this.compiledAvroSchemaRepository = compiledAvroSchemaRepository; + } - public CompiledSchema getLatestAvroSchema(Topic topic) { - SchemaVersion latestVersion = schemaVersionsRepository.latestSchemaVersion(topic) - .orElseThrow(() -> new SchemaNotFoundException(topic)); - if (!schemaVersionsRepository.schemaVersionExists(topic, latestVersion)) { - throw new SchemaNotFoundException(topic, latestVersion); - } - return getCompiledSchemaAtVersion(topic, latestVersion); + public CompiledSchema getLatestAvroSchema(Topic topic) { + SchemaVersion latestVersion = + schemaVersionsRepository + .latestSchemaVersion(topic) + .orElseThrow(() -> new SchemaNotFoundException(topic)); + if (!schemaVersionsRepository.schemaVersionExists(topic, latestVersion)) { + throw new SchemaNotFoundException(topic, latestVersion); } + return getCompiledSchemaAtVersion(topic, latestVersion); + } - public CompiledSchema getAvroSchema(Topic topic, SchemaVersion version) { - SchemaVersionsResult result = schemaVersionsRepository.versions(topic); - if (result.isFailure()) { - throw new SchemaNotFoundException(topic, version); - } - if (!result.versionExists(version)) { - throw new SchemaVersionDoesNotExistException(topic, version); - } - return getCompiledSchemaAtVersion(topic, version); + public CompiledSchema getAvroSchema(Topic topic, SchemaVersion version) { + SchemaVersionsResult result = schemaVersionsRepository.versions(topic); + if (result.isFailure()) { + throw new SchemaNotFoundException(topic, version); } + if (!result.versionExists(version)) { + throw new SchemaVersionDoesNotExistException(topic, version); + } + return getCompiledSchemaAtVersion(topic, version); + } - public CompiledSchema getAvroSchema(Topic topic, SchemaId id) { - CompiledSchema schema = compiledAvroSchemaRepository.getSchema(topic, id); - - if (schema == null) { - throw new SchemaNotFoundException(id); - } + public CompiledSchema getAvroSchema(Topic topic, SchemaId id) { + CompiledSchema schema = compiledAvroSchemaRepository.getSchema(topic, id); - return schema; + if (schema == null) { + throw new SchemaNotFoundException(id); } - /** - * This method should be used only for cache-backed repository implementations - * where we have possibly stale versions-cache and are 100% sure the requested version exists. - * If it does not exist, each method call will try to load the requested version - * from underlying schema repository. - */ - public CompiledSchema getKnownAvroSchemaVersion(Topic topic, SchemaVersion version) { - return getCompiledSchemaAtVersion(topic, version); - } + return schema; + } - public void refreshVersions(Topic topic) { - schemaVersionsRepository.versions(topic, ONLINE); - } + /** + * This method should be used only for cache-backed repository implementations where we have + * possibly stale versions-cache and are 100% sure the requested version exists. If it does not + * exist, each method call will try to load the requested version from underlying schema + * repository. + */ + public CompiledSchema getKnownAvroSchemaVersion(Topic topic, SchemaVersion version) { + return getCompiledSchemaAtVersion(topic, version); + } + + public void refreshVersions(Topic topic) { + schemaVersionsRepository.versions(topic, ONLINE); + } - private CompiledSchema getCompiledSchemaAtVersion(Topic topic, SchemaVersion latestVersion) { - try { - return compiledAvroSchemaRepository.getSchema(topic, latestVersion); - } catch (CouldNotLoadSchemaException e) { - throw e; - } catch (Exception e) { - throw new CouldNotLoadSchemaException(topic, latestVersion, e); - } + private CompiledSchema getCompiledSchemaAtVersion( + Topic topic, SchemaVersion latestVersion) { + try { + return compiledAvroSchemaRepository.getSchema(topic, latestVersion); + } catch (CouldNotLoadSchemaException e) { + throw e; + } catch (Exception e) { + throw new CouldNotLoadSchemaException(topic, latestVersion, e); } + } } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaVersion.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaVersion.java index 80ccb485c2..12b498f00f 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaVersion.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaVersion.java @@ -1,51 +1,52 @@ package pl.allegro.tech.hermes.schema; import com.google.common.base.Joiner; - import java.util.List; import java.util.Objects; import java.util.stream.Collectors; public final class SchemaVersion { - private final int value; - - private SchemaVersion(int value) { - this.value = value; - } - - public static SchemaVersion valueOf(int version) { - return new SchemaVersion(version); - } + private final int value; - public int value() { - return value; - } + private SchemaVersion(int value) { + this.value = value; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - SchemaVersion that = (SchemaVersion) o; - return value == that.value; - } + public static SchemaVersion valueOf(int version) { + return new SchemaVersion(version); + } - @Override - public int hashCode() { - return Objects.hash(value); - } + public int value() { + return value; + } - @Override - public String toString() { - return "SchemaVersion(" + value + ")"; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - public static String toString(List versions) { - return "[" + Joiner.on(',').join(versions.stream().map(SchemaVersion::value).collect(Collectors.toList())) + "]"; + if (o == null || getClass() != o.getClass()) { + return false; } - + SchemaVersion that = (SchemaVersion) o; + return value == that.value; + } + + @Override + public int hashCode() { + return Objects.hash(value); + } + + @Override + public String toString() { + return "SchemaVersion(" + value + ")"; + } + + public static String toString(List versions) { + return "[" + + Joiner.on(',') + .join(versions.stream().map(SchemaVersion::value).collect(Collectors.toList())) + + "]"; + } } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaVersionDoesNotExistException.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaVersionDoesNotExistException.java index c0c3f06c75..8f58174652 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaVersionDoesNotExistException.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaVersionDoesNotExistException.java @@ -1,25 +1,28 @@ package pl.allegro.tech.hermes.schema; +import static java.lang.String.format; + import pl.allegro.tech.hermes.api.ErrorCode; import pl.allegro.tech.hermes.api.Topic; -import static java.lang.String.format; - public class SchemaVersionDoesNotExistException extends SchemaException { - private final SchemaVersion schemaVersion; + private final SchemaVersion schemaVersion; - SchemaVersionDoesNotExistException(Topic topic, SchemaVersion schemaVersion) { - super(format("Schema version %s for topic %s does not exist", schemaVersion.value(), topic.getQualifiedName())); - this.schemaVersion = schemaVersion; - } + SchemaVersionDoesNotExistException(Topic topic, SchemaVersion schemaVersion) { + super( + format( + "Schema version %s for topic %s does not exist", + schemaVersion.value(), topic.getQualifiedName())); + this.schemaVersion = schemaVersion; + } - public SchemaVersion getSchemaVersion() { - return schemaVersion; - } + public SchemaVersion getSchemaVersion() { + return schemaVersion; + } - @Override - public ErrorCode getCode() { - return ErrorCode.OTHER; - } + @Override + public ErrorCode getCode() { + return ErrorCode.OTHER; + } } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaVersionsRepository.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaVersionsRepository.java index d4ca2d98f2..0a27f89858 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaVersionsRepository.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaVersionsRepository.java @@ -1,29 +1,28 @@ package pl.allegro.tech.hermes.schema; -import pl.allegro.tech.hermes.api.Topic; - import java.util.Comparator; import java.util.Optional; +import pl.allegro.tech.hermes.api.Topic; public interface SchemaVersionsRepository { - default boolean schemaVersionExists(Topic topic, SchemaVersion version) { - return versions(topic).get().contains(version); - } + default boolean schemaVersionExists(Topic topic, SchemaVersion version) { + return versions(topic).get().contains(version); + } - default Optional latestSchemaVersion(Topic topic) { - return versions(topic).get().stream().max(Comparator.comparingInt(SchemaVersion::value)); - } + default Optional latestSchemaVersion(Topic topic) { + return versions(topic).get().stream().max(Comparator.comparingInt(SchemaVersion::value)); + } - default Optional onlineLatestSchemaVersion(Topic topic) { - return versions(topic, true).get().stream().max(Comparator.comparingInt(SchemaVersion::value)); - } + default Optional onlineLatestSchemaVersion(Topic topic) { + return versions(topic, true).get().stream().max(Comparator.comparingInt(SchemaVersion::value)); + } - default SchemaVersionsResult versions(Topic topic) { - return versions(topic, false); - } + default SchemaVersionsResult versions(Topic topic) { + return versions(topic, false); + } - SchemaVersionsResult versions(Topic topic, boolean online); + SchemaVersionsResult versions(Topic topic, boolean online); - void close(); + void close(); } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaVersionsResult.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaVersionsResult.java index 20632c328d..4302732e3a 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaVersionsResult.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SchemaVersionsResult.java @@ -1,44 +1,44 @@ package pl.allegro.tech.hermes.schema; -import java.util.List; - import static java.util.Collections.emptyList; +import java.util.List; + public class SchemaVersionsResult { - private final Status status; - private final List versions; - - public static SchemaVersionsResult succeeded(List versions) { - return new SchemaVersionsResult(Status.SUCCESS, versions); - } - - public static SchemaVersionsResult failed() { - return new SchemaVersionsResult(Status.FAILURE, emptyList()); - } - - private SchemaVersionsResult(Status status, List versions) { - this.status = status; - this.versions = versions; - } - - public List get() { - return versions; - } - - public boolean isSuccess() { - return status == Status.SUCCESS; - } - - public boolean isFailure() { - return status == Status.FAILURE; - } - - public boolean versionExists(SchemaVersion version) { - return versions.contains(version); - } - - enum Status { - SUCCESS, - FAILURE - } + private final Status status; + private final List versions; + + public static SchemaVersionsResult succeeded(List versions) { + return new SchemaVersionsResult(Status.SUCCESS, versions); + } + + public static SchemaVersionsResult failed() { + return new SchemaVersionsResult(Status.FAILURE, emptyList()); + } + + private SchemaVersionsResult(Status status, List versions) { + this.status = status; + this.versions = versions; + } + + public List get() { + return versions; + } + + public boolean isSuccess() { + return status == Status.SUCCESS; + } + + public boolean isFailure() { + return status == Status.FAILURE; + } + + public boolean versionExists(SchemaVersion version) { + return versions.contains(version); + } + + enum Status { + SUCCESS, + FAILURE + } } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SubjectNamingStrategy.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SubjectNamingStrategy.java index b1c37fea78..c8261a57cd 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SubjectNamingStrategy.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/SubjectNamingStrategy.java @@ -4,33 +4,33 @@ public interface SubjectNamingStrategy { - class Namespace { - private final String value; - private final String separator; - - public Namespace(String value, String separator) { - this.value = value; - this.separator = separator; - } - - public String getValue() { - return value; - } - - public String apply(String name) { - return value.isEmpty() ? name : value + separator + name; - } - } - - String apply(TopicName topic); + class Namespace { + private final String value; + private final String separator; - SubjectNamingStrategy qualifiedName = TopicName::qualifiedName; + public Namespace(String value, String separator) { + this.value = value; + this.separator = separator; + } - default SubjectNamingStrategy withNamespacePrefixIf(boolean enabled, Namespace namespace) { - return enabled ? topicName -> namespace.apply(this.apply(topicName)) : this; + public String getValue() { + return value; } - default SubjectNamingStrategy withValueSuffixIf(boolean valueSuffixEnabled) { - return valueSuffixEnabled ? topicName -> this.apply(topicName) + "-value" : this; + public String apply(String name) { + return value.isEmpty() ? name : value + separator + name; } + } + + String apply(TopicName topic); + + SubjectNamingStrategy qualifiedName = TopicName::qualifiedName; + + default SubjectNamingStrategy withNamespacePrefixIf(boolean enabled, Namespace namespace) { + return enabled ? topicName -> namespace.apply(this.apply(topicName)) : this; + } + + default SubjectNamingStrategy withValueSuffixIf(boolean valueSuffixEnabled) { + return valueSuffixEnabled ? topicName -> this.apply(topicName) + "-value" : this; + } } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryCompatibilityResponse.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryCompatibilityResponse.java index 4f1a4d7605..471046b77d 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryCompatibilityResponse.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryCompatibilityResponse.java @@ -2,36 +2,35 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.Objects; class SchemaRegistryCompatibilityResponse { - private final boolean compatible; + private final boolean compatible; - @JsonCreator - SchemaRegistryCompatibilityResponse(@JsonProperty("is_compatible") boolean compatible) { - this.compatible = compatible; - } + @JsonCreator + SchemaRegistryCompatibilityResponse(@JsonProperty("is_compatible") boolean compatible) { + this.compatible = compatible; + } - public boolean isCompatible() { - return compatible; - } + public boolean isCompatible() { + return compatible; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - SchemaRegistryCompatibilityResponse that = (SchemaRegistryCompatibilityResponse) o; - return compatible == that.compatible; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(compatible); + if (o == null || getClass() != o.getClass()) { + return false; } + SchemaRegistryCompatibilityResponse that = (SchemaRegistryCompatibilityResponse) o; + return compatible == that.compatible; + } + + @Override + public int hashCode() { + return Objects.hash(compatible); + } } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryRawSchemaClient.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryRawSchemaClient.java index ced558137c..9330243078 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryRawSchemaClient.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryRawSchemaClient.java @@ -1,9 +1,19 @@ package pl.allegro.tech.hermes.schema.confluent; +import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; +import static jakarta.ws.rs.core.Response.Status.Family.SUCCESSFUL; + import com.fasterxml.jackson.databind.ObjectMapper; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.RawSchema; @@ -17,339 +27,392 @@ import pl.allegro.tech.hermes.schema.SubjectNamingStrategy; import pl.allegro.tech.hermes.schema.resolver.SchemaRepositoryInstanceResolver; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; - -import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; -import static jakarta.ws.rs.core.Response.Status.Family.SUCCESSFUL; - /** - * This implementation of RawSchemaClient is compatible with Confluent Schema Registry API - * except for the deleteAllSchemaVersions and validation endpoint which are not fully supported by the Confluent project. + * This implementation of RawSchemaClient is compatible with Confluent Schema Registry API except + * for the deleteAllSchemaVersions and validation endpoint which are not fully supported by the + * Confluent project. */ public class SchemaRegistryRawSchemaClient implements RawSchemaClient { - private static final Logger logger = LoggerFactory.getLogger(SchemaRegistryRawSchemaClient.class); - - private static final String SCHEMA_REPO_CONTENT_TYPE = "application/vnd.schemaregistry.v1+json"; - - private final SchemaRepositoryInstanceResolver schemaRepositoryInstanceResolver; - - private final ObjectMapper objectMapper; - private final boolean validationEndpointEnabled; - private final String deleteSchemaPathSuffix; - private final SubjectNamingStrategy subjectNamingStrategy; - - public SchemaRegistryRawSchemaClient(SchemaRepositoryInstanceResolver schemaRepositoryInstanceResolver, - ObjectMapper objectMapper, - SubjectNamingStrategy subjectNamingStrategy) { - this(schemaRepositoryInstanceResolver, objectMapper, false, "versions", subjectNamingStrategy); - } - - public SchemaRegistryRawSchemaClient(SchemaRepositoryInstanceResolver schemaRepositoryInstanceResolver, - ObjectMapper objectMapper, - boolean validationEndpointEnabled, - String deleteSchemaPathSuffix, - SubjectNamingStrategy subjectNamingStrategy) { - this.schemaRepositoryInstanceResolver = schemaRepositoryInstanceResolver; - this.validationEndpointEnabled = validationEndpointEnabled; - this.deleteSchemaPathSuffix = deleteSchemaPathSuffix; - this.objectMapper = objectMapper; - this.subjectNamingStrategy = subjectNamingStrategy; - } - - @Override - public Optional getRawSchemaWithMetadata(TopicName topic, SchemaVersion schemaVersion) { - String version = Integer.toString(schemaVersion.value()); - String subject = subjectNamingStrategy.apply(topic); - Response response = getRawSchemaWithMetadataResponse(subject, version); - return extractRawSchemaWithMetadata(subject, version, response); - } - - @Override - public Optional getRawSchemaWithMetadata(TopicName topic, SchemaId schemaId) { - String subject = subjectNamingStrategy.apply(topic); - Optional schema = getRawSchema(subject, schemaId); - - return schema - .map(sc -> getRawSchemaWithMetadataResponse(subject, sc)) - .map(response -> extractRawSchemaWithMetadata(subject, schemaId, response)) - .map(Optional::get); - } - - @Override - public Optional getLatestRawSchemaWithMetadata(TopicName topic) { - final String version = "latest"; - String subject = subjectNamingStrategy.apply(topic); - Response response = getRawSchemaWithMetadataResponse(subject, version); - return extractRawSchemaWithMetadata(subject, version, response); - } - - private Response getRawSchemaWithMetadataResponse(String subject, String version) { - return schemaRepositoryInstanceResolver.resolve(subject) - .path("subjects") - .path(subject) - .path("versions") - .path(version) - .request() - .get(); - } - - private Response getRawSchemaWithMetadataResponse(String subject, RawSchema schema) { - return schemaRepositoryInstanceResolver.resolve(subject) - .path("subjects") - .path(subject) - .request() - .accept(MediaType.APPLICATION_JSON_TYPE) - .post(Entity.entity(SchemaRegistryRequestResponse.fromRawSchema(schema), SCHEMA_REPO_CONTENT_TYPE)); + private static final Logger logger = LoggerFactory.getLogger(SchemaRegistryRawSchemaClient.class); + + private static final String SCHEMA_REPO_CONTENT_TYPE = "application/vnd.schemaregistry.v1+json"; + + private final SchemaRepositoryInstanceResolver schemaRepositoryInstanceResolver; + + private final ObjectMapper objectMapper; + private final boolean validationEndpointEnabled; + private final String deleteSchemaPathSuffix; + private final SubjectNamingStrategy subjectNamingStrategy; + + public SchemaRegistryRawSchemaClient( + SchemaRepositoryInstanceResolver schemaRepositoryInstanceResolver, + ObjectMapper objectMapper, + SubjectNamingStrategy subjectNamingStrategy) { + this(schemaRepositoryInstanceResolver, objectMapper, false, "versions", subjectNamingStrategy); + } + + public SchemaRegistryRawSchemaClient( + SchemaRepositoryInstanceResolver schemaRepositoryInstanceResolver, + ObjectMapper objectMapper, + boolean validationEndpointEnabled, + String deleteSchemaPathSuffix, + SubjectNamingStrategy subjectNamingStrategy) { + this.schemaRepositoryInstanceResolver = schemaRepositoryInstanceResolver; + this.validationEndpointEnabled = validationEndpointEnabled; + this.deleteSchemaPathSuffix = deleteSchemaPathSuffix; + this.objectMapper = objectMapper; + this.subjectNamingStrategy = subjectNamingStrategy; + } + + @Override + public Optional getRawSchemaWithMetadata( + TopicName topic, SchemaVersion schemaVersion) { + String version = Integer.toString(schemaVersion.value()); + String subject = subjectNamingStrategy.apply(topic); + Response response = getRawSchemaWithMetadataResponse(subject, version); + return extractRawSchemaWithMetadata(subject, version, response); + } + + @Override + public Optional getRawSchemaWithMetadata( + TopicName topic, SchemaId schemaId) { + String subject = subjectNamingStrategy.apply(topic); + Optional schema = getRawSchema(subject, schemaId); + + return schema + .map(sc -> getRawSchemaWithMetadataResponse(subject, sc)) + .map(response -> extractRawSchemaWithMetadata(subject, schemaId, response)) + .map(Optional::get); + } + + @Override + public Optional getLatestRawSchemaWithMetadata(TopicName topic) { + final String version = "latest"; + String subject = subjectNamingStrategy.apply(topic); + Response response = getRawSchemaWithMetadataResponse(subject, version); + return extractRawSchemaWithMetadata(subject, version, response); + } + + private Response getRawSchemaWithMetadataResponse(String subject, String version) { + return schemaRepositoryInstanceResolver + .resolve(subject) + .path("subjects") + .path(subject) + .path("versions") + .path(version) + .request() + .get(); + } + + private Response getRawSchemaWithMetadataResponse(String subject, RawSchema schema) { + return schemaRepositoryInstanceResolver + .resolve(subject) + .path("subjects") + .path(subject) + .request() + .accept(MediaType.APPLICATION_JSON_TYPE) + .post( + Entity.entity( + SchemaRegistryRequestResponse.fromRawSchema(schema), SCHEMA_REPO_CONTENT_TYPE)); + } + + public Optional getRawSchema(String subject, SchemaId schemaId) { + String idString = Integer.toString(schemaId.value()); + + Response response = + schemaRepositoryInstanceResolver + .resolve(subject) + .path("schemas") + .path("ids") + .path(idString) + .request() + .get(); + + return extractSchema(response, subject, schemaId); + } + + private Optional extractSchema(Response response, String subject, SchemaId schemaId) { + switch (response.getStatusInfo().getFamily()) { + case SUCCESSFUL: + logger.info("Found schema for subject {} and id {}", subject, schemaId.value()); + SchemaRegistryRequestResponse schemaRegistryResponse = + response.readEntity(SchemaRegistryRequestResponse.class); + return Optional.of(RawSchema.valueOf(schemaRegistryResponse.getSchema())); + case CLIENT_ERROR: + logger.error( + "Could not find schema for subject {} and id {}, reason: {}", + subject, + schemaId.value(), + response.getStatus()); + return Optional.empty(); + case SERVER_ERROR: + default: + logger.error( + "Could not find schema for subject {} and id {}, reason: {}", + subject, + schemaId.value(), + response.getStatus()); + throw new InternalSchemaRepositoryException(subject, response); } - - public Optional getRawSchema(String subject, SchemaId schemaId) { - String idString = Integer.toString(schemaId.value()); - - Response response = schemaRepositoryInstanceResolver.resolve(subject) - .path("schemas") - .path("ids") - .path(idString) - .request() - .get(); - - return extractSchema(response, subject, schemaId); - } - - private Optional extractSchema(Response response, String subject, SchemaId schemaId) { - switch (response.getStatusInfo().getFamily()) { - case SUCCESSFUL: - logger.info("Found schema for subject {} and id {}", subject, schemaId.value()); - SchemaRegistryRequestResponse schemaRegistryResponse = response.readEntity(SchemaRegistryRequestResponse.class); - return Optional.of(RawSchema.valueOf(schemaRegistryResponse.getSchema())); - case CLIENT_ERROR: - logger.error("Could not find schema for subject {} and id {}, reason: {}", subject, schemaId.value(), response.getStatus()); - return Optional.empty(); - case SERVER_ERROR: - default: - logger.error("Could not find schema for subject {} and id {}, reason: {}", subject, schemaId.value(), response.getStatus()); - throw new InternalSchemaRepositoryException(subject, response); - } - } - - private Optional extractRawSchemaWithMetadata(String subject, String version, Response response) { - switch (response.getStatusInfo().getFamily()) { - case SUCCESSFUL: - logger.info("Found schema metadata for subject {} at version {}", subject, version); - SchemaRegistryResponse schemaRegistryResponse = response.readEntity(SchemaRegistryResponse.class); - return Optional.of(schemaRegistryResponse.toRawSchemaWithMetadata()); - case CLIENT_ERROR: - logger.error("Could not find schema metadata for subject {} at version {}, reason: {}", - subject, - version, - response.getStatus()); - return Optional.empty(); - case SERVER_ERROR: - default: - logger.error("Could not find schema metadata for subject {} at version {}, reason: {}", - subject, - version, - response.getStatus()); - throw new InternalSchemaRepositoryException(subject, response); - } - } - - private Optional extractRawSchemaWithMetadata(String subject, SchemaId schemaId, Response response) { - Integer id = schemaId.value(); - switch (response.getStatusInfo().getFamily()) { - case SUCCESSFUL: - logger.info("Found schema metadata for subject {} and id {}", subject, id); - SchemaRegistryResponse schemaRegistryResponse = response.readEntity(SchemaRegistryResponse.class); - return Optional.of(schemaRegistryResponse.toRawSchemaWithMetadata()); - case CLIENT_ERROR: - logger.error("Could not find schema metadata for subject {} and id {}, reason: {}", subject, id, response.getStatus()); - return Optional.empty(); - case SERVER_ERROR: - default: - logger.error("Could not find schema metadata for subject {} and id {}, reason: {}", subject, id, response.getStatus()); - throw new InternalSchemaRepositoryException(subject, response); - } - } - - @Override - public List getVersions(TopicName topic) { - String subject = subjectNamingStrategy.apply(topic); - Response response = schemaRepositoryInstanceResolver.resolve(subject) - .path("subjects") - .path(subject) - .path("versions") - .request() - .get(); - return extractSchemaVersions(subject, response); - } - - private List extractSchemaVersions(String subject, Response response) { - switch (response.getStatusInfo().getFamily()) { - case SUCCESSFUL: - return Arrays.stream(response.readEntity(Integer[].class)) - .sorted(Comparator.reverseOrder()) - .map(SchemaVersion::valueOf) - .collect(Collectors.toList()); - case CLIENT_ERROR: - logger.error("Could not find schema versions for subject {}, reason: {} {}", subject, response.getStatus(), - response.readEntity(String.class)); - return Collections.emptyList(); - case SERVER_ERROR: - default: - throw new InternalSchemaRepositoryException(subject, response); - } - } - - @Override - public void registerSchema(TopicName topic, RawSchema rawSchema) { - String subject = subjectNamingStrategy.apply(topic); - Response response = schemaRepositoryInstanceResolver.resolve(subject) - .path("subjects") - .path(subject) - .path("versions") - .request() - .accept(MediaType.APPLICATION_JSON_TYPE) - .post(Entity.entity(SchemaRegistryRequestResponse.fromRawSchema(rawSchema), SCHEMA_REPO_CONTENT_TYPE)); - checkSchemaRegistration(subject, response); + } + + private Optional extractRawSchemaWithMetadata( + String subject, String version, Response response) { + switch (response.getStatusInfo().getFamily()) { + case SUCCESSFUL: + logger.info("Found schema metadata for subject {} at version {}", subject, version); + SchemaRegistryResponse schemaRegistryResponse = + response.readEntity(SchemaRegistryResponse.class); + return Optional.of(schemaRegistryResponse.toRawSchemaWithMetadata()); + case CLIENT_ERROR: + logger.error( + "Could not find schema metadata for subject {} at version {}, reason: {}", + subject, + version, + response.getStatus()); + return Optional.empty(); + case SERVER_ERROR: + default: + logger.error( + "Could not find schema metadata for subject {} at version {}, reason: {}", + subject, + version, + response.getStatus()); + throw new InternalSchemaRepositoryException(subject, response); } - - private void checkSchemaRegistration(String subject, Response response) { - switch (response.getStatusInfo().getFamily()) { - case SUCCESSFUL: - logger.info("Successful write to schema registry for subject {}", subject); - break; - case CLIENT_ERROR: - throw new BadSchemaRequestException(subject, response); - case SERVER_ERROR: - default: - throw new InternalSchemaRepositoryException(subject, response); - } + } + + private Optional extractRawSchemaWithMetadata( + String subject, SchemaId schemaId, Response response) { + Integer id = schemaId.value(); + switch (response.getStatusInfo().getFamily()) { + case SUCCESSFUL: + logger.info("Found schema metadata for subject {} and id {}", subject, id); + SchemaRegistryResponse schemaRegistryResponse = + response.readEntity(SchemaRegistryResponse.class); + return Optional.of(schemaRegistryResponse.toRawSchemaWithMetadata()); + case CLIENT_ERROR: + logger.error( + "Could not find schema metadata for subject {} and id {}, reason: {}", + subject, + id, + response.getStatus()); + return Optional.empty(); + case SERVER_ERROR: + default: + logger.error( + "Could not find schema metadata for subject {} and id {}, reason: {}", + subject, + id, + response.getStatus()); + throw new InternalSchemaRepositoryException(subject, response); } - - @Override - public void deleteAllSchemaVersions(TopicName topic) { - String subject = subjectNamingStrategy.apply(topic); - Response response = schemaRepositoryInstanceResolver.resolve(subject) - .path("subjects") - .path(subject) - .path(deleteSchemaPathSuffix) - .request() - .delete(); - checkSchemaRemoval(subject, response); + } + + @Override + public List getVersions(TopicName topic) { + String subject = subjectNamingStrategy.apply(topic); + Response response = + schemaRepositoryInstanceResolver + .resolve(subject) + .path("subjects") + .path(subject) + .path("versions") + .request() + .get(); + return extractSchemaVersions(subject, response); + } + + private List extractSchemaVersions(String subject, Response response) { + switch (response.getStatusInfo().getFamily()) { + case SUCCESSFUL: + return Arrays.stream(response.readEntity(Integer[].class)) + .sorted(Comparator.reverseOrder()) + .map(SchemaVersion::valueOf) + .collect(Collectors.toList()); + case CLIENT_ERROR: + logger.error( + "Could not find schema versions for subject {}, reason: {} {}", + subject, + response.getStatus(), + response.readEntity(String.class)); + return Collections.emptyList(); + case SERVER_ERROR: + default: + throw new InternalSchemaRepositoryException(subject, response); } - - private void checkSchemaRemoval(String subject, Response response) { - switch (response.getStatusInfo().getFamily()) { - case SUCCESSFUL: - logger.info("Successful removed schema subject {}", subject); - break; - case CLIENT_ERROR: - throw new BadSchemaRequestException(subject, response); - case SERVER_ERROR: - default: - int statusCode = response.getStatus(); - String responseBody = response.readEntity(String.class); - logger.warn("Could not remove schema of subject {}. Reason: {} {}", subject, statusCode, responseBody); - throw new InternalSchemaRepositoryException(subject, statusCode, responseBody); - } + } + + @Override + public void registerSchema(TopicName topic, RawSchema rawSchema) { + String subject = subjectNamingStrategy.apply(topic); + Response response = + schemaRepositoryInstanceResolver + .resolve(subject) + .path("subjects") + .path(subject) + .path("versions") + .request() + .accept(MediaType.APPLICATION_JSON_TYPE) + .post( + Entity.entity( + SchemaRegistryRequestResponse.fromRawSchema(rawSchema), + SCHEMA_REPO_CONTENT_TYPE)); + checkSchemaRegistration(subject, response); + } + + private void checkSchemaRegistration(String subject, Response response) { + switch (response.getStatusInfo().getFamily()) { + case SUCCESSFUL: + logger.info("Successful write to schema registry for subject {}", subject); + break; + case CLIENT_ERROR: + throw new BadSchemaRequestException(subject, response); + case SERVER_ERROR: + default: + throw new InternalSchemaRepositoryException(subject, response); } - - @Override - public void validateSchema(TopicName topic, RawSchema schema) { - String subject = subjectNamingStrategy.apply(topic); - checkCompatibility(subject, schema); - if (validationEndpointEnabled) { - checkValidation(subject, schema); - } + } + + @Override + public void deleteAllSchemaVersions(TopicName topic) { + String subject = subjectNamingStrategy.apply(topic); + Response response = + schemaRepositoryInstanceResolver + .resolve(subject) + .path("subjects") + .path(subject) + .path(deleteSchemaPathSuffix) + .request() + .delete(); + checkSchemaRemoval(subject, response); + } + + private void checkSchemaRemoval(String subject, Response response) { + switch (response.getStatusInfo().getFamily()) { + case SUCCESSFUL: + logger.info("Successful removed schema subject {}", subject); + break; + case CLIENT_ERROR: + throw new BadSchemaRequestException(subject, response); + case SERVER_ERROR: + default: + int statusCode = response.getStatus(); + String responseBody = response.readEntity(String.class); + logger.warn( + "Could not remove schema of subject {}. Reason: {} {}", + subject, + statusCode, + responseBody); + throw new InternalSchemaRepositoryException(subject, statusCode, responseBody); } - - private void checkCompatibility(String subject, RawSchema schema) { - Response response = schemaRepositoryInstanceResolver.resolve(subject) - .path("compatibility") - .path("subjects") - .path(subject) - .path("versions") - .path("latest") - .request() - .accept(MediaType.APPLICATION_JSON_TYPE) - .post(Entity.entity(SchemaRegistryRequestResponse.fromRawSchema(schema), SCHEMA_REPO_CONTENT_TYPE)); - checkSchemaCompatibilityResponse(subject, response); + } + + @Override + public void validateSchema(TopicName topic, RawSchema schema) { + String subject = subjectNamingStrategy.apply(topic); + checkCompatibility(subject, schema); + if (validationEndpointEnabled) { + checkValidation(subject, schema); } - - private void checkValidation(String subject, RawSchema schema) { - Response response = schemaRepositoryInstanceResolver.resolve(subject) - .path("subjects") - .path(subject) - .path("validation") - .request() - .accept(MediaType.APPLICATION_JSON_TYPE) - .post(Entity.entity(SchemaRegistryRequestResponse.fromRawSchema(schema), SCHEMA_REPO_CONTENT_TYPE)); - - checkValidationResponse(subject, response); + } + + private void checkCompatibility(String subject, RawSchema schema) { + Response response = + schemaRepositoryInstanceResolver + .resolve(subject) + .path("compatibility") + .path("subjects") + .path(subject) + .path("versions") + .path("latest") + .request() + .accept(MediaType.APPLICATION_JSON_TYPE) + .post( + Entity.entity( + SchemaRegistryRequestResponse.fromRawSchema(schema), SCHEMA_REPO_CONTENT_TYPE)); + checkSchemaCompatibilityResponse(subject, response); + } + + private void checkValidation(String subject, RawSchema schema) { + Response response = + schemaRepositoryInstanceResolver + .resolve(subject) + .path("subjects") + .path(subject) + .path("validation") + .request() + .accept(MediaType.APPLICATION_JSON_TYPE) + .post( + Entity.entity( + SchemaRegistryRequestResponse.fromRawSchema(schema), SCHEMA_REPO_CONTENT_TYPE)); + + checkValidationResponse(subject, response); + } + + private void checkValidationResponse(String subject, Response response) { + if (response.getStatusInfo().getFamily() == SUCCESSFUL) { + validateSuccessfulValidationResult(subject, response); + } else { + handleErrorResponse(subject, response); } + } - private void checkValidationResponse(String subject, Response response) { - if (response.getStatusInfo().getFamily() == SUCCESSFUL) { - validateSuccessfulValidationResult(subject, response); - } else { - handleErrorResponse(subject, response); - } - } - - private void validateSuccessfulValidationResult(String subject, Response response) { - SchemaRegistryValidationResponse validationResponse = response.readEntity(SchemaRegistryValidationResponse.class); + private void validateSuccessfulValidationResult(String subject, Response response) { + SchemaRegistryValidationResponse validationResponse = + response.readEntity(SchemaRegistryValidationResponse.class); - if (!validationResponse.isValid()) { - throw new BadSchemaRequestException(subject, BAD_REQUEST.getStatusCode(), - validationResponse.getErrorsMessage()); - } + if (!validationResponse.isValid()) { + throw new BadSchemaRequestException( + subject, BAD_REQUEST.getStatusCode(), validationResponse.getErrorsMessage()); } + } - private void checkSchemaCompatibilityResponse(String subject, Response response) { - if (response.getStatusInfo().getFamily() == SUCCESSFUL) { - validateSuccessfulCompatibilityResult(subject, response); - } else { - handleErrorResponse(subject, response); - } + private void checkSchemaCompatibilityResponse(String subject, Response response) { + if (response.getStatusInfo().getFamily() == SUCCESSFUL) { + validateSuccessfulCompatibilityResult(subject, response); + } else { + handleErrorResponse(subject, response); } + } - private void handleErrorResponse(String subject, Response response) { - switch (response.getStatusInfo().getFamily()) { - case CLIENT_ERROR: - if (response.getStatus() == 422) { // for other cases we assume the schema is valid - throw new BadSchemaRequestException(subject, response); - } - break; - case SERVER_ERROR: - default: - int statusCode = response.getStatus(); - String responseBody = response.readEntity(String.class); - logger.warn("Could not validate schema of subject {}. Reason: {} {}", subject, statusCode, responseBody); - throw new InternalSchemaRepositoryException(subject, statusCode, responseBody); + private void handleErrorResponse(String subject, Response response) { + switch (response.getStatusInfo().getFamily()) { + case CLIENT_ERROR: + if (response.getStatus() == 422) { // for other cases we assume the schema is valid + throw new BadSchemaRequestException(subject, response); } + break; + case SERVER_ERROR: + default: + int statusCode = response.getStatus(); + String responseBody = response.readEntity(String.class); + logger.warn( + "Could not validate schema of subject {}. Reason: {} {}", + subject, + statusCode, + responseBody); + throw new InternalSchemaRepositoryException(subject, statusCode, responseBody); } - - private void validateSuccessfulCompatibilityResult(String subject, Response response) { - String validationResultStr = response.readEntity(String.class); - SchemaRegistryCompatibilityResponse validationResponse = - toSchemaRegistryValidationResponse(subject, validationResultStr, response.getStatus()); - if (!validationResponse.isCompatible()) { - throw new BadSchemaRequestException(subject, response.getStatus(), validationResultStr); - } + } + + private void validateSuccessfulCompatibilityResult(String subject, Response response) { + String validationResultStr = response.readEntity(String.class); + SchemaRegistryCompatibilityResponse validationResponse = + toSchemaRegistryValidationResponse(subject, validationResultStr, response.getStatus()); + if (!validationResponse.isCompatible()) { + throw new BadSchemaRequestException(subject, response.getStatus(), validationResultStr); } - - private SchemaRegistryCompatibilityResponse toSchemaRegistryValidationResponse(String subject, String validationResultStr, int status) { - try { - return objectMapper.readValue(validationResultStr, SchemaRegistryCompatibilityResponse.class); - } catch (IOException e) { - logger.error("Could not parse schema validation response from schema registry", e); - throw new InternalSchemaRepositoryException(subject, status, validationResultStr); - } + } + + private SchemaRegistryCompatibilityResponse toSchemaRegistryValidationResponse( + String subject, String validationResultStr, int status) { + try { + return objectMapper.readValue(validationResultStr, SchemaRegistryCompatibilityResponse.class); + } catch (IOException e) { + logger.error("Could not parse schema validation response from schema registry", e); + throw new InternalSchemaRepositoryException(subject, status, validationResultStr); } + } } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryRequestResponse.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryRequestResponse.java index f7319ee17f..46422d808f 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryRequestResponse.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryRequestResponse.java @@ -2,41 +2,40 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import pl.allegro.tech.hermes.api.RawSchema; - import java.util.Objects; +import pl.allegro.tech.hermes.api.RawSchema; class SchemaRegistryRequestResponse { - private final String schema; + private final String schema; - @JsonCreator - SchemaRegistryRequestResponse(@JsonProperty("schema") String schema) { - this.schema = schema; - } + @JsonCreator + SchemaRegistryRequestResponse(@JsonProperty("schema") String schema) { + this.schema = schema; + } - static SchemaRegistryRequestResponse fromRawSchema(RawSchema rawSchema) { - return new SchemaRegistryRequestResponse(rawSchema.value()); - } + static SchemaRegistryRequestResponse fromRawSchema(RawSchema rawSchema) { + return new SchemaRegistryRequestResponse(rawSchema.value()); + } - public String getSchema() { - return schema; - } + public String getSchema() { + return schema; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - SchemaRegistryRequestResponse that = (SchemaRegistryRequestResponse) o; - return Objects.equals(schema, that.schema); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(schema); + if (o == null || getClass() != o.getClass()) { + return false; } + SchemaRegistryRequestResponse that = (SchemaRegistryRequestResponse) o; + return Objects.equals(schema, that.schema); + } + + @Override + public int hashCode() { + return Objects.hash(schema); + } } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryResponse.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryResponse.java index bfc4133e1e..60f30535e3 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryResponse.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryResponse.java @@ -3,72 +3,68 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.constraints.NotNull; -import pl.allegro.tech.hermes.api.RawSchemaWithMetadata; - import java.util.Objects; +import pl.allegro.tech.hermes.api.RawSchemaWithMetadata; class SchemaRegistryResponse { - @NotNull - private final String subject; + @NotNull private final String subject; - @NotNull - private final Integer id; + @NotNull private final Integer id; - @NotNull - private final Integer version; + @NotNull private final Integer version; - @NotNull - private final String schema; + @NotNull private final String schema; - @JsonCreator - SchemaRegistryResponse(@JsonProperty("subject") String subject, - @JsonProperty("id") Integer id, - @JsonProperty("version") Integer version, - @JsonProperty("schema") String schema) { - this.subject = subject; - this.version = version; - this.id = id; - this.schema = schema; - } + @JsonCreator + SchemaRegistryResponse( + @JsonProperty("subject") String subject, + @JsonProperty("id") Integer id, + @JsonProperty("version") Integer version, + @JsonProperty("schema") String schema) { + this.subject = subject; + this.version = version; + this.id = id; + this.schema = schema; + } - public String getSubject() { - return subject; - } + public String getSubject() { + return subject; + } - public Integer getVersion() { - return version; - } + public Integer getVersion() { + return version; + } - public Integer getId() { - return id; - } + public Integer getId() { + return id; + } - public String getSchema() { - return schema; - } + public String getSchema() { + return schema; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - SchemaRegistryResponse that = (SchemaRegistryResponse) o; - return Objects.equals(subject, that.subject) - && Objects.equals(version, that.version) - && Objects.equals(id, that.id) - && Objects.equals(schema, that.schema); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(subject, version, id, schema); + if (o == null || getClass() != o.getClass()) { + return false; } + SchemaRegistryResponse that = (SchemaRegistryResponse) o; + return Objects.equals(subject, that.subject) + && Objects.equals(version, that.version) + && Objects.equals(id, that.id) + && Objects.equals(schema, that.schema); + } - RawSchemaWithMetadata toRawSchemaWithMetadata() { - return RawSchemaWithMetadata.of(schema, id, version); - } + @Override + public int hashCode() { + return Objects.hash(subject, version, id, schema); + } + + RawSchemaWithMetadata toRawSchemaWithMetadata() { + return RawSchemaWithMetadata.of(schema, id, version); + } } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryValidationError.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryValidationError.java index 946ca40a16..765f000f82 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryValidationError.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryValidationError.java @@ -3,36 +3,35 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.Objects; @JsonIgnoreProperties(ignoreUnknown = true) class SchemaRegistryValidationError { - private final String message; + private final String message; - @JsonCreator - SchemaRegistryValidationError(@JsonProperty("message") String message) { - this.message = message; - } + @JsonCreator + SchemaRegistryValidationError(@JsonProperty("message") String message) { + this.message = message; + } - public String getMessage() { - return message; - } + public String getMessage() { + return message; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof SchemaRegistryValidationError)) { - return false; - } - SchemaRegistryValidationError that = (SchemaRegistryValidationError) o; - return Objects.equals(message, that.message); + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(message); + if (!(o instanceof SchemaRegistryValidationError)) { + return false; } + SchemaRegistryValidationError that = (SchemaRegistryValidationError) o; + return Objects.equals(message, that.message); + } + + @Override + public int hashCode() { + return Objects.hash(message); + } } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryValidationResponse.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryValidationResponse.java index 1fb6832a78..36a32d3b3d 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryValidationResponse.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/confluent/SchemaRegistryValidationResponse.java @@ -3,30 +3,30 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.List; import java.util.stream.Collectors; class SchemaRegistryValidationResponse { - private final boolean valid; - private final List errors; + private final boolean valid; + private final List errors; - @JsonCreator - SchemaRegistryValidationResponse(@JsonProperty("is_valid") boolean valid, - @JsonProperty("errors") List errors) { - this.valid = valid; - this.errors = errors; - } + @JsonCreator + SchemaRegistryValidationResponse( + @JsonProperty("is_valid") boolean valid, + @JsonProperty("errors") List errors) { + this.valid = valid; + this.errors = errors; + } - public boolean isValid() { - return valid; - } + public boolean isValid() { + return valid; + } - @JsonIgnore - public String getErrorsMessage() { - return errors.stream() - .map(SchemaRegistryValidationError::getMessage) - .collect(Collectors.joining(". ")); - } + @JsonIgnore + public String getErrorsMessage() { + return errors.stream() + .map(SchemaRegistryValidationError::getMessage) + .collect(Collectors.joining(". ")); + } } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/resolver/DefaultSchemaRepositoryInstanceResolver.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/resolver/DefaultSchemaRepositoryInstanceResolver.java index a3f9c31f9a..f823eaa055 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/resolver/DefaultSchemaRepositoryInstanceResolver.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/resolver/DefaultSchemaRepositoryInstanceResolver.java @@ -2,19 +2,18 @@ import jakarta.ws.rs.client.Client; import jakarta.ws.rs.client.WebTarget; - import java.net.URI; public class DefaultSchemaRepositoryInstanceResolver implements SchemaRepositoryInstanceResolver { - private final WebTarget target; + private final WebTarget target; - public DefaultSchemaRepositoryInstanceResolver(Client client, URI schemaRegistryServerUri) { - this.target = client.target(schemaRegistryServerUri); - } + public DefaultSchemaRepositoryInstanceResolver(Client client, URI schemaRegistryServerUri) { + this.target = client.target(schemaRegistryServerUri); + } - @Override - public WebTarget resolve(String subject) { - return target; - } + @Override + public WebTarget resolve(String subject) { + return target; + } } diff --git a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/resolver/SchemaRepositoryInstanceResolver.java b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/resolver/SchemaRepositoryInstanceResolver.java index 10c00b13c9..56bd2e6d83 100644 --- a/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/resolver/SchemaRepositoryInstanceResolver.java +++ b/hermes-schema/src/main/java/pl/allegro/tech/hermes/schema/resolver/SchemaRepositoryInstanceResolver.java @@ -4,5 +4,5 @@ public interface SchemaRepositoryInstanceResolver { - WebTarget resolve(String subject); + WebTarget resolve(String subject); } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/avro/AvroUser.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/avro/AvroUser.java index a72ed634f5..a2a9e45303 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/avro/AvroUser.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/avro/AvroUser.java @@ -1,119 +1,121 @@ package pl.allegro.tech.hermes.test.helper.avro; +import static pl.allegro.tech.hermes.test.helper.avro.RecordToBytesConverter.recordToBytes; + import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Optional; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; import pl.allegro.tech.hermes.schema.CompiledSchema; import pl.allegro.tech.hermes.test.helper.message.TestMessage; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.Optional; - -import static pl.allegro.tech.hermes.test.helper.avro.RecordToBytesConverter.recordToBytes; - public class AvroUser { - private static final String METADATA_FIELD = "__metadata"; - private static final String NAME_FIELD = "name"; - private static final String AGE_FIELD = "age"; - private static final String FAVORITE_COLOR_FIELD = "favoriteColor"; - - private final CompiledSchema schema; - - private final GenericRecord record; - - public AvroUser() { - this("defaultName", 0, "defaultColor"); - } - - public AvroUser(CompiledSchema schema) { - this(schema, "defaultName", 0, "defaultColor"); - } - - public AvroUser(String name, int age, String favouriteColour) { - this(CompiledSchema.of(AvroUserSchemaLoader.load(), 1, 1), name, age, favouriteColour); - } - - public AvroUser(CompiledSchema schema, String name, int age, String favouriteColour) { - this.schema = schema; - this.record = create(name, age, favouriteColour); - } - - public AvroUser(CompiledSchema schema, byte[] bytes) throws IOException { - this.schema = schema; - this.record = RecordToBytesConverter.bytesToRecord(bytes, schema.getSchema()); - } - - public Schema getSchema() { - return schema.getSchema(); - } - - public String getSchemaAsString() { - return schema.getSchema().toString(); - } - - public String getName() { - return record.get(NAME_FIELD).toString(); - } - - public int getAge() { - return Integer.parseInt(record.get(AGE_FIELD).toString()); - } - - public String getFavoriteColor() { - return record.get(FAVORITE_COLOR_FIELD).toString(); - } - - public byte[] asBytes() { - try { - return recordToBytes(record, schema.getSchema()); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - public String asJson() { - return asTestMessage().toString(); - } - - public String asAvroEncodedJson() { - return asAvroEncodedTestMessage().toString(); - } - - public TestMessage asTestMessage() { - return TestMessage.of(NAME_FIELD, getName()).append(AGE_FIELD, getAge()).append(FAVORITE_COLOR_FIELD, getFavoriteColor()); - } - - public TestMessage asAvroEncodedTestMessage() { - Object favoriteColorType = Optional.ofNullable(getFavoriteColor()) - .map(color -> (Object) ImmutableMap.of("string", color)) - .orElse("null"); - return TestMessage.of(METADATA_FIELD, null) - .append(NAME_FIELD, getName()) - .append(AGE_FIELD, getAge()) - .append(FAVORITE_COLOR_FIELD, favoriteColorType); - } - - public static AvroUser create(CompiledSchema schema, byte[] bytes) { - try { - return new AvroUser(schema, bytes); - } catch (IOException ex) { - throw new UncheckedIOException(ex); - } - } - - private GenericRecord create(String name, int age, String favoriteColor) { - GenericRecord user = new GenericData.Record(schema.getSchema()); - user.put(NAME_FIELD, name); - user.put(AGE_FIELD, age); - user.put(FAVORITE_COLOR_FIELD, favoriteColor); - - return user; - } - - public CompiledSchema getCompiledSchema() { - return schema; - } + private static final String METADATA_FIELD = "__metadata"; + private static final String NAME_FIELD = "name"; + private static final String AGE_FIELD = "age"; + private static final String FAVORITE_COLOR_FIELD = "favoriteColor"; + + private final CompiledSchema schema; + + private final GenericRecord record; + + public AvroUser() { + this("defaultName", 0, "defaultColor"); + } + + public AvroUser(CompiledSchema schema) { + this(schema, "defaultName", 0, "defaultColor"); + } + + public AvroUser(String name, int age, String favouriteColour) { + this(CompiledSchema.of(AvroUserSchemaLoader.load(), 1, 1), name, age, favouriteColour); + } + + public AvroUser(CompiledSchema schema, String name, int age, String favouriteColour) { + this.schema = schema; + this.record = create(name, age, favouriteColour); + } + + public AvroUser(CompiledSchema schema, byte[] bytes) throws IOException { + this.schema = schema; + this.record = RecordToBytesConverter.bytesToRecord(bytes, schema.getSchema()); + } + + public Schema getSchema() { + return schema.getSchema(); + } + + public String getSchemaAsString() { + return schema.getSchema().toString(); + } + + public String getName() { + return record.get(NAME_FIELD).toString(); + } + + public int getAge() { + return Integer.parseInt(record.get(AGE_FIELD).toString()); + } + + public String getFavoriteColor() { + return record.get(FAVORITE_COLOR_FIELD).toString(); + } + + public byte[] asBytes() { + try { + return recordToBytes(record, schema.getSchema()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public String asJson() { + return asTestMessage().toString(); + } + + public String asAvroEncodedJson() { + return asAvroEncodedTestMessage().toString(); + } + + public TestMessage asTestMessage() { + return TestMessage.of(NAME_FIELD, getName()) + .append(AGE_FIELD, getAge()) + .append(FAVORITE_COLOR_FIELD, getFavoriteColor()); + } + + public TestMessage asAvroEncodedTestMessage() { + Object favoriteColorType = + Optional.ofNullable(getFavoriteColor()) + .map(color -> (Object) ImmutableMap.of("string", color)) + .orElse("null"); + return TestMessage.of(METADATA_FIELD, null) + .append(NAME_FIELD, getName()) + .append(AGE_FIELD, getAge()) + .append(FAVORITE_COLOR_FIELD, favoriteColorType); + } + + public static AvroUser create(CompiledSchema schema, byte[] bytes) { + try { + return new AvroUser(schema, bytes); + } catch (IOException ex) { + throw new UncheckedIOException(ex); + } + } + + private GenericRecord create(String name, int age, String favoriteColor) { + GenericRecord user = new GenericData.Record(schema.getSchema()); + user.put(NAME_FIELD, name); + user.put(AGE_FIELD, age); + user.put(FAVORITE_COLOR_FIELD, favoriteColor); + + return user; + } + + public CompiledSchema getCompiledSchema() { + return schema; + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/avro/AvroUserSchemaLoader.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/avro/AvroUserSchemaLoader.java index d6b43891c3..bef52bd5bb 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/avro/AvroUserSchemaLoader.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/avro/AvroUserSchemaLoader.java @@ -1,22 +1,21 @@ package pl.allegro.tech.hermes.test.helper.avro; -import org.apache.avro.Schema; - import java.io.IOException; import java.io.UncheckedIOException; +import org.apache.avro.Schema; public class AvroUserSchemaLoader { - public static Schema load() { - return load("/schema/user.avsc"); - } + public static Schema load() { + return load("/schema/user.avsc"); + } - public static Schema load(String schemaResourceName) { - try { - return new Schema.Parser().parse(AvroUserSchemaLoader.class.getResourceAsStream(schemaResourceName)); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + public static Schema load(String schemaResourceName) { + try { + return new Schema.Parser() + .parse(AvroUserSchemaLoader.class.getResourceAsStream(schemaResourceName)); + } catch (IOException e) { + throw new UncheckedIOException(e); } - + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/avro/RecordToBytesConverter.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/avro/RecordToBytesConverter.java index 759cb836cb..3e9e221f51 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/avro/RecordToBytesConverter.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/avro/RecordToBytesConverter.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.test.helper.avro; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import org.apache.avro.Schema; import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.generic.GenericDatumWriter; @@ -10,25 +12,22 @@ import org.apache.avro.io.DecoderFactory; import org.apache.avro.io.EncoderFactory; -import java.io.ByteArrayOutputStream; -import java.io.IOException; - public interface RecordToBytesConverter { - static byte[] recordToBytes(GenericRecord record, Schema schema) throws IOException { - try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { - BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null); - DatumWriter writer = new GenericDatumWriter<>(schema); + static byte[] recordToBytes(GenericRecord record, Schema schema) throws IOException { + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null); + DatumWriter writer = new GenericDatumWriter<>(schema); - writer.write(record, encoder); - encoder.flush(); - return out.toByteArray(); - } + writer.write(record, encoder); + encoder.flush(); + return out.toByteArray(); } + } - static GenericRecord bytesToRecord(byte [] data, Schema schema) throws IOException { - GenericDatumReader reader = new GenericDatumReader<>(schema); - BinaryDecoder binaryDecoder = DecoderFactory.get().binaryDecoder(data, null); - return reader.read(null, binaryDecoder); - } + static GenericRecord bytesToRecord(byte[] data, Schema schema) throws IOException { + GenericDatumReader reader = new GenericDatumReader<>(schema); + BinaryDecoder binaryDecoder = DecoderFactory.get().binaryDecoder(data, null); + return reader.read(null, binaryDecoder); + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/builder/GroupBuilder.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/builder/GroupBuilder.java index f93033081d..39228970fa 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/builder/GroupBuilder.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/builder/GroupBuilder.java @@ -1,36 +1,37 @@ package pl.allegro.tech.hermes.test.helper.builder; -import pl.allegro.tech.hermes.api.Group; - import java.util.concurrent.atomic.AtomicLong; +import pl.allegro.tech.hermes.api.Group; public class GroupBuilder { - private static final AtomicLong sequence = new AtomicLong(); + private static final AtomicLong sequence = new AtomicLong(); - private final String groupName; + private final String groupName; - private GroupBuilder(String name) { - this.groupName = name; - } + private GroupBuilder(String name) { + this.groupName = name; + } - public static GroupBuilder group(String name) { - return new GroupBuilder(name); - } + public static GroupBuilder group(String name) { + return new GroupBuilder(name); + } - public Group build() { - return new Group(groupName); - } + public Group build() { + return new Group(groupName); + } - public static GroupBuilder groupWithRandomName() { - return groupWithRandomNameEndedWith(""); - } + public static GroupBuilder groupWithRandomName() { + return groupWithRandomNameEndedWith(""); + } - public static GroupBuilder groupWithRandomNameEndedWith(String suffix) { - return group(GroupBuilder.class.getSimpleName() + "Group" + sequence.incrementAndGet() + suffix); - } + public static GroupBuilder groupWithRandomNameEndedWith(String suffix) { + return group( + GroupBuilder.class.getSimpleName() + "Group" + sequence.incrementAndGet() + suffix); + } - public static GroupBuilder groupWithRandomNameContaining(String string) { - return group(GroupBuilder.class.getSimpleName() + "Group" + string + sequence.incrementAndGet()); - } + public static GroupBuilder groupWithRandomNameContaining(String string) { + return group( + GroupBuilder.class.getSimpleName() + "Group" + string + sequence.incrementAndGet()); + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/builder/OAuthProviderBuilder.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/builder/OAuthProviderBuilder.java index a5922b224f..7e51a805a6 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/builder/OAuthProviderBuilder.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/builder/OAuthProviderBuilder.java @@ -4,67 +4,74 @@ public class OAuthProviderBuilder { - private final String name; + private final String name; - private String tokenEndpoint = "http://example.com/token"; + private String tokenEndpoint = "http://example.com/token"; - private String clientId = "testClient123"; + private String clientId = "testClient123"; - private String clientSecret = "testPassword123"; + private String clientSecret = "testPassword123"; - private int tokenRequestInitialDelay = 1; + private int tokenRequestInitialDelay = 1; - private int tokenRequestMaxDelay = 8; + private int tokenRequestMaxDelay = 8; - private int requestTimeout = 500; + private int requestTimeout = 500; - private int socketTimeout = 0; + private int socketTimeout = 0; - public OAuthProviderBuilder(String name) { - this.name = name; - } + public OAuthProviderBuilder(String name) { + this.name = name; + } - public static OAuthProviderBuilder oAuthProvider(String name) { - return new OAuthProviderBuilder(name); - } + public static OAuthProviderBuilder oAuthProvider(String name) { + return new OAuthProviderBuilder(name); + } - public OAuthProvider build() { - return new OAuthProvider(name, tokenEndpoint, clientId, clientSecret, tokenRequestInitialDelay, - tokenRequestMaxDelay, requestTimeout, socketTimeout); - } + public OAuthProvider build() { + return new OAuthProvider( + name, + tokenEndpoint, + clientId, + clientSecret, + tokenRequestInitialDelay, + tokenRequestMaxDelay, + requestTimeout, + socketTimeout); + } - public OAuthProviderBuilder withTokenEndpoint(String tokenEndpoint) { - this.tokenEndpoint = tokenEndpoint; - return this; - } + public OAuthProviderBuilder withTokenEndpoint(String tokenEndpoint) { + this.tokenEndpoint = tokenEndpoint; + return this; + } - public OAuthProviderBuilder withClientId(String clientId) { - this.clientId = clientId; - return this; - } + public OAuthProviderBuilder withClientId(String clientId) { + this.clientId = clientId; + return this; + } - public OAuthProviderBuilder withClientSecret(String clientSecret) { - this.clientSecret = clientSecret; - return this; - } + public OAuthProviderBuilder withClientSecret(String clientSecret) { + this.clientSecret = clientSecret; + return this; + } - public OAuthProviderBuilder withTokenRequestInitialDelay(int tokenRequestInitialDelay) { - this.tokenRequestInitialDelay = tokenRequestInitialDelay; - return this; - } + public OAuthProviderBuilder withTokenRequestInitialDelay(int tokenRequestInitialDelay) { + this.tokenRequestInitialDelay = tokenRequestInitialDelay; + return this; + } - public OAuthProviderBuilder withTokenRequestMaxDelay(int tokenRequestMaxDelay) { - this.tokenRequestMaxDelay = tokenRequestMaxDelay; - return this; - } + public OAuthProviderBuilder withTokenRequestMaxDelay(int tokenRequestMaxDelay) { + this.tokenRequestMaxDelay = tokenRequestMaxDelay; + return this; + } - public OAuthProviderBuilder withRequestTimeout(int requestTimeout) { - this.requestTimeout = requestTimeout; - return this; - } + public OAuthProviderBuilder withRequestTimeout(int requestTimeout) { + this.requestTimeout = requestTimeout; + return this; + } - public OAuthProviderBuilder withSocketTimeout(int socketTimeout) { - this.socketTimeout = socketTimeout; - return this; - } + public OAuthProviderBuilder withSocketTimeout(int socketTimeout) { + this.socketTimeout = socketTimeout; + return this; + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/builder/SubscriptionBuilder.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/builder/SubscriptionBuilder.java index 63a4ecedff..0408f25601 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/builder/SubscriptionBuilder.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/builder/SubscriptionBuilder.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.test.helper.builder; +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicLong; import pl.allegro.tech.hermes.api.BatchSubscriptionPolicy; import pl.allegro.tech.hermes.api.ContentType; import pl.allegro.tech.hermes.api.DeliveryType; @@ -18,250 +23,294 @@ import pl.allegro.tech.hermes.api.TopicName; import pl.allegro.tech.hermes.api.TrackingMode; -import java.net.URI; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicLong; - public class SubscriptionBuilder { - private static final AtomicLong sequence = new AtomicLong(); - - private final TopicName topicName; - - private final String name; - - private Subscription.State state = Subscription.State.PENDING; - - private EndpointAddress endpoint = EndpointAddress.of("http://localhost:12345"); - - private ContentType contentType = ContentType.JSON; - - private String description = "description"; - - private SubscriptionPolicy serialSubscriptionPolicy = new SubscriptionPolicy(100, 10, 1000, 1000, false, 100, null, 0, 1, 600); - - private BatchSubscriptionPolicy batchSubscriptionPolicy; - - private boolean trackingEnabled = false; - - private TrackingMode trackingMode = TrackingMode.TRACKING_OFF; + private static final AtomicLong sequence = new AtomicLong(); - private boolean http2Enabled = false; + private final TopicName topicName; - private boolean profilingEnabled = false; + private final String name; - private long profilingThresholdMs = 0; + private Subscription.State state = Subscription.State.PENDING; - private OwnerId owner = new OwnerId("Plaintext", "some team"); + private EndpointAddress endpoint = EndpointAddress.of("http://localhost:12345"); - private MonitoringDetails monitoringDetails = MonitoringDetails.EMPTY; + private ContentType contentType = ContentType.JSON; - private DeliveryType deliveryType = DeliveryType.SERIAL; + private String description = "description"; - private final List filters = new ArrayList<>(); + private SubscriptionPolicy serialSubscriptionPolicy = + new SubscriptionPolicy(100, 10, 1000, 1000, false, 100, null, 0, 1, 600); - private SubscriptionMode mode = SubscriptionMode.ANYCAST; + private BatchSubscriptionPolicy batchSubscriptionPolicy; - private final List
headers = new ArrayList<>(); + private boolean trackingEnabled = false; - private EndpointAddressResolverMetadata metadata = EndpointAddressResolverMetadata.empty(); - private SubscriptionOAuthPolicy oAuthPolicy; + private TrackingMode trackingMode = TrackingMode.TRACKING_OFF; - private boolean attachingIdentityHeadersEnabled = false; + private boolean http2Enabled = false; - private boolean autoDeleteWithTopicEnabled = false; + private boolean profilingEnabled = false; - private SubscriptionBuilder(TopicName topicName, String subscriptionName, EndpointAddress endpoint) { - this.topicName = topicName; - this.name = subscriptionName; - this.endpoint = endpoint; - } + private long profilingThresholdMs = 0; - private SubscriptionBuilder(TopicName topicName, String subscriptionName) { - this.topicName = topicName; - this.name = subscriptionName; - } + private OwnerId owner = new OwnerId("Plaintext", "some team"); - public static SubscriptionBuilder subscriptionWithRandomName(TopicName topicName, String endpoint) { - return new SubscriptionBuilder(topicName, "subscription" + sequence.incrementAndGet(), EndpointAddress.of(endpoint)); - } + private MonitoringDetails monitoringDetails = MonitoringDetails.EMPTY; - public static SubscriptionBuilder subscriptionWithRandomName(TopicName topicName) { - return new SubscriptionBuilder(topicName, UUID.randomUUID().toString()); - } + private DeliveryType deliveryType = DeliveryType.SERIAL; - public static SubscriptionBuilder subscription(SubscriptionName subscriptionName) { - return new SubscriptionBuilder(subscriptionName.getTopicName(), subscriptionName.getName()); - } + private final List filters = new ArrayList<>(); - public static SubscriptionBuilder subscription(TopicName topicName, String subscriptionName) { - return new SubscriptionBuilder(topicName, subscriptionName); - } + private SubscriptionMode mode = SubscriptionMode.ANYCAST; - public static SubscriptionBuilder subscription(Topic topic, String subscriptionName) { - return new SubscriptionBuilder(topic.getName(), subscriptionName); - } + private final List
headers = new ArrayList<>(); - public static SubscriptionBuilder subscription(TopicName topicName, String subscriptionName, EndpointAddress endpoint) { - return new SubscriptionBuilder(topicName, subscriptionName, endpoint); - } + private EndpointAddressResolverMetadata metadata = EndpointAddressResolverMetadata.empty(); + private SubscriptionOAuthPolicy oAuthPolicy; - public static SubscriptionBuilder subscription(String topicQualifiedName, String subscriptionName) { - return new SubscriptionBuilder(TopicName.fromQualifiedName(topicQualifiedName), subscriptionName); - } - - public static SubscriptionBuilder subscription(String topicQualifiedName, String subscriptionName, String endpoint) { - return subscription(TopicName.fromQualifiedName(topicQualifiedName), subscriptionName, EndpointAddress.of(endpoint)); - } - - public static SubscriptionBuilder subscription(String topicQualifiedName, String subscriptionName, URI endpoint) { - return subscription(TopicName.fromQualifiedName(topicQualifiedName), subscriptionName, EndpointAddress.of(endpoint)); - } - - public static SubscriptionBuilder subscription(String topicQualifiedName, String subscriptionName, EndpointAddress endpoint) { - return subscription(TopicName.fromQualifiedName(topicQualifiedName), subscriptionName, endpoint); - } - - public Subscription build() { - if (deliveryType == DeliveryType.SERIAL) { - return Subscription.createSerialSubscription( - topicName, name, endpoint, state, description, - serialSubscriptionPolicy, trackingEnabled, - trackingMode, owner, monitoringDetails, contentType, - filters, mode, headers, metadata, oAuthPolicy, http2Enabled, profilingEnabled, - profilingThresholdMs, attachingIdentityHeadersEnabled, autoDeleteWithTopicEnabled - ); - } else { - return Subscription.createBatchSubscription( - topicName, name, endpoint, state, description, - batchSubscriptionPolicy, trackingEnabled, - trackingMode, owner, monitoringDetails, contentType, - filters, headers, metadata, oAuthPolicy, http2Enabled, - attachingIdentityHeadersEnabled, autoDeleteWithTopicEnabled - ); - } - } - - public SubscriptionBuilder withEndpoint(EndpointAddress endpoint) { - this.endpoint = endpoint; - return this; - } - - public SubscriptionBuilder withEndpoint(String endpoint) { - this.endpoint = EndpointAddress.of(endpoint); - return this; - } + private boolean attachingIdentityHeadersEnabled = false; - public SubscriptionBuilder withEndpoint(URI endpoint) { - this.endpoint = EndpointAddress.of(endpoint.toString()); - return this; - } - - public SubscriptionBuilder withState(Subscription.State state) { - this.state = state; - return this; - } - - public SubscriptionBuilder withDescription(String description) { - this.description = description; - return this; - } - - public SubscriptionBuilder withSubscriptionPolicy(BatchSubscriptionPolicy subscriptionPolicy) { - this.batchSubscriptionPolicy = subscriptionPolicy; - this.deliveryType = DeliveryType.BATCH; - return this; - } - - public SubscriptionBuilder withSubscriptionPolicy(SubscriptionPolicy subscriptionPolicy) { - this.serialSubscriptionPolicy = subscriptionPolicy; - this.deliveryType = DeliveryType.SERIAL; - return this; - } - - public SubscriptionBuilder withRequestTimeout(int timeout) { - SubscriptionPolicy policy = this.serialSubscriptionPolicy; - this.serialSubscriptionPolicy = SubscriptionPolicy.Builder.subscriptionPolicy().withRate(policy.getRate()) - .withMessageTtl(policy.getMessageTtl()).withMessageBackoff(policy.getMessageBackoff()) - .withRequestTimeout(timeout).build(); - return this; - } - - public SubscriptionBuilder withTrackingMode(TrackingMode trackingMode) { - this.trackingMode = trackingMode; - this.trackingEnabled = trackingMode != TrackingMode.TRACKING_OFF; - return this; - } - - public SubscriptionBuilder withHttp2Enabled(boolean http2Enabled) { - this.http2Enabled = http2Enabled; - return this; - } - - public SubscriptionBuilder withProfilingEnabled(boolean profilingEnabled) { - this.profilingEnabled = profilingEnabled; - return this; - } - - public SubscriptionBuilder withProfilingThresholdMs(long profilingThresholdMs) { - this.profilingThresholdMs = profilingThresholdMs; - return this; - } - - public SubscriptionBuilder withOwner(OwnerId owner) { - this.owner = owner; - return this; - } - - public SubscriptionBuilder withMonitoringDetails(MonitoringDetails monitoringDetails) { - this.monitoringDetails = monitoringDetails; - return this; - } - - public SubscriptionBuilder withDeliveryType(DeliveryType deliveryType) { - this.deliveryType = deliveryType; - return this; - } - - public SubscriptionBuilder withContentType(ContentType contentType) { - this.contentType = contentType; - return this; - } - - public SubscriptionBuilder withFilter(MessageFilterSpecification filter) { - this.filters.add(filter); - return this; - } - - public SubscriptionBuilder withMode(SubscriptionMode mode) { - this.mode = mode; - return this; - } - - public SubscriptionBuilder withHeader(String name, String value) { - this.headers.add(new Header(name, value)); - return this; - } - - public SubscriptionBuilder withEndpointAddressResolverMetadata(EndpointAddressResolverMetadata metadata) { - this.metadata = metadata; - return this; - } - - public SubscriptionBuilder withOAuthPolicy(SubscriptionOAuthPolicy oAuthPolicy) { - this.oAuthPolicy = oAuthPolicy; - return this; - } - - public SubscriptionBuilder withAttachingIdentityHeadersEnabled(boolean attachingIdentityHeadersEnabled) { - this.attachingIdentityHeadersEnabled = attachingIdentityHeadersEnabled; - return this; - } - - public SubscriptionBuilder withAutoDeleteWithTopicEnabled(boolean autoDeleteWithTopicEnabled) { - this.autoDeleteWithTopicEnabled = autoDeleteWithTopicEnabled; - return this; - } + private boolean autoDeleteWithTopicEnabled = false; + + private SubscriptionBuilder( + TopicName topicName, String subscriptionName, EndpointAddress endpoint) { + this.topicName = topicName; + this.name = subscriptionName; + this.endpoint = endpoint; + } + + private SubscriptionBuilder(TopicName topicName, String subscriptionName) { + this.topicName = topicName; + this.name = subscriptionName; + } + + public static SubscriptionBuilder subscriptionWithRandomName( + TopicName topicName, String endpoint) { + return new SubscriptionBuilder( + topicName, "subscription" + sequence.incrementAndGet(), EndpointAddress.of(endpoint)); + } + + public static SubscriptionBuilder subscriptionWithRandomName(TopicName topicName) { + return new SubscriptionBuilder(topicName, UUID.randomUUID().toString()); + } + + public static SubscriptionBuilder subscription(SubscriptionName subscriptionName) { + return new SubscriptionBuilder(subscriptionName.getTopicName(), subscriptionName.getName()); + } + + public static SubscriptionBuilder subscription(TopicName topicName, String subscriptionName) { + return new SubscriptionBuilder(topicName, subscriptionName); + } + + public static SubscriptionBuilder subscription(Topic topic, String subscriptionName) { + return new SubscriptionBuilder(topic.getName(), subscriptionName); + } + + public static SubscriptionBuilder subscription( + TopicName topicName, String subscriptionName, EndpointAddress endpoint) { + return new SubscriptionBuilder(topicName, subscriptionName, endpoint); + } + + public static SubscriptionBuilder subscription( + String topicQualifiedName, String subscriptionName) { + return new SubscriptionBuilder( + TopicName.fromQualifiedName(topicQualifiedName), subscriptionName); + } + + public static SubscriptionBuilder subscription( + String topicQualifiedName, String subscriptionName, String endpoint) { + return subscription( + TopicName.fromQualifiedName(topicQualifiedName), + subscriptionName, + EndpointAddress.of(endpoint)); + } + + public static SubscriptionBuilder subscription( + String topicQualifiedName, String subscriptionName, URI endpoint) { + return subscription( + TopicName.fromQualifiedName(topicQualifiedName), + subscriptionName, + EndpointAddress.of(endpoint)); + } + + public static SubscriptionBuilder subscription( + String topicQualifiedName, String subscriptionName, EndpointAddress endpoint) { + return subscription( + TopicName.fromQualifiedName(topicQualifiedName), subscriptionName, endpoint); + } + + public Subscription build() { + if (deliveryType == DeliveryType.SERIAL) { + return Subscription.createSerialSubscription( + topicName, + name, + endpoint, + state, + description, + serialSubscriptionPolicy, + trackingEnabled, + trackingMode, + owner, + monitoringDetails, + contentType, + filters, + mode, + headers, + metadata, + oAuthPolicy, + http2Enabled, + profilingEnabled, + profilingThresholdMs, + attachingIdentityHeadersEnabled, + autoDeleteWithTopicEnabled); + } else { + return Subscription.createBatchSubscription( + topicName, + name, + endpoint, + state, + description, + batchSubscriptionPolicy, + trackingEnabled, + trackingMode, + owner, + monitoringDetails, + contentType, + filters, + headers, + metadata, + oAuthPolicy, + http2Enabled, + attachingIdentityHeadersEnabled, + autoDeleteWithTopicEnabled); + } + } + + public SubscriptionBuilder withEndpoint(EndpointAddress endpoint) { + this.endpoint = endpoint; + return this; + } + + public SubscriptionBuilder withEndpoint(String endpoint) { + this.endpoint = EndpointAddress.of(endpoint); + return this; + } + + public SubscriptionBuilder withEndpoint(URI endpoint) { + this.endpoint = EndpointAddress.of(endpoint.toString()); + return this; + } + + public SubscriptionBuilder withState(Subscription.State state) { + this.state = state; + return this; + } + + public SubscriptionBuilder withDescription(String description) { + this.description = description; + return this; + } + + public SubscriptionBuilder withSubscriptionPolicy(BatchSubscriptionPolicy subscriptionPolicy) { + this.batchSubscriptionPolicy = subscriptionPolicy; + this.deliveryType = DeliveryType.BATCH; + return this; + } + + public SubscriptionBuilder withSubscriptionPolicy(SubscriptionPolicy subscriptionPolicy) { + this.serialSubscriptionPolicy = subscriptionPolicy; + this.deliveryType = DeliveryType.SERIAL; + return this; + } + + public SubscriptionBuilder withRequestTimeout(int timeout) { + SubscriptionPolicy policy = this.serialSubscriptionPolicy; + this.serialSubscriptionPolicy = + SubscriptionPolicy.Builder.subscriptionPolicy() + .withRate(policy.getRate()) + .withMessageTtl(policy.getMessageTtl()) + .withMessageBackoff(policy.getMessageBackoff()) + .withRequestTimeout(timeout) + .build(); + return this; + } + + public SubscriptionBuilder withTrackingMode(TrackingMode trackingMode) { + this.trackingMode = trackingMode; + this.trackingEnabled = trackingMode != TrackingMode.TRACKING_OFF; + return this; + } + + public SubscriptionBuilder withHttp2Enabled(boolean http2Enabled) { + this.http2Enabled = http2Enabled; + return this; + } + + public SubscriptionBuilder withProfilingEnabled(boolean profilingEnabled) { + this.profilingEnabled = profilingEnabled; + return this; + } + + public SubscriptionBuilder withProfilingThresholdMs(long profilingThresholdMs) { + this.profilingThresholdMs = profilingThresholdMs; + return this; + } + + public SubscriptionBuilder withOwner(OwnerId owner) { + this.owner = owner; + return this; + } + + public SubscriptionBuilder withMonitoringDetails(MonitoringDetails monitoringDetails) { + this.monitoringDetails = monitoringDetails; + return this; + } + + public SubscriptionBuilder withDeliveryType(DeliveryType deliveryType) { + this.deliveryType = deliveryType; + return this; + } + + public SubscriptionBuilder withContentType(ContentType contentType) { + this.contentType = contentType; + return this; + } + + public SubscriptionBuilder withFilter(MessageFilterSpecification filter) { + this.filters.add(filter); + return this; + } + + public SubscriptionBuilder withMode(SubscriptionMode mode) { + this.mode = mode; + return this; + } + + public SubscriptionBuilder withHeader(String name, String value) { + this.headers.add(new Header(name, value)); + return this; + } + + public SubscriptionBuilder withEndpointAddressResolverMetadata( + EndpointAddressResolverMetadata metadata) { + this.metadata = metadata; + return this; + } + + public SubscriptionBuilder withOAuthPolicy(SubscriptionOAuthPolicy oAuthPolicy) { + this.oAuthPolicy = oAuthPolicy; + return this; + } + + public SubscriptionBuilder withAttachingIdentityHeadersEnabled( + boolean attachingIdentityHeadersEnabled) { + this.attachingIdentityHeadersEnabled = attachingIdentityHeadersEnabled; + return this; + } + + public SubscriptionBuilder withAutoDeleteWithTopicEnabled(boolean autoDeleteWithTopicEnabled) { + this.autoDeleteWithTopicEnabled = autoDeleteWithTopicEnabled; + return this; + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/builder/TopicBuilder.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/builder/TopicBuilder.java index cc15900358..8d54d6aa86 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/builder/TopicBuilder.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/builder/TopicBuilder.java @@ -1,5 +1,12 @@ package pl.allegro.tech.hermes.test.helper.builder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import pl.allegro.tech.hermes.api.ContentType; import pl.allegro.tech.hermes.api.OfflineRetentionTime; import pl.allegro.tech.hermes.api.OwnerId; @@ -11,195 +18,199 @@ import pl.allegro.tech.hermes.api.TopicLabel; import pl.allegro.tech.hermes.api.TopicName; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - public class TopicBuilder { - private static final AtomicLong sequence = new AtomicLong(); - - private final TopicName name; - - private String description = "description"; - - private OwnerId owner = new OwnerId("Plaintext", "some team"); - - private boolean jsonToAvroDryRunEnabled = false; - - private Topic.Ack ack = Topic.Ack.LEADER; + private static final AtomicLong sequence = new AtomicLong(); - private boolean fallbackToRemoteDatacenterEnabled = false; + private final TopicName name; - private PublishingChaosPolicy chaos = PublishingChaosPolicy.disabled(); + private String description = "description"; - private ContentType contentType = ContentType.JSON; + private OwnerId owner = new OwnerId("Plaintext", "some team"); - private RetentionTime retentionTime = RetentionTime.of(1, TimeUnit.DAYS); + private boolean jsonToAvroDryRunEnabled = false; - private boolean trackingEnabled = false; + private Topic.Ack ack = Topic.Ack.LEADER; - private boolean migratedFromJsonType = false; + private boolean fallbackToRemoteDatacenterEnabled = false; - private boolean schemaIdAwareSerialization = false; + private PublishingChaosPolicy chaos = PublishingChaosPolicy.disabled(); - private int maxMessageSize = 1024 * 1024; + private ContentType contentType = ContentType.JSON; - private final List publishers = new ArrayList<>(); + private RetentionTime retentionTime = RetentionTime.of(1, TimeUnit.DAYS); - private boolean authEnabled = false; + private boolean trackingEnabled = false; - private boolean unauthenticatedAccessEnabled = true; + private boolean migratedFromJsonType = false; - private boolean subscribingRestricted = false; + private boolean schemaIdAwareSerialization = false; - private TopicDataOfflineStorage offlineStorage = TopicDataOfflineStorage.defaultOfflineStorage(); + private int maxMessageSize = 1024 * 1024; - private Set labels = Collections.emptySet(); + private final List publishers = new ArrayList<>(); - private TopicBuilder(TopicName topicName) { - this.name = topicName; - } + private boolean authEnabled = false; - public static TopicBuilder topicWithRandomName() { - return topicWithRandomNameEndedWith(""); - } + private boolean unauthenticatedAccessEnabled = true; - public static TopicBuilder topicWithRandomNameContaining(String string) { - return topic( - TopicBuilder.class.getSimpleName() + "Group" + sequence.incrementAndGet(), - TopicBuilder.class.getSimpleName() + "Topic" + string + sequence.incrementAndGet() - ); - } + private boolean subscribingRestricted = false; - public static TopicBuilder topicWithRandomNameEndedWith(String suffix) { - return topic( - TopicBuilder.class.getSimpleName() + "Group" + sequence.incrementAndGet(), - TopicBuilder.class.getSimpleName() + "Topic" + sequence.incrementAndGet() + suffix - ); - } + private TopicDataOfflineStorage offlineStorage = TopicDataOfflineStorage.defaultOfflineStorage(); - public static TopicBuilder randomTopic(String group, String topicNamePrefix) { - return topic(group, topicNamePrefix + "-" + UUID.randomUUID()); - } + private Set labels = Collections.emptySet(); - public static TopicBuilder topic(TopicName topicName) { - return new TopicBuilder(topicName); - } + private TopicBuilder(TopicName topicName) { + this.name = topicName; + } - public static TopicBuilder topic(String groupName, String topicName) { - return new TopicBuilder(new TopicName(groupName, topicName)); - } + public static TopicBuilder topicWithRandomName() { + return topicWithRandomNameEndedWith(""); + } - public static TopicBuilder topic(String qualifiedName) { - return new TopicBuilder(TopicName.fromQualifiedName(qualifiedName)); - } + public static TopicBuilder topicWithRandomNameContaining(String string) { + return topic( + TopicBuilder.class.getSimpleName() + "Group" + sequence.incrementAndGet(), + TopicBuilder.class.getSimpleName() + "Topic" + string + sequence.incrementAndGet()); + } - public Topic build() { - return new Topic( - name, description, owner, retentionTime, migratedFromJsonType, ack, fallbackToRemoteDatacenterEnabled, - chaos, trackingEnabled, contentType, jsonToAvroDryRunEnabled, schemaIdAwareSerialization, maxMessageSize, - new PublishingAuth(publishers, authEnabled, unauthenticatedAccessEnabled), subscribingRestricted, - offlineStorage, labels, null, null - ); - } + public static TopicBuilder topicWithRandomNameEndedWith(String suffix) { + return topic( + TopicBuilder.class.getSimpleName() + "Group" + sequence.incrementAndGet(), + TopicBuilder.class.getSimpleName() + "Topic" + sequence.incrementAndGet() + suffix); + } - public TopicBuilder withDescription(String description) { - this.description = description; - return this; - } + public static TopicBuilder randomTopic(String group, String topicNamePrefix) { + return topic(group, topicNamePrefix + "-" + UUID.randomUUID()); + } - public TopicBuilder withOwner(OwnerId owner) { - this.owner = owner; - return this; - } + public static TopicBuilder topic(TopicName topicName) { + return new TopicBuilder(topicName); + } - public TopicBuilder withRetentionTime(RetentionTime retentionTime) { - this.retentionTime = retentionTime; - return this; - } + public static TopicBuilder topic(String groupName, String topicName) { + return new TopicBuilder(new TopicName(groupName, topicName)); + } - public TopicBuilder withRetentionTime(int retentionTime, TimeUnit unit) { - this.retentionTime = new RetentionTime(retentionTime, unit); - return this; - } + public static TopicBuilder topic(String qualifiedName) { + return new TopicBuilder(TopicName.fromQualifiedName(qualifiedName)); + } - public TopicBuilder withJsonToAvroDryRun(boolean enabled) { - this.jsonToAvroDryRunEnabled = enabled; - return this; - } + public Topic build() { + return new Topic( + name, + description, + owner, + retentionTime, + migratedFromJsonType, + ack, + fallbackToRemoteDatacenterEnabled, + chaos, + trackingEnabled, + contentType, + jsonToAvroDryRunEnabled, + schemaIdAwareSerialization, + maxMessageSize, + new PublishingAuth(publishers, authEnabled, unauthenticatedAccessEnabled), + subscribingRestricted, + offlineStorage, + labels, + null, + null); + } - public TopicBuilder withAck(Topic.Ack ack) { - this.ack = ack; - return this; - } - - public TopicBuilder withFallbackToRemoteDatacenterEnabled() { - this.fallbackToRemoteDatacenterEnabled = true; - return this; - } - - public TopicBuilder withTrackingEnabled(boolean enabled) { - this.trackingEnabled = enabled; - return this; - } - - public TopicBuilder withContentType(ContentType contentType) { - this.contentType = contentType; - return this; - } - - public TopicBuilder migratedFromJsonType() { - this.migratedFromJsonType = true; - return this; - } - - public TopicBuilder withSchemaIdAwareSerialization() { - this.schemaIdAwareSerialization = true; - return this; - } - - public TopicBuilder withSubscribingRestricted() { - this.subscribingRestricted = true; - return this; - } - - public TopicBuilder withMaxMessageSize(int size) { - this.maxMessageSize = size; - return this; - } - - public TopicBuilder withPublisher(String serviceName) { - this.publishers.add(serviceName); - return this; - } - - public TopicBuilder withAuthEnabled() { - this.authEnabled = true; - return this; - } - - public TopicBuilder withUnauthenticatedAccessDisabled() { - this.unauthenticatedAccessEnabled = false; - return this; - } - - public TopicBuilder withOfflineStorage(int days) { - this.offlineStorage = new TopicDataOfflineStorage(true, OfflineRetentionTime.of(days)); - return this; - } - - public TopicBuilder withLabels(Set labels) { - this.labels = labels; - return this; - } - - public TopicBuilder withPublishingChaosPolicy(PublishingChaosPolicy chaos) { - this.chaos = chaos; - return this; - } + public TopicBuilder withDescription(String description) { + this.description = description; + return this; + } + + public TopicBuilder withOwner(OwnerId owner) { + this.owner = owner; + return this; + } + + public TopicBuilder withRetentionTime(RetentionTime retentionTime) { + this.retentionTime = retentionTime; + return this; + } + + public TopicBuilder withRetentionTime(int retentionTime, TimeUnit unit) { + this.retentionTime = new RetentionTime(retentionTime, unit); + return this; + } + + public TopicBuilder withJsonToAvroDryRun(boolean enabled) { + this.jsonToAvroDryRunEnabled = enabled; + return this; + } + + public TopicBuilder withAck(Topic.Ack ack) { + this.ack = ack; + return this; + } + + public TopicBuilder withFallbackToRemoteDatacenterEnabled() { + this.fallbackToRemoteDatacenterEnabled = true; + return this; + } + + public TopicBuilder withTrackingEnabled(boolean enabled) { + this.trackingEnabled = enabled; + return this; + } + + public TopicBuilder withContentType(ContentType contentType) { + this.contentType = contentType; + return this; + } + + public TopicBuilder migratedFromJsonType() { + this.migratedFromJsonType = true; + return this; + } + + public TopicBuilder withSchemaIdAwareSerialization() { + this.schemaIdAwareSerialization = true; + return this; + } + + public TopicBuilder withSubscribingRestricted() { + this.subscribingRestricted = true; + return this; + } + + public TopicBuilder withMaxMessageSize(int size) { + this.maxMessageSize = size; + return this; + } + + public TopicBuilder withPublisher(String serviceName) { + this.publishers.add(serviceName); + return this; + } + + public TopicBuilder withAuthEnabled() { + this.authEnabled = true; + return this; + } + + public TopicBuilder withUnauthenticatedAccessDisabled() { + this.unauthenticatedAccessEnabled = false; + return this; + } + + public TopicBuilder withOfflineStorage(int days) { + this.offlineStorage = new TopicDataOfflineStorage(true, OfflineRetentionTime.of(days)); + return this; + } + + public TopicBuilder withLabels(Set labels) { + this.labels = labels; + return this; + } + + public TopicBuilder withPublishingChaosPolicy(PublishingChaosPolicy chaos) { + this.chaos = chaos; + return this; + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/cache/FakeTicker.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/cache/FakeTicker.java index 2536143568..279c582435 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/cache/FakeTicker.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/cache/FakeTicker.java @@ -1,18 +1,17 @@ package pl.allegro.tech.hermes.test.helper.cache; import com.google.common.base.Ticker; - import java.time.Duration; public class FakeTicker extends Ticker { - private long currentNanos = 0; + private long currentNanos = 0; - @Override - public long read() { - return currentNanos; - } + @Override + public long read() { + return currentNanos; + } - public void advance(Duration duration) { - currentNanos += duration.toNanos(); - } + public void advance(Duration duration) { + currentNanos += duration.toNanos(); + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/OAuth2AuthenticationFeature.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/OAuth2AuthenticationFeature.java index de7666924f..bcc65a3484 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/OAuth2AuthenticationFeature.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/OAuth2AuthenticationFeature.java @@ -1,20 +1,22 @@ package pl.allegro.tech.hermes.test.helper.client; -import java.io.IOException; -import java.util.function.Function; import jakarta.ws.rs.client.ClientRequestContext; import jakarta.ws.rs.client.ClientRequestFilter; +import java.io.IOException; +import java.util.function.Function; public class OAuth2AuthenticationFeature implements ClientRequestFilter { - private final Function authTokenSupplier; + private final Function authTokenSupplier; - public OAuth2AuthenticationFeature(Function authTokenSupplier) { - this.authTokenSupplier = authTokenSupplier; - } + public OAuth2AuthenticationFeature(Function authTokenSupplier) { + this.authTokenSupplier = authTokenSupplier; + } - @Override - public void filter(ClientRequestContext requestContext) throws IOException { - requestContext.getHeaders().add("Authorization", String.format("Token %s", authTokenSupplier.apply(requestContext))); - } + @Override + public void filter(ClientRequestContext requestContext) throws IOException { + requestContext + .getHeaders() + .add("Authorization", String.format("Token %s", authTokenSupplier.apply(requestContext))); + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/PasswordAuthenticationFeature.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/PasswordAuthenticationFeature.java index ec871bdddf..d20f726ce6 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/PasswordAuthenticationFeature.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/PasswordAuthenticationFeature.java @@ -1,19 +1,19 @@ package pl.allegro.tech.hermes.test.helper.client; -import java.io.IOException; import jakarta.ws.rs.client.ClientRequestContext; import jakarta.ws.rs.client.ClientRequestFilter; +import java.io.IOException; public class PasswordAuthenticationFeature implements ClientRequestFilter { - private final String password; + private final String password; - public PasswordAuthenticationFeature(String password) { - this.password = password; - } + public PasswordAuthenticationFeature(String password) { + this.password = password; + } - @Override - public void filter(ClientRequestContext requestContext) throws IOException { - requestContext.getHeaders().add("Hermes-Admin-Password", password); - } + @Override + public void filter(ClientRequestContext requestContext) throws IOException { + requestContext.getHeaders().add("Hermes-Admin-Password", password); + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/integration/ConsumerTestClient.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/integration/ConsumerTestClient.java index 84a908f6dd..a901134c1f 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/integration/ConsumerTestClient.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/integration/ConsumerTestClient.java @@ -5,35 +5,33 @@ class ConsumerTestClient { - private static final String STATUS_SUBSCRIPTIONS = "/status/subscriptions"; - - private static final String METRICS_PATH = "/status/prometheus"; - - private final WebTestClient webTestClient; - private final String consumerContainerUrl; - - public ConsumerTestClient(int consumerPort) { - this.consumerContainerUrl = "http://localhost:" + consumerPort; - this.webTestClient = WebTestClient - .bindToServer() - .baseUrl(consumerContainerUrl) - .codecs(configurer -> configurer - .defaultCodecs() - .maxInMemorySize(16 * 1024 * 1024)) - .build(); - } - - public WebTestClient.ResponseSpec getRunningSubscriptionsStatus() { - return webTestClient.get().uri(UriBuilder.fromUri(consumerContainerUrl) - .path(STATUS_SUBSCRIPTIONS) - .build()) - .exchange(); - } - - public WebTestClient.ResponseSpec getMetrics() { - return webTestClient.get().uri(UriBuilder.fromUri(consumerContainerUrl) - .path(METRICS_PATH) - .build()) - .exchange(); - } + private static final String STATUS_SUBSCRIPTIONS = "/status/subscriptions"; + + private static final String METRICS_PATH = "/status/prometheus"; + + private final WebTestClient webTestClient; + private final String consumerContainerUrl; + + public ConsumerTestClient(int consumerPort) { + this.consumerContainerUrl = "http://localhost:" + consumerPort; + this.webTestClient = + WebTestClient.bindToServer() + .baseUrl(consumerContainerUrl) + .codecs(configurer -> configurer.defaultCodecs().maxInMemorySize(16 * 1024 * 1024)) + .build(); + } + + public WebTestClient.ResponseSpec getRunningSubscriptionsStatus() { + return webTestClient + .get() + .uri(UriBuilder.fromUri(consumerContainerUrl).path(STATUS_SUBSCRIPTIONS).build()) + .exchange(); + } + + public WebTestClient.ResponseSpec getMetrics() { + return webTestClient + .get() + .uri(UriBuilder.fromUri(consumerContainerUrl).path(METRICS_PATH).build()) + .exchange(); + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/integration/FrontendSlowClient.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/integration/FrontendSlowClient.java index 26684c4475..1ccd877810 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/integration/FrontendSlowClient.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/integration/FrontendSlowClient.java @@ -1,8 +1,5 @@ package pl.allegro.tech.hermes.test.helper.client.integration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.BufferedReader; import java.io.DataOutputStream; import java.io.IOException; @@ -12,110 +9,135 @@ import java.net.SocketTimeoutException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class FrontendSlowClient { - public static final String MSG_BODY = "{\"field\": \"value\"}"; - - public String msgHeadWithContentLenght(String topicName) { - return "POST /topics/" + topicName + " HTTP/1.1\n" - + "Host: " + host + ":" + port + "\n" - + "Content-Type: application/json\n" - + "Content-Length: " + MSG_BODY.length() + "\r\n\r\n"; - } - - private String msgHeadWithChunkedEncoding(String topicName) { - return "POST /topics/" + topicName + " HTTP/1.1\n" - + "Host: " + host + ":" + port + "\n" - + "Content-Type: application/json\n" - + "Transfer-Encoding: chunked \r\n\r\n"; + public static final String MSG_BODY = "{\"field\": \"value\"}"; + + public String msgHeadWithContentLenght(String topicName) { + return "POST /topics/" + + topicName + + " HTTP/1.1\n" + + "Host: " + + host + + ":" + + port + + "\n" + + "Content-Type: application/json\n" + + "Content-Length: " + + MSG_BODY.length() + + "\r\n\r\n"; + } + + private String msgHeadWithChunkedEncoding(String topicName) { + return "POST /topics/" + + topicName + + " HTTP/1.1\n" + + "Host: " + + host + + ":" + + port + + "\n" + + "Content-Type: application/json\n" + + "Transfer-Encoding: chunked \r\n\r\n"; + } + + private static final Logger LOGGER = LoggerFactory.getLogger(FrontendSlowClient.class); + + private final String host; + private final int port; + + public FrontendSlowClient(String host, int port) { + this.host = host; + this.port = port; + } + + private final ExecutorService executor = Executors.newSingleThreadExecutor(); + + public String slowEvent( + int clientTimeout, + int pauseTimeBetweenChunks, + int delayBeforeSendingFirstData, + String topicName, + boolean chunkedEncoding) + throws IOException, InterruptedException { + + Socket clientSocket = new Socket(host, port); + + Thread.sleep(delayBeforeSendingFirstData); + + DataOutputStream outToServer = new DataOutputStream(clientSocket.getOutputStream()); + BufferedReader inFromServer = + new BufferedReader(new InputStreamReader(clientSocket.getInputStream())); + + clientSocket.setSoTimeout(clientTimeout); + + if (chunkedEncoding) { + outToServer.writeBytes(msgHeadWithChunkedEncoding(topicName)); + outToServer.flush(); + } else { + outToServer.writeBytes(msgHeadWithContentLenght(topicName)); + outToServer.flush(); } - - private static final Logger LOGGER = LoggerFactory.getLogger(FrontendSlowClient.class); - - private final String host; - private final int port; - - public FrontendSlowClient(String host, int port) { - this.host = host; - this.port = port; + sendBodyInChunks(outToServer, MSG_BODY, pauseTimeBetweenChunks, chunkedEncoding); + + String response; + try { + response = readResponse(inFromServer); + LOGGER.info("Response: {}", response); + } catch (SocketTimeoutException e) { + LOGGER.warn("client timeout"); + clientSocket.close(); + throw e; } - private final ExecutorService executor = Executors.newSingleThreadExecutor(); - - public String slowEvent(int clientTimeout, int pauseTimeBetweenChunks, int delayBeforeSendingFirstData, - String topicName, boolean chunkedEncoding) - throws IOException, InterruptedException { - - Socket clientSocket = new Socket(host, port); - - Thread.sleep(delayBeforeSendingFirstData); - - DataOutputStream outToServer = new DataOutputStream(clientSocket.getOutputStream()); - BufferedReader inFromServer = new BufferedReader(new InputStreamReader(clientSocket.getInputStream())); - - clientSocket.setSoTimeout(clientTimeout); - - if (chunkedEncoding) { - outToServer.writeBytes(msgHeadWithChunkedEncoding(topicName)); - outToServer.flush(); - } else { - outToServer.writeBytes(msgHeadWithContentLenght(topicName)); - outToServer.flush(); - } - sendBodyInChunks(outToServer, MSG_BODY, pauseTimeBetweenChunks, chunkedEncoding); - - String response; - try { - response = readResponse(inFromServer); - LOGGER.info("Response: {}", response); - } catch (SocketTimeoutException e) { - LOGGER.warn("client timeout"); - clientSocket.close(); - throw e; - } - - clientSocket.close(); - - return response; - } - - private void sendBodyInChunks(DataOutputStream outToServer, String msgBody, int pauseTime, boolean encoded) { - executor.execute(() -> { - try { - for (int index = 0; index < msgBody.length(); index++) { - outToServer.writeBytes(prepareChunk(msgBody, index, encoded)); - outToServer.flush(); - LOGGER.info("Sent chunk"); - if (pauseTime > 0) { - Thread.sleep(pauseTime); - } - } - if (encoded) { - outToServer.writeBytes("0\r\n\r\n"); - outToServer.flush(); - LOGGER.info("Finished chunked encoding"); - } - } catch (SocketException e) { - LOGGER.warn("Socket closed"); - } catch (InterruptedException | IOException e) { - LOGGER.error("Something went wrong while sending data", e); + clientSocket.close(); + + return response; + } + + private void sendBodyInChunks( + DataOutputStream outToServer, String msgBody, int pauseTime, boolean encoded) { + executor.execute( + () -> { + try { + for (int index = 0; index < msgBody.length(); index++) { + outToServer.writeBytes(prepareChunk(msgBody, index, encoded)); + outToServer.flush(); + LOGGER.info("Sent chunk"); + if (pauseTime > 0) { + Thread.sleep(pauseTime); + } + } + if (encoded) { + outToServer.writeBytes("0\r\n\r\n"); + outToServer.flush(); + LOGGER.info("Finished chunked encoding"); } + } catch (SocketException e) { + LOGGER.warn("Socket closed"); + } catch (InterruptedException | IOException e) { + LOGGER.error("Something went wrong while sending data", e); + } }); - } + } - private String prepareChunk(String msg, int index, boolean encoded) { - return encoded ? String.format("1\n%c\r\n", msg.charAt(index)) : String.valueOf(msg.charAt(index)); - } - - private String readResponse(BufferedReader bufferedReader) throws IOException { - String line; - StringBuilder response = new StringBuilder(); + private String prepareChunk(String msg, int index, boolean encoded) { + return encoded + ? String.format("1\n%c\r\n", msg.charAt(index)) + : String.valueOf(msg.charAt(index)); + } - while (!(line = bufferedReader.readLine()).isEmpty()) { - response.append(line); - } + private String readResponse(BufferedReader bufferedReader) throws IOException { + String line; + StringBuilder response = new StringBuilder(); - return response.toString(); + while (!(line = bufferedReader.readLine()).isEmpty()) { + response.append(line); } + + return response.toString(); + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/integration/FrontendTestClient.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/integration/FrontendTestClient.java index bb29c99df1..e8c425effa 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/integration/FrontendTestClient.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/integration/FrontendTestClient.java @@ -1,9 +1,18 @@ package pl.allegro.tech.hermes.test.helper.client.integration; +import static jakarta.ws.rs.client.ClientBuilder.newClient; +import static org.awaitility.Awaitility.waitAtMost; +import static org.glassfish.jersey.client.ClientProperties.REQUEST_ENTITY_PROCESSING; +import static org.glassfish.jersey.client.RequestEntityProcessing.CHUNKED; +import static pl.allegro.tech.hermes.api.AvroMediaType.AVRO_BINARY; + import jakarta.ws.rs.client.Client; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.UriBuilder; +import java.io.IOException; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicInteger; import org.glassfish.jersey.client.ClientConfig; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; @@ -12,200 +21,199 @@ import org.springframework.util.MultiValueMap; import reactor.core.publisher.Mono; -import java.io.IOException; -import java.time.Duration; -import java.util.concurrent.atomic.AtomicInteger; - -import static jakarta.ws.rs.client.ClientBuilder.newClient; -import static org.awaitility.Awaitility.waitAtMost; -import static org.glassfish.jersey.client.ClientProperties.REQUEST_ENTITY_PROCESSING; -import static org.glassfish.jersey.client.RequestEntityProcessing.CHUNKED; -import static pl.allegro.tech.hermes.api.AvroMediaType.AVRO_BINARY; - public class FrontendTestClient { - private static final String TOPIC_PATH = "/topics/{topicName}"; - private static final String STATUS_HEALTH_PATH = "/status/health"; - private static final String STATUS_READY_PATH = "/status/ready"; - private static final String STATUS_PING_PATH = "/status/ping"; - private static final String METRICS_PATH = "/status/prometheus"; - - private final WebTestClient webTestClient; - private final FrontendSlowClient slowTestClient; - private final String frontendContainerUrl; - private final Client chunkedClient; - - public FrontendTestClient(int frontendPort) { - this.frontendContainerUrl = "http://localhost:" + frontendPort; - this.webTestClient = WebTestClient - .bindToServer(new JdkClientHttpConnector()) - .baseUrl(frontendContainerUrl) - .codecs(configurer -> configurer - .defaultCodecs() - .maxInMemorySize(16 * 1024 * 1024)) - .build(); - this.slowTestClient = new FrontendSlowClient("localhost", frontendPort); - this.chunkedClient = newClient(new ClientConfig().property(REQUEST_ENTITY_PROCESSING, CHUNKED)); - } - - public int publishUntilSuccess(String topicQualifiedName, String body) { - AtomicInteger attempts = new AtomicInteger(0); - waitAtMost(Duration.ofSeconds(10)) - .untilAsserted(() -> { - attempts.getAndIncrement(); - publish(topicQualifiedName, body).expectStatus().isCreated(); - }); - return attempts.get(); - } - - public Response publishChunked(String topicQualifiedName, String body) { - return chunkedClient.target(UriBuilder - .fromUri(frontendContainerUrl) - .path(TOPIC_PATH) - .build(topicQualifiedName)) - .request().post(Entity.text(body)); - - } - - public int publishUntilSuccess(String topicQualifiedName, String body, MultiValueMap headers) { - AtomicInteger attempts = new AtomicInteger(0); - waitAtMost(Duration.ofSeconds(10)) - .untilAsserted(() -> { - attempts.getAndIncrement(); - publish(topicQualifiedName, body, headers).expectStatus().isCreated(); - }); - return attempts.get(); - } - - public int publishJSONUntilSuccess(String topicQualifiedName, String body, MultiValueMap headers) { - AtomicInteger attempts = new AtomicInteger(0); - waitAtMost(Duration.ofSeconds(10)) - .untilAsserted(() -> { - attempts.getAndIncrement(); - publishJSON(topicQualifiedName, body, headers).expectStatus().isCreated(); - }); - return attempts.get(); - } - - public int publishAvroUntilSuccess(String topicQualifiedName, byte[] body) { - return publishAvroUntilSuccess(topicQualifiedName, body, new HttpHeaders()); - } - - public int publishAvroUntilSuccess(String topicQualifiedName, byte[] body, MultiValueMap headers) { - AtomicInteger attempts = new AtomicInteger(0); - waitAtMost(Duration.ofSeconds(10)) - .untilAsserted(() -> { - attempts.getAndIncrement(); - publishAvro(topicQualifiedName, body, headers).expectStatus().isCreated(); - }); - return attempts.get(); - } - - public int publishUntilStatus(String topicQualifiedName, String body, int statusCode) { - AtomicInteger attempts = new AtomicInteger(0); - waitAtMost(Duration.ofSeconds(10)) - .untilAsserted(() -> { - attempts.getAndIncrement(); - publish(topicQualifiedName, body).expectStatus().isEqualTo(statusCode); - }); - return attempts.get(); - } - - public WebTestClient.ResponseSpec publish(String topicQualifiedName, String body) { - return webTestClient.post().uri(UriBuilder - .fromUri(frontendContainerUrl) - .path(TOPIC_PATH) - .build(topicQualifiedName)) - .body(Mono.just(body), String.class) - .exchange(); - } - - public WebTestClient.ResponseSpec publish(String topicQualifiedName, String body, MultiValueMap headers) { - return webTestClient.post().uri(UriBuilder - .fromUri(frontendContainerUrl) - .path(TOPIC_PATH) - .build(topicQualifiedName)) - .body(Mono.just(body), String.class) - .headers(requestHeaders -> requestHeaders.addAll(headers)) - .exchange(); - } - - public WebTestClient.ResponseSpec publish(String topicQualifiedName, byte[] body, MultiValueMap headers) { - return webTestClient.post().uri(UriBuilder - .fromUri(frontendContainerUrl) - .path(TOPIC_PATH) - .build(topicQualifiedName)) - .body(Mono.just(body), byte[].class) - .headers(requestHeaders -> requestHeaders.addAll(headers)) - .exchange(); - } - - WebTestClient.ResponseSpec publishJSON(String topicQualifiedName, String body, MultiValueMap headers) { - return webTestClient.post().uri(UriBuilder - .fromUri(frontendContainerUrl) - .path(TOPIC_PATH) - .build(topicQualifiedName)) - .contentType(MediaType.APPLICATION_JSON) - .body(Mono.just(body), String.class) - .headers(requestHeaders -> requestHeaders.addAll(headers)) - .exchange(); - } - - public WebTestClient.ResponseSpec publishAvro(String topicQualifiedName, byte[] body, MultiValueMap headers) { - return webTestClient.post().uri(UriBuilder - .fromUri(frontendContainerUrl) - .path(TOPIC_PATH) - .build(topicQualifiedName)) - .header("Content-Type", AVRO_BINARY) - .headers(requestHeaders -> requestHeaders.addAll(headers)) - .body(Mono.just(body), byte[].class) - .exchange(); - } - - WebTestClient.ResponseSpec publishWithHeaders(String topicQualifiedName, String body, MultiValueMap headers) { - return webTestClient.post().uri(UriBuilder - .fromUri(frontendContainerUrl) - .path(TOPIC_PATH) - .build(topicQualifiedName)) - .headers(it -> it.addAll(headers)) - .body(Mono.just(body), String.class) - .exchange(); - } - - String publishSlowly(int clientTimeout, int pauseTimeBetweenChunks, int delayBeforeSendingFirstData, - String topicName, boolean chunkedEncoding) throws IOException, InterruptedException { - return slowTestClient.slowEvent(clientTimeout, pauseTimeBetweenChunks, delayBeforeSendingFirstData, topicName, chunkedEncoding); - } - - public WebTestClient.ResponseSpec getStatusHealth() { - return webTestClient.get().uri(UriBuilder - .fromUri(frontendContainerUrl) - .path(STATUS_HEALTH_PATH) - .build()) - .exchange(); - } - - public WebTestClient.ResponseSpec getStatusReady() { - return webTestClient.get().uri(UriBuilder - .fromUri(frontendContainerUrl) - .path(STATUS_READY_PATH) - .build()) - .exchange(); - } - - public WebTestClient.ResponseSpec getStatusPing() { - return webTestClient.get().uri(UriBuilder - .fromUri(frontendContainerUrl) - .path(STATUS_PING_PATH) - .build()) - .exchange(); - } - - public WebTestClient.ResponseSpec getMetrics() { - return webTestClient.get().uri(UriBuilder - .fromUri(frontendContainerUrl) - .path(METRICS_PATH) - .build()) - .exchange(); - } + private static final String TOPIC_PATH = "/topics/{topicName}"; + private static final String STATUS_HEALTH_PATH = "/status/health"; + private static final String STATUS_READY_PATH = "/status/ready"; + private static final String STATUS_PING_PATH = "/status/ping"; + private static final String METRICS_PATH = "/status/prometheus"; + + private final WebTestClient webTestClient; + private final FrontendSlowClient slowTestClient; + private final String frontendContainerUrl; + private final Client chunkedClient; + + public FrontendTestClient(int frontendPort) { + this.frontendContainerUrl = "http://localhost:" + frontendPort; + this.webTestClient = + WebTestClient.bindToServer(new JdkClientHttpConnector()) + .baseUrl(frontendContainerUrl) + .codecs(configurer -> configurer.defaultCodecs().maxInMemorySize(16 * 1024 * 1024)) + .build(); + this.slowTestClient = new FrontendSlowClient("localhost", frontendPort); + this.chunkedClient = newClient(new ClientConfig().property(REQUEST_ENTITY_PROCESSING, CHUNKED)); + } + + public int publishUntilSuccess(String topicQualifiedName, String body) { + AtomicInteger attempts = new AtomicInteger(0); + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + attempts.getAndIncrement(); + publish(topicQualifiedName, body).expectStatus().isCreated(); + }); + return attempts.get(); + } + + public Response publishChunked(String topicQualifiedName, String body) { + return chunkedClient + .target(UriBuilder.fromUri(frontendContainerUrl).path(TOPIC_PATH).build(topicQualifiedName)) + .request() + .post(Entity.text(body)); + } + + public int publishUntilSuccess( + String topicQualifiedName, String body, MultiValueMap headers) { + AtomicInteger attempts = new AtomicInteger(0); + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + attempts.getAndIncrement(); + publish(topicQualifiedName, body, headers).expectStatus().isCreated(); + }); + return attempts.get(); + } + + public int publishJSONUntilSuccess( + String topicQualifiedName, String body, MultiValueMap headers) { + AtomicInteger attempts = new AtomicInteger(0); + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + attempts.getAndIncrement(); + publishJSON(topicQualifiedName, body, headers).expectStatus().isCreated(); + }); + return attempts.get(); + } + + public int publishAvroUntilSuccess(String topicQualifiedName, byte[] body) { + return publishAvroUntilSuccess(topicQualifiedName, body, new HttpHeaders()); + } + + public int publishAvroUntilSuccess( + String topicQualifiedName, byte[] body, MultiValueMap headers) { + AtomicInteger attempts = new AtomicInteger(0); + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + attempts.getAndIncrement(); + publishAvro(topicQualifiedName, body, headers).expectStatus().isCreated(); + }); + return attempts.get(); + } + + public int publishUntilStatus(String topicQualifiedName, String body, int statusCode) { + AtomicInteger attempts = new AtomicInteger(0); + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + attempts.getAndIncrement(); + publish(topicQualifiedName, body).expectStatus().isEqualTo(statusCode); + }); + return attempts.get(); + } + + public WebTestClient.ResponseSpec publish(String topicQualifiedName, String body) { + return webTestClient + .post() + .uri(UriBuilder.fromUri(frontendContainerUrl).path(TOPIC_PATH).build(topicQualifiedName)) + .body(Mono.just(body), String.class) + .exchange(); + } + + public WebTestClient.ResponseSpec publish( + String topicQualifiedName, String body, MultiValueMap headers) { + return webTestClient + .post() + .uri(UriBuilder.fromUri(frontendContainerUrl).path(TOPIC_PATH).build(topicQualifiedName)) + .body(Mono.just(body), String.class) + .headers(requestHeaders -> requestHeaders.addAll(headers)) + .exchange(); + } + + public WebTestClient.ResponseSpec publish( + String topicQualifiedName, byte[] body, MultiValueMap headers) { + return webTestClient + .post() + .uri(UriBuilder.fromUri(frontendContainerUrl).path(TOPIC_PATH).build(topicQualifiedName)) + .body(Mono.just(body), byte[].class) + .headers(requestHeaders -> requestHeaders.addAll(headers)) + .exchange(); + } + + WebTestClient.ResponseSpec publishJSON( + String topicQualifiedName, String body, MultiValueMap headers) { + return webTestClient + .post() + .uri(UriBuilder.fromUri(frontendContainerUrl).path(TOPIC_PATH).build(topicQualifiedName)) + .contentType(MediaType.APPLICATION_JSON) + .body(Mono.just(body), String.class) + .headers(requestHeaders -> requestHeaders.addAll(headers)) + .exchange(); + } + + public WebTestClient.ResponseSpec publishAvro( + String topicQualifiedName, byte[] body, MultiValueMap headers) { + return webTestClient + .post() + .uri(UriBuilder.fromUri(frontendContainerUrl).path(TOPIC_PATH).build(topicQualifiedName)) + .header("Content-Type", AVRO_BINARY) + .headers(requestHeaders -> requestHeaders.addAll(headers)) + .body(Mono.just(body), byte[].class) + .exchange(); + } + + WebTestClient.ResponseSpec publishWithHeaders( + String topicQualifiedName, String body, MultiValueMap headers) { + return webTestClient + .post() + .uri(UriBuilder.fromUri(frontendContainerUrl).path(TOPIC_PATH).build(topicQualifiedName)) + .headers(it -> it.addAll(headers)) + .body(Mono.just(body), String.class) + .exchange(); + } + + String publishSlowly( + int clientTimeout, + int pauseTimeBetweenChunks, + int delayBeforeSendingFirstData, + String topicName, + boolean chunkedEncoding) + throws IOException, InterruptedException { + return slowTestClient.slowEvent( + clientTimeout, + pauseTimeBetweenChunks, + delayBeforeSendingFirstData, + topicName, + chunkedEncoding); + } + + public WebTestClient.ResponseSpec getStatusHealth() { + return webTestClient + .get() + .uri(UriBuilder.fromUri(frontendContainerUrl).path(STATUS_HEALTH_PATH).build()) + .exchange(); + } + + public WebTestClient.ResponseSpec getStatusReady() { + return webTestClient + .get() + .uri(UriBuilder.fromUri(frontendContainerUrl).path(STATUS_READY_PATH).build()) + .exchange(); + } + + public WebTestClient.ResponseSpec getStatusPing() { + return webTestClient + .get() + .uri(UriBuilder.fromUri(frontendContainerUrl).path(STATUS_PING_PATH).build()) + .exchange(); + } + + public WebTestClient.ResponseSpec getMetrics() { + return webTestClient + .get() + .uri(UriBuilder.fromUri(frontendContainerUrl).path(METRICS_PATH).build()) + .exchange(); + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/integration/HermesInitHelper.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/integration/HermesInitHelper.java index d90c793d07..6dfb3b05dd 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/integration/HermesInitHelper.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/integration/HermesInitHelper.java @@ -1,5 +1,8 @@ package pl.allegro.tech.hermes.test.helper.client.integration; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.waitAtMost; + import java.time.Duration; import pl.allegro.tech.hermes.api.Group; import pl.allegro.tech.hermes.api.OAuthProvider; @@ -7,96 +10,88 @@ import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.api.TopicWithSchema; -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.waitAtMost; - public class HermesInitHelper { - private final ManagementTestClient managementTestClient; - - public HermesInitHelper(int managementPort) { - managementTestClient = new ManagementTestClient(managementPort); - } - - public HermesInitHelper(int managementPort, String defaultHeaderName, String defaultHeaderValue) { - managementTestClient = new ManagementTestClient(managementPort, defaultHeaderName, defaultHeaderValue); - } - - public Topic createTopic(Topic topic) { - createGroupIfMissing(Group.from(topic.getName().getGroupName())); - managementTestClient.createTopic(TopicWithSchema.topicWithSchema(topic, null)) - .expectStatus() - .is2xxSuccessful(); - waitUntilTopicCreated(topic.getQualifiedName()); - return topic; - } - - public Topic createTopicWithSchema(TopicWithSchema topic) { - createGroupIfMissing(Group.from(topic.getName().getGroupName())); - managementTestClient.createTopic(topic) - .expectStatus() - .is2xxSuccessful(); - waitUntilTopicCreated(topic.getQualifiedName()); - return topic; + private final ManagementTestClient managementTestClient; + + public HermesInitHelper(int managementPort) { + managementTestClient = new ManagementTestClient(managementPort); + } + + public HermesInitHelper(int managementPort, String defaultHeaderName, String defaultHeaderValue) { + managementTestClient = + new ManagementTestClient(managementPort, defaultHeaderName, defaultHeaderValue); + } + + public Topic createTopic(Topic topic) { + createGroupIfMissing(Group.from(topic.getName().getGroupName())); + managementTestClient + .createTopic(TopicWithSchema.topicWithSchema(topic, null)) + .expectStatus() + .is2xxSuccessful(); + waitUntilTopicCreated(topic.getQualifiedName()); + return topic; + } + + public Topic createTopicWithSchema(TopicWithSchema topic) { + createGroupIfMissing(Group.from(topic.getName().getGroupName())); + managementTestClient.createTopic(topic).expectStatus().is2xxSuccessful(); + waitUntilTopicCreated(topic.getQualifiedName()); + return topic; + } + + public Group createGroupIfMissing(Group group) { + if (managementTestClient.getGroups().contains(group.getGroupName())) { + return group; } - public Group createGroupIfMissing(Group group) { - if (managementTestClient.getGroups().contains(group.getGroupName())) { - return group; - } - - managementTestClient.createGroup(group) - .expectStatus() - .is2xxSuccessful(); - waitUntilGroupCreated(group.getGroupName()); - return group; - } - - public Group createGroup(Group group) { - managementTestClient.createGroup(group) - .expectStatus() - .is2xxSuccessful(); - waitUntilGroupCreated(group.getGroupName()); - return group; - } - - private void waitUntilGroupCreated(String groupName) { - waitAtMost(Duration.ofMinutes(1)) - .until(() -> managementTestClient.getGroups().contains(groupName)); - } - - private void waitUntilTopicCreated(String topicQualifiedName) { - waitAtMost(Duration.ofMinutes(1)) - .untilAsserted(() -> managementTestClient.getTopic(topicQualifiedName) - .expectStatus() - .is2xxSuccessful()); - } - - public Subscription createSubscription(Subscription subscription) { - managementTestClient.createSubscription(subscription) - .expectStatus() - .is2xxSuccessful(); - waitUntilSubscriptionIsActive(subscription); - return subscription; - } - - public void waitUntilSubscriptionIsActive(Subscription subscription) { - waitAtMost(Duration.ofSeconds(10)) - .untilAsserted(() -> { - Subscription sub = managementTestClient.getSubscription(subscription.getQualifiedTopicName(), subscription.getName()) - .expectStatus() - .is2xxSuccessful() - .expectBody(Subscription.class) - .returnResult() - .getResponseBody(); - assertThat(sub.getState()).isEqualTo(Subscription.State.ACTIVE); + managementTestClient.createGroup(group).expectStatus().is2xxSuccessful(); + waitUntilGroupCreated(group.getGroupName()); + return group; + } + + public Group createGroup(Group group) { + managementTestClient.createGroup(group).expectStatus().is2xxSuccessful(); + waitUntilGroupCreated(group.getGroupName()); + return group; + } + + private void waitUntilGroupCreated(String groupName) { + waitAtMost(Duration.ofMinutes(1)) + .until(() -> managementTestClient.getGroups().contains(groupName)); + } + + private void waitUntilTopicCreated(String topicQualifiedName) { + waitAtMost(Duration.ofMinutes(1)) + .untilAsserted( + () -> + managementTestClient.getTopic(topicQualifiedName).expectStatus().is2xxSuccessful()); + } + + public Subscription createSubscription(Subscription subscription) { + managementTestClient.createSubscription(subscription).expectStatus().is2xxSuccessful(); + waitUntilSubscriptionIsActive(subscription); + return subscription; + } + + public void waitUntilSubscriptionIsActive(Subscription subscription) { + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + Subscription sub = + managementTestClient + .getSubscription(subscription.getQualifiedTopicName(), subscription.getName()) + .expectStatus() + .is2xxSuccessful() + .expectBody(Subscription.class) + .returnResult() + .getResponseBody(); + assertThat(sub.getState()).isEqualTo(Subscription.State.ACTIVE); }); - } + } - public OAuthProvider createOAuthProvider(OAuthProvider provider) { - managementTestClient.createOAuthProvider(provider) - .expectStatus() - .is2xxSuccessful(); - return provider; - } + public OAuthProvider createOAuthProvider(OAuthProvider provider) { + managementTestClient.createOAuthProvider(provider).expectStatus().is2xxSuccessful(); + return provider; + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/integration/HermesTestClient.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/integration/HermesTestClient.java index 1f646f1e76..c682343ed0 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/integration/HermesTestClient.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/integration/HermesTestClient.java @@ -1,7 +1,15 @@ package pl.allegro.tech.hermes.test.helper.client.integration; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.waitAtMost; +import static pl.allegro.tech.hermes.test.helper.endpoint.TimeoutAdjuster.adjust; + import jakarta.ws.rs.core.Response; +import java.io.IOException; import java.time.Duration; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicLong; import org.springframework.http.HttpHeaders; import org.springframework.test.web.reactive.server.WebTestClient; import org.springframework.util.MultiValueMap; @@ -18,460 +26,527 @@ import pl.allegro.tech.hermes.api.TopicWithSchema; import pl.allegro.tech.hermes.consumers.supervisor.process.RunningSubscriptionStatus; -import java.io.IOException; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.atomic.AtomicLong; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.waitAtMost; -import static pl.allegro.tech.hermes.test.helper.endpoint.TimeoutAdjuster.adjust; - public class HermesTestClient { - private final ManagementTestClient managementTestClient; - private final FrontendTestClient frontendTestClient; - private final ConsumerTestClient consumerTestClient; - - public HermesTestClient(int managementPort, int frontendPort, int consumerPort) { - this.managementTestClient = new ManagementTestClient(managementPort); - this.frontendTestClient = new FrontendTestClient(frontendPort); - this.consumerTestClient = new ConsumerTestClient(consumerPort); - } - - public HermesTestClient(int managementPort, int frontendPort, int consumerPort, String defaultHeaderName, String defaultHeaderValue) { - this.managementTestClient = new ManagementTestClient(managementPort, defaultHeaderName, defaultHeaderValue); - this.frontendTestClient = new FrontendTestClient(frontendPort); - this.consumerTestClient = new ConsumerTestClient(consumerPort); - } - - public WebTestClient.ResponseSpec createGroup(Group group) { - return managementTestClient.createGroup(group); - } - - public WebTestClient.ResponseSpec createTopic(TopicWithSchema topicWithSchema) { - return managementTestClient.createTopic(topicWithSchema); - } - - public WebTestClient.ResponseSpec getTopicResponse(String topicQualifiedName) { - return managementTestClient.getTopic(topicQualifiedName); - } - - public WebTestClient.ResponseSpec saveSchema(String topicQualifiedName, String schema) { - return managementTestClient.saveSchema(topicQualifiedName, schema); - - } - - public void ensureSchemaSaved(String topicQualifiedName, boolean validate, String schema) { - managementTestClient.saveSchema(topicQualifiedName, validate, schema) - .expectStatus().isCreated(); - waitAtMost(adjust(Duration.ofMinutes(1))).untilAsserted(() -> - managementTestClient.getSchema(topicQualifiedName).expectStatus().isOk() - ); - } - - public WebTestClient.ResponseSpec saveSchema(String topicQualifiedName, boolean validate, String schema) { - return managementTestClient.saveSchema(topicQualifiedName, validate, schema); - } - - public WebTestClient.ResponseSpec getSchema(String topicQualifiedName) { - return managementTestClient.getSchema(topicQualifiedName); - } - - public WebTestClient.ResponseSpec deleteSchema(String topicQualifiedName) { - return managementTestClient.deleteSchema(topicQualifiedName); - } - - public WebTestClient.ResponseSpec updateTopic(String qualifiedTopicName, PatchData patch) { - return managementTestClient.updateTopic(qualifiedTopicName, patch); - } - - public Subscription getSubscription(String topicQualifiedName, String subscriptionName) { - return getSubscriptionResponse(topicQualifiedName, subscriptionName) - .expectStatus() - .is2xxSuccessful() - .expectBody(Subscription.class) - .returnResult() - .getResponseBody(); - } - - public WebTestClient.ResponseSpec getSubscriptionResponse(String topicQualifiedName, String subscriptionName) { - return managementTestClient.getSubscription(topicQualifiedName, subscriptionName); - } - - public WebTestClient.ResponseSpec getSubscriptionMetrics(String topicQualifiedName, String subscriptionName) { - return managementTestClient.getSubscriptionMetrics(topicQualifiedName, subscriptionName); - } - - public WebTestClient.ResponseSpec suspendSubscription(Topic topic, String subscription) { - return managementTestClient.updateSubscriptionState(topic, subscription, Subscription.State.SUSPENDED) - .expectStatus() - .is2xxSuccessful(); - } - - public void waitUntilSubscriptionActivated(String topicQualifiedName, String subscriptionName) { - waitAtMost(Duration.ofSeconds(10)) - .untilAsserted(() -> { - assertThat(managementTestClient.getSubscription(topicQualifiedName, subscriptionName) - .expectStatus() - .is2xxSuccessful() - .expectBody(Subscription.class) - .returnResult().getResponseBody().getState()) - .isEqualTo(Subscription.State.ACTIVE); - assertThat(managementTestClient.getConsumerGroupsDescription(topicQualifiedName, subscriptionName) - .expectBodyList(ConsumerGroup.class).returnResult().getResponseBody() - .get(0) - .getState()) - .isEqualTo("Stable"); - } - ); - } - - public void waitUntilSubscriptionSuspended(String topicQualifiedName, String subscriptionName) { - waitAtMost(Duration.ofSeconds(10)) - .untilAsserted(() -> { - assertThat(managementTestClient.getSubscription(topicQualifiedName, subscriptionName) - .expectStatus() - .is2xxSuccessful() - .expectBody(Subscription.class) - .returnResult().getResponseBody().getState()) - .isEqualTo(Subscription.State.SUSPENDED); - assertThat(managementTestClient.getConsumerGroupsDescription(topicQualifiedName, subscriptionName) - .expectBodyList(ConsumerGroup.class).returnResult().getResponseBody() - .get(0) - .getState()) - .isEqualTo("Empty"); - } - ); - } - - public void waitUntilConsumerCommitsOffset(String topicQualifiedName, String subscriptionName) { - long committedMessagesCount = calculateCommittedMessages(topicQualifiedName, subscriptionName); - waitAtMost(adjust(Duration.ofMinutes(1))).untilAsserted(() -> { - long currentCommittedMessagesCount = calculateCommittedMessages(topicQualifiedName, subscriptionName); - assertThat(currentCommittedMessagesCount).isGreaterThan(committedMessagesCount); - } - ); - } - - private long calculateCommittedMessages(String topicQualifiedName, String subscription) { - AtomicLong messagesCommittedCount = new AtomicLong(0); - List consumerGroups = getConsumerGroupsDescription(topicQualifiedName, subscription) - .expectBodyList(ConsumerGroup.class) - .returnResult().getResponseBody(); - Objects.requireNonNull(consumerGroups).forEach(consumerGroup -> - consumerGroup.getMembers().forEach(member -> - member.getPartitions().forEach(partition -> - messagesCommittedCount.addAndGet(partition.getCurrentOffset()) - ))); - return messagesCommittedCount.get(); - } - - public int publishUntilSuccess(String topicQualifiedName, String body) { - return frontendTestClient.publishUntilSuccess(topicQualifiedName, body); - } - - public int publishUntilStatus(String topicQualifiedName, String body, int statusCode) { - return frontendTestClient.publishUntilStatus(topicQualifiedName, body, statusCode); - } - - public int publishUntilSuccess(String topicQualifiedName, String body, MultiValueMap headers) { - return frontendTestClient.publishUntilSuccess(topicQualifiedName, body, headers); - } - - public int publishJSONUntilSuccess(String topicQualifiedName, String body) { - return frontendTestClient.publishJSONUntilSuccess(topicQualifiedName, body, new HttpHeaders()); - } - - public int publishAvroUntilSuccess(String topicQualifiedName, byte[] body) { - return frontendTestClient.publishAvroUntilSuccess(topicQualifiedName, body); - } - - public int publishAvroUntilSuccess(String topicQualifiedName, byte[] body, MultiValueMap headers) { - return frontendTestClient.publishAvroUntilSuccess(topicQualifiedName, body, headers); - } - - public WebTestClient.ResponseSpec updateSubscription(Topic topic, String subscription, PatchData patch) { - return managementTestClient.updateSubscription(topic, subscription, patch); - } - - public WebTestClient.ResponseSpec publish(String topicQualifiedName, String body, MultiValueMap headers) { - return frontendTestClient.publishWithHeaders(topicQualifiedName, body, headers); - } - - public WebTestClient.ResponseSpec publish(String topicQualifiedName, String body) { - return frontendTestClient.publish(topicQualifiedName, body); - } - - public WebTestClient.ResponseSpec publishAvro(String topicQualifiedName, byte[] body) { - return frontendTestClient.publishAvro(topicQualifiedName, body, new HttpHeaders()); - } - - public WebTestClient.ResponseSpec publishAvro(String topicQualifiedName, byte[] body, MultiValueMap headers) { - return frontendTestClient.publishAvro(topicQualifiedName, body, headers); - } - - public WebTestClient.ResponseSpec publishJSON(String topicQualifiedName, String body) { - return frontendTestClient.publishJSON(topicQualifiedName, body, new HttpHeaders()); - } - - public Response publishChunked(String topicQualifiedName, String body) { - return frontendTestClient.publishChunked(topicQualifiedName, body); - } - - public String publishSlowly(int clientTimeout, int pauseTimeBetweenChunks, int delayBeforeSendingFirstData, - String topicName, boolean chunkedEncoding) throws IOException, InterruptedException { - return frontendTestClient.publishSlowly(clientTimeout, pauseTimeBetweenChunks, delayBeforeSendingFirstData, topicName, chunkedEncoding); - } - - public String publishSlowly(int clientTimeout, int pauseTimeBetweenChunks, int delayBeforeSendingFirstData, String topicName) - throws IOException, InterruptedException { - return publishSlowly(clientTimeout, pauseTimeBetweenChunks, delayBeforeSendingFirstData, topicName, false); - } - - public void blacklistTopic(String topicQualifiedName) { - managementTestClient.blacklistTopic(topicQualifiedName).expectStatus().is2xxSuccessful(); - } - - public WebTestClient.ResponseSpec blacklistTopicResponse(String topicQualifiedName) { - return managementTestClient.blacklistTopic(topicQualifiedName); - } - - public void unblacklistTopic(String topicQualifiedName) { - managementTestClient.unblacklistTopic(topicQualifiedName).expectStatus().is2xxSuccessful(); - } - - public BlacklistStatus isTopicBlacklisted(String topicQualifiedName) { - return managementTestClient.isTopicBlacklisted(topicQualifiedName) - .expectStatus() - .is2xxSuccessful() - .expectBody(BlacklistStatus.class) - .returnResult() - .getResponseBody(); - } - - public WebTestClient.ResponseSpec unblacklistTopicResponse(String topicQualifiedName) { - return managementTestClient.unblacklistTopic(topicQualifiedName); - } - - public WebTestClient.ResponseSpec getLatestUndeliveredMessage(String topicQualifiedName, String subscriptionName) { - return managementTestClient.getLatestUndeliveredMessage(topicQualifiedName, subscriptionName); - } - - public List getRunningSubscriptionsStatus() { - return consumerTestClient.getRunningSubscriptionsStatus() - .expectStatus() - .is2xxSuccessful() - .expectBodyList(RunningSubscriptionStatus.class) - .returnResult() - .getResponseBody(); - } - - public WebTestClient.ResponseSpec retransmit(String qualifiedName, String subscriptionName, OffsetRetransmissionDate retransmissionDate, boolean dryRun) { - return managementTestClient.retransmit(qualifiedName, subscriptionName, retransmissionDate, dryRun); - } - - public WebTestClient.ResponseSpec getPreview(String qualifiedTopicName, String primaryKafkaClusterName, int partition, long offset) { - return managementTestClient.getPreview(qualifiedTopicName, primaryKafkaClusterName, partition, offset); - } - - public WebTestClient.ResponseSpec getPreview(String qualifiedTopicName) { - return managementTestClient.getPreview(qualifiedTopicName); - } - - public WebTestClient.ResponseSpec getTopicMetrics(String qualifiedName) { - return managementTestClient.getTopicMetrics(qualifiedName); - } - - public WebTestClient.ResponseSpec listSubscriptions(String qualifiedName) { - return managementTestClient.listSubscriptions(qualifiedName, false); - } - - public WebTestClient.ResponseSpec listTopics(String groupName) { - return managementTestClient.listTopics(groupName, false); - } - - public WebTestClient.ResponseSpec getConsumersMetrics() { - return consumerTestClient.getMetrics(); - } - - public WebTestClient.ResponseSpec getFrontendMetrics() { - return frontendTestClient.getMetrics(); - } - - public WebTestClient.ResponseSpec verifyFilters(String qualifiedTopicName, - MessageFiltersVerificationInput input) { - return managementTestClient.verifyFilters(qualifiedTopicName, input); - } - - public WebTestClient.ResponseSpec getManagementHealth() { - return managementTestClient.getStatusHealth(); - } - - public WebTestClient.ResponseSpec getManagementStats() { - return managementTestClient.getStats(); - } - - public WebTestClient.ResponseSpec setReadiness(String dc, boolean state) { - return managementTestClient.setReadiness(dc, state); - } - - public WebTestClient.ResponseSpec getReadiness() { - return managementTestClient.getReadiness(); - } - - public WebTestClient.ResponseSpec getFrontendReadiness() { - return frontendTestClient.getStatusReady(); - } - - public WebTestClient.ResponseSpec getAllTopicClients(String topicQualifiedName) { - return managementTestClient.getAllTopicClients(topicQualifiedName); - } - - public WebTestClient.ResponseSpec getSubscriptionsForOwner(String source, String ownerId) { - return managementTestClient.getSubscriptionsForOwner(source, ownerId); - } - - public WebTestClient.ResponseSpec deleteSubscription(String topicQualifiedName, String subscriptionName) { - return managementTestClient.deleteSubscription(topicQualifiedName, subscriptionName); - } - - public WebTestClient.ResponseSpec getTopicsForOwner(String source, String ownerId) { - return managementTestClient.getTopicsForOwner(source, ownerId); - } - - public WebTestClient.ResponseSpec deleteTopic(String topicQualifiedName) { - return managementTestClient.deleteTopic(topicQualifiedName); - } - - public WebTestClient.ResponseSpec listTrackedTopics(String groupName) { - return managementTestClient.listTopics(groupName, true); - } - - public WebTestClient.ResponseSpec queryTopics(String group, String query) { - return managementTestClient.queryTopics(group, query); - } - - public WebTestClient.ResponseSpec queryGroups(String query) { - return managementTestClient.queryGroups(query); - } - - public WebTestClient.ResponseSpec queryTopics(String query) { - return managementTestClient.queryTopics(query); - } - - public WebTestClient.ResponseSpec queryTopicMetrics(String query) { - return managementTestClient.queryTopicMetrics(query); - } - - public WebTestClient.ResponseSpec querySubscriptionMetrics(String query) { - return managementTestClient.querySubscriptionMetrics(query); - } - - public WebTestClient.ResponseSpec querySubscriptions(String query) { - return managementTestClient.querySubscriptions(query); - } - - public WebTestClient.ResponseSpec listUnhealthy() { - return managementTestClient.listUnhealthy(); - } - - public WebTestClient.ResponseSpec listUnhealthyAsPlainText() { - return managementTestClient.listUnhealthyAsPlainText(); - } - - public WebTestClient.ResponseSpec listUnhealthyForOwner(String ownerId) { - return managementTestClient.listUnhealthy(ownerId); - } - - public WebTestClient.ResponseSpec listUnhealthyForOwnerAsPlainText(String ownerId) { - return managementTestClient.listUnhealthyAsPlainText(ownerId); - } - - public WebTestClient.ResponseSpec listUnhealthyForTopic(String qualifiedName) { - return managementTestClient.listUnhealthyForTopic(qualifiedName); - } - - public WebTestClient.ResponseSpec listUnhealthyForTopicAsPlainText(String qualifiedName) { - return managementTestClient.listUnhealthyForTopicAsPlainText(qualifiedName); - } - - public WebTestClient.ResponseSpec listUnhealthyForSubscription(String topicQualifiedName, String subscriptionName) { - return managementTestClient.listUnhealthyForSubscription(topicQualifiedName, subscriptionName); - } - - public WebTestClient.ResponseSpec listUnhealthyForSubscriptionAsPlainText(String topicQualifiedName, String subscriptionName) { - return managementTestClient.listUnhealthyForSubscriptionAsPlainText(topicQualifiedName, subscriptionName); - } - - public WebTestClient.ResponseSpec createOAuthProvider(OAuthProvider provider) { - return managementTestClient.createOAuthProvider(provider); - } - - public WebTestClient.ResponseSpec getOAuthProvider(String name) { - return managementTestClient.getOAuthProvider(name); - } - - public WebTestClient.ResponseSpec removeOAuthProvider(String name) { - return managementTestClient.removeOAuthProvider(name); - } - - public WebTestClient.ResponseSpec listOAuthProvider() { - return managementTestClient.listOAuthProvider(); - } - - public WebTestClient.ResponseSpec updateOAuthProvider(String name, PatchData patch) { - return managementTestClient.updateOAuthProvider(name, patch); - } - - public WebTestClient.ResponseSpec setMode(String mode) { - return managementTestClient.setMode(mode); - } - - public WebTestClient.ResponseSpec getOfflineRetransmissionTasks() { - return managementTestClient.getOfflineRetransmissionTasks(); - } - - public WebTestClient.ResponseSpec deleteOfflineRetransmissionTask(String taskId) { - return managementTestClient.deleteOfflineRetransmissionTask(taskId); - } - - public WebTestClient.ResponseSpec createOfflineRetransmissionTask(OfflineRetransmissionRequest request) { - return managementTestClient.createOfflineRetransmissionTask(request); - } - - public WebTestClient.ResponseSpec createSubscription(Subscription subscription) { - return managementTestClient.createSubscription(subscription); - } - - public WebTestClient.ResponseSpec listTrackedSubscriptions(String qualifiedName) { - return managementTestClient.listSubscriptions(qualifiedName, true); - } - - public WebTestClient.ResponseSpec querySubscriptions(String qualifiedName, String query) { - return managementTestClient.querySubscriptions(qualifiedName, query); - } - - public WebTestClient.ResponseSpec getSubscriptionHealth(String qualifiedTopicName, String name) { - return managementTestClient.getSubscriptionHealth(qualifiedTopicName, name); - } - - public WebTestClient.ResponseSpec getConsumerGroupsDescription(String qualifiedTopicName, String subscriptionName) { - return managementTestClient.getConsumerGroupsDescription(qualifiedTopicName, subscriptionName); - } - - public WebTestClient.ResponseSpec deleteGroup(String groupName) { - return managementTestClient.deleteGroup(groupName); - } - - public WebTestClient.ResponseSpec updateGroup(String groupName, Group group) { - return managementTestClient.updateGroup(groupName, group); - } - - public List getGroups() { - return managementTestClient.getGroups(); - } - - public WebTestClient.ResponseSpec moveOffsetsToTheEnd(String topicQualifiedName, String subscriptionName) { - return managementTestClient.moveOffsetsToTheEnd(topicQualifiedName, subscriptionName); - } + private final ManagementTestClient managementTestClient; + private final FrontendTestClient frontendTestClient; + private final ConsumerTestClient consumerTestClient; + + public HermesTestClient(int managementPort, int frontendPort, int consumerPort) { + this.managementTestClient = new ManagementTestClient(managementPort); + this.frontendTestClient = new FrontendTestClient(frontendPort); + this.consumerTestClient = new ConsumerTestClient(consumerPort); + } + + public HermesTestClient( + int managementPort, + int frontendPort, + int consumerPort, + String defaultHeaderName, + String defaultHeaderValue) { + this.managementTestClient = + new ManagementTestClient(managementPort, defaultHeaderName, defaultHeaderValue); + this.frontendTestClient = new FrontendTestClient(frontendPort); + this.consumerTestClient = new ConsumerTestClient(consumerPort); + } + + public WebTestClient.ResponseSpec createGroup(Group group) { + return managementTestClient.createGroup(group); + } + + public WebTestClient.ResponseSpec createTopic(TopicWithSchema topicWithSchema) { + return managementTestClient.createTopic(topicWithSchema); + } + + public WebTestClient.ResponseSpec getTopicResponse(String topicQualifiedName) { + return managementTestClient.getTopic(topicQualifiedName); + } + + public WebTestClient.ResponseSpec saveSchema(String topicQualifiedName, String schema) { + return managementTestClient.saveSchema(topicQualifiedName, schema); + } + + public void ensureSchemaSaved(String topicQualifiedName, boolean validate, String schema) { + managementTestClient + .saveSchema(topicQualifiedName, validate, schema) + .expectStatus() + .isCreated(); + waitAtMost(adjust(Duration.ofMinutes(1))) + .untilAsserted( + () -> managementTestClient.getSchema(topicQualifiedName).expectStatus().isOk()); + } + + public WebTestClient.ResponseSpec saveSchema( + String topicQualifiedName, boolean validate, String schema) { + return managementTestClient.saveSchema(topicQualifiedName, validate, schema); + } + + public WebTestClient.ResponseSpec getSchema(String topicQualifiedName) { + return managementTestClient.getSchema(topicQualifiedName); + } + + public WebTestClient.ResponseSpec deleteSchema(String topicQualifiedName) { + return managementTestClient.deleteSchema(topicQualifiedName); + } + + public WebTestClient.ResponseSpec updateTopic(String qualifiedTopicName, PatchData patch) { + return managementTestClient.updateTopic(qualifiedTopicName, patch); + } + + public Subscription getSubscription(String topicQualifiedName, String subscriptionName) { + return getSubscriptionResponse(topicQualifiedName, subscriptionName) + .expectStatus() + .is2xxSuccessful() + .expectBody(Subscription.class) + .returnResult() + .getResponseBody(); + } + + public WebTestClient.ResponseSpec getSubscriptionResponse( + String topicQualifiedName, String subscriptionName) { + return managementTestClient.getSubscription(topicQualifiedName, subscriptionName); + } + + public WebTestClient.ResponseSpec getSubscriptionMetrics( + String topicQualifiedName, String subscriptionName) { + return managementTestClient.getSubscriptionMetrics(topicQualifiedName, subscriptionName); + } + + public WebTestClient.ResponseSpec suspendSubscription(Topic topic, String subscription) { + return managementTestClient + .updateSubscriptionState(topic, subscription, Subscription.State.SUSPENDED) + .expectStatus() + .is2xxSuccessful(); + } + + public void waitUntilSubscriptionActivated(String topicQualifiedName, String subscriptionName) { + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + assertThat( + managementTestClient + .getSubscription(topicQualifiedName, subscriptionName) + .expectStatus() + .is2xxSuccessful() + .expectBody(Subscription.class) + .returnResult() + .getResponseBody() + .getState()) + .isEqualTo(Subscription.State.ACTIVE); + assertThat( + managementTestClient + .getConsumerGroupsDescription(topicQualifiedName, subscriptionName) + .expectBodyList(ConsumerGroup.class) + .returnResult() + .getResponseBody() + .get(0) + .getState()) + .isEqualTo("Stable"); + }); + } + + public void waitUntilSubscriptionSuspended(String topicQualifiedName, String subscriptionName) { + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + assertThat( + managementTestClient + .getSubscription(topicQualifiedName, subscriptionName) + .expectStatus() + .is2xxSuccessful() + .expectBody(Subscription.class) + .returnResult() + .getResponseBody() + .getState()) + .isEqualTo(Subscription.State.SUSPENDED); + assertThat( + managementTestClient + .getConsumerGroupsDescription(topicQualifiedName, subscriptionName) + .expectBodyList(ConsumerGroup.class) + .returnResult() + .getResponseBody() + .get(0) + .getState()) + .isEqualTo("Empty"); + }); + } + + public void waitUntilConsumerCommitsOffset(String topicQualifiedName, String subscriptionName) { + long committedMessagesCount = calculateCommittedMessages(topicQualifiedName, subscriptionName); + waitAtMost(adjust(Duration.ofMinutes(1))) + .untilAsserted( + () -> { + long currentCommittedMessagesCount = + calculateCommittedMessages(topicQualifiedName, subscriptionName); + assertThat(currentCommittedMessagesCount).isGreaterThan(committedMessagesCount); + }); + } + + private long calculateCommittedMessages(String topicQualifiedName, String subscription) { + AtomicLong messagesCommittedCount = new AtomicLong(0); + List consumerGroups = + getConsumerGroupsDescription(topicQualifiedName, subscription) + .expectBodyList(ConsumerGroup.class) + .returnResult() + .getResponseBody(); + Objects.requireNonNull(consumerGroups) + .forEach( + consumerGroup -> + consumerGroup + .getMembers() + .forEach( + member -> + member + .getPartitions() + .forEach( + partition -> + messagesCommittedCount.addAndGet( + partition.getCurrentOffset())))); + return messagesCommittedCount.get(); + } + + public int publishUntilSuccess(String topicQualifiedName, String body) { + return frontendTestClient.publishUntilSuccess(topicQualifiedName, body); + } + + public int publishUntilStatus(String topicQualifiedName, String body, int statusCode) { + return frontendTestClient.publishUntilStatus(topicQualifiedName, body, statusCode); + } + + public int publishUntilSuccess( + String topicQualifiedName, String body, MultiValueMap headers) { + return frontendTestClient.publishUntilSuccess(topicQualifiedName, body, headers); + } + + public int publishJSONUntilSuccess(String topicQualifiedName, String body) { + return frontendTestClient.publishJSONUntilSuccess(topicQualifiedName, body, new HttpHeaders()); + } + + public int publishAvroUntilSuccess(String topicQualifiedName, byte[] body) { + return frontendTestClient.publishAvroUntilSuccess(topicQualifiedName, body); + } + + public int publishAvroUntilSuccess( + String topicQualifiedName, byte[] body, MultiValueMap headers) { + return frontendTestClient.publishAvroUntilSuccess(topicQualifiedName, body, headers); + } + + public WebTestClient.ResponseSpec updateSubscription( + Topic topic, String subscription, PatchData patch) { + return managementTestClient.updateSubscription(topic, subscription, patch); + } + + public WebTestClient.ResponseSpec publish( + String topicQualifiedName, String body, MultiValueMap headers) { + return frontendTestClient.publishWithHeaders(topicQualifiedName, body, headers); + } + + public WebTestClient.ResponseSpec publish(String topicQualifiedName, String body) { + return frontendTestClient.publish(topicQualifiedName, body); + } + + public WebTestClient.ResponseSpec publishAvro(String topicQualifiedName, byte[] body) { + return frontendTestClient.publishAvro(topicQualifiedName, body, new HttpHeaders()); + } + + public WebTestClient.ResponseSpec publishAvro( + String topicQualifiedName, byte[] body, MultiValueMap headers) { + return frontendTestClient.publishAvro(topicQualifiedName, body, headers); + } + + public WebTestClient.ResponseSpec publishJSON(String topicQualifiedName, String body) { + return frontendTestClient.publishJSON(topicQualifiedName, body, new HttpHeaders()); + } + + public Response publishChunked(String topicQualifiedName, String body) { + return frontendTestClient.publishChunked(topicQualifiedName, body); + } + + public String publishSlowly( + int clientTimeout, + int pauseTimeBetweenChunks, + int delayBeforeSendingFirstData, + String topicName, + boolean chunkedEncoding) + throws IOException, InterruptedException { + return frontendTestClient.publishSlowly( + clientTimeout, + pauseTimeBetweenChunks, + delayBeforeSendingFirstData, + topicName, + chunkedEncoding); + } + + public String publishSlowly( + int clientTimeout, + int pauseTimeBetweenChunks, + int delayBeforeSendingFirstData, + String topicName) + throws IOException, InterruptedException { + return publishSlowly( + clientTimeout, pauseTimeBetweenChunks, delayBeforeSendingFirstData, topicName, false); + } + + public void blacklistTopic(String topicQualifiedName) { + managementTestClient.blacklistTopic(topicQualifiedName).expectStatus().is2xxSuccessful(); + } + + public WebTestClient.ResponseSpec blacklistTopicResponse(String topicQualifiedName) { + return managementTestClient.blacklistTopic(topicQualifiedName); + } + + public void unblacklistTopic(String topicQualifiedName) { + managementTestClient.unblacklistTopic(topicQualifiedName).expectStatus().is2xxSuccessful(); + } + + public BlacklistStatus isTopicBlacklisted(String topicQualifiedName) { + return managementTestClient + .isTopicBlacklisted(topicQualifiedName) + .expectStatus() + .is2xxSuccessful() + .expectBody(BlacklistStatus.class) + .returnResult() + .getResponseBody(); + } + + public WebTestClient.ResponseSpec unblacklistTopicResponse(String topicQualifiedName) { + return managementTestClient.unblacklistTopic(topicQualifiedName); + } + + public WebTestClient.ResponseSpec getLatestUndeliveredMessage( + String topicQualifiedName, String subscriptionName) { + return managementTestClient.getLatestUndeliveredMessage(topicQualifiedName, subscriptionName); + } + + public List getRunningSubscriptionsStatus() { + return consumerTestClient + .getRunningSubscriptionsStatus() + .expectStatus() + .is2xxSuccessful() + .expectBodyList(RunningSubscriptionStatus.class) + .returnResult() + .getResponseBody(); + } + + public WebTestClient.ResponseSpec retransmit( + String qualifiedName, + String subscriptionName, + OffsetRetransmissionDate retransmissionDate, + boolean dryRun) { + return managementTestClient.retransmit( + qualifiedName, subscriptionName, retransmissionDate, dryRun); + } + + public WebTestClient.ResponseSpec getPreview( + String qualifiedTopicName, String primaryKafkaClusterName, int partition, long offset) { + return managementTestClient.getPreview( + qualifiedTopicName, primaryKafkaClusterName, partition, offset); + } + + public WebTestClient.ResponseSpec getPreview(String qualifiedTopicName) { + return managementTestClient.getPreview(qualifiedTopicName); + } + + public WebTestClient.ResponseSpec getTopicMetrics(String qualifiedName) { + return managementTestClient.getTopicMetrics(qualifiedName); + } + + public WebTestClient.ResponseSpec listSubscriptions(String qualifiedName) { + return managementTestClient.listSubscriptions(qualifiedName, false); + } + + public WebTestClient.ResponseSpec listTopics(String groupName) { + return managementTestClient.listTopics(groupName, false); + } + + public WebTestClient.ResponseSpec getConsumersMetrics() { + return consumerTestClient.getMetrics(); + } + + public WebTestClient.ResponseSpec getFrontendMetrics() { + return frontendTestClient.getMetrics(); + } + + public WebTestClient.ResponseSpec verifyFilters( + String qualifiedTopicName, MessageFiltersVerificationInput input) { + return managementTestClient.verifyFilters(qualifiedTopicName, input); + } + + public WebTestClient.ResponseSpec getManagementHealth() { + return managementTestClient.getStatusHealth(); + } + + public WebTestClient.ResponseSpec getManagementStats() { + return managementTestClient.getStats(); + } + + public WebTestClient.ResponseSpec setReadiness(String dc, boolean state) { + return managementTestClient.setReadiness(dc, state); + } + + public WebTestClient.ResponseSpec getReadiness() { + return managementTestClient.getReadiness(); + } + + public WebTestClient.ResponseSpec getFrontendReadiness() { + return frontendTestClient.getStatusReady(); + } + + public WebTestClient.ResponseSpec getAllTopicClients(String topicQualifiedName) { + return managementTestClient.getAllTopicClients(topicQualifiedName); + } + + public WebTestClient.ResponseSpec getSubscriptionsForOwner(String source, String ownerId) { + return managementTestClient.getSubscriptionsForOwner(source, ownerId); + } + + public WebTestClient.ResponseSpec deleteSubscription( + String topicQualifiedName, String subscriptionName) { + return managementTestClient.deleteSubscription(topicQualifiedName, subscriptionName); + } + + public WebTestClient.ResponseSpec getTopicsForOwner(String source, String ownerId) { + return managementTestClient.getTopicsForOwner(source, ownerId); + } + + public WebTestClient.ResponseSpec deleteTopic(String topicQualifiedName) { + return managementTestClient.deleteTopic(topicQualifiedName); + } + + public WebTestClient.ResponseSpec listTrackedTopics(String groupName) { + return managementTestClient.listTopics(groupName, true); + } + + public WebTestClient.ResponseSpec queryTopics(String group, String query) { + return managementTestClient.queryTopics(group, query); + } + + public WebTestClient.ResponseSpec queryGroups(String query) { + return managementTestClient.queryGroups(query); + } + + public WebTestClient.ResponseSpec queryTopics(String query) { + return managementTestClient.queryTopics(query); + } + + public WebTestClient.ResponseSpec queryTopicMetrics(String query) { + return managementTestClient.queryTopicMetrics(query); + } + + public WebTestClient.ResponseSpec querySubscriptionMetrics(String query) { + return managementTestClient.querySubscriptionMetrics(query); + } + + public WebTestClient.ResponseSpec querySubscriptions(String query) { + return managementTestClient.querySubscriptions(query); + } + + public WebTestClient.ResponseSpec listUnhealthy() { + return managementTestClient.listUnhealthy(); + } + + public WebTestClient.ResponseSpec listUnhealthyAsPlainText() { + return managementTestClient.listUnhealthyAsPlainText(); + } + + public WebTestClient.ResponseSpec listUnhealthyForOwner(String ownerId) { + return managementTestClient.listUnhealthy(ownerId); + } + + public WebTestClient.ResponseSpec listUnhealthyForOwnerAsPlainText(String ownerId) { + return managementTestClient.listUnhealthyAsPlainText(ownerId); + } + + public WebTestClient.ResponseSpec listUnhealthyForTopic(String qualifiedName) { + return managementTestClient.listUnhealthyForTopic(qualifiedName); + } + + public WebTestClient.ResponseSpec listUnhealthyForTopicAsPlainText(String qualifiedName) { + return managementTestClient.listUnhealthyForTopicAsPlainText(qualifiedName); + } + + public WebTestClient.ResponseSpec listUnhealthyForSubscription( + String topicQualifiedName, String subscriptionName) { + return managementTestClient.listUnhealthyForSubscription(topicQualifiedName, subscriptionName); + } + + public WebTestClient.ResponseSpec listUnhealthyForSubscriptionAsPlainText( + String topicQualifiedName, String subscriptionName) { + return managementTestClient.listUnhealthyForSubscriptionAsPlainText( + topicQualifiedName, subscriptionName); + } + + public WebTestClient.ResponseSpec createOAuthProvider(OAuthProvider provider) { + return managementTestClient.createOAuthProvider(provider); + } + + public WebTestClient.ResponseSpec getOAuthProvider(String name) { + return managementTestClient.getOAuthProvider(name); + } + + public WebTestClient.ResponseSpec removeOAuthProvider(String name) { + return managementTestClient.removeOAuthProvider(name); + } + + public WebTestClient.ResponseSpec listOAuthProvider() { + return managementTestClient.listOAuthProvider(); + } + + public WebTestClient.ResponseSpec updateOAuthProvider(String name, PatchData patch) { + return managementTestClient.updateOAuthProvider(name, patch); + } + + public WebTestClient.ResponseSpec setMode(String mode) { + return managementTestClient.setMode(mode); + } + + public WebTestClient.ResponseSpec getOfflineRetransmissionTasks() { + return managementTestClient.getOfflineRetransmissionTasks(); + } + + public WebTestClient.ResponseSpec deleteOfflineRetransmissionTask(String taskId) { + return managementTestClient.deleteOfflineRetransmissionTask(taskId); + } + + public WebTestClient.ResponseSpec createOfflineRetransmissionTask( + OfflineRetransmissionRequest request) { + return managementTestClient.createOfflineRetransmissionTask(request); + } + + public WebTestClient.ResponseSpec createSubscription(Subscription subscription) { + return managementTestClient.createSubscription(subscription); + } + + public WebTestClient.ResponseSpec listTrackedSubscriptions(String qualifiedName) { + return managementTestClient.listSubscriptions(qualifiedName, true); + } + + public WebTestClient.ResponseSpec querySubscriptions(String qualifiedName, String query) { + return managementTestClient.querySubscriptions(qualifiedName, query); + } + + public WebTestClient.ResponseSpec getSubscriptionHealth(String qualifiedTopicName, String name) { + return managementTestClient.getSubscriptionHealth(qualifiedTopicName, name); + } + + public WebTestClient.ResponseSpec getConsumerGroupsDescription( + String qualifiedTopicName, String subscriptionName) { + return managementTestClient.getConsumerGroupsDescription(qualifiedTopicName, subscriptionName); + } + + public WebTestClient.ResponseSpec deleteGroup(String groupName) { + return managementTestClient.deleteGroup(groupName); + } + + public WebTestClient.ResponseSpec updateGroup(String groupName, Group group) { + return managementTestClient.updateGroup(groupName, group); + } + + public List getGroups() { + return managementTestClient.getGroups(); + } + + public WebTestClient.ResponseSpec moveOffsetsToTheEnd( + String topicQualifiedName, String subscriptionName) { + return managementTestClient.moveOffsetsToTheEnd(topicQualifiedName, subscriptionName); + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/integration/ManagementTestClient.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/integration/ManagementTestClient.java index d3dfcc5e5c..2735b5f710 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/integration/ManagementTestClient.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/client/integration/ManagementTestClient.java @@ -4,6 +4,8 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import jakarta.ws.rs.core.UriBuilder; +import java.time.Duration; +import java.util.List; import org.eclipse.jetty.client.HttpClient; import org.springframework.http.MediaType; import org.springframework.http.client.reactive.ClientHttpConnector; @@ -21,722 +23,796 @@ import pl.allegro.tech.hermes.api.TopicWithSchema; import reactor.core.publisher.Mono; -import java.time.Duration; -import java.util.List; - public class ManagementTestClient { - private static final String TOPICS_PATH = "/topics"; - - private static final String TOPIC_PATH = "/topics/{topicName}"; - - private static final String SUBSCRIPTIONS_PATH = "/topics/{topicName}/subscriptions"; - - private static final String SUBSCRIPTION_PATH = "/topics/{topicName}/subscriptions/{subscriptionName}"; - - private static final String SUBSCRIPTION_STATE_PATH = "/topics/{topicName}/subscriptions/{subscriptionName}/state"; - - private static final String SUBSCRIPTION_METRICS_PATH = "/topics/{topicName}/subscriptions/{subscriptionName}/metrics"; - - private static final String GROUPS_PATH = "/groups"; - - private static final String GROUP_PATH = "/groups/{groupName}"; - - private static final String RETRANSMISSION_PATH = "/topics/{topicName}/subscriptions/{subscriptionName}/retransmission"; - - private static final String BLACKLIST_TOPICS_PATH = "/blacklist/topics"; - - private static final String BLACKLIST_TOPIC_PATH = "/blacklist/topics/{topicName}"; - - private static final String LATEST_UNDELIVERED_MESSAGE = "/topics/{topicName}/subscriptions/{subscriptionName}/undelivered"; - - private static final String TOPIC_PREVIEW = "/topics/{topicName}/preview"; - - private static final String TOPIC_PREVIEW_OFFSET = "/topics/{topicName}/preview/cluster/{brokersClusterName}/partition/{partition}/offset/{offset}"; - - private static final String MOVE_SUBSCRIPTION_OFFSETS = "/topics/{topicName}/subscriptions/{subscriptionName}/moveOffsetsToTheEnd"; - - private static final String SET_READINESS = "/readiness/datacenters/{dc}"; - - private static final String GET_READINESS = "/readiness/datacenters"; - - private static final String TOPIC_SCHEMA = "/topics/{topicName}/schema"; - - private static final String ALL_TOPIC_CLIENTS = "/topics/{topicName}/clients"; - - private static final String SUBSCRIPTIONS_BY_OWNER = "/subscriptions/owner/{source}/{ownerId}"; - - private static final String TOPICS_BY_OWNER = "/topics/owner/{source}/{ownerId}"; - - private static final String TOPIC_METRICS_PATH = "/topics/{topicName}/metrics"; - - private static final String FILTERS = "/filters/{topicName}"; - - private static final String STATUS_HEALTH = "/status/health"; - - private static final String STATS = "/stats"; - - private static final String QUERY_GROUPS = "/query/groups"; - - private static final String QUERY_TOPICS = "/query/topics"; - - private static final String QUERY_SUBSCRIPTIONS = "/query/subscriptions"; - - private static final String QUERY_TOPIC_METRICS = "/query/topics/metrics"; - - private static final String QUERY_SUBSCRIPTION_METRICS = "/query/subscriptions/metrics"; - - private static final String OAUTH_PROVIDERS_PATH = "/oauth/providers"; - - private static final String SUBSCRIPTIONS_QUERY = "/topics/{topicName}/subscriptions/query"; - - private static final String OAUTH_PROVIDER_PATH = "/oauth/providers/{oAuthProviderName}"; - - private static final String UNHEALTHY_PATH = "/unhealthy"; - - private static final String TOPICS_QUERY = "/topics/query"; - - private static final String MODE = "/mode"; - - private static final String OFFLINE_RETRANSMISSION_TASKS = "/offline-retransmission/tasks"; - - private static final String OFFLINE_RETRANSMISSION_TASK = "/offline-retransmission/tasks/{taskId}"; - - private static final String SUBSCRIPTION_HEALTH = "/topics/{topicName}/subscriptions/{subscription}/health"; - - private static final String CONSUMER_GROUPS = "/topics/{topicName}/subscriptions/{subscription}/consumer-groups"; - - private static final String OWNERS_SEARCH_PATH = "/owners/sources/{source}"; - - private final WebTestClient webTestClient; - - private final String managementContainerUrl; - - private final ObjectMapper objectMapper; - - public ManagementTestClient(int managementPort) { - this.managementContainerUrl = "http://localhost:" + managementPort; - this.webTestClient = configureWebTestClient().build(); - this.objectMapper = new ObjectMapper(); - } - - public ManagementTestClient(int managementPort, String defaultHeaderName, String defaultHeaderValue) { - this.managementContainerUrl = "http://localhost:" + managementPort; - this.webTestClient = configureWebTestClient() - .defaultHeader(defaultHeaderName, defaultHeaderValue) - .build(); - this.objectMapper = new ObjectMapper(); - } - - private WebTestClient.Builder configureWebTestClient() { - return WebTestClient - .bindToServer(clientHttpConnector()) - .responseTimeout(Duration.ofSeconds(30)) - .baseUrl(managementContainerUrl) - .codecs(configurer -> configurer - .defaultCodecs() - .maxInMemorySize(16 * 1024 * 1024)); - } - - private static ClientHttpConnector clientHttpConnector() { - HttpClient httpClient = new HttpClient(); - httpClient.setMaxConnectionsPerDestination(256); - return new JettyClientHttpConnector(httpClient); - } - - public WebTestClient.ResponseSpec createGroup(Group group) { - return sendCreateGroupRequest(group); - } - - public List getGroups() { - String jsonString = webTestClient.get().uri(GROUPS_PATH) - .exchange() - .expectBody(String.class) - .returnResult() - .getResponseBody(); - - return mapStringJsonToListOfString(jsonString); - } - - public WebTestClient.ResponseSpec createTopic(TopicWithSchema topicWithSchema) { - return sendCreateTopicRequest(topicWithSchema); - } - - public WebTestClient.ResponseSpec getTopic(String topicQualifiedName) { - return getSingleTopic(topicQualifiedName); - } - - public WebTestClient.ResponseSpec createSubscription(Subscription subscription) { - return sendCreateSubscriptionRequest(subscription); - } - - public WebTestClient.ResponseSpec updateSubscription(Topic topic, String subscription, PatchData patch) { - return webTestClient.put().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(SUBSCRIPTION_PATH) - .build(topic.getQualifiedName(), subscription)) - .body(Mono.just(patch), PatchData.class) - .exchange(); - } - - public WebTestClient.ResponseSpec updateSubscriptionState(Topic topic, String subscription, Subscription.State state) { - return webTestClient.put().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(SUBSCRIPTION_STATE_PATH) - .build(topic.getQualifiedName(), subscription)) - .body(Mono.just(state), Subscription.State.class) - .exchange(); - } - - public WebTestClient.ResponseSpec getSubscription(String topicQualifiedName, String subscriptionName) { - return getSingleSubscription(topicQualifiedName, subscriptionName); - } - - public WebTestClient.ResponseSpec getSubscriptionMetrics(String topicQualifiedName, String subscriptionName) { - return webTestClient.get().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(SUBSCRIPTION_METRICS_PATH) - .build(topicQualifiedName, subscriptionName)) - .exchange(); - } + private static final String TOPICS_PATH = "/topics"; - private WebTestClient.ResponseSpec getSingleTopic(String topicQualifiedName) { - return webTestClient.get().uri( - UriBuilder.fromUri(managementContainerUrl) - .path(TOPIC_PATH) - .build(topicQualifiedName)) - .exchange(); - } + private static final String TOPIC_PATH = "/topics/{topicName}"; - private WebTestClient.ResponseSpec getSingleSubscription(String topicQualifiedName, String subscriptionName) { - return webTestClient.get().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(SUBSCRIPTION_PATH) - .build(topicQualifiedName, subscriptionName)) - .exchange(); - } + private static final String SUBSCRIPTIONS_PATH = "/topics/{topicName}/subscriptions"; - private WebTestClient.ResponseSpec sendCreateTopicRequest(TopicWithSchema topicWithSchema) { - return webTestClient.post().uri(TOPICS_PATH) - .body(Mono.just(topicWithSchema), TopicWithSchema.class) - .exchange(); - } + private static final String SUBSCRIPTION_PATH = + "/topics/{topicName}/subscriptions/{subscriptionName}"; - WebTestClient.ResponseSpec retransmit(String topicName, String subscriptionName, OffsetRetransmissionDate retransmissionDate, boolean dryRun) { - return webTestClient.put().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(RETRANSMISSION_PATH) - .queryParam("dryRun", dryRun) - .build(topicName, subscriptionName)) - .body(Mono.just(retransmissionDate), OffsetRetransmissionDate.class) - .exchange(); - } + private static final String SUBSCRIPTION_STATE_PATH = + "/topics/{topicName}/subscriptions/{subscriptionName}/state"; - private WebTestClient.ResponseSpec sendCreateSubscriptionRequest(Subscription subscription) { - return webTestClient.post().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(SUBSCRIPTIONS_PATH) - .build(subscription.getQualifiedTopicName())) - .body(Mono.just(subscription), Subscription.class) - .exchange(); - } + private static final String SUBSCRIPTION_METRICS_PATH = + "/topics/{topicName}/subscriptions/{subscriptionName}/metrics"; - private WebTestClient.ResponseSpec sendCreateGroupRequest(Group group) { - return webTestClient.post().uri(GROUPS_PATH) - .body(Mono.just(group), Group.class) - .exchange(); - } + private static final String GROUPS_PATH = "/groups"; - private List mapStringJsonToListOfString(String jsonString) { - try { - return objectMapper.readValue(jsonString, new TypeReference<>() { - }); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } + private static final String GROUP_PATH = "/groups/{groupName}"; - public WebTestClient.ResponseSpec blacklistTopic(String topicQualifiedName) { - return webTestClient.post().uri(BLACKLIST_TOPICS_PATH) - .body(Mono.just(List.of(topicQualifiedName)), List.class) - .exchange(); - } + private static final String RETRANSMISSION_PATH = + "/topics/{topicName}/subscriptions/{subscriptionName}/retransmission"; - public WebTestClient.ResponseSpec unblacklistTopic(String topicQualifiedName) { - return webTestClient.delete().uri(UriBuilder.fromUri(managementContainerUrl) - .path(BLACKLIST_TOPIC_PATH) - .build(topicQualifiedName)) - .exchange(); - } + private static final String BLACKLIST_TOPICS_PATH = "/blacklist/topics"; - public WebTestClient.ResponseSpec isTopicBlacklisted(String topicQualifiedName) { - return webTestClient.get().uri(UriBuilder.fromUri(managementContainerUrl) - .path(BLACKLIST_TOPIC_PATH) - .build(topicQualifiedName)) - .exchange(); - } + private static final String BLACKLIST_TOPIC_PATH = "/blacklist/topics/{topicName}"; - public WebTestClient.ResponseSpec getLatestUndeliveredMessage(String topicQualifiedName, String subscriptionName) { - return webTestClient.get().uri(UriBuilder.fromUri(managementContainerUrl) - .path(LATEST_UNDELIVERED_MESSAGE) - .build(topicQualifiedName, subscriptionName)) - .exchange(); - } + private static final String LATEST_UNDELIVERED_MESSAGE = + "/topics/{topicName}/subscriptions/{subscriptionName}/undelivered"; - public WebTestClient.ResponseSpec getPreview(String qualifiedTopicName, String primaryKafkaClusterName, int partition, long offset) { - return webTestClient.get().uri(UriBuilder.fromUri(managementContainerUrl) - .path(TOPIC_PREVIEW_OFFSET) - .build(qualifiedTopicName, primaryKafkaClusterName, partition, offset)) - .exchange(); - } - - public WebTestClient.ResponseSpec getPreview(String qualifiedTopicName) { - return webTestClient.get().uri(UriBuilder.fromUri(managementContainerUrl) - .path(TOPIC_PREVIEW) - .build(qualifiedTopicName)) - .exchange(); - } - - public WebTestClient.ResponseSpec updateTopic(String qualifiedTopicName, PatchData patch) { - return webTestClient.put().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(TOPIC_PATH) - .build(qualifiedTopicName)) - .body(Mono.just(patch), PatchData.class) - .exchange(); - } - - public WebTestClient.ResponseSpec getTopicMetrics(String qualifiedTopicName) { - return webTestClient.get().uri(UriBuilder.fromUri(managementContainerUrl) - .path(TOPIC_METRICS_PATH) - .build(qualifiedTopicName)) - .exchange(); - } - - public WebTestClient.ResponseSpec listSubscriptions(String qualifiedTopicName, boolean tracked) { - return webTestClient.get().uri(UriBuilder.fromUri(managementContainerUrl) - .path(SUBSCRIPTIONS_PATH) - .queryParam("tracked", tracked) - .build(qualifiedTopicName)) - .exchange(); - } - - public WebTestClient.ResponseSpec listTopics(String groupName, boolean tracked) { - return webTestClient.get().uri(UriBuilder.fromUri(managementContainerUrl) - .path(TOPICS_PATH) - .queryParam("groupName", groupName) - .queryParam("tracked", tracked) - .build()) - .exchange(); - } - - public WebTestClient.ResponseSpec setReadiness(String dc, boolean state) { - return webTestClient.post().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(SET_READINESS) - .build(dc)) - .body(Mono.just(new Readiness(state)), Readiness.class) - .exchange(); - } - - public WebTestClient.ResponseSpec getReadiness() { - return webTestClient.get().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(GET_READINESS) - .build()) - .exchange(); - } - - public WebTestClient.ResponseSpec saveSchema(String qualifiedTopicName, boolean validate, String schema) { - return webTestClient.post().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(TOPIC_SCHEMA) - .queryParam("validate", validate) - .build(qualifiedTopicName)) - .header("Content-Type", "application/json") - .body(Mono.just(schema), String.class) - .exchange(); - } - - public WebTestClient.ResponseSpec saveSchema(String qualifiedTopicName, String schema) { - return webTestClient.post().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(TOPIC_SCHEMA) - .build(qualifiedTopicName)) - .header("Content-Type", "application/json") - .body(Mono.just(schema), String.class) - .exchange(); - } - - public WebTestClient.ResponseSpec getSchema(String qualifiedTopicName) { - return webTestClient.get().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(TOPIC_SCHEMA) - .build(qualifiedTopicName)) - .exchange(); - } - - public WebTestClient.ResponseSpec deleteSchema(String qualifiedTopicName) { - return webTestClient.delete().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(TOPIC_SCHEMA) - .build(qualifiedTopicName)) - .exchange(); - } - - public WebTestClient.ResponseSpec verifyFilters(String qualifiedTopicName, - MessageFiltersVerificationInput input) { - return webTestClient.post().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(FILTERS) - .build(qualifiedTopicName) - ).contentType(MediaType.APPLICATION_JSON) - .body(Mono.just(input), MessageFiltersVerificationInput.class) - .exchange(); - } + private static final String TOPIC_PREVIEW = "/topics/{topicName}/preview"; - public WebTestClient.ResponseSpec getStatusHealth() { - return webTestClient.get().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(STATUS_HEALTH) - .build()) - .exchange(); - } + private static final String TOPIC_PREVIEW_OFFSET = + "/topics/{topicName}/preview/cluster/{brokersClusterName}/partition/{partition}/offset/{offset}"; - public WebTestClient.ResponseSpec getStats() { - return webTestClient.get().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(STATS) - .build()) - .exchange(); - } + private static final String MOVE_SUBSCRIPTION_OFFSETS = + "/topics/{topicName}/subscriptions/{subscriptionName}/moveOffsetsToTheEnd"; - public WebTestClient.ResponseSpec createOAuthProvider(OAuthProvider provider) { - return webTestClient.post().uri(OAUTH_PROVIDERS_PATH) - .body(Mono.just(provider), OAuthProvider.class) - .exchange(); - } + private static final String SET_READINESS = "/readiness/datacenters/{dc}"; - public WebTestClient.ResponseSpec getOAuthProvider(String name) { - return webTestClient.get().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(OAUTH_PROVIDER_PATH) - .build(name)) - .exchange(); - } + private static final String GET_READINESS = "/readiness/datacenters"; - public WebTestClient.ResponseSpec removeOAuthProvider(String name) { - return webTestClient.delete().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(OAUTH_PROVIDER_PATH) - .build(name)) - .exchange(); - } + private static final String TOPIC_SCHEMA = "/topics/{topicName}/schema"; - public WebTestClient.ResponseSpec listOAuthProvider() { - return webTestClient.get().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(OAUTH_PROVIDERS_PATH) - .build()) - .exchange(); - } + private static final String ALL_TOPIC_CLIENTS = "/topics/{topicName}/clients"; - public WebTestClient.ResponseSpec updateOAuthProvider(String name, PatchData patch) { - return webTestClient.put().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(OAUTH_PROVIDER_PATH) - .build(name)) - .body(Mono.just(patch), PatchData.class) - .exchange(); - } + private static final String SUBSCRIPTIONS_BY_OWNER = "/subscriptions/owner/{source}/{ownerId}"; - public WebTestClient.ResponseSpec getAllTopicClients(String topicQualifiedName) { - return webTestClient.get().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(ALL_TOPIC_CLIENTS) - .build(topicQualifiedName)) - .exchange(); - } + private static final String TOPICS_BY_OWNER = "/topics/owner/{source}/{ownerId}"; - public WebTestClient.ResponseSpec getSubscriptionsForOwner(String source, String ownerId) { - return webTestClient.get().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(SUBSCRIPTIONS_BY_OWNER) - .build(source, ownerId)) - .exchange(); - } + private static final String TOPIC_METRICS_PATH = "/topics/{topicName}/metrics"; - public WebTestClient.ResponseSpec deleteSubscription(String topicQualifiedName, String subscriptionName) { - return webTestClient.delete().uri(UriBuilder.fromUri(managementContainerUrl) - .path(SUBSCRIPTION_PATH) - .build(topicQualifiedName, subscriptionName)) - .exchange(); - } + private static final String FILTERS = "/filters/{topicName}"; - public WebTestClient.ResponseSpec getTopicsForOwner(String source, String ownerId) { - return webTestClient.get().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(TOPICS_BY_OWNER) - .build(source, ownerId)) - .exchange(); - } + private static final String STATUS_HEALTH = "/status/health"; - public WebTestClient.ResponseSpec deleteTopic(String topicQualifiedName) { - return webTestClient.delete().uri(UriBuilder.fromUri(managementContainerUrl) - .path(TOPIC_PATH) - .build(topicQualifiedName)) - .exchange(); - } + private static final String STATS = "/stats"; - public WebTestClient.ResponseSpec queryTopics(String group, String query) { - return webTestClient.post().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(TOPICS_QUERY) - .queryParam("groupName", group) - .build()) - .contentType(MediaType.APPLICATION_JSON) - .body(Mono.just(query), String.class) - .exchange(); - } + private static final String QUERY_GROUPS = "/query/groups"; - public WebTestClient.ResponseSpec queryGroups(String query) { - return webTestClient.post().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(QUERY_GROUPS) - .build()) - .contentType(MediaType.APPLICATION_JSON) - .body(Mono.just(query), String.class) - .exchange(); - } + private static final String QUERY_TOPICS = "/query/topics"; - public WebTestClient.ResponseSpec queryTopics(String query) { - return webTestClient.post().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(QUERY_TOPICS) - .build()) - .contentType(MediaType.APPLICATION_JSON) - .body(Mono.just(query), String.class) - .exchange(); - } + private static final String QUERY_SUBSCRIPTIONS = "/query/subscriptions"; - public WebTestClient.ResponseSpec queryTopicMetrics(String query) { - return webTestClient.post().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(QUERY_TOPIC_METRICS) - .build()) - .contentType(MediaType.APPLICATION_JSON) - .body(Mono.just(query), String.class) - .exchange(); - } + private static final String QUERY_TOPIC_METRICS = "/query/topics/metrics"; - public WebTestClient.ResponseSpec querySubscriptionMetrics(String query) { - return webTestClient.post().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(QUERY_SUBSCRIPTION_METRICS) - .build()) - .contentType(MediaType.APPLICATION_JSON) - .body(Mono.just(query), String.class) - .exchange(); - } + private static final String QUERY_SUBSCRIPTION_METRICS = "/query/subscriptions/metrics"; - public WebTestClient.ResponseSpec querySubscriptions(String query) { - return webTestClient.post().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(QUERY_SUBSCRIPTIONS) - .build()) - .contentType(MediaType.APPLICATION_JSON) - .body(Mono.just(query), String.class) - .exchange(); - } + private static final String OAUTH_PROVIDERS_PATH = "/oauth/providers"; - public WebTestClient.ResponseSpec listUnhealthy() { - return webTestClient.get().uri( - UriBuilder.fromUri(managementContainerUrl) - .path(UNHEALTHY_PATH) - .queryParam("respectMonitoringSeverity", "false") - .build() - ) - .header("Accept", "application/json") - .exchange(); - } + private static final String SUBSCRIPTIONS_QUERY = "/topics/{topicName}/subscriptions/query"; - public WebTestClient.ResponseSpec listUnhealthyAsPlainText() { - return webTestClient.get().uri( - UriBuilder.fromUri(managementContainerUrl) - .path(UNHEALTHY_PATH) - .queryParam("respectMonitoringSeverity", "false") - .build() - ) - .header("Accept", "text/plain") - .exchange(); - } + private static final String OAUTH_PROVIDER_PATH = "/oauth/providers/{oAuthProviderName}"; - public WebTestClient.ResponseSpec listUnhealthy(String ownerId) { - return webTestClient.get().uri( - UriBuilder.fromUri(managementContainerUrl) - .path(UNHEALTHY_PATH) - .queryParam("ownerSourceName", "Plaintext") - .queryParam("ownerId", ownerId) - .queryParam("respectMonitoringSeverity", "false") - .build() - ) - .header("Accept", "application/json") - .exchange(); - } + private static final String UNHEALTHY_PATH = "/unhealthy"; - public WebTestClient.ResponseSpec listUnhealthyAsPlainText(String ownerId) { - return webTestClient.get().uri( - UriBuilder.fromUri(managementContainerUrl) - .path(UNHEALTHY_PATH) - .queryParam("ownerSourceName", "Plaintext") - .queryParam("ownerId", ownerId) - .queryParam("respectMonitoringSeverity", "false") - .build() - ) - .header("Accept", "text/plain") - .exchange(); - } + private static final String TOPICS_QUERY = "/topics/query"; - public WebTestClient.ResponseSpec listUnhealthyForTopic(String qualifiedName) { - return webTestClient.get().uri( - UriBuilder.fromUri(managementContainerUrl) - .path(UNHEALTHY_PATH) - .queryParam("qualifiedTopicNames", qualifiedName) - .queryParam("respectMonitoringSeverity", "false") - .build() - ) - .header("Accept", "application/json") - .exchange(); - } + private static final String MODE = "/mode"; - public WebTestClient.ResponseSpec listUnhealthyForTopicAsPlainText(String qualifiedName) { - return webTestClient.get().uri( - UriBuilder.fromUri(managementContainerUrl) - .path(UNHEALTHY_PATH) - .queryParam("qualifiedTopicNames", qualifiedName) - .build() - ) - .header("Accept", "text/plain") - .exchange(); - } + private static final String OFFLINE_RETRANSMISSION_TASKS = "/offline-retransmission/tasks"; - public WebTestClient.ResponseSpec listUnhealthyForSubscription(String topicQualifiedName, String subscriptionName) { - return webTestClient.get().uri( - UriBuilder.fromUri(managementContainerUrl) - .path(UNHEALTHY_PATH) - .queryParam("subscriptionNames", subscriptionName) - .queryParam("qualifiedTopicNames", topicQualifiedName) - .queryParam("respectMonitoringSeverity", "false") - .build() - ) - .header("Accept", "application/json") - .exchange(); - } + private static final String OFFLINE_RETRANSMISSION_TASK = + "/offline-retransmission/tasks/{taskId}"; - public WebTestClient.ResponseSpec listUnhealthyForSubscriptionAsPlainText(String topicQualifiedName, String subscriptionName) { - return webTestClient.get().uri( - UriBuilder.fromUri(managementContainerUrl) - .path(UNHEALTHY_PATH) - .queryParam("subscriptionNames", subscriptionName) - .queryParam("qualifiedTopicNames", topicQualifiedName) - .queryParam("respectMonitoringSeverity", "false") - .build() - ) - .header("Accept", "text/plain") - .exchange(); - } + private static final String SUBSCRIPTION_HEALTH = + "/topics/{topicName}/subscriptions/{subscription}/health"; - public WebTestClient.ResponseSpec searchOwners(String source, String searchString) { - return webTestClient.get().uri( - UriBuilder.fromUri(managementContainerUrl) - .path(OWNERS_SEARCH_PATH) - .queryParam("search", searchString) - .build(source) - ) - .exchange(); - } + private static final String CONSUMER_GROUPS = + "/topics/{topicName}/subscriptions/{subscription}/consumer-groups"; - public WebTestClient.ResponseSpec setMode(String mode) { - return webTestClient.post().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(MODE) - .queryParam("mode", mode) - .build()) - .exchange(); - } + private static final String OWNERS_SEARCH_PATH = "/owners/sources/{source}"; - public WebTestClient.ResponseSpec getOfflineRetransmissionTasks() { - return webTestClient.get().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(OFFLINE_RETRANSMISSION_TASKS) - .build()) - .exchange(); - } + private final WebTestClient webTestClient; - public WebTestClient.ResponseSpec deleteOfflineRetransmissionTask(String taskId) { - return webTestClient.delete().uri(UriBuilder.fromUri(managementContainerUrl) - .path(OFFLINE_RETRANSMISSION_TASK) - .build(taskId)) - .exchange(); - } + private final String managementContainerUrl; - public WebTestClient.ResponseSpec createOfflineRetransmissionTask(OfflineRetransmissionRequest request) { - return webTestClient.post().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(OFFLINE_RETRANSMISSION_TASKS) - .build()) - .header("Content-Type", "application/json") - .body(Mono.just(request), OfflineRetransmissionRequest.class) - .exchange(); - } + private final ObjectMapper objectMapper; - public WebTestClient.ResponseSpec querySubscriptions(String qualifiedTopicName, String query) { - return webTestClient.post().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(SUBSCRIPTIONS_QUERY) - .build(qualifiedTopicName)) - .contentType(MediaType.APPLICATION_JSON) - .body(Mono.just(query), String.class) - .exchange(); - } + public ManagementTestClient(int managementPort) { + this.managementContainerUrl = "http://localhost:" + managementPort; + this.webTestClient = configureWebTestClient().build(); + this.objectMapper = new ObjectMapper(); + } - public WebTestClient.ResponseSpec getSubscriptionHealth(String qualifiedTopicName, String name) { - return webTestClient.get().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(SUBSCRIPTION_HEALTH) - .build(qualifiedTopicName, name)) - .exchange(); - } + public ManagementTestClient( + int managementPort, String defaultHeaderName, String defaultHeaderValue) { + this.managementContainerUrl = "http://localhost:" + managementPort; + this.webTestClient = + configureWebTestClient().defaultHeader(defaultHeaderName, defaultHeaderValue).build(); + this.objectMapper = new ObjectMapper(); + } - public WebTestClient.ResponseSpec getConsumerGroupsDescription(String qualifiedTopicName, String subscriptionName) { - return webTestClient.get().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(CONSUMER_GROUPS) - .build(qualifiedTopicName, subscriptionName)) - .exchange(); - } + private WebTestClient.Builder configureWebTestClient() { + return WebTestClient.bindToServer(clientHttpConnector()) + .responseTimeout(Duration.ofSeconds(30)) + .baseUrl(managementContainerUrl) + .codecs(configurer -> configurer.defaultCodecs().maxInMemorySize(16 * 1024 * 1024)); + } - public WebTestClient.ResponseSpec deleteGroup(String groupName) { - return webTestClient.delete().uri(UriBuilder.fromUri(managementContainerUrl) - .path(GROUP_PATH) - .build(groupName)) - .exchange(); - } + private static ClientHttpConnector clientHttpConnector() { + HttpClient httpClient = new HttpClient(); + httpClient.setMaxConnectionsPerDestination(256); + return new JettyClientHttpConnector(httpClient); + } - public WebTestClient.ResponseSpec updateGroup(String groupName, Group group) { - return webTestClient.put().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(GROUP_PATH) - .build(groupName)) - .body(Mono.just(group), Group.class) - .exchange(); - } + public WebTestClient.ResponseSpec createGroup(Group group) { + return sendCreateGroupRequest(group); + } - public WebTestClient.ResponseSpec moveOffsetsToTheEnd(String topicQualifiedName, String subscriptionName) { - return webTestClient.post().uri(UriBuilder - .fromUri(managementContainerUrl) - .path(MOVE_SUBSCRIPTION_OFFSETS) - .build(topicQualifiedName, subscriptionName)) - .exchange(); - } + public List getGroups() { + String jsonString = + webTestClient + .get() + .uri(GROUPS_PATH) + .exchange() + .expectBody(String.class) + .returnResult() + .getResponseBody(); + + return mapStringJsonToListOfString(jsonString); + } + + public WebTestClient.ResponseSpec createTopic(TopicWithSchema topicWithSchema) { + return sendCreateTopicRequest(topicWithSchema); + } + + public WebTestClient.ResponseSpec getTopic(String topicQualifiedName) { + return getSingleTopic(topicQualifiedName); + } + + public WebTestClient.ResponseSpec createSubscription(Subscription subscription) { + return sendCreateSubscriptionRequest(subscription); + } + + public WebTestClient.ResponseSpec updateSubscription( + Topic topic, String subscription, PatchData patch) { + return webTestClient + .put() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(SUBSCRIPTION_PATH) + .build(topic.getQualifiedName(), subscription)) + .body(Mono.just(patch), PatchData.class) + .exchange(); + } + + public WebTestClient.ResponseSpec updateSubscriptionState( + Topic topic, String subscription, Subscription.State state) { + return webTestClient + .put() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(SUBSCRIPTION_STATE_PATH) + .build(topic.getQualifiedName(), subscription)) + .body(Mono.just(state), Subscription.State.class) + .exchange(); + } + + public WebTestClient.ResponseSpec getSubscription( + String topicQualifiedName, String subscriptionName) { + return getSingleSubscription(topicQualifiedName, subscriptionName); + } + + public WebTestClient.ResponseSpec getSubscriptionMetrics( + String topicQualifiedName, String subscriptionName) { + return webTestClient + .get() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(SUBSCRIPTION_METRICS_PATH) + .build(topicQualifiedName, subscriptionName)) + .exchange(); + } + + private WebTestClient.ResponseSpec getSingleTopic(String topicQualifiedName) { + return webTestClient + .get() + .uri(UriBuilder.fromUri(managementContainerUrl).path(TOPIC_PATH).build(topicQualifiedName)) + .exchange(); + } + + private WebTestClient.ResponseSpec getSingleSubscription( + String topicQualifiedName, String subscriptionName) { + return webTestClient + .get() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(SUBSCRIPTION_PATH) + .build(topicQualifiedName, subscriptionName)) + .exchange(); + } + + private WebTestClient.ResponseSpec sendCreateTopicRequest(TopicWithSchema topicWithSchema) { + return webTestClient + .post() + .uri(TOPICS_PATH) + .body(Mono.just(topicWithSchema), TopicWithSchema.class) + .exchange(); + } + + WebTestClient.ResponseSpec retransmit( + String topicName, + String subscriptionName, + OffsetRetransmissionDate retransmissionDate, + boolean dryRun) { + return webTestClient + .put() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(RETRANSMISSION_PATH) + .queryParam("dryRun", dryRun) + .build(topicName, subscriptionName)) + .body(Mono.just(retransmissionDate), OffsetRetransmissionDate.class) + .exchange(); + } + + private WebTestClient.ResponseSpec sendCreateSubscriptionRequest(Subscription subscription) { + return webTestClient + .post() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(SUBSCRIPTIONS_PATH) + .build(subscription.getQualifiedTopicName())) + .body(Mono.just(subscription), Subscription.class) + .exchange(); + } + + private WebTestClient.ResponseSpec sendCreateGroupRequest(Group group) { + return webTestClient.post().uri(GROUPS_PATH).body(Mono.just(group), Group.class).exchange(); + } + + private List mapStringJsonToListOfString(String jsonString) { + try { + return objectMapper.readValue(jsonString, new TypeReference<>() {}); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + public WebTestClient.ResponseSpec blacklistTopic(String topicQualifiedName) { + return webTestClient + .post() + .uri(BLACKLIST_TOPICS_PATH) + .body(Mono.just(List.of(topicQualifiedName)), List.class) + .exchange(); + } + + public WebTestClient.ResponseSpec unblacklistTopic(String topicQualifiedName) { + return webTestClient + .delete() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(BLACKLIST_TOPIC_PATH) + .build(topicQualifiedName)) + .exchange(); + } + + public WebTestClient.ResponseSpec isTopicBlacklisted(String topicQualifiedName) { + return webTestClient + .get() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(BLACKLIST_TOPIC_PATH) + .build(topicQualifiedName)) + .exchange(); + } + + public WebTestClient.ResponseSpec getLatestUndeliveredMessage( + String topicQualifiedName, String subscriptionName) { + return webTestClient + .get() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(LATEST_UNDELIVERED_MESSAGE) + .build(topicQualifiedName, subscriptionName)) + .exchange(); + } + + public WebTestClient.ResponseSpec getPreview( + String qualifiedTopicName, String primaryKafkaClusterName, int partition, long offset) { + return webTestClient + .get() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(TOPIC_PREVIEW_OFFSET) + .build(qualifiedTopicName, primaryKafkaClusterName, partition, offset)) + .exchange(); + } + + public WebTestClient.ResponseSpec getPreview(String qualifiedTopicName) { + return webTestClient + .get() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(TOPIC_PREVIEW) + .build(qualifiedTopicName)) + .exchange(); + } + + public WebTestClient.ResponseSpec updateTopic(String qualifiedTopicName, PatchData patch) { + return webTestClient + .put() + .uri(UriBuilder.fromUri(managementContainerUrl).path(TOPIC_PATH).build(qualifiedTopicName)) + .body(Mono.just(patch), PatchData.class) + .exchange(); + } + + public WebTestClient.ResponseSpec getTopicMetrics(String qualifiedTopicName) { + return webTestClient + .get() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(TOPIC_METRICS_PATH) + .build(qualifiedTopicName)) + .exchange(); + } + + public WebTestClient.ResponseSpec listSubscriptions(String qualifiedTopicName, boolean tracked) { + return webTestClient + .get() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(SUBSCRIPTIONS_PATH) + .queryParam("tracked", tracked) + .build(qualifiedTopicName)) + .exchange(); + } + + public WebTestClient.ResponseSpec listTopics(String groupName, boolean tracked) { + return webTestClient + .get() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(TOPICS_PATH) + .queryParam("groupName", groupName) + .queryParam("tracked", tracked) + .build()) + .exchange(); + } + + public WebTestClient.ResponseSpec setReadiness(String dc, boolean state) { + return webTestClient + .post() + .uri(UriBuilder.fromUri(managementContainerUrl).path(SET_READINESS).build(dc)) + .body(Mono.just(new Readiness(state)), Readiness.class) + .exchange(); + } + + public WebTestClient.ResponseSpec getReadiness() { + return webTestClient + .get() + .uri(UriBuilder.fromUri(managementContainerUrl).path(GET_READINESS).build()) + .exchange(); + } + + public WebTestClient.ResponseSpec saveSchema( + String qualifiedTopicName, boolean validate, String schema) { + return webTestClient + .post() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(TOPIC_SCHEMA) + .queryParam("validate", validate) + .build(qualifiedTopicName)) + .header("Content-Type", "application/json") + .body(Mono.just(schema), String.class) + .exchange(); + } + + public WebTestClient.ResponseSpec saveSchema(String qualifiedTopicName, String schema) { + return webTestClient + .post() + .uri( + UriBuilder.fromUri(managementContainerUrl).path(TOPIC_SCHEMA).build(qualifiedTopicName)) + .header("Content-Type", "application/json") + .body(Mono.just(schema), String.class) + .exchange(); + } + + public WebTestClient.ResponseSpec getSchema(String qualifiedTopicName) { + return webTestClient + .get() + .uri( + UriBuilder.fromUri(managementContainerUrl).path(TOPIC_SCHEMA).build(qualifiedTopicName)) + .exchange(); + } + + public WebTestClient.ResponseSpec deleteSchema(String qualifiedTopicName) { + return webTestClient + .delete() + .uri( + UriBuilder.fromUri(managementContainerUrl).path(TOPIC_SCHEMA).build(qualifiedTopicName)) + .exchange(); + } + + public WebTestClient.ResponseSpec verifyFilters( + String qualifiedTopicName, MessageFiltersVerificationInput input) { + return webTestClient + .post() + .uri(UriBuilder.fromUri(managementContainerUrl).path(FILTERS).build(qualifiedTopicName)) + .contentType(MediaType.APPLICATION_JSON) + .body(Mono.just(input), MessageFiltersVerificationInput.class) + .exchange(); + } + + public WebTestClient.ResponseSpec getStatusHealth() { + return webTestClient + .get() + .uri(UriBuilder.fromUri(managementContainerUrl).path(STATUS_HEALTH).build()) + .exchange(); + } + + public WebTestClient.ResponseSpec getStats() { + return webTestClient + .get() + .uri(UriBuilder.fromUri(managementContainerUrl).path(STATS).build()) + .exchange(); + } + + public WebTestClient.ResponseSpec createOAuthProvider(OAuthProvider provider) { + return webTestClient + .post() + .uri(OAUTH_PROVIDERS_PATH) + .body(Mono.just(provider), OAuthProvider.class) + .exchange(); + } + + public WebTestClient.ResponseSpec getOAuthProvider(String name) { + return webTestClient + .get() + .uri(UriBuilder.fromUri(managementContainerUrl).path(OAUTH_PROVIDER_PATH).build(name)) + .exchange(); + } + + public WebTestClient.ResponseSpec removeOAuthProvider(String name) { + return webTestClient + .delete() + .uri(UriBuilder.fromUri(managementContainerUrl).path(OAUTH_PROVIDER_PATH).build(name)) + .exchange(); + } + + public WebTestClient.ResponseSpec listOAuthProvider() { + return webTestClient + .get() + .uri(UriBuilder.fromUri(managementContainerUrl).path(OAUTH_PROVIDERS_PATH).build()) + .exchange(); + } + + public WebTestClient.ResponseSpec updateOAuthProvider(String name, PatchData patch) { + return webTestClient + .put() + .uri(UriBuilder.fromUri(managementContainerUrl).path(OAUTH_PROVIDER_PATH).build(name)) + .body(Mono.just(patch), PatchData.class) + .exchange(); + } + + public WebTestClient.ResponseSpec getAllTopicClients(String topicQualifiedName) { + return webTestClient + .get() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(ALL_TOPIC_CLIENTS) + .build(topicQualifiedName)) + .exchange(); + } + + public WebTestClient.ResponseSpec getSubscriptionsForOwner(String source, String ownerId) { + return webTestClient + .get() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(SUBSCRIPTIONS_BY_OWNER) + .build(source, ownerId)) + .exchange(); + } + + public WebTestClient.ResponseSpec deleteSubscription( + String topicQualifiedName, String subscriptionName) { + return webTestClient + .delete() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(SUBSCRIPTION_PATH) + .build(topicQualifiedName, subscriptionName)) + .exchange(); + } + + public WebTestClient.ResponseSpec getTopicsForOwner(String source, String ownerId) { + return webTestClient + .get() + .uri( + UriBuilder.fromUri(managementContainerUrl).path(TOPICS_BY_OWNER).build(source, ownerId)) + .exchange(); + } + + public WebTestClient.ResponseSpec deleteTopic(String topicQualifiedName) { + return webTestClient + .delete() + .uri(UriBuilder.fromUri(managementContainerUrl).path(TOPIC_PATH).build(topicQualifiedName)) + .exchange(); + } + + public WebTestClient.ResponseSpec queryTopics(String group, String query) { + return webTestClient + .post() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(TOPICS_QUERY) + .queryParam("groupName", group) + .build()) + .contentType(MediaType.APPLICATION_JSON) + .body(Mono.just(query), String.class) + .exchange(); + } + + public WebTestClient.ResponseSpec queryGroups(String query) { + return webTestClient + .post() + .uri(UriBuilder.fromUri(managementContainerUrl).path(QUERY_GROUPS).build()) + .contentType(MediaType.APPLICATION_JSON) + .body(Mono.just(query), String.class) + .exchange(); + } + + public WebTestClient.ResponseSpec queryTopics(String query) { + return webTestClient + .post() + .uri(UriBuilder.fromUri(managementContainerUrl).path(QUERY_TOPICS).build()) + .contentType(MediaType.APPLICATION_JSON) + .body(Mono.just(query), String.class) + .exchange(); + } + + public WebTestClient.ResponseSpec queryTopicMetrics(String query) { + return webTestClient + .post() + .uri(UriBuilder.fromUri(managementContainerUrl).path(QUERY_TOPIC_METRICS).build()) + .contentType(MediaType.APPLICATION_JSON) + .body(Mono.just(query), String.class) + .exchange(); + } + + public WebTestClient.ResponseSpec querySubscriptionMetrics(String query) { + return webTestClient + .post() + .uri(UriBuilder.fromUri(managementContainerUrl).path(QUERY_SUBSCRIPTION_METRICS).build()) + .contentType(MediaType.APPLICATION_JSON) + .body(Mono.just(query), String.class) + .exchange(); + } + + public WebTestClient.ResponseSpec querySubscriptions(String query) { + return webTestClient + .post() + .uri(UriBuilder.fromUri(managementContainerUrl).path(QUERY_SUBSCRIPTIONS).build()) + .contentType(MediaType.APPLICATION_JSON) + .body(Mono.just(query), String.class) + .exchange(); + } + + public WebTestClient.ResponseSpec listUnhealthy() { + return webTestClient + .get() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(UNHEALTHY_PATH) + .queryParam("respectMonitoringSeverity", "false") + .build()) + .header("Accept", "application/json") + .exchange(); + } + + public WebTestClient.ResponseSpec listUnhealthyAsPlainText() { + return webTestClient + .get() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(UNHEALTHY_PATH) + .queryParam("respectMonitoringSeverity", "false") + .build()) + .header("Accept", "text/plain") + .exchange(); + } + + public WebTestClient.ResponseSpec listUnhealthy(String ownerId) { + return webTestClient + .get() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(UNHEALTHY_PATH) + .queryParam("ownerSourceName", "Plaintext") + .queryParam("ownerId", ownerId) + .queryParam("respectMonitoringSeverity", "false") + .build()) + .header("Accept", "application/json") + .exchange(); + } + + public WebTestClient.ResponseSpec listUnhealthyAsPlainText(String ownerId) { + return webTestClient + .get() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(UNHEALTHY_PATH) + .queryParam("ownerSourceName", "Plaintext") + .queryParam("ownerId", ownerId) + .queryParam("respectMonitoringSeverity", "false") + .build()) + .header("Accept", "text/plain") + .exchange(); + } + + public WebTestClient.ResponseSpec listUnhealthyForTopic(String qualifiedName) { + return webTestClient + .get() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(UNHEALTHY_PATH) + .queryParam("qualifiedTopicNames", qualifiedName) + .queryParam("respectMonitoringSeverity", "false") + .build()) + .header("Accept", "application/json") + .exchange(); + } + + public WebTestClient.ResponseSpec listUnhealthyForTopicAsPlainText(String qualifiedName) { + return webTestClient + .get() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(UNHEALTHY_PATH) + .queryParam("qualifiedTopicNames", qualifiedName) + .build()) + .header("Accept", "text/plain") + .exchange(); + } + + public WebTestClient.ResponseSpec listUnhealthyForSubscription( + String topicQualifiedName, String subscriptionName) { + return webTestClient + .get() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(UNHEALTHY_PATH) + .queryParam("subscriptionNames", subscriptionName) + .queryParam("qualifiedTopicNames", topicQualifiedName) + .queryParam("respectMonitoringSeverity", "false") + .build()) + .header("Accept", "application/json") + .exchange(); + } + + public WebTestClient.ResponseSpec listUnhealthyForSubscriptionAsPlainText( + String topicQualifiedName, String subscriptionName) { + return webTestClient + .get() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(UNHEALTHY_PATH) + .queryParam("subscriptionNames", subscriptionName) + .queryParam("qualifiedTopicNames", topicQualifiedName) + .queryParam("respectMonitoringSeverity", "false") + .build()) + .header("Accept", "text/plain") + .exchange(); + } + + public WebTestClient.ResponseSpec searchOwners(String source, String searchString) { + return webTestClient + .get() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(OWNERS_SEARCH_PATH) + .queryParam("search", searchString) + .build(source)) + .exchange(); + } + + public WebTestClient.ResponseSpec setMode(String mode) { + return webTestClient + .post() + .uri(UriBuilder.fromUri(managementContainerUrl).path(MODE).queryParam("mode", mode).build()) + .exchange(); + } + + public WebTestClient.ResponseSpec getOfflineRetransmissionTasks() { + return webTestClient + .get() + .uri(UriBuilder.fromUri(managementContainerUrl).path(OFFLINE_RETRANSMISSION_TASKS).build()) + .exchange(); + } + + public WebTestClient.ResponseSpec deleteOfflineRetransmissionTask(String taskId) { + return webTestClient + .delete() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(OFFLINE_RETRANSMISSION_TASK) + .build(taskId)) + .exchange(); + } + + public WebTestClient.ResponseSpec createOfflineRetransmissionTask( + OfflineRetransmissionRequest request) { + return webTestClient + .post() + .uri(UriBuilder.fromUri(managementContainerUrl).path(OFFLINE_RETRANSMISSION_TASKS).build()) + .header("Content-Type", "application/json") + .body(Mono.just(request), OfflineRetransmissionRequest.class) + .exchange(); + } + + public WebTestClient.ResponseSpec querySubscriptions(String qualifiedTopicName, String query) { + return webTestClient + .post() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(SUBSCRIPTIONS_QUERY) + .build(qualifiedTopicName)) + .contentType(MediaType.APPLICATION_JSON) + .body(Mono.just(query), String.class) + .exchange(); + } + + public WebTestClient.ResponseSpec getSubscriptionHealth(String qualifiedTopicName, String name) { + return webTestClient + .get() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(SUBSCRIPTION_HEALTH) + .build(qualifiedTopicName, name)) + .exchange(); + } + + public WebTestClient.ResponseSpec getConsumerGroupsDescription( + String qualifiedTopicName, String subscriptionName) { + return webTestClient + .get() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(CONSUMER_GROUPS) + .build(qualifiedTopicName, subscriptionName)) + .exchange(); + } + + public WebTestClient.ResponseSpec deleteGroup(String groupName) { + return webTestClient + .delete() + .uri(UriBuilder.fromUri(managementContainerUrl).path(GROUP_PATH).build(groupName)) + .exchange(); + } + + public WebTestClient.ResponseSpec updateGroup(String groupName, Group group) { + return webTestClient + .put() + .uri(UriBuilder.fromUri(managementContainerUrl).path(GROUP_PATH).build(groupName)) + .body(Mono.just(group), Group.class) + .exchange(); + } + + public WebTestClient.ResponseSpec moveOffsetsToTheEnd( + String topicQualifiedName, String subscriptionName) { + return webTestClient + .post() + .uri( + UriBuilder.fromUri(managementContainerUrl) + .path(MOVE_SUBSCRIPTION_OFFSETS) + .build(topicQualifiedName, subscriptionName)) + .exchange(); + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/concurrent/ManuallyTriggeredScheduledExecutorService.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/concurrent/ManuallyTriggeredScheduledExecutorService.java index 249eb557a0..da3adf2519 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/concurrent/ManuallyTriggeredScheduledExecutorService.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/concurrent/ManuallyTriggeredScheduledExecutorService.java @@ -1,5 +1,6 @@ package pl.allegro.tech.hermes.test.helper.concurrent; +import jakarta.annotation.Nonnull; import java.util.Collection; import java.util.List; import java.util.concurrent.Callable; @@ -8,116 +9,122 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; -import jakarta.annotation.Nonnull; public class ManuallyTriggeredScheduledExecutorService implements ScheduledExecutorService { - private final ConcurrentLinkedQueue> scheduledTasks = new ConcurrentLinkedQueue<>(); - private boolean shutdown; - - @Override - public ScheduledFuture scheduleAtFixedRate(@Nonnull Runnable command, long initialDelay, long period, @Nonnull TimeUnit unit) { - return insertTask(command, initialDelay, unit); - } - - @Override - public ScheduledFuture scheduleWithFixedDelay(@Nonnull Runnable command, long initialDelay, long delay, @Nonnull TimeUnit unit) { - return insertTask(command, initialDelay, unit); - } - - @Override - public void shutdown() { - shutdown = true; - } - - @Override - public List shutdownNow() { - shutdown(); - return List.of(); - } - - @Override - public boolean isShutdown() { - return false; - } - - @Override - public boolean isTerminated() { - return shutdown; - } - - @Override - public boolean awaitTermination(long timeout, @Nonnull TimeUnit unit) { - return true; - } - - @Override - public ScheduledFuture schedule(@Nonnull Runnable command, long delay, @Nonnull TimeUnit unit) { - throw new UnsupportedOperationException(); - } - - @Override - public ScheduledFuture schedule(@Nonnull Callable callable, long delay, @Nonnull TimeUnit unit) { - throw new UnsupportedOperationException(); - } - - @Override - public void execute(@Nonnull Runnable command) { - throw new UnsupportedOperationException(); - } - - @Override - public Future submit(@Nonnull Callable task) { - throw new UnsupportedOperationException(); - } - - @Override - public Future submit(@Nonnull Runnable task, T result) { - throw new UnsupportedOperationException(); - } - - @Override - public Future submit(@Nonnull Runnable task) { - throw new UnsupportedOperationException(); - } - - @Override - public List> invokeAll(@Nonnull Collection> tasks) { - throw new UnsupportedOperationException(); - } - - @Override - public List> invokeAll(@Nonnull Collection> tasks, long timeout, @Nonnull TimeUnit unit) { - throw new UnsupportedOperationException(); - } - - @Override - public T invokeAny(@Nonnull Collection> tasks) { - throw new UnsupportedOperationException(); - } - - @Override - public T invokeAny(@Nonnull Collection> tasks, long timeout, @Nonnull TimeUnit unit) { - throw new UnsupportedOperationException(); - } - - public void triggerScheduledTasks() { - for (ScheduledTask scheduledTask : scheduledTasks) { - if (!scheduledTask.isCancelled()) { - scheduledTask.execute(); - } - } - } - - private ScheduledFuture insertTask(Runnable command, long delay, TimeUnit unit) { - ScheduledTask scheduledTask = new ScheduledTask<>( - () -> { - command.run(); - return null; - }, - unit.convert(delay, TimeUnit.MILLISECONDS) - ); - scheduledTasks.offer(scheduledTask); - return scheduledTask; - } + private final ConcurrentLinkedQueue> scheduledTasks = + new ConcurrentLinkedQueue<>(); + private boolean shutdown; + + @Override + public ScheduledFuture scheduleAtFixedRate( + @Nonnull Runnable command, long initialDelay, long period, @Nonnull TimeUnit unit) { + return insertTask(command, initialDelay, unit); + } + + @Override + public ScheduledFuture scheduleWithFixedDelay( + @Nonnull Runnable command, long initialDelay, long delay, @Nonnull TimeUnit unit) { + return insertTask(command, initialDelay, unit); + } + + @Override + public void shutdown() { + shutdown = true; + } + + @Override + public List shutdownNow() { + shutdown(); + return List.of(); + } + + @Override + public boolean isShutdown() { + return false; + } + + @Override + public boolean isTerminated() { + return shutdown; + } + + @Override + public boolean awaitTermination(long timeout, @Nonnull TimeUnit unit) { + return true; + } + + @Override + public ScheduledFuture schedule( + @Nonnull Runnable command, long delay, @Nonnull TimeUnit unit) { + throw new UnsupportedOperationException(); + } + + @Override + public ScheduledFuture schedule( + @Nonnull Callable callable, long delay, @Nonnull TimeUnit unit) { + throw new UnsupportedOperationException(); + } + + @Override + public void execute(@Nonnull Runnable command) { + throw new UnsupportedOperationException(); + } + + @Override + public Future submit(@Nonnull Callable task) { + throw new UnsupportedOperationException(); + } + + @Override + public Future submit(@Nonnull Runnable task, T result) { + throw new UnsupportedOperationException(); + } + + @Override + public Future submit(@Nonnull Runnable task) { + throw new UnsupportedOperationException(); + } + + @Override + public List> invokeAll(@Nonnull Collection> tasks) { + throw new UnsupportedOperationException(); + } + + @Override + public List> invokeAll( + @Nonnull Collection> tasks, long timeout, @Nonnull TimeUnit unit) { + throw new UnsupportedOperationException(); + } + + @Override + public T invokeAny(@Nonnull Collection> tasks) { + throw new UnsupportedOperationException(); + } + + @Override + public T invokeAny( + @Nonnull Collection> tasks, long timeout, @Nonnull TimeUnit unit) { + throw new UnsupportedOperationException(); + } + + public void triggerScheduledTasks() { + for (ScheduledTask scheduledTask : scheduledTasks) { + if (!scheduledTask.isCancelled()) { + scheduledTask.execute(); + } + } + } + + private ScheduledFuture insertTask(Runnable command, long delay, TimeUnit unit) { + ScheduledTask scheduledTask = + new ScheduledTask<>( + () -> { + command.run(); + return null; + }, + unit.convert(delay, TimeUnit.MILLISECONDS)); + scheduledTasks.offer(scheduledTask); + return scheduledTask; + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/concurrent/ScheduledTask.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/concurrent/ScheduledTask.java index 4e9fcf7cb6..b101a29673 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/concurrent/ScheduledTask.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/concurrent/ScheduledTask.java @@ -1,5 +1,6 @@ package pl.allegro.tech.hermes.test.helper.concurrent; +import jakarta.annotation.Nonnull; import java.util.Objects; import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; @@ -8,62 +9,62 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import jakarta.annotation.Nonnull; class ScheduledTask implements ScheduledFuture { - private final Callable callable; - private final long delay; - private final CompletableFuture result; + private final Callable callable; + private final long delay; + private final CompletableFuture result; - public ScheduledTask(Callable callable, long delay) { - this.callable = Objects.requireNonNull(callable); - this.result = new CompletableFuture<>(); - this.delay = delay; - } + public ScheduledTask(Callable callable, long delay) { + this.callable = Objects.requireNonNull(callable); + this.result = new CompletableFuture<>(); + this.delay = delay; + } - public void execute() { - if (!result.isDone()) { - try { - callable.call(); - } catch (Exception e) { - result.completeExceptionally(e); - } - } + public void execute() { + if (!result.isDone()) { + try { + callable.call(); + } catch (Exception e) { + result.completeExceptionally(e); + } } + } - @Override - public long getDelay(TimeUnit unit) { - return unit.convert(delay, TimeUnit.MILLISECONDS); - } + @Override + public long getDelay(TimeUnit unit) { + return unit.convert(delay, TimeUnit.MILLISECONDS); + } - @Override - public int compareTo(Delayed o) { - return Long.compare(getDelay(TimeUnit.MILLISECONDS), o.getDelay(TimeUnit.MILLISECONDS)); - } + @Override + public int compareTo(Delayed o) { + return Long.compare(getDelay(TimeUnit.MILLISECONDS), o.getDelay(TimeUnit.MILLISECONDS)); + } - @Override - public boolean cancel(boolean mayInterruptIfRunning) { - return result.cancel(mayInterruptIfRunning); - } + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + return result.cancel(mayInterruptIfRunning); + } - @Override - public boolean isCancelled() { - return result.isCancelled(); - } + @Override + public boolean isCancelled() { + return result.isCancelled(); + } - @Override - public boolean isDone() { - return result.isDone(); - } + @Override + public boolean isDone() { + return result.isDone(); + } - @Override - public T get() throws InterruptedException, ExecutionException { - return result.get(); - } + @Override + public T get() throws InterruptedException, ExecutionException { + return result.get(); + } - @Override - public T get(long timeout, @Nonnull TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { - return result.get(timeout, unit); - } + @Override + public T get(long timeout, @Nonnull TimeUnit unit) + throws InterruptedException, ExecutionException, TimeoutException { + return result.get(timeout, unit); + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/concurrent/TestExecutorServiceFactory.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/concurrent/TestExecutorServiceFactory.java index 19b3fedda1..f4a61c13a8 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/concurrent/TestExecutorServiceFactory.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/concurrent/TestExecutorServiceFactory.java @@ -1,19 +1,18 @@ package pl.allegro.tech.hermes.test.helper.concurrent; -import pl.allegro.tech.hermes.common.concurrent.ExecutorServiceFactory; - import java.util.concurrent.ScheduledExecutorService; +import pl.allegro.tech.hermes.common.concurrent.ExecutorServiceFactory; public class TestExecutorServiceFactory implements ExecutorServiceFactory { - private final ScheduledExecutorService scheduledExecutorService; + private final ScheduledExecutorService scheduledExecutorService; - public TestExecutorServiceFactory(ScheduledExecutorService scheduledExecutorService) { - this.scheduledExecutorService = scheduledExecutorService; - } + public TestExecutorServiceFactory(ScheduledExecutorService scheduledExecutorService) { + this.scheduledExecutorService = scheduledExecutorService; + } - @Override - public ScheduledExecutorService createSingleThreadScheduledExecutor(String nameFormat) { - return scheduledExecutorService; - } + @Override + public ScheduledExecutorService createSingleThreadScheduledExecutor(String nameFormat) { + return scheduledExecutorService; + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/BrokerId.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/BrokerId.java index 9d866ce164..b7c40b547c 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/BrokerId.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/BrokerId.java @@ -3,26 +3,26 @@ import java.util.Objects; public class BrokerId { - private final int id; + private final int id; - public BrokerId(int id) { - this.id = id; - } + public BrokerId(int id) { + this.id = id; + } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - BrokerId brokerId = (BrokerId) o; - return id == brokerId.id; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - @Override - public int hashCode() { - return Objects.hash(id); + if (o == null || getClass() != o.getClass()) { + return false; } + BrokerId brokerId = (BrokerId) o; + return id == brokerId.id; + } + + @Override + public int hashCode() { + return Objects.hash(id); + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/ConfluentSchemaRegistryContainer.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/ConfluentSchemaRegistryContainer.java index c7ebb866d7..deddc6bb03 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/ConfluentSchemaRegistryContainer.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/ConfluentSchemaRegistryContainer.java @@ -1,31 +1,35 @@ package pl.allegro.tech.hermes.test.helper.containers; +import static java.lang.String.format; + import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.utility.DockerImageName; -import static java.lang.String.format; - -public class ConfluentSchemaRegistryContainer extends GenericContainer { - private static final DockerImageName DEFAULT_SCHEMA_REGISTRY_IMAGE_NAME = DockerImageName.parse("confluentinc/cp-schema-registry") - .withTag(ImageTags.confluentImagesTag()); - private static final int SCHEMA_REGISTRY_PORT = 8081; +public class ConfluentSchemaRegistryContainer + extends GenericContainer { + private static final DockerImageName DEFAULT_SCHEMA_REGISTRY_IMAGE_NAME = + DockerImageName.parse("confluentinc/cp-schema-registry") + .withTag(ImageTags.confluentImagesTag()); + private static final int SCHEMA_REGISTRY_PORT = 8081; - public ConfluentSchemaRegistryContainer() { - super(DEFAULT_SCHEMA_REGISTRY_IMAGE_NAME); - addEnv("SCHEMA_REGISTRY_HOST_NAME", "localhost"); - withExposedPorts(SCHEMA_REGISTRY_PORT); - withNetworkAliases("schema-registry"); - waitingFor(Wait.forHttp("/subjects")); - } + public ConfluentSchemaRegistryContainer() { + super(DEFAULT_SCHEMA_REGISTRY_IMAGE_NAME); + addEnv("SCHEMA_REGISTRY_HOST_NAME", "localhost"); + withExposedPorts(SCHEMA_REGISTRY_PORT); + withNetworkAliases("schema-registry"); + waitingFor(Wait.forHttp("/subjects")); + } - public ConfluentSchemaRegistryContainer withKafkaCluster(KafkaContainerCluster cluster) { - withNetwork(cluster.getNetwork()); - withEnv("SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS", "PLAINTEXT://" + cluster.getBootstrapServersForInternalClients()); - return self(); - } + public ConfluentSchemaRegistryContainer withKafkaCluster(KafkaContainerCluster cluster) { + withNetwork(cluster.getNetwork()); + withEnv( + "SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS", + "PLAINTEXT://" + cluster.getBootstrapServersForInternalClients()); + return self(); + } - public String getUrl() { - return format("http://%s:%d", getHost(), getMappedPort(SCHEMA_REGISTRY_PORT)); - } + public String getUrl() { + return format("http://%s:%d", getHost(), getMappedPort(SCHEMA_REGISTRY_PORT)); + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/GooglePubSubContainer.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/GooglePubSubContainer.java index ba4722ab3c..78c6cc61e6 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/GooglePubSubContainer.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/GooglePubSubContainer.java @@ -5,9 +5,10 @@ public class GooglePubSubContainer extends PubSubEmulatorContainer { - private static final String DOCKER_IMAGE = "gcr.io/google.com/cloudsdktool/cloud-sdk:367.0.0-emulators"; + private static final String DOCKER_IMAGE = + "gcr.io/google.com/cloudsdktool/cloud-sdk:367.0.0-emulators"; - public GooglePubSubContainer() { - super(DockerImageName.parse(DOCKER_IMAGE)); - } + public GooglePubSubContainer() { + super(DockerImageName.parse(DOCKER_IMAGE)); + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/ImageTags.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/ImageTags.java index 3c2751bc42..cf21b92454 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/ImageTags.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/ImageTags.java @@ -1,10 +1,10 @@ package pl.allegro.tech.hermes.test.helper.containers; public class ImageTags { - public static String confluentImagesTag() { - if (System.getProperty("os.arch").equals("aarch64")) { - return System.getProperty("confluentImagesTag", "7.6.1"); - } - return System.getProperty("confluentImagesTag", "6.1.0"); + public static String confluentImagesTag() { + if (System.getProperty("os.arch").equals("aarch64")) { + return System.getProperty("confluentImagesTag", "7.6.1"); } + return System.getProperty("confluentImagesTag", "6.1.0"); + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/KafkaContainer.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/KafkaContainer.java index a608318764..a5fdbd3093 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/KafkaContainer.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/KafkaContainer.java @@ -1,149 +1,161 @@ package pl.allegro.tech.hermes.test.helper.containers; +import static pl.allegro.tech.hermes.test.helper.containers.TestcontainersUtils.copyScriptToContainer; +import static pl.allegro.tech.hermes.test.helper.containers.TestcontainersUtils.readFileFromClasspath; + import com.github.dockerjava.api.command.InspectContainerResponse; +import java.io.IOException; +import java.time.Duration; +import java.util.concurrent.TimeUnit; import org.rnorth.ducttape.unreliables.Unreliables; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.Network; import org.testcontainers.utility.DockerImageName; -import java.io.IOException; -import java.time.Duration; -import java.util.concurrent.TimeUnit; - -import static pl.allegro.tech.hermes.test.helper.containers.TestcontainersUtils.copyScriptToContainer; -import static pl.allegro.tech.hermes.test.helper.containers.TestcontainersUtils.readFileFromClasspath; - class KafkaContainer extends GenericContainer { - private static final String START_STOP_SCRIPT = "/kafka_start_stop_wrapper.sh"; - private static final String STARTER_SCRIPT = "/kafka_start.sh"; - private static final int KAFKA_PORT = 9093; - private static final int KAFKA_INTERNAL_CLIENT_PORT = 9094; - private static final Duration READINESS_CHECK_TIMEOUT = Duration.ofMinutes(360); - - private final String networkAlias; - private final BrokerId brokerId; - private String externalZookeeperConnect = null; - private int advertisedPort; - private boolean running = false; - - KafkaContainer(DockerImageName dockerImageName, Network network, int brokerNum) { - super(dockerImageName); - withNetwork(network); - withExposedPorts(KAFKA_PORT); - this.brokerId = new BrokerId(brokerNum); - this.networkAlias = "broker-" + brokerNum; - withNetworkAliases(networkAlias); - withEnv("KAFKA_BROKER_ID", "" + brokerNum); - withEnv("KAFKA_LISTENERS", - "PLAINTEXT://0.0.0.0:" + KAFKA_PORT + ",BROKER://0.0.0.0:9092,INTERNAL_CLIENT://0.0.0.0:" + KAFKA_INTERNAL_CLIENT_PORT - ); - withEnv("KAFKA_LISTENER_SECURITY_PROTOCOL_MAP", "BROKER:PLAINTEXT,PLAINTEXT:PLAINTEXT,INTERNAL_CLIENT:PLAINTEXT"); - withEnv("KAFKA_INTER_BROKER_LISTENER_NAME", "BROKER"); - withEnv("KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR", "1"); - withEnv("KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS", "1"); - withEnv("KAFKA_LOG_FLUSH_INTERVAL_MESSAGES", Long.MAX_VALUE + ""); - withEnv("KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS", "0"); - } - - BrokerId getBrokerId() { - return brokerId; + private static final String START_STOP_SCRIPT = "/kafka_start_stop_wrapper.sh"; + private static final String STARTER_SCRIPT = "/kafka_start.sh"; + private static final int KAFKA_PORT = 9093; + private static final int KAFKA_INTERNAL_CLIENT_PORT = 9094; + private static final Duration READINESS_CHECK_TIMEOUT = Duration.ofMinutes(360); + + private final String networkAlias; + private final BrokerId brokerId; + private String externalZookeeperConnect = null; + private int advertisedPort; + private boolean running = false; + + KafkaContainer(DockerImageName dockerImageName, Network network, int brokerNum) { + super(dockerImageName); + withNetwork(network); + withExposedPorts(KAFKA_PORT); + this.brokerId = new BrokerId(brokerNum); + this.networkAlias = "broker-" + brokerNum; + withNetworkAliases(networkAlias); + withEnv("KAFKA_BROKER_ID", "" + brokerNum); + withEnv( + "KAFKA_LISTENERS", + "PLAINTEXT://0.0.0.0:" + + KAFKA_PORT + + ",BROKER://0.0.0.0:9092,INTERNAL_CLIENT://0.0.0.0:" + + KAFKA_INTERNAL_CLIENT_PORT); + withEnv( + "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP", + "BROKER:PLAINTEXT,PLAINTEXT:PLAINTEXT,INTERNAL_CLIENT:PLAINTEXT"); + withEnv("KAFKA_INTER_BROKER_LISTENER_NAME", "BROKER"); + withEnv("KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR", "1"); + withEnv("KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS", "1"); + withEnv("KAFKA_LOG_FLUSH_INTERVAL_MESSAGES", Long.MAX_VALUE + ""); + withEnv("KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS", "0"); + } + + BrokerId getBrokerId() { + return brokerId; + } + + boolean isKafkaRunning() { + return running; + } + + void stopKafka() { + try { + ExecResult execResult = execInContainer("sh", "-c", "touch /tmp/stop"); + if (execResult.getExitCode() != 0) { + throw new Exception(execResult.getStderr()); + } + Unreliables.retryUntilTrue( + (int) READINESS_CHECK_TIMEOUT.getSeconds(), TimeUnit.SECONDS, this::isStopped); + running = false; + } catch (Exception ex) { + throw new RuntimeException(ex); } - - boolean isKafkaRunning() { - return running; - } - - void stopKafka() { - try { - ExecResult execResult = execInContainer("sh", "-c", "touch /tmp/stop"); - if (execResult.getExitCode() != 0) { - throw new Exception(execResult.getStderr()); - } - Unreliables.retryUntilTrue((int) READINESS_CHECK_TIMEOUT.getSeconds(), TimeUnit.SECONDS, this::isStopped); - running = false; - } catch (Exception ex) { - throw new RuntimeException(ex); - } - } - - void startKafka() { - try { - ExecResult execResult = execInContainer("sh", "-c", "touch /tmp/start"); - if (execResult.getExitCode() != 0) { - throw new Exception(execResult.getStderr()); - } - Unreliables.retryUntilTrue((int) READINESS_CHECK_TIMEOUT.getSeconds(), TimeUnit.SECONDS, this::isStarted); - running = true; - } catch (Exception ex) { - throw new RuntimeException(ex); - } + } + + void startKafka() { + try { + ExecResult execResult = execInContainer("sh", "-c", "touch /tmp/start"); + if (execResult.getExitCode() != 0) { + throw new Exception(execResult.getStderr()); + } + Unreliables.retryUntilTrue( + (int) READINESS_CHECK_TIMEOUT.getSeconds(), TimeUnit.SECONDS, this::isStarted); + running = true; + } catch (Exception ex) { + throw new RuntimeException(ex); } - - private boolean isStopped() throws IOException, InterruptedException { - String command = "ps aux | grep kafka"; - ExecResult result = execInContainer("sh", "-c", command); - return result.getExitCode() == 0 && !result.getStdout().contains("java"); - } - - private boolean isStarted() throws IOException, InterruptedException { - ExecResult result = execInContainer("sh", "-c", "kafka-topics --bootstrap-server localhost:9092 --list"); - return result.getExitCode() == 0; - } - - KafkaContainer withExternalZookeeper(String connectString) { - externalZookeeperConnect = connectString; - return self(); - } - - KafkaContainer withAdvertisedPort(int advertisedPort) { - this.advertisedPort = advertisedPort; - return self(); - } - - String getAddressForExternalClients() { - return String.format("%s:%s", getHost(), advertisedPort); - } - - String getAddressForInternalClients() { - return String.format("%s:%s", networkAlias, KAFKA_INTERNAL_CLIENT_PORT); - } - - @Override - protected void doStart() { - withCommand("sh", "-c", "while [ ! -f " + START_STOP_SCRIPT + " ]; do sleep 0.1; done; " + START_STOP_SCRIPT); - super.doStart(); - } - - @Override - protected void containerIsStarting(InspectContainerResponse containerInfo, boolean reused) { - try { - super.containerIsStarting(containerInfo, reused); - if (reused) { - return; - } - String startScript = readFileFromClasspath("testcontainers/kafka_start.sh") - .replaceAll("", advertisedPort + "") - .replaceAll("", KAFKA_INTERNAL_CLIENT_PORT + "") - .replaceAll("", networkAlias) - .replaceAll("", externalZookeeperConnect); - copyScriptToContainer(startScript, this, STARTER_SCRIPT); - String wrapperScript = readFileFromClasspath("testcontainers/kafka_start_stop_wrapper.sh") - .replaceAll("", STARTER_SCRIPT); - copyScriptToContainer(wrapperScript, this, START_STOP_SCRIPT); - } catch (Exception ex) { - throw new RuntimeException(ex); - } - } - - @Override - protected void containerIsStarted(InspectContainerResponse containerInfo) { - super.containerIsStarted(containerInfo); - running = true; - } - - @Override - protected void containerIsStopped(InspectContainerResponse containerInfo) { - super.containerIsStopped(containerInfo); - running = false; + } + + private boolean isStopped() throws IOException, InterruptedException { + String command = "ps aux | grep kafka"; + ExecResult result = execInContainer("sh", "-c", command); + return result.getExitCode() == 0 && !result.getStdout().contains("java"); + } + + private boolean isStarted() throws IOException, InterruptedException { + ExecResult result = + execInContainer("sh", "-c", "kafka-topics --bootstrap-server localhost:9092 --list"); + return result.getExitCode() == 0; + } + + KafkaContainer withExternalZookeeper(String connectString) { + externalZookeeperConnect = connectString; + return self(); + } + + KafkaContainer withAdvertisedPort(int advertisedPort) { + this.advertisedPort = advertisedPort; + return self(); + } + + String getAddressForExternalClients() { + return String.format("%s:%s", getHost(), advertisedPort); + } + + String getAddressForInternalClients() { + return String.format("%s:%s", networkAlias, KAFKA_INTERNAL_CLIENT_PORT); + } + + @Override + protected void doStart() { + withCommand( + "sh", + "-c", + "while [ ! -f " + START_STOP_SCRIPT + " ]; do sleep 0.1; done; " + START_STOP_SCRIPT); + super.doStart(); + } + + @Override + protected void containerIsStarting(InspectContainerResponse containerInfo, boolean reused) { + try { + super.containerIsStarting(containerInfo, reused); + if (reused) { + return; + } + String startScript = + readFileFromClasspath("testcontainers/kafka_start.sh") + .replaceAll("", advertisedPort + "") + .replaceAll("", KAFKA_INTERNAL_CLIENT_PORT + "") + .replaceAll("", networkAlias) + .replaceAll("", externalZookeeperConnect); + copyScriptToContainer(startScript, this, STARTER_SCRIPT); + String wrapperScript = + readFileFromClasspath("testcontainers/kafka_start_stop_wrapper.sh") + .replaceAll("", STARTER_SCRIPT); + copyScriptToContainer(wrapperScript, this, START_STOP_SCRIPT); + } catch (Exception ex) { + throw new RuntimeException(ex); } + } + + @Override + protected void containerIsStarted(InspectContainerResponse containerInfo) { + super.containerIsStarted(containerInfo); + running = true; + } + + @Override + protected void containerIsStopped(InspectContainerResponse containerInfo) { + super.containerIsStopped(containerInfo); + running = false; + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/KafkaContainerCluster.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/KafkaContainerCluster.java index 3524c40956..0af027a1ff 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/KafkaContainerCluster.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/KafkaContainerCluster.java @@ -1,13 +1,11 @@ package pl.allegro.tech.hermes.test.helper.containers; -import org.testcontainers.containers.Container.ExecResult; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.containers.Network; -import org.testcontainers.containers.ToxiproxyContainer; -import org.testcontainers.containers.ToxiproxyContainer.ContainerProxy; -import org.testcontainers.lifecycle.Startable; -import org.testcontainers.lifecycle.Startables; -import org.testcontainers.utility.DockerImageName; +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.concurrent.TimeUnit.SECONDS; +import static java.util.stream.Collectors.toList; +import static org.rnorth.ducttape.unreliables.Unreliables.retryUntilTrue; +import static pl.allegro.tech.hermes.test.helper.containers.TestcontainersUtils.copyScriptToContainer; +import static pl.allegro.tech.hermes.test.helper.containers.TestcontainersUtils.readFileFromClasspath; import java.io.IOException; import java.time.Duration; @@ -17,203 +15,203 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.util.concurrent.TimeUnit.SECONDS; -import static java.util.stream.Collectors.toList; -import static org.rnorth.ducttape.unreliables.Unreliables.retryUntilTrue; -import static pl.allegro.tech.hermes.test.helper.containers.TestcontainersUtils.copyScriptToContainer; -import static pl.allegro.tech.hermes.test.helper.containers.TestcontainersUtils.readFileFromClasspath; +import org.testcontainers.containers.Container.ExecResult; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.ToxiproxyContainer; +import org.testcontainers.containers.ToxiproxyContainer.ContainerProxy; +import org.testcontainers.lifecycle.Startable; +import org.testcontainers.lifecycle.Startables; +import org.testcontainers.utility.DockerImageName; public class KafkaContainerCluster implements Startable { - private static final DockerImageName ZOOKEEPER_IMAGE_NAME = DockerImageName.parse("confluentinc/cp-zookeeper") - .withTag(ImageTags.confluentImagesTag()); - private static final DockerImageName KAFKA_IMAGE_NAME = DockerImageName.parse("confluentinc/cp-kafka") - .withTag(ImageTags.confluentImagesTag()); - private static final DockerImageName TOXIPROXY_IMAGE_NAME = DockerImageName.parse("ghcr.io/shopify/toxiproxy") - .withTag("2.4.0").asCompatibleSubstituteFor("shopify/toxiproxy"); - - private static final Duration CLUSTER_START_TIMEOUT = Duration.ofMinutes(360); - private static final String ZOOKEEPER_NETWORK_ALIAS = "zookeeper"; - private static final String READINESS_CHECK_SCRIPT = "/kafka_readiness_check.sh"; - private static final int ZOOKEEPER_PORT = 2181; - - private final List proxies = new ArrayList<>(); - private final int brokersNum; - private final int minInSyncReplicas; - private final ZookeeperContainer zookeeper; - private final ToxiproxyContainer toxiproxy; - private final List brokers = new ArrayList<>(); - - public KafkaContainerCluster(int brokersNum) { - checkArgument(brokersNum > 0, "brokersNum '" + brokersNum + "' must be greater than 0"); - this.brokersNum = brokersNum; - this.minInSyncReplicas = Math.max(brokersNum - 1, 1); - this.zookeeper = createZookeeper(Network.newNetwork()); - this.toxiproxy = new ToxiproxyContainer(TOXIPROXY_IMAGE_NAME) - .withNetwork(zookeeper.getNetwork()); - int internalTopicsRf = Math.max(brokersNum - 1, 1); - for (int brokerId = 0; brokerId < brokersNum; brokerId++) { - KafkaContainer container = new KafkaContainer(KAFKA_IMAGE_NAME, zookeeper.getNetwork(), brokerId) - .dependsOn(zookeeper) - .withExternalZookeeper(ZOOKEEPER_NETWORK_ALIAS + ":" + ZOOKEEPER_PORT) - .withEnv("KAFKA_BROKER_ID", brokerId + "") - .withEnv("KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR", internalTopicsRf + "") - .withEnv("KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS", internalTopicsRf + "") - .withEnv("KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR", internalTopicsRf + "") - .withEnv("KAFKA_TRANSACTION_STATE_LOG_MIN_ISR", internalTopicsRf + "") - .withEnv("KAFKA_MIN_INSYNC_REPLICAS", minInSyncReplicas + ""); - brokers.add(container); - } - } - - private ZookeeperContainer createZookeeper(Network network) { - return new ZookeeperContainer(ZOOKEEPER_IMAGE_NAME, ZOOKEEPER_PORT) - .withNetwork(network) - .withNetworkAliases(ZOOKEEPER_NETWORK_ALIAS); - } - - public List getAllBrokers() { - return brokers.stream().map(KafkaContainer::getBrokerId).collect(toList()); - } - - public int getMinInSyncReplicas() { - return minInSyncReplicas; - } - - public String getBootstrapServersForExternalClients() { - return brokers.stream() - .map(KafkaContainer::getAddressForExternalClients) - .collect(Collectors.joining(",")); - } - - public String getBootstrapServersForInternalClients() { - return brokers.stream() - .map(KafkaContainer::getAddressForInternalClients) - .collect(Collectors.joining(",")); - } - - public Network getNetwork() { - return zookeeper.getNetwork(); - } - - @Override - public void start() { - try { - startToxiproxy(); - Startables.deepStart(brokers) - .get(CLUSTER_START_TIMEOUT.getSeconds(), SECONDS); - String readinessScript = readFileFromClasspath("testcontainers/kafka_readiness_check.sh"); - for (KafkaContainer kafkaContainer : brokers) { - copyScriptToContainer(readinessScript, kafkaContainer, READINESS_CHECK_SCRIPT); - } - waitForClusterFormation(); - } catch (Exception ex) { - throw new RuntimeException(ex); - } - } - - private void startToxiproxy() { - toxiproxy.start(); - for (KafkaContainer kafkaContainer : brokers) { - ContainerProxy proxy = toxiproxy.getProxy(kafkaContainer, kafkaContainer.getExposedPorts().get(0)); - proxies.add(proxy); - kafkaContainer.withAdvertisedPort(proxy.getProxyPort()); - } - } - - public void stop(List brokerIds) { - select(brokerIds).stream() - .parallel() - .forEach(KafkaContainer::stopKafka); - } - - public void start(List brokerIds) { - select(brokerIds).stream() - .parallel() - .forEach(KafkaContainer::startKafka); - } - - private Set select(List brokerIds) { - return brokers.stream() - .filter(b -> brokerIds.contains(b.getBrokerId())) - .collect(Collectors.toSet()); - } - - @Override - public void stop() { - toxiproxy.stop(); - Stream.concat(brokers.stream(), Stream.of(zookeeper)) - .parallel() - .forEach(GenericContainer::stop); - } - - public void cutOffConnectionsBetweenBrokersAndClients() { - proxies.forEach(proxy -> proxy.setConnectionCut(true)); - } - - public void restoreConnectionsBetweenBrokersAndClients() { - proxies.forEach(proxy -> proxy.setConnectionCut(false)); - } - - public void startAllStoppedBrokers() { - brokers.stream() - .filter(broker -> !broker.isKafkaRunning()) - .parallel() - .forEach(KafkaContainer::startKafka); - waitForClusterFormation(); - } - - private void waitForClusterFormation() { - retryUntilTrue((int) CLUSTER_START_TIMEOUT.getSeconds(), TimeUnit.SECONDS, this::isZookeeperReady); - retryUntilTrue((int) CLUSTER_START_TIMEOUT.getSeconds(), TimeUnit.SECONDS, this::isKafkaReady); - } - - private boolean isZookeeperReady() throws IOException, InterruptedException { - ExecResult result = zookeeper.execInContainer( - "sh", - "-c", - "zookeeper-shell " + ZOOKEEPER_NETWORK_ALIAS + ":" + ZOOKEEPER_PORT + " ls /brokers/ids | tail -n 1" - ); - String brokers = result.getStdout(); - return brokers != null && brokers.split(",").length == brokersNum; - } - - private boolean isKafkaReady() throws IOException, InterruptedException { - KafkaContainer firstBroker = selectFirstRunningBroker(); - ExecResult result = firstBroker.execInContainer( - "sh", - "-c", - READINESS_CHECK_SCRIPT + " " + brokersNum - ); - return result.getExitCode() == 0; - } - - public int countOfflinePartitions() throws IOException, InterruptedException { - return countPartitions("--unavailable-partitions"); - } - - public int countUnderReplicatedPartitions() throws IOException, InterruptedException { - return countPartitions("--under-replicated-partitions"); - } - - private int countPartitions(String option) throws IOException, InterruptedException { - KafkaContainer firstBroker = selectFirstRunningBroker(); - ExecResult result = firstBroker.execInContainer( - "sh", - "-c", - "kafka-topics --bootstrap-server localhost:9092 --describe " + option + " | wc -l" - ); - String sanitizedOutput = result.getStdout() - .replaceAll("\"", "") - .replaceAll("\\s+", ""); - return Integer.parseInt(sanitizedOutput); - } - - private KafkaContainer selectFirstRunningBroker() { - return brokers.stream() - .filter(KafkaContainer::isKafkaRunning) - .findFirst() - .orElseThrow(() -> new IllegalStateException("There is no running broker")); - } + private static final DockerImageName ZOOKEEPER_IMAGE_NAME = + DockerImageName.parse("confluentinc/cp-zookeeper").withTag(ImageTags.confluentImagesTag()); + private static final DockerImageName KAFKA_IMAGE_NAME = + DockerImageName.parse("confluentinc/cp-kafka").withTag(ImageTags.confluentImagesTag()); + private static final DockerImageName TOXIPROXY_IMAGE_NAME = + DockerImageName.parse("ghcr.io/shopify/toxiproxy") + .withTag("2.4.0") + .asCompatibleSubstituteFor("shopify/toxiproxy"); + + private static final Duration CLUSTER_START_TIMEOUT = Duration.ofMinutes(360); + private static final String ZOOKEEPER_NETWORK_ALIAS = "zookeeper"; + private static final String READINESS_CHECK_SCRIPT = "/kafka_readiness_check.sh"; + private static final int ZOOKEEPER_PORT = 2181; + + private final List proxies = new ArrayList<>(); + private final int brokersNum; + private final int minInSyncReplicas; + private final ZookeeperContainer zookeeper; + private final ToxiproxyContainer toxiproxy; + private final List brokers = new ArrayList<>(); + + public KafkaContainerCluster(int brokersNum) { + checkArgument(brokersNum > 0, "brokersNum '" + brokersNum + "' must be greater than 0"); + this.brokersNum = brokersNum; + this.minInSyncReplicas = Math.max(brokersNum - 1, 1); + this.zookeeper = createZookeeper(Network.newNetwork()); + this.toxiproxy = + new ToxiproxyContainer(TOXIPROXY_IMAGE_NAME).withNetwork(zookeeper.getNetwork()); + int internalTopicsRf = Math.max(brokersNum - 1, 1); + for (int brokerId = 0; brokerId < brokersNum; brokerId++) { + KafkaContainer container = + new KafkaContainer(KAFKA_IMAGE_NAME, zookeeper.getNetwork(), brokerId) + .dependsOn(zookeeper) + .withExternalZookeeper(ZOOKEEPER_NETWORK_ALIAS + ":" + ZOOKEEPER_PORT) + .withEnv("KAFKA_BROKER_ID", brokerId + "") + .withEnv("KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR", internalTopicsRf + "") + .withEnv("KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS", internalTopicsRf + "") + .withEnv("KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR", internalTopicsRf + "") + .withEnv("KAFKA_TRANSACTION_STATE_LOG_MIN_ISR", internalTopicsRf + "") + .withEnv("KAFKA_MIN_INSYNC_REPLICAS", minInSyncReplicas + ""); + brokers.add(container); + } + } + + private ZookeeperContainer createZookeeper(Network network) { + return new ZookeeperContainer(ZOOKEEPER_IMAGE_NAME, ZOOKEEPER_PORT) + .withNetwork(network) + .withNetworkAliases(ZOOKEEPER_NETWORK_ALIAS); + } + + public List getAllBrokers() { + return brokers.stream().map(KafkaContainer::getBrokerId).collect(toList()); + } + + public int getMinInSyncReplicas() { + return minInSyncReplicas; + } + + public String getBootstrapServersForExternalClients() { + return brokers.stream() + .map(KafkaContainer::getAddressForExternalClients) + .collect(Collectors.joining(",")); + } + + public String getBootstrapServersForInternalClients() { + return brokers.stream() + .map(KafkaContainer::getAddressForInternalClients) + .collect(Collectors.joining(",")); + } + + public Network getNetwork() { + return zookeeper.getNetwork(); + } + + @Override + public void start() { + try { + startToxiproxy(); + Startables.deepStart(brokers).get(CLUSTER_START_TIMEOUT.getSeconds(), SECONDS); + String readinessScript = readFileFromClasspath("testcontainers/kafka_readiness_check.sh"); + for (KafkaContainer kafkaContainer : brokers) { + copyScriptToContainer(readinessScript, kafkaContainer, READINESS_CHECK_SCRIPT); + } + waitForClusterFormation(); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + private void startToxiproxy() { + toxiproxy.start(); + for (KafkaContainer kafkaContainer : brokers) { + ContainerProxy proxy = + toxiproxy.getProxy(kafkaContainer, kafkaContainer.getExposedPorts().get(0)); + proxies.add(proxy); + kafkaContainer.withAdvertisedPort(proxy.getProxyPort()); + } + } + + public void stop(List brokerIds) { + select(brokerIds).stream().parallel().forEach(KafkaContainer::stopKafka); + } + + public void start(List brokerIds) { + select(brokerIds).stream().parallel().forEach(KafkaContainer::startKafka); + } + + private Set select(List brokerIds) { + return brokers.stream() + .filter(b -> brokerIds.contains(b.getBrokerId())) + .collect(Collectors.toSet()); + } + + @Override + public void stop() { + toxiproxy.stop(); + Stream.concat(brokers.stream(), Stream.of(zookeeper)) + .parallel() + .forEach(GenericContainer::stop); + } + + public void cutOffConnectionsBetweenBrokersAndClients() { + proxies.forEach(proxy -> proxy.setConnectionCut(true)); + } + + public void restoreConnectionsBetweenBrokersAndClients() { + proxies.forEach(proxy -> proxy.setConnectionCut(false)); + } + + public void startAllStoppedBrokers() { + brokers.stream() + .filter(broker -> !broker.isKafkaRunning()) + .parallel() + .forEach(KafkaContainer::startKafka); + waitForClusterFormation(); + } + + private void waitForClusterFormation() { + retryUntilTrue( + (int) CLUSTER_START_TIMEOUT.getSeconds(), TimeUnit.SECONDS, this::isZookeeperReady); + retryUntilTrue((int) CLUSTER_START_TIMEOUT.getSeconds(), TimeUnit.SECONDS, this::isKafkaReady); + } + + private boolean isZookeeperReady() throws IOException, InterruptedException { + ExecResult result = + zookeeper.execInContainer( + "sh", + "-c", + "zookeeper-shell " + + ZOOKEEPER_NETWORK_ALIAS + + ":" + + ZOOKEEPER_PORT + + " ls /brokers/ids | tail -n 1"); + String brokers = result.getStdout(); + return brokers != null && brokers.split(",").length == brokersNum; + } + + private boolean isKafkaReady() throws IOException, InterruptedException { + KafkaContainer firstBroker = selectFirstRunningBroker(); + ExecResult result = + firstBroker.execInContainer("sh", "-c", READINESS_CHECK_SCRIPT + " " + brokersNum); + return result.getExitCode() == 0; + } + + public int countOfflinePartitions() throws IOException, InterruptedException { + return countPartitions("--unavailable-partitions"); + } + + public int countUnderReplicatedPartitions() throws IOException, InterruptedException { + return countPartitions("--under-replicated-partitions"); + } + + private int countPartitions(String option) throws IOException, InterruptedException { + KafkaContainer firstBroker = selectFirstRunningBroker(); + ExecResult result = + firstBroker.execInContainer( + "sh", + "-c", + "kafka-topics --bootstrap-server localhost:9092 --describe " + option + " | wc -l"); + String sanitizedOutput = result.getStdout().replaceAll("\"", "").replaceAll("\\s+", ""); + return Integer.parseInt(sanitizedOutput); + } + + private KafkaContainer selectFirstRunningBroker() { + return brokers.stream() + .filter(KafkaContainer::isKafkaRunning) + .findFirst() + .orElseThrow(() -> new IllegalStateException("There is no running broker")); + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/TestcontainersUtils.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/TestcontainersUtils.java index 7286a46d36..6fb010caf7 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/TestcontainersUtils.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/TestcontainersUtils.java @@ -1,27 +1,29 @@ package pl.allegro.tech.hermes.test.helper.containers; -import org.testcontainers.containers.ContainerState; -import org.testcontainers.images.builder.Transferable; -import org.testcontainers.shaded.org.apache.commons.io.IOUtils; +import static java.nio.charset.StandardCharsets.UTF_8; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; - -import static java.nio.charset.StandardCharsets.UTF_8; +import org.testcontainers.containers.ContainerState; +import org.testcontainers.images.builder.Transferable; +import org.testcontainers.shaded.org.apache.commons.io.IOUtils; public class TestcontainersUtils { - public static void copyScriptToContainer(String content, ContainerState containerState, String target) { - containerState.copyFileToContainer(Transferable.of(content.getBytes(StandardCharsets.UTF_8), 0777), target); - } + public static void copyScriptToContainer( + String content, ContainerState containerState, String target) { + containerState.copyFileToContainer( + Transferable.of(content.getBytes(StandardCharsets.UTF_8), 0777), target); + } - public static String readFileFromClasspath(String fileName) throws IOException { - InputStream inputStream = TestcontainersUtils.class.getClassLoader().getResourceAsStream(fileName); - if (inputStream == null) { - throw new FileNotFoundException("File '" + fileName + "' does not exist"); - } - return IOUtils.toString(inputStream, UTF_8); + public static String readFileFromClasspath(String fileName) throws IOException { + InputStream inputStream = + TestcontainersUtils.class.getClassLoader().getResourceAsStream(fileName); + if (inputStream == null) { + throw new FileNotFoundException("File '" + fileName + "' does not exist"); } + return IOUtils.toString(inputStream, UTF_8); + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/ZookeeperContainer.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/ZookeeperContainer.java index 787b8a364c..817e327831 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/ZookeeperContainer.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/containers/ZookeeperContainer.java @@ -1,5 +1,6 @@ package pl.allegro.tech.hermes.test.helper.containers; +import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.containers.GenericContainer; @@ -7,52 +8,50 @@ import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.utility.DockerImageName; -import java.util.Optional; - public class ZookeeperContainer extends GenericContainer { - private static final DockerImageName DEFAULT_ZOOKEEPER_IMAGE_NAME = DockerImageName.parse("confluentinc/cp-zookeeper") - .withTag(ImageTags.confluentImagesTag()); - private static final int DEFAULT_ZOOKEEPER_PORT = 2181; - private Logger logger; - - private final int clientPort; - - public ZookeeperContainer() { - this(DEFAULT_ZOOKEEPER_IMAGE_NAME, DEFAULT_ZOOKEEPER_PORT); - } - - public ZookeeperContainer(String loggerName) { - this(DEFAULT_ZOOKEEPER_IMAGE_NAME, DEFAULT_ZOOKEEPER_PORT, Optional.of(loggerName)); - } - - public ZookeeperContainer(DockerImageName zooKeeperImage, int clientPort) { - this(zooKeeperImage, clientPort, Optional.empty()); + private static final DockerImageName DEFAULT_ZOOKEEPER_IMAGE_NAME = + DockerImageName.parse("confluentinc/cp-zookeeper").withTag(ImageTags.confluentImagesTag()); + private static final int DEFAULT_ZOOKEEPER_PORT = 2181; + private Logger logger; + + private final int clientPort; + + public ZookeeperContainer() { + this(DEFAULT_ZOOKEEPER_IMAGE_NAME, DEFAULT_ZOOKEEPER_PORT); + } + + public ZookeeperContainer(String loggerName) { + this(DEFAULT_ZOOKEEPER_IMAGE_NAME, DEFAULT_ZOOKEEPER_PORT, Optional.of(loggerName)); + } + + public ZookeeperContainer(DockerImageName zooKeeperImage, int clientPort) { + this(zooKeeperImage, clientPort, Optional.empty()); + } + + private ZookeeperContainer( + DockerImageName zooKeeperImage, int clientPort, Optional maybeLoggerName) { + super(zooKeeperImage); + maybeLoggerName.ifPresent(s -> this.logger = LoggerFactory.getLogger(s)); + withExposedPorts(clientPort); + withEnv("ZOOKEEPER_CLIENT_PORT", String.valueOf(clientPort)); + this.clientPort = clientPort; + } + + @Override + public void start() { + super.start(); + waitingFor(Wait.forHealthcheck()); + setupLogger(); + } + + private void setupLogger() { + if (this.logger != null) { + Slf4jLogConsumer logConsumer = new Slf4jLogConsumer(this.logger); + followOutput(logConsumer); } + } - private ZookeeperContainer(DockerImageName zooKeeperImage, int clientPort, Optional maybeLoggerName) { - super(zooKeeperImage); - maybeLoggerName.ifPresent(s -> this.logger = LoggerFactory.getLogger(s)); - withExposedPorts(clientPort); - withEnv("ZOOKEEPER_CLIENT_PORT", String.valueOf(clientPort)); - this.clientPort = clientPort; - - } - - @Override - public void start() { - super.start(); - waitingFor(Wait.forHealthcheck()); - setupLogger(); - } - - private void setupLogger() { - if (this.logger != null) { - Slf4jLogConsumer logConsumer = new Slf4jLogConsumer(this.logger); - followOutput(logConsumer); - } - } - - public String getConnectionString() { - return String.format("%s:%s", getHost(), getMappedPort(clientPort)); - } + public String getConnectionString() { + return String.format("%s:%s", getHost(), getMappedPort(clientPort)); + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/endpoint/MultiUrlEndpointAddressResolver.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/endpoint/MultiUrlEndpointAddressResolver.java index 31ac30d651..efb1c07bf5 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/endpoint/MultiUrlEndpointAddressResolver.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/endpoint/MultiUrlEndpointAddressResolver.java @@ -1,6 +1,10 @@ package pl.allegro.tech.hermes.test.helper.endpoint; import com.google.common.base.Throwables; +import java.net.URI; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; import pl.allegro.tech.hermes.api.EndpointAddress; import pl.allegro.tech.hermes.api.EndpointAddressResolverMetadata; import pl.allegro.tech.hermes.consumers.consumer.Message; @@ -10,40 +14,38 @@ import pl.allegro.tech.hermes.consumers.consumer.sender.resolver.EndpointAddressResolver; import pl.allegro.tech.hermes.consumers.consumer.sender.resolver.InterpolatingEndpointAddressResolver; -import java.net.URI; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - public class MultiUrlEndpointAddressResolver implements EndpointAddressResolver { - private final EndpointAddressResolver delegate = new InterpolatingEndpointAddressResolver(new MessageBodyInterpolator()); - - @Override - public List resolveAll(EndpointAddress address, Message message, EndpointAddressResolverMetadata metadata) { - return Stream.of(address.getEndpoint().split(";")) - .map(url -> safeResolve(message, url, metadata)) - .collect(Collectors.toList()); - } - - @Override - public URI resolve(EndpointAddress address, Message message, EndpointAddressResolverMetadata metadata) - throws EndpointAddressResolutionException { - return delegate.resolve(address, message, metadata); - } - - @Override - public URI resolve(EndpointAddress address, MessageBatch batch, EndpointAddressResolverMetadata metadata) - throws EndpointAddressResolutionException { - return delegate.resolve(address, batch, metadata); - } - - private URI safeResolve(Message message, String url, EndpointAddressResolverMetadata metadata) { - try { - return delegate.resolve(EndpointAddress.of(url), message, metadata); - } catch (EndpointAddressResolutionException e) { - throw Throwables.propagate(e); - } + private final EndpointAddressResolver delegate = + new InterpolatingEndpointAddressResolver(new MessageBodyInterpolator()); + + @Override + public List resolveAll( + EndpointAddress address, Message message, EndpointAddressResolverMetadata metadata) { + return Stream.of(address.getEndpoint().split(";")) + .map(url -> safeResolve(message, url, metadata)) + .collect(Collectors.toList()); + } + + @Override + public URI resolve( + EndpointAddress address, Message message, EndpointAddressResolverMetadata metadata) + throws EndpointAddressResolutionException { + return delegate.resolve(address, message, metadata); + } + + @Override + public URI resolve( + EndpointAddress address, MessageBatch batch, EndpointAddressResolverMetadata metadata) + throws EndpointAddressResolutionException { + return delegate.resolve(address, batch, metadata); + } + + private URI safeResolve(Message message, String url, EndpointAddressResolverMetadata metadata) { + try { + return delegate.resolve(EndpointAddress.of(url), message, metadata); + } catch (EndpointAddressResolutionException e) { + throw Throwables.propagate(e); } - + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/endpoint/RemoteServiceEndpoint.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/endpoint/RemoteServiceEndpoint.java index 64adf9918a..edd75353e6 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/endpoint/RemoteServiceEndpoint.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/endpoint/RemoteServiceEndpoint.java @@ -1,255 +1,266 @@ package pl.allegro.tech.hermes.test.helper.endpoint; +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.stubbing.Scenario.STARTED; +import static jakarta.ws.rs.core.Response.Status.MOVED_PERMANENTLY; +import static java.util.stream.Collectors.toList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static pl.allegro.tech.hermes.test.helper.endpoint.TimeoutAdjuster.adjust; + import com.github.tomakehurst.wiremock.WireMockServer; import com.github.tomakehurst.wiremock.client.WireMock; import com.github.tomakehurst.wiremock.verification.LoggedRequest; import com.google.common.collect.Iterables; -import java.time.Duration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import pl.allegro.tech.hermes.test.helper.message.TestMessage; - import java.net.URI; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.NoSuchElementException; -import java.util.concurrent.TimeUnit; import java.util.function.Consumer; - -import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; -import static com.github.tomakehurst.wiremock.client.WireMock.post; -import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; -import static com.github.tomakehurst.wiremock.stubbing.Scenario.STARTED; -import static java.util.stream.Collectors.toList; -import static jakarta.ws.rs.core.Response.Status.MOVED_PERMANENTLY; -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; -import static pl.allegro.tech.hermes.test.helper.endpoint.TimeoutAdjuster.adjust; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import pl.allegro.tech.hermes.test.helper.message.TestMessage; public class RemoteServiceEndpoint { - private static final Logger logger = LoggerFactory.getLogger(RemoteServiceEndpoint.class); + private static final Logger logger = LoggerFactory.getLogger(RemoteServiceEndpoint.class); - private final List receivedRequests = Collections.synchronizedList(new ArrayList<>()); - private final String path; - private final URI url; + private final List receivedRequests = + Collections.synchronizedList(new ArrayList<>()); + private final String path; + private final URI url; - private final WireMock listener; - private final WireMockServer service; + private final WireMock listener; + private final WireMockServer service; - private List expectedMessages = new ArrayList<>(); + private List expectedMessages = new ArrayList<>(); - private int returnedStatusCode = 200; - private int delay = 0; + private int returnedStatusCode = 200; + private int delay = 0; - public RemoteServiceEndpoint(WireMockServer service) { - this(service, "/"); - } + public RemoteServiceEndpoint(WireMockServer service) { + this(service, "/"); + } - public RemoteServiceEndpoint(WireMockServer service, final String path) { - this.listener = new WireMock("localhost", service.port()); - this.path = path; - this.url = URI.create(String.format("http://localhost:%d%s", service.port(), path)); - this.service = service; + public RemoteServiceEndpoint(WireMockServer service, final String path) { + this.listener = new WireMock("localhost", service.port()); + this.path = path; + this.url = URI.create(String.format("http://localhost:%d%s", service.port(), path)); + this.service = service; - service.resetMappings(); - service.resetRequests(); - service.resetScenarios(); + service.resetMappings(); + service.resetRequests(); + service.resetScenarios(); - service.addMockServiceRequestListener((request, response) -> { - if (path.equals(request.getUrl())) { - receivedRequests.add(LoggedRequest.createFrom(request)); - } + service.addMockServiceRequestListener( + (request, response) -> { + if (path.equals(request.getUrl())) { + receivedRequests.add(LoggedRequest.createFrom(request)); + } }); - } - - public void expectMessages(TestMessage... messages) { - expectMessages(Arrays.stream(messages).map(TestMessage::body).collect(toList())); - } - - public void expectMessages(String... messages) { - expectMessages(Arrays.asList(messages)); - } - - public void expectMessages(List messages) { - receivedRequests.clear(); - expectedMessages = messages; - messages.forEach(m -> listener - .register( - post(urlEqualTo(path)) - .willReturn(aResponse().withStatus(returnedStatusCode).withFixedDelay(delay)))); - } - - public void redirectMessage(String message) { - receivedRequests.clear(); - - expectedMessages = Collections.singletonList(message); - - listener.register( + } + + public void expectMessages(TestMessage... messages) { + expectMessages(Arrays.stream(messages).map(TestMessage::body).collect(toList())); + } + + public void expectMessages(String... messages) { + expectMessages(Arrays.asList(messages)); + } + + public void expectMessages(List messages) { + receivedRequests.clear(); + expectedMessages = messages; + messages.forEach( + m -> + listener.register( post(urlEqualTo(path)) - .willReturn(aResponse() - .withStatus(MOVED_PERMANENTLY.getStatusCode()) - .withHeader("Location", "http://localhost:" + service.port()))); + .willReturn(aResponse().withStatus(returnedStatusCode).withFixedDelay(delay)))); + } + + public void redirectMessage(String message) { + receivedRequests.clear(); + + expectedMessages = Collections.singletonList(message); + + listener.register( + post(urlEqualTo(path)) + .willReturn( + aResponse() + .withStatus(MOVED_PERMANENTLY.getStatusCode()) + .withHeader("Location", "http://localhost:" + service.port()))); + } + + public void retryMessage(String message, int delay) { + receivedRequests.clear(); + expectedMessages = Arrays.asList(message, message); + int retryStatusCode = 503; + listener.register( + post(urlEqualTo(path)) + .inScenario("retrying") + .whenScenarioStateIs(STARTED) + .willSetStateTo("retried") + .willReturn( + aResponse() + .withStatus(retryStatusCode) + .withHeader("Retry-After", Integer.toString(delay)) + .withFixedDelay(delay))); + listener.register( + post(urlEqualTo(path)) + .inScenario("retrying") + .whenScenarioStateIs("retried") + .willReturn(aResponse().withStatus(returnedStatusCode).withFixedDelay(delay))); + } + + public void slowThenFastMessage(String message, int chunks, int delayMs) { + receivedRequests.clear(); + expectedMessages = Arrays.asList(message, message); + + listener.register( + post(urlEqualTo(path)) + .inScenario("slowAndFast") + .whenScenarioStateIs(STARTED) + .willSetStateTo("slow") + .willReturn( + aResponse() + .withStatus(returnedStatusCode) + .withBody("I am very slow!") + .withChunkedDribbleDelay(chunks, delayMs))); + + listener.register( + post(urlEqualTo(path)) + .inScenario("slowAndFast") + .whenScenarioStateIs("slow") + .willReturn(aResponse().withStatus(returnedStatusCode).withFixedDelay(0))); + } + + public void setReturnedStatusCode(int statusCode) { + returnedStatusCode = statusCode; + } + + public void waitUntilReceived(long seconds) { + logger.info("Expecting to receive {} messages", expectedMessages.size()); + await() + .atMost(adjust(Duration.ofSeconds(seconds))) + .untilAsserted( + () -> + assertThat(receivedRequests.size()) + .isGreaterThanOrEqualTo(expectedMessages.size())); + synchronized (receivedRequests) { + assertThat(receivedRequests.stream().map(LoggedRequest::getBodyAsString).collect(toList())) + .containsAll(expectedMessages); } - - public void retryMessage(String message, int delay) { - receivedRequests.clear(); - expectedMessages = Arrays.asList(message, message); - int retryStatusCode = 503; - listener.register( - post(urlEqualTo(path)) - .inScenario("retrying") - .whenScenarioStateIs(STARTED) - .willSetStateTo("retried") - .willReturn(aResponse() - .withStatus(retryStatusCode) - .withHeader("Retry-After", Integer.toString(delay)) - .withFixedDelay(delay))); - listener.register( - post(urlEqualTo(path)) - .inScenario("retrying") - .whenScenarioStateIs("retried") - .willReturn(aResponse().withStatus(returnedStatusCode).withFixedDelay(delay))); - } - - public void slowThenFastMessage(String message, int chunks, int delayMs) { - receivedRequests.clear(); - expectedMessages = Arrays.asList(message, message); - - listener.register( - post(urlEqualTo(path)) - .inScenario("slowAndFast") - .whenScenarioStateIs(STARTED) - .willSetStateTo("slow") - .willReturn( - aResponse() - .withStatus(returnedStatusCode) - .withBody("I am very slow!") - .withChunkedDribbleDelay(chunks, delayMs) - ) - ); - - listener.register( - post(urlEqualTo(path)) - .inScenario("slowAndFast") - .whenScenarioStateIs("slow") - .willReturn( - aResponse() - .withStatus(returnedStatusCode) - .withFixedDelay(0) - ) - ); - } - - public void setReturnedStatusCode(int statusCode) { - returnedStatusCode = statusCode; + } + + public void waitUntilReceived(long seconds, int numberOfExpectedMessages) { + waitUntilReceived(seconds, numberOfExpectedMessages, body -> {}); + } + + public void waitUntilReceived(Consumer requestBodyConsumer) { + waitUntilRequestReceived( + loggedRequest -> requestBodyConsumer.accept(loggedRequest.getBodyAsString())); + } + + public void waitUntilRequestReceived(Consumer requestConsumer) { + waitUntilReceived(60, 1, requestConsumer); + } + + public void waitUntilReceived( + long seconds, int numberOfExpectedMessages, Consumer requestBodyConsumer) { + logger.info("Expecting to receive {} messages", numberOfExpectedMessages); + await() + .atMost(adjust(Duration.ofSeconds(seconds))) + .untilAsserted( + () -> + assertThat(receivedRequests.size()) + .isGreaterThanOrEqualTo(numberOfExpectedMessages)); + synchronized (receivedRequests) { + receivedRequests.forEach(requestBodyConsumer); } - - public void waitUntilReceived(long seconds) { - logger.info("Expecting to receive {} messages", expectedMessages.size()); - await().atMost(adjust(Duration.ofSeconds(seconds))).untilAsserted(() -> - assertThat(receivedRequests.size()).isGreaterThanOrEqualTo(expectedMessages.size())); - synchronized (receivedRequests) { - assertThat(receivedRequests.stream().map(LoggedRequest::getBodyAsString).collect(toList())).containsAll(expectedMessages); - } - } - - public void waitUntilReceived(long seconds, int numberOfExpectedMessages) { - waitUntilReceived(seconds, numberOfExpectedMessages, body -> { - }); - } - - public void waitUntilReceived(Consumer requestBodyConsumer) { - waitUntilRequestReceived(loggedRequest -> requestBodyConsumer.accept(loggedRequest.getBodyAsString())); + } + + public void waitUntilReceived( + Duration duration, + int numberOfExpectedMessages, + Consumer requestBodyConsumer) { + logger.info("Expecting to receive {} messages", numberOfExpectedMessages); + await() + .atMost(duration) + .untilAsserted( + () -> assertThat(receivedRequests.size()).isEqualTo(numberOfExpectedMessages)); + synchronized (receivedRequests) { + receivedRequests.forEach(requestBodyConsumer); } + } - public void waitUntilRequestReceived(Consumer requestConsumer) { - waitUntilReceived(60, 1, requestConsumer); - } + public void waitUntilReceived() { + this.waitUntilReceived(60); + } - public void waitUntilReceived(long seconds, int numberOfExpectedMessages, Consumer requestBodyConsumer) { - logger.info("Expecting to receive {} messages", numberOfExpectedMessages); - await().atMost(adjust(Duration.ofSeconds(seconds))).untilAsserted(() -> - assertThat(receivedRequests.size()).isGreaterThanOrEqualTo(numberOfExpectedMessages)); - synchronized (receivedRequests) { - receivedRequests.forEach(requestBodyConsumer); - } - } - - public void waitUntilReceived(Duration duration, int numberOfExpectedMessages, Consumer requestBodyConsumer) { - logger.info("Expecting to receive {} messages", numberOfExpectedMessages); - await().atMost(duration).untilAsserted(() -> assertThat(receivedRequests.size()).isEqualTo(numberOfExpectedMessages)); - synchronized (receivedRequests) { - receivedRequests.forEach(requestBodyConsumer); - } - } + public void makeSureNoneReceived() { + logger.info("Expecting to receive no messages"); + assertThat(receivedRequests).isEmpty(); + } - public void waitUntilReceived() { - this.waitUntilReceived(60); + public LoggedRequest getLastReceivedRequest() { + synchronized (receivedRequests) { + return Iterables.getLast(receivedRequests); } + } - public void makeSureNoneReceived() { - logger.info("Expecting to receive no messages"); - assertThat(receivedRequests).isEmpty(); + public LoggedRequest getFirstReceivedRequest() { + LoggedRequest item = Iterables.getFirst(receivedRequests, null); + if (item == null) { + throw new NoSuchElementException(); } + return item; + } - public LoggedRequest getLastReceivedRequest() { - synchronized (receivedRequests) { - return Iterables.getLast(receivedRequests); - } + public boolean receivedMessageWithHeader(String header, String value) { + synchronized (receivedRequests) { + return receivedRequests.stream().anyMatch(r -> r.header(header).containsValue(value)); } - - public LoggedRequest getFirstReceivedRequest() { - LoggedRequest item = Iterables.getFirst(receivedRequests, null); - if (item == null) { - throw new NoSuchElementException(); - } - return item; - } - - public boolean receivedMessageWithHeader(String header, String value) { - synchronized (receivedRequests) { - return receivedRequests.stream().anyMatch(r -> r.header(header).containsValue(value)); - } - } - - public java.time.Duration durationBetweenFirstAndLastRequest() { - return java.time.Duration.between( - getFirstReceivedRequest().getLoggedDate().toInstant(), - getLastReceivedRequest().getLoggedDate().toInstant()); - } - - public LoggedRequest waitAndGetLastRequest() { - waitUntilReceived(); - return getLastReceivedRequest(); - } - - public void reset() { - delay = 0; - returnedStatusCode = 200; - receivedRequests.clear(); - listener.resetMappings(); - service.resetMappings(); - } - - public RemoteServiceEndpoint setDelay(int delay) { - this.delay = delay; - return this; - } - - public URI getUrl() { - return url; - } - - public int getServicePort() { - return this.service.port(); - } - - public void stop() { - listener.shutdown(); - service.shutdown(); - } - + } + + public java.time.Duration durationBetweenFirstAndLastRequest() { + return java.time.Duration.between( + getFirstReceivedRequest().getLoggedDate().toInstant(), + getLastReceivedRequest().getLoggedDate().toInstant()); + } + + public LoggedRequest waitAndGetLastRequest() { + waitUntilReceived(); + return getLastReceivedRequest(); + } + + public void reset() { + delay = 0; + returnedStatusCode = 200; + receivedRequests.clear(); + listener.resetMappings(); + service.resetMappings(); + } + + public RemoteServiceEndpoint setDelay(int delay) { + this.delay = delay; + return this; + } + + public URI getUrl() { + return url; + } + + public int getServicePort() { + return this.service.port(); + } + + public void stop() { + listener.shutdown(); + service.shutdown(); + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/endpoint/TimeoutAdjuster.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/endpoint/TimeoutAdjuster.java index 2d06f09c44..295d5a3f03 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/endpoint/TimeoutAdjuster.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/endpoint/TimeoutAdjuster.java @@ -1,17 +1,18 @@ package pl.allegro.tech.hermes.test.helper.endpoint; -import java.time.Duration; - import static java.lang.Double.parseDouble; +import java.time.Duration; + public class TimeoutAdjuster { - private static final double timeoutMultiplier = parseDouble(System.getProperty("tests.timeout.multiplier", "1")); + private static final double timeoutMultiplier = + parseDouble(System.getProperty("tests.timeout.multiplier", "1")); - public static long adjust(long value) { - return (long) Math.floor(value * timeoutMultiplier); - } + public static long adjust(long value) { + return (long) Math.floor(value * timeoutMultiplier); + } - public static Duration adjust(Duration duration) { - return Duration.ofMillis(adjust(duration.toMillis())); - } + public static Duration adjust(Duration duration) { + return Duration.ofMillis(adjust(duration.toMillis())); + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/environment/HermesTestApp.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/environment/HermesTestApp.java index e3da3b2fa0..85b960fbd2 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/environment/HermesTestApp.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/environment/HermesTestApp.java @@ -2,13 +2,13 @@ public interface HermesTestApp { - HermesTestApp start(); + HermesTestApp start(); - void stop(); + void stop(); - boolean shouldBeRestarted(); + boolean shouldBeRestarted(); - void restoreDefaultSettings(); + void restoreDefaultSettings(); - int getPort(); + int getPort(); } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/environment/Starter.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/environment/Starter.java index 1291fe3d18..1cc7872bb4 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/environment/Starter.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/environment/Starter.java @@ -2,9 +2,9 @@ public interface Starter { - void start() throws Exception; + void start() throws Exception; - void stop() throws Exception; + void stop() throws Exception; - T instance(); + T instance(); } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/message/TestMessage.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/message/TestMessage.java index 91ed2bdcf4..4896e7c2ae 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/message/TestMessage.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/message/TestMessage.java @@ -2,66 +2,64 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import pl.allegro.tech.hermes.common.message.wrapper.AvroMetadataMarker; - import java.util.LinkedHashMap; import java.util.Map; import java.util.UUID; +import pl.allegro.tech.hermes.common.message.wrapper.AvroMetadataMarker; public final class TestMessage { - private static final ObjectMapper objectMapper = new ObjectMapper(); + private static final ObjectMapper objectMapper = new ObjectMapper(); - private final Map content = new LinkedHashMap<>(); + private final Map content = new LinkedHashMap<>(); - private TestMessage() { - } + private TestMessage() {} - public static TestMessage of(String key, Object value) { - return new TestMessage().append(key, value); - } - - public static TestMessage[] simpleMessages(int n) { - TestMessage[] simpleMessages = new TestMessage[n]; + public static TestMessage of(String key, Object value) { + return new TestMessage().append(key, value); + } - for (int i = 0; i < n; i++) { - simpleMessages[i] = simple(); - } + public static TestMessage[] simpleMessages(int n) { + TestMessage[] simpleMessages = new TestMessage[n]; - return simpleMessages; + for (int i = 0; i < n; i++) { + simpleMessages[i] = simple(); } - public static TestMessage simple() { - return new TestMessage().append("hello", "world"); - } + return simpleMessages; + } - public static TestMessage random() { - return new TestMessage().append("random", UUID.randomUUID().toString()); - } + public static TestMessage simple() { + return new TestMessage().append("hello", "world"); + } - public TestMessage append(String key, Object value) { - content.put(key, value); - return this; - } + public static TestMessage random() { + return new TestMessage().append("random", UUID.randomUUID().toString()); + } - public TestMessage withEmptyAvroMetadata() { - return append(AvroMetadataMarker.METADATA_MARKER, null); - } + public TestMessage append(String key, Object value) { + content.put(key, value); + return this; + } - public String body() { - return toString(); - } + public TestMessage withEmptyAvroMetadata() { + return append(AvroMetadataMarker.METADATA_MARKER, null); + } - public Map getContent() { - return content; - } + public String body() { + return toString(); + } + + public Map getContent() { + return content; + } - @Override - public String toString() { - try { - return objectMapper.writeValueAsString(content); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } + @Override + public String toString() { + try { + return objectMapper.writeValueAsString(content); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); } + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/metrics/MicrometerUtils.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/metrics/MicrometerUtils.java index 759ec0363c..8852008ffe 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/metrics/MicrometerUtils.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/metrics/MicrometerUtils.java @@ -4,16 +4,25 @@ import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Tags; import io.micrometer.core.instrument.search.Search; - import java.util.Optional; import java.util.function.Function; public class MicrometerUtils { - public static Optional metricValue(MeterRegistry meterRegistry, String metricName, Tags tags, Function searchMapper, Function metricValueMapper) { - return Optional.ofNullable(searchMapper.apply(meterRegistry.find(metricName).tags(tags))).map(metricValueMapper); - } + public static Optional metricValue( + MeterRegistry meterRegistry, + String metricName, + Tags tags, + Function searchMapper, + Function metricValueMapper) { + return Optional.ofNullable(searchMapper.apply(meterRegistry.find(metricName).tags(tags))) + .map(metricValueMapper); + } - public static Optional metricValue(MeterRegistry meterRegistry, String metricName, Function searchMapper, Function metricValueMapper) { - return metricValue(meterRegistry, metricName, Tags.empty(), searchMapper, metricValueMapper); - } + public static Optional metricValue( + MeterRegistry meterRegistry, + String metricName, + Function searchMapper, + Function metricValueMapper) { + return metricValue(meterRegistry, metricName, Tags.empty(), searchMapper, metricValueMapper); + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/metrics/TestMetricsFacadeFactory.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/metrics/TestMetricsFacadeFactory.java index 8037c05db3..c474ee36e6 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/metrics/TestMetricsFacadeFactory.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/metrics/TestMetricsFacadeFactory.java @@ -5,7 +5,7 @@ public class TestMetricsFacadeFactory { - public static MetricsFacade create() { - return new MetricsFacade(new SimpleMeterRegistry()); - } + public static MetricsFacade create() { + return new MetricsFacade(new SimpleMeterRegistry()); + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/oauth/server/OAuthClient.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/oauth/server/OAuthClient.java index a45baeb443..d02aa3b69a 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/oauth/server/OAuthClient.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/oauth/server/OAuthClient.java @@ -1,5 +1,3 @@ package pl.allegro.tech.hermes.test.helper.oauth.server; -public record OAuthClient(String clientId, String secret) { - -} +public record OAuthClient(String clientId, String secret) {} diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/oauth/server/OAuthResourceOwner.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/oauth/server/OAuthResourceOwner.java index 79b1e53463..f221c03d99 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/oauth/server/OAuthResourceOwner.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/oauth/server/OAuthResourceOwner.java @@ -1,4 +1,3 @@ package pl.allegro.tech.hermes.test.helper.oauth.server; -public record OAuthResourceOwner(String username, String password) { -} +public record OAuthResourceOwner(String username, String password) {} diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/oauth/server/OAuthTestServer.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/oauth/server/OAuthTestServer.java index acb3b9e767..334f727a63 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/oauth/server/OAuthTestServer.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/oauth/server/OAuthTestServer.java @@ -1,94 +1,80 @@ package pl.allegro.tech.hermes.test.helper.oauth.server; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.github.tomakehurst.wiremock.WireMockServer; - -import java.util.Map; -import java.util.UUID; - import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; import static com.github.tomakehurst.wiremock.client.WireMock.post; import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; import static org.apache.hc.core5.http.HttpStatus.SC_OK; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.tomakehurst.wiremock.WireMockServer; +import java.util.Map; +import java.util.UUID; + public class OAuthTestServer { - private static final String OAUTH2_TOKEN_ENDPOINT = "/oauth2/token"; + private static final String OAUTH2_TOKEN_ENDPOINT = "/oauth2/token"; - private final ObjectMapper objectMapper = new ObjectMapper(); - private final WireMockServer wireMockServer; + private final ObjectMapper objectMapper = new ObjectMapper(); + private final WireMockServer wireMockServer; - public OAuthTestServer() { - wireMockServer = new WireMockServer(0); - } + public OAuthTestServer() { + wireMockServer = new WireMockServer(0); + } - public String getTokenEndpoint() { - return String.format("http://localhost:%s%s", wireMockServer.port(), OAUTH2_TOKEN_ENDPOINT); - } + public String getTokenEndpoint() { + return String.format("http://localhost:%s%s", wireMockServer.port(), OAUTH2_TOKEN_ENDPOINT); + } - public String stubAccessTokenForPasswordGrant(OAuthClient client, OAuthResourceOwner resourceOwner) { - String token = UUID.randomUUID().toString(); - Map params = Map.of( - "access_token", token, - "token_type", "Bearer" - ); - try { - String body = objectMapper.writeValueAsString(params); - wireMockServer.addStubMapping( - post(urlPathEqualTo(OAUTH2_TOKEN_ENDPOINT)) - .withQueryParam("grant_type", equalTo("password")) - .withQueryParam("client_id", equalTo(client.clientId())) - .withQueryParam("client_secret", equalTo(client.secret())) - .withQueryParam("username", equalTo(resourceOwner.username())) - .withQueryParam("password", equalTo(resourceOwner.password())) - .willReturn( - aResponse() - .withBody(body) - .withStatus(SC_OK) - ) - .build()); - return token; - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } + public String stubAccessTokenForPasswordGrant( + OAuthClient client, OAuthResourceOwner resourceOwner) { + String token = UUID.randomUUID().toString(); + Map params = Map.of("access_token", token, "token_type", "Bearer"); + try { + String body = objectMapper.writeValueAsString(params); + wireMockServer.addStubMapping( + post(urlPathEqualTo(OAUTH2_TOKEN_ENDPOINT)) + .withQueryParam("grant_type", equalTo("password")) + .withQueryParam("client_id", equalTo(client.clientId())) + .withQueryParam("client_secret", equalTo(client.secret())) + .withQueryParam("username", equalTo(resourceOwner.username())) + .withQueryParam("password", equalTo(resourceOwner.password())) + .willReturn(aResponse().withBody(body).withStatus(SC_OK)) + .build()); + return token; + } catch (JsonProcessingException e) { + throw new RuntimeException(e); } + } - public String stubAccessTokenForClientCredentials(OAuthClient client) { - String token = UUID.randomUUID().toString(); - Map params = Map.of( - "access_token", token, - "token_type", "Bearer" - ); - try { - String body = objectMapper.writeValueAsString(params); - wireMockServer.addStubMapping( - post(urlPathEqualTo(OAUTH2_TOKEN_ENDPOINT)) - .withQueryParam("grant_type", equalTo("client_credentials")) - .withQueryParam("client_id", equalTo(client.clientId())) - .withQueryParam("client_secret", equalTo(client.secret())) - .willReturn( - aResponse() - .withBody(body) - .withStatus(SC_OK) - ) - .build()); - return token; - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } + public String stubAccessTokenForClientCredentials(OAuthClient client) { + String token = UUID.randomUUID().toString(); + Map params = Map.of("access_token", token, "token_type", "Bearer"); + try { + String body = objectMapper.writeValueAsString(params); + wireMockServer.addStubMapping( + post(urlPathEqualTo(OAUTH2_TOKEN_ENDPOINT)) + .withQueryParam("grant_type", equalTo("client_credentials")) + .withQueryParam("client_id", equalTo(client.clientId())) + .withQueryParam("client_secret", equalTo(client.secret())) + .willReturn(aResponse().withBody(body).withStatus(SC_OK)) + .build()); + return token; + } catch (JsonProcessingException e) { + throw new RuntimeException(e); } + } - public void reset() { - wireMockServer.resetAll(); - } + public void reset() { + wireMockServer.resetAll(); + } - public void start() { - wireMockServer.start(); - } + public void start() { + wireMockServer.start(); + } - public void stop() { - wireMockServer.stop(); - } + public void stop() { + wireMockServer.stop(); + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/time/ModifiableClock.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/time/ModifiableClock.java index a1be85dc1b..0967facbac 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/time/ModifiableClock.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/time/ModifiableClock.java @@ -1,37 +1,37 @@ package pl.allegro.tech.hermes.test.helper.time; +import static java.time.Instant.now; + import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.time.ZoneId; import java.time.temporal.ChronoUnit; -import static java.time.Instant.now; - public class ModifiableClock extends Clock { - private Clock clock = fixed(now(systemDefaultZone()), ZoneId.systemDefault()); + private Clock clock = fixed(now(systemDefaultZone()), ZoneId.systemDefault()); - public void advanceMinutes(int minutes) { - clock = fixed(now(clock).plus(minutes, ChronoUnit.MINUTES), ZoneId.systemDefault()); - } + public void advanceMinutes(int minutes) { + clock = fixed(now(clock).plus(minutes, ChronoUnit.MINUTES), ZoneId.systemDefault()); + } - public void advance(Duration step) { - clock = fixed(now(clock).plus(step), ZoneId.systemDefault()); - } + public void advance(Duration step) { + clock = fixed(now(clock).plus(step), ZoneId.systemDefault()); + } - @Override - public ZoneId getZone() { - return clock.getZone(); - } + @Override + public ZoneId getZone() { + return clock.getZone(); + } - @Override - public Clock withZone(ZoneId zone) { - return clock.withZone(zone); - } + @Override + public Clock withZone(ZoneId zone) { + return clock.withZone(zone); + } - @Override - public Instant instant() { - return clock.instant(); - } + @Override + public Instant instant() { + return clock.instant(); + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/util/Ports.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/util/Ports.java index a2ab3df979..420703ee94 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/util/Ports.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/util/Ports.java @@ -1,73 +1,74 @@ package pl.allegro.tech.hermes.test.helper.util; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.net.ConnectException; import java.net.DatagramSocket; import java.net.ServerSocket; import java.net.Socket; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public final class Ports { - private static final Logger logger = LoggerFactory.getLogger(Ports.class); + private static final Logger logger = LoggerFactory.getLogger(Ports.class); - private Ports() { - } + private Ports() {} - public static int nextAvailable() { - try { - ServerSocket socket = new ServerSocket(0); - socket.setReuseAddress(true); - int port = socket.getLocalPort(); - socket.getLocalSocketAddress(); - socket.close(); + public static int nextAvailable() { + try { + ServerSocket socket = new ServerSocket(0); + socket.setReuseAddress(true); + int port = socket.getLocalPort(); + socket.getLocalSocketAddress(); + socket.close(); - // second check whether the port is available as on some dynamic environments it can be still in use - try (Socket ignore = new Socket("127.0.0.1", port)) { - logger.warn("Connected to randomly selected port {} meaning it is still in use. Drawing next port.", port); - return nextAvailable(); - } catch (ConnectException ex) { - // expected exception as on provided port no one should listen - return port; - } - } catch (IOException exception) { - throw new NoAvailablePortException(exception); - } + // second check whether the port is available as on some dynamic environments it can be still + // in use + try (Socket ignore = new Socket("127.0.0.1", port)) { + logger.warn( + "Connected to randomly selected port {} meaning it is still in use. Drawing next port.", + port); + return nextAvailable(); + } catch (ConnectException ex) { + // expected exception as on provided port no one should listen + return port; + } + } catch (IOException exception) { + throw new NoAvailablePortException(exception); } + } - public static int nextAvailable(int min, int max) { - for (int port = min; port <= max; ++port) { - if (isPortAvailable(port)) { - return port; - } - } - throw new NoAvailablePortException("Problem finding port in a scope: " + min + " " + max); + public static int nextAvailable(int min, int max) { + for (int port = min; port <= max; ++port) { + if (isPortAvailable(port)) { + return port; + } } + throw new NoAvailablePortException("Problem finding port in a scope: " + min + " " + max); + } - public static boolean isPortAvailable(int port) { - try { - try (ServerSocket socket = new ServerSocket(port)) { - socket.setReuseAddress(true); + public static boolean isPortAvailable(int port) { + try { + try (ServerSocket socket = new ServerSocket(port)) { + socket.setReuseAddress(true); - try (DatagramSocket datagramSocket = new DatagramSocket(port)) { - datagramSocket.setReuseAddress(true); - return true; - } - } - } catch (IOException e) { - return false; + try (DatagramSocket datagramSocket = new DatagramSocket(port)) { + datagramSocket.setReuseAddress(true); + return true; } + } + } catch (IOException e) { + return false; } + } - public static class NoAvailablePortException extends RuntimeException { - public NoAvailablePortException(Throwable cause) { - super(cause); - } + public static class NoAvailablePortException extends RuntimeException { + public NoAvailablePortException(Throwable cause) { + super(cause); + } - public NoAvailablePortException(String message) { - super(message); - } + public NoAvailablePortException(String message) { + super(message); } + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/zookeeper/ZookeeperBaseTest.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/zookeeper/ZookeeperBaseTest.java index e72bab2ac8..73b2330472 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/zookeeper/ZookeeperBaseTest.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/zookeeper/ZookeeperBaseTest.java @@ -11,58 +11,59 @@ @SuppressFBWarnings("MS_PKGPROTECT") public abstract class ZookeeperBaseTest { - protected static TestingServer zookeeperServer; + protected static TestingServer zookeeperServer; - protected static CuratorFramework zookeeperClient; + protected static CuratorFramework zookeeperClient; - protected static ZookeeperWaiter wait; + protected static ZookeeperWaiter wait; - protected ZookeeperBaseTest() { - } + protected ZookeeperBaseTest() {} - @BeforeClass - public static void beforeZookeeperClass() throws Exception { - zookeeperServer = new TestingServer(45678); - zookeeperClient = CuratorFrameworkFactory.builder() - .connectString(zookeeperServer.getConnectString()) - .retryPolicy(new ExponentialBackoffRetry(1000, 3)) - .build(); - zookeeperClient.start(); - wait = new ZookeeperWaiter(zookeeperClient); + @BeforeClass + public static void beforeZookeeperClass() throws Exception { + zookeeperServer = new TestingServer(45678); + zookeeperClient = + CuratorFrameworkFactory.builder() + .connectString(zookeeperServer.getConnectString()) + .retryPolicy(new ExponentialBackoffRetry(1000, 3)) + .build(); + zookeeperClient.start(); + wait = new ZookeeperWaiter(zookeeperClient); - wait.untilZookeeperClientStarted(); - } + wait.untilZookeeperClientStarted(); + } - protected static CuratorFramework newClient() { - CuratorFramework newClient = CuratorFrameworkFactory.builder() - .connectString(zookeeperServer.getConnectString()) - .retryPolicy(new ExponentialBackoffRetry(1000, 3)) - .build(); - newClient.start(); - wait.untilZookeeperClientStarted(newClient); - return newClient; - } + protected static CuratorFramework newClient() { + CuratorFramework newClient = + CuratorFrameworkFactory.builder() + .connectString(zookeeperServer.getConnectString()) + .retryPolicy(new ExponentialBackoffRetry(1000, 3)) + .build(); + newClient.start(); + wait.untilZookeeperClientStarted(newClient); + return newClient; + } - @AfterClass - public static void tearDown() throws Exception { - zookeeperServer.stop(); - } + @AfterClass + public static void tearDown() throws Exception { + zookeeperServer.stop(); + } - protected void createPath(String path) throws Exception { - if (zookeeperClient.checkExists().forPath(path) == null) { - zookeeperClient.create().creatingParentsIfNeeded().forPath(path); - } + protected void createPath(String path) throws Exception { + if (zookeeperClient.checkExists().forPath(path) == null) { + zookeeperClient.create().creatingParentsIfNeeded().forPath(path); } + } - protected void deleteData(String path) throws Exception { - if (zookeeperClient.checkExists().forPath(path) != null) { - zookeeperClient.delete().deletingChildrenIfNeeded().forPath(path); - } + protected void deleteData(String path) throws Exception { + if (zookeeperClient.checkExists().forPath(path) != null) { + zookeeperClient.delete().deletingChildrenIfNeeded().forPath(path); } + } - protected void deleteAllNodes() throws Exception { - if (zookeeperClient.checkExists().forPath("/hermes") != null) { - zookeeperClient.delete().guaranteed().deletingChildrenIfNeeded().forPath("/hermes"); - } + protected void deleteAllNodes() throws Exception { + if (zookeeperClient.checkExists().forPath("/hermes") != null) { + zookeeperClient.delete().guaranteed().deletingChildrenIfNeeded().forPath("/hermes"); } + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/zookeeper/ZookeeperResource.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/zookeeper/ZookeeperResource.java index 67e66a50fa..151fea1c79 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/zookeeper/ZookeeperResource.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/zookeeper/ZookeeperResource.java @@ -1,100 +1,100 @@ package pl.allegro.tech.hermes.test.helper.zookeeper; +import java.io.IOException; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.RetryOneTime; import org.apache.curator.test.TestingServer; import org.junit.rules.ExternalResource; -import java.io.IOException; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; - public class ZookeeperResource extends ExternalResource { - private static Starter zookeeperStarter; + private static Starter zookeeperStarter; + + private final int curatorPort; + + private final boolean initializeOnce; + + private final Consumer initializer; + + public ZookeeperResource(int curatorPort, boolean initializeOnce, Consumer initializer) { + this.curatorPort = curatorPort; + this.initializeOnce = initializeOnce; + this.initializer = initializer; + } + + public CuratorFramework curator() { + return zookeeperStarter.curator(); + } + + @Override + protected void before() throws Throwable { + if (initializeOnce) { + if (zookeeperStarter == null) { + zookeeperStarter = new Starter(curatorPort, initializer); + zookeeperStarter.start(); + } + } else { + zookeeperStarter = new Starter(curatorPort, initializer); + zookeeperStarter.start(); + } + } + + @Override + protected void after() { + if (!initializeOnce) { + zookeeperStarter.stop(); + } else { + zookeeperStarter.registerShutdownHook(); + } + } + + public static final class Starter { private final int curatorPort; - - private final boolean initializeOnce; private final Consumer initializer; - - public ZookeeperResource(int curatorPort, boolean initializeOnce, Consumer initializer) { - this.curatorPort = curatorPort; - this.initializeOnce = initializeOnce; - this.initializer = initializer; + + private TestingServer server; + + private CuratorFramework curator; + + Starter(int curatorPort, Consumer initializer) { + this.curatorPort = curatorPort; + this.initializer = initializer; } public CuratorFramework curator() { - return zookeeperStarter.curator(); + return curator; } - @Override - protected void before() throws Throwable { - if (initializeOnce) { - if (zookeeperStarter == null) { - zookeeperStarter = new Starter(curatorPort, initializer); - zookeeperStarter.start(); - } - } else { - zookeeperStarter = new Starter(curatorPort, initializer); - zookeeperStarter.start(); - } + void start() { + try { + server = new TestingServer(curatorPort, true); + this.curator = + CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1000)); + this.curator.start(); + + this.curator.blockUntilConnected(10, TimeUnit.SECONDS); + initializer.accept(this); + } catch (Exception ex) { + throw new IllegalStateException("Failed to start Zookeeper", ex); + } } - @Override - protected void after() { - if (!initializeOnce) { - zookeeperStarter.stop(); - } else { - zookeeperStarter.registerShutdownHook(); - } + void stop() { + try { + this.curator.close(); + this.server.close(); + } catch (IOException ex) { + throw new IllegalStateException("Failed to stop zookeeper", ex); + } } - - public static final class Starter { - - private final int curatorPort; - - private final Consumer initializer; - - private TestingServer server; - - private CuratorFramework curator; - - Starter(int curatorPort, Consumer initializer) { - this.curatorPort = curatorPort; - this.initializer = initializer; - } - - public CuratorFramework curator() { - return curator; - } - - void start() { - try { - server = new TestingServer(curatorPort, true); - this.curator = CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1000)); - this.curator.start(); - - this.curator.blockUntilConnected(10, TimeUnit.SECONDS); - initializer.accept(this); - } catch (Exception ex) { - throw new IllegalStateException("Failed to start Zookeeper", ex); - } - } - - void stop() { - try { - this.curator.close(); - this.server.close(); - } catch (IOException ex) { - throw new IllegalStateException("Failed to stop zookeeper", ex); - } - } - - void registerShutdownHook() { - Runtime.getRuntime().addShutdownHook(new Thread(Starter.this::stop)); - } + + void registerShutdownHook() { + Runtime.getRuntime().addShutdownHook(new Thread(Starter.this::stop)); } + } } diff --git a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/zookeeper/ZookeeperWaiter.java b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/zookeeper/ZookeeperWaiter.java index 3e9fcb0bea..fa1ec5237e 100644 --- a/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/zookeeper/ZookeeperWaiter.java +++ b/hermes-test-helper/src/main/java/pl/allegro/tech/hermes/test/helper/zookeeper/ZookeeperWaiter.java @@ -1,34 +1,34 @@ package pl.allegro.tech.hermes.test.helper.zookeeper; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.imps.CuratorFrameworkState; - -import java.util.concurrent.TimeUnit; - import static org.awaitility.Awaitility.await; +import java.util.concurrent.TimeUnit; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.imps.CuratorFrameworkState; public class ZookeeperWaiter { - private final CuratorFramework zookeeper; + private final CuratorFramework zookeeper; - public ZookeeperWaiter(CuratorFramework zookeeper) { - this.zookeeper = zookeeper; - } + public ZookeeperWaiter(CuratorFramework zookeeper) { + this.zookeeper = zookeeper; + } - public void untilZookeeperClientStarted() { - this.untilZookeeperClientStarted(zookeeper); - } + public void untilZookeeperClientStarted() { + this.untilZookeeperClientStarted(zookeeper); + } - public void untilZookeeperClientStarted(CuratorFramework client) { - await().atMost(2, TimeUnit.SECONDS).until(() -> client.getState() == CuratorFrameworkState.STARTED); - } + public void untilZookeeperClientStarted(CuratorFramework client) { + await() + .atMost(2, TimeUnit.SECONDS) + .until(() -> client.getState() == CuratorFrameworkState.STARTED); + } - public void untilZookeeperPathIsCreated(final String path) { - await().atMost(2, TimeUnit.SECONDS).until(() -> zookeeper.getData().forPath(path) != null); - } + public void untilZookeeperPathIsCreated(final String path) { + await().atMost(2, TimeUnit.SECONDS).until(() -> zookeeper.getData().forPath(path) != null); + } - public void untilZookeeperPathNotExists(final String path) { - await().atMost(2, TimeUnit.SECONDS).until(() -> zookeeper.checkExists().forPath(path) == null); - } + public void untilZookeeperPathNotExists(final String path) { + await().atMost(2, TimeUnit.SECONDS).until(() -> zookeeper.checkExists().forPath(path) == null); + } } diff --git a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/DailyIndexFactory.java b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/DailyIndexFactory.java index 5aff871ef7..f91ac51ae9 100644 --- a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/DailyIndexFactory.java +++ b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/DailyIndexFactory.java @@ -6,21 +6,21 @@ public abstract class DailyIndexFactory implements IndexFactory { - private final String basePath; - private final Clock clock; - private final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy_MM_dd"); + private final String basePath; + private final Clock clock; + private final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy_MM_dd"); - public DailyIndexFactory(String basePath) { - this(basePath, Clock.systemUTC()); - } + public DailyIndexFactory(String basePath) { + this(basePath, Clock.systemUTC()); + } - public DailyIndexFactory(String basePath, Clock clock) { - this.basePath = basePath; - this.clock = clock; - } + public DailyIndexFactory(String basePath, Clock clock) { + this.basePath = basePath; + this.clock = clock; + } - @Override - public String createIndex() { - return basePath + "_" + dateTimeFormatter.format(LocalDate.now(clock)); - } + @Override + public String createIndex() { + return basePath + "_" + dateTimeFormatter.format(LocalDate.now(clock)); + } } diff --git a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/ElasticsearchClientFactory.java b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/ElasticsearchClientFactory.java index 47ce58655a..1e44846690 100644 --- a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/ElasticsearchClientFactory.java +++ b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/ElasticsearchClientFactory.java @@ -1,38 +1,37 @@ package pl.allegro.tech.hermes.tracker.elasticsearch; +import java.net.InetAddress; +import java.net.UnknownHostException; import org.elasticsearch.client.Client; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.transport.client.PreBuiltTransportClient; -import java.net.InetAddress; -import java.net.UnknownHostException; - public class ElasticsearchClientFactory { - private final TransportClient client; + private final TransportClient client; - public ElasticsearchClientFactory(int port, String clusterName, String... hosts) { - client = new PreBuiltTransportClient( - Settings.builder().put("cluster.name", clusterName).build()); + public ElasticsearchClientFactory(int port, String clusterName, String... hosts) { + client = + new PreBuiltTransportClient(Settings.builder().put("cluster.name", clusterName).build()); - for (String host : hosts) { - try { - client.addTransportAddress(new TransportAddress(InetAddress.getByName(host), port)); - } catch (UnknownHostException e) { - throw new RuntimeException("Unknown host", e); - } - } + for (String host : hosts) { + try { + client.addTransportAddress(new TransportAddress(InetAddress.getByName(host), port)); + } catch (UnknownHostException e) { + throw new RuntimeException("Unknown host", e); + } } + } - public Client client() { - return client; - } + public Client client() { + return client; + } - public void close() { - if (client != null) { - client.close(); - } + public void close() { + if (client != null) { + client.close(); } + } } diff --git a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/ElasticsearchDocument.java b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/ElasticsearchDocument.java index ec6d3b364e..02d9d9be53 100644 --- a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/ElasticsearchDocument.java +++ b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/ElasticsearchDocument.java @@ -1,27 +1,26 @@ package pl.allegro.tech.hermes.tracker.elasticsearch; +import java.util.concurrent.Callable; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentBuilder; -import java.util.concurrent.Callable; - public class ElasticsearchDocument { - private final BytesReference bytesReference; + private final BytesReference bytesReference; - private ElasticsearchDocument(BytesReference bytesReference) { - this.bytesReference = bytesReference; - } + private ElasticsearchDocument(BytesReference bytesReference) { + this.bytesReference = bytesReference; + } - public BytesReference bytes() { - return bytesReference; - } + public BytesReference bytes() { + return bytesReference; + } - public static ElasticsearchDocument build(Callable builder) { - try { - return new ElasticsearchDocument(BytesReference.bytes(builder.call())); - } catch (Exception e) { - throw new ElasticsearchRepositoryException(e); - } + public static ElasticsearchDocument build(Callable builder) { + try { + return new ElasticsearchDocument(BytesReference.bytes(builder.call())); + } catch (Exception e) { + throw new ElasticsearchRepositoryException(e); } + } } diff --git a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/ElasticsearchQueueCommitter.java b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/ElasticsearchQueueCommitter.java index 8e5e12d232..69d60dd017 100644 --- a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/ElasticsearchQueueCommitter.java +++ b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/ElasticsearchQueueCommitter.java @@ -1,53 +1,62 @@ package pl.allegro.tech.hermes.tracker.elasticsearch; +import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ThreadFactory; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.client.Client; import org.elasticsearch.common.xcontent.XContentType; import pl.allegro.tech.hermes.metrics.HermesTimer; import pl.allegro.tech.hermes.tracker.QueueCommitter; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ThreadFactory; - -import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; -import static java.util.concurrent.TimeUnit.MILLISECONDS; - public class ElasticsearchQueueCommitter extends QueueCommitter { - private final IndexFactory indexFactory; - private final Client client; - private final String typeName; - - public ElasticsearchQueueCommitter(BlockingQueue queue, - HermesTimer timer, - IndexFactory indexFactory, - String typeName, - Client client) { - super(queue, timer); - this.indexFactory = indexFactory; - this.typeName = typeName; - this.client = client; - } - - @Override - protected void processBatch(List batch) throws ExecutionException, InterruptedException { - BulkRequestBuilder bulk = client.prepareBulk(); - batch.forEach(entry -> bulk.add( - client.prepareIndex(indexFactory.createIndex(), typeName).setSource(entry.bytes(), XContentType.JSON))); - bulk.execute().get(); - } - - public static void scheduleCommitAtFixedRate(BlockingQueue queue, - IndexFactory indexFactory, - String typeName, - Client client, - HermesTimer timer, - int interval) { - ElasticsearchQueueCommitter committer = new ElasticsearchQueueCommitter(queue, timer, indexFactory, typeName, client); - ThreadFactory factory = new ThreadFactoryBuilder().setNameFormat("elasticsearch-queue-committer-%d").build(); - newSingleThreadScheduledExecutor(factory).scheduleAtFixedRate(committer, interval, interval, MILLISECONDS); - } + private final IndexFactory indexFactory; + private final Client client; + private final String typeName; + + public ElasticsearchQueueCommitter( + BlockingQueue queue, + HermesTimer timer, + IndexFactory indexFactory, + String typeName, + Client client) { + super(queue, timer); + this.indexFactory = indexFactory; + this.typeName = typeName; + this.client = client; + } + + @Override + protected void processBatch(List batch) + throws ExecutionException, InterruptedException { + BulkRequestBuilder bulk = client.prepareBulk(); + batch.forEach( + entry -> + bulk.add( + client + .prepareIndex(indexFactory.createIndex(), typeName) + .setSource(entry.bytes(), XContentType.JSON))); + bulk.execute().get(); + } + + public static void scheduleCommitAtFixedRate( + BlockingQueue queue, + IndexFactory indexFactory, + String typeName, + Client client, + HermesTimer timer, + int interval) { + ElasticsearchQueueCommitter committer = + new ElasticsearchQueueCommitter(queue, timer, indexFactory, typeName, client); + ThreadFactory factory = + new ThreadFactoryBuilder().setNameFormat("elasticsearch-queue-committer-%d").build(); + newSingleThreadScheduledExecutor(factory) + .scheduleAtFixedRate(committer, interval, interval, MILLISECONDS); + } } diff --git a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/ElasticsearchRepositoryException.java b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/ElasticsearchRepositoryException.java index fac38704d4..ece926224a 100644 --- a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/ElasticsearchRepositoryException.java +++ b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/ElasticsearchRepositoryException.java @@ -1,7 +1,7 @@ package pl.allegro.tech.hermes.tracker.elasticsearch; public class ElasticsearchRepositoryException extends RuntimeException { - public ElasticsearchRepositoryException(Throwable cause) { - super(cause); - } + public ElasticsearchRepositoryException(Throwable cause) { + super(cause); + } } diff --git a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/IndexFactory.java b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/IndexFactory.java index 5507be19ee..ada42aaa03 100644 --- a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/IndexFactory.java +++ b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/IndexFactory.java @@ -2,5 +2,5 @@ public interface IndexFactory { - String createIndex(); + String createIndex(); } diff --git a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/LogSchemaAware.java b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/LogSchemaAware.java index 2f9ae339b4..dcee42c102 100644 --- a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/LogSchemaAware.java +++ b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/LogSchemaAware.java @@ -2,21 +2,20 @@ public interface LogSchemaAware { - String MESSAGE_ID = "messageId"; - String BATCH_ID = "batchId"; - String TIMESTAMP = "timestamp"; - String TIMESTAMP_SECONDS = "timestamp_seconds"; - String PUBLISH_TIMESTAMP = "publish_timestamp"; - String STATUS = "status"; - String TOPIC_NAME = "topicName"; - String SUBSCRIPTION = "subscription"; - String PARTITION = "partition"; - String OFFSET = "offset"; - String REASON = "reason"; - String CLUSTER = "cluster"; - String SOURCE_HOSTNAME = "hostname"; - String REMOTE_HOSTNAME = "remote_hostname"; - String EXTRA_REQUEST_HEADERS = "extra_request_headers"; - String STORAGE_DATACENTER = "storageDc"; - + String MESSAGE_ID = "messageId"; + String BATCH_ID = "batchId"; + String TIMESTAMP = "timestamp"; + String TIMESTAMP_SECONDS = "timestamp_seconds"; + String PUBLISH_TIMESTAMP = "publish_timestamp"; + String STATUS = "status"; + String TOPIC_NAME = "topicName"; + String SUBSCRIPTION = "subscription"; + String PARTITION = "partition"; + String OFFSET = "offset"; + String REASON = "reason"; + String CLUSTER = "cluster"; + String SOURCE_HOSTNAME = "hostname"; + String REMOTE_HOSTNAME = "remote_hostname"; + String EXTRA_REQUEST_HEADERS = "extra_request_headers"; + String STORAGE_DATACENTER = "storageDc"; } diff --git a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/SchemaManager.java b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/SchemaManager.java index ea71237cd8..64bd9c6be1 100644 --- a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/SchemaManager.java +++ b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/SchemaManager.java @@ -1,20 +1,5 @@ package pl.allegro.tech.hermes.tracker.elasticsearch; -import org.elasticsearch.action.admin.indices.alias.Alias; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.xcontent.XContentBuilder; -import pl.allegro.tech.hermes.tracker.elasticsearch.consumers.ConsumersDailyIndexFactory; -import pl.allegro.tech.hermes.tracker.elasticsearch.consumers.ConsumersIndexFactory; -import pl.allegro.tech.hermes.tracker.elasticsearch.frontend.FrontendDailyIndexFactory; -import pl.allegro.tech.hermes.tracker.elasticsearch.frontend.FrontendIndexFactory; - -import java.io.IOException; -import java.util.Arrays; -import java.util.function.Function; - import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static pl.allegro.tech.hermes.tracker.elasticsearch.LogSchemaAware.BATCH_ID; import static pl.allegro.tech.hermes.tracker.elasticsearch.LogSchemaAware.CLUSTER; @@ -33,122 +18,211 @@ import static pl.allegro.tech.hermes.tracker.elasticsearch.LogSchemaAware.TIMESTAMP_SECONDS; import static pl.allegro.tech.hermes.tracker.elasticsearch.LogSchemaAware.TOPIC_NAME; -public class SchemaManager { - - public static final String PUBLISHED_INDEX = "published_messages"; - public static final String PUBLISHED_TYPE = "published_message"; - public static final String PUBLISHED_ALIAS_NAME = "alias_published_messages"; - public static final String PUBLISHED_TEMPLATE_NAME = "template_published_messages"; - public static final String PUBLISHED_INDICES_REG_EXP = "published_messages_*"; - - public static final String SENT_INDEX = "sent_messages"; - public static final String SENT_TYPE = "sent_message"; - public static final String SENT_ALIAS_NAME = "alias_sent_messages"; - public static final String SENT_TEMPLATE_NAME = "template_sent_messages"; - public static final String SENT_INDICES_REG_EXP = "sent_messages_*"; - - private final Client client; - private final FrontendIndexFactory frontendIndexFactory; - private final ConsumersIndexFactory consumersIndexFactory; - private final boolean dynamicMappingEnabled; - - public static SchemaManager schemaManagerWithDailyIndexes(Client elasticClient) { - return new SchemaManager(elasticClient, new FrontendDailyIndexFactory(), new ConsumersDailyIndexFactory()); - } - - public SchemaManager(Client client, FrontendIndexFactory frontendIndexFactory, ConsumersIndexFactory consumersIndexFactory) { - this(client, frontendIndexFactory, consumersIndexFactory, true); - } - - public SchemaManager(Client client, FrontendIndexFactory frontendIndexFactory, ConsumersIndexFactory consumersIndexFactory, - boolean dynamicMappingEnabled) { - this.client = client; - this.frontendIndexFactory = frontendIndexFactory; - this.consumersIndexFactory = consumersIndexFactory; - this.dynamicMappingEnabled = dynamicMappingEnabled; - } - - public void ensureSchema() { - createTemplate(PUBLISHED_TEMPLATE_NAME, PUBLISHED_TYPE, PUBLISHED_INDICES_REG_EXP, PUBLISHED_ALIAS_NAME, - preparePublishedMapping()); - createTemplate(SENT_TEMPLATE_NAME, SENT_TYPE, SENT_INDICES_REG_EXP, SENT_ALIAS_NAME, prepareSentMapping()); - - createIndexIfNeeded(frontendIndexFactory); - createIndexIfNeeded(consumersIndexFactory); - - createAlias(frontendIndexFactory, PUBLISHED_ALIAS_NAME); - createAlias(consumersIndexFactory, SENT_ALIAS_NAME); - } - - private void createIndexIfNeeded(IndexFactory indexFactory) { - IndicesExistsResponse response = - client.admin().indices().exists(new IndicesExistsRequest(indexFactory.createIndex())).actionGet(); - - if (response.isExists()) { - return; - } - - client.admin().indices().prepareCreate(indexFactory.createIndex()).execute().actionGet(); - } - - private void createAlias(IndexFactory indexFactory, String alias) { - client.admin().indices().prepareAliases() - .addAlias(indexFactory.createIndex(), alias) - .execute().actionGet(); - } - - private void createTemplate(String templateName, String indexType, String indicesRegExp, String aliasName, - XContentBuilder templateMapping) { - - PutIndexTemplateRequest publishedTemplateRequest = new PutIndexTemplateRequest(templateName) - .patterns(Arrays.asList(indicesRegExp)) - .mapping(indexType, templateMapping) - .alias(new Alias(aliasName)); +import java.io.IOException; +import java.util.Arrays; +import java.util.function.Function; +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.xcontent.XContentBuilder; +import pl.allegro.tech.hermes.tracker.elasticsearch.consumers.ConsumersDailyIndexFactory; +import pl.allegro.tech.hermes.tracker.elasticsearch.consumers.ConsumersIndexFactory; +import pl.allegro.tech.hermes.tracker.elasticsearch.frontend.FrontendDailyIndexFactory; +import pl.allegro.tech.hermes.tracker.elasticsearch.frontend.FrontendIndexFactory; - client.admin().indices().putTemplate(publishedTemplateRequest).actionGet(); - } +public class SchemaManager { - private XContentBuilder preparePublishedMapping() { - return prepareMapping(PUBLISHED_TYPE, Function.identity()); + public static final String PUBLISHED_INDEX = "published_messages"; + public static final String PUBLISHED_TYPE = "published_message"; + public static final String PUBLISHED_ALIAS_NAME = "alias_published_messages"; + public static final String PUBLISHED_TEMPLATE_NAME = "template_published_messages"; + public static final String PUBLISHED_INDICES_REG_EXP = "published_messages_*"; + + public static final String SENT_INDEX = "sent_messages"; + public static final String SENT_TYPE = "sent_message"; + public static final String SENT_ALIAS_NAME = "alias_sent_messages"; + public static final String SENT_TEMPLATE_NAME = "template_sent_messages"; + public static final String SENT_INDICES_REG_EXP = "sent_messages_*"; + + private final Client client; + private final FrontendIndexFactory frontendIndexFactory; + private final ConsumersIndexFactory consumersIndexFactory; + private final boolean dynamicMappingEnabled; + + public static SchemaManager schemaManagerWithDailyIndexes(Client elasticClient) { + return new SchemaManager( + elasticClient, new FrontendDailyIndexFactory(), new ConsumersDailyIndexFactory()); + } + + public SchemaManager( + Client client, + FrontendIndexFactory frontendIndexFactory, + ConsumersIndexFactory consumersIndexFactory) { + this(client, frontendIndexFactory, consumersIndexFactory, true); + } + + public SchemaManager( + Client client, + FrontendIndexFactory frontendIndexFactory, + ConsumersIndexFactory consumersIndexFactory, + boolean dynamicMappingEnabled) { + this.client = client; + this.frontendIndexFactory = frontendIndexFactory; + this.consumersIndexFactory = consumersIndexFactory; + this.dynamicMappingEnabled = dynamicMappingEnabled; + } + + public void ensureSchema() { + createTemplate( + PUBLISHED_TEMPLATE_NAME, + PUBLISHED_TYPE, + PUBLISHED_INDICES_REG_EXP, + PUBLISHED_ALIAS_NAME, + preparePublishedMapping()); + createTemplate( + SENT_TEMPLATE_NAME, SENT_TYPE, SENT_INDICES_REG_EXP, SENT_ALIAS_NAME, prepareSentMapping()); + + createIndexIfNeeded(frontendIndexFactory); + createIndexIfNeeded(consumersIndexFactory); + + createAlias(frontendIndexFactory, PUBLISHED_ALIAS_NAME); + createAlias(consumersIndexFactory, SENT_ALIAS_NAME); + } + + private void createIndexIfNeeded(IndexFactory indexFactory) { + IndicesExistsResponse response = + client + .admin() + .indices() + .exists(new IndicesExistsRequest(indexFactory.createIndex())) + .actionGet(); + + if (response.isExists()) { + return; } - private XContentBuilder prepareSentMapping() { - return prepareMapping(SENT_TYPE, contentBuilder -> { - try { - return contentBuilder - .startObject(SUBSCRIPTION).field("type", "keyword").field("norms", false).endObject() - .startObject(PUBLISH_TIMESTAMP).field("type", "date").field("index", false).endObject() - .startObject(BATCH_ID).field("type", "keyword").field("norms", false).endObject() - .startObject(OFFSET).field("type", "long").endObject() - .startObject(PARTITION).field("type", "integer").endObject(); - } catch (IOException e) { - throw new ElasticsearchRepositoryException(e); - } + client.admin().indices().prepareCreate(indexFactory.createIndex()).execute().actionGet(); + } + + private void createAlias(IndexFactory indexFactory, String alias) { + client + .admin() + .indices() + .prepareAliases() + .addAlias(indexFactory.createIndex(), alias) + .execute() + .actionGet(); + } + + private void createTemplate( + String templateName, + String indexType, + String indicesRegExp, + String aliasName, + XContentBuilder templateMapping) { + + PutIndexTemplateRequest publishedTemplateRequest = + new PutIndexTemplateRequest(templateName) + .patterns(Arrays.asList(indicesRegExp)) + .mapping(indexType, templateMapping) + .alias(new Alias(aliasName)); + + client.admin().indices().putTemplate(publishedTemplateRequest).actionGet(); + } + + private XContentBuilder preparePublishedMapping() { + return prepareMapping(PUBLISHED_TYPE, Function.identity()); + } + + private XContentBuilder prepareSentMapping() { + return prepareMapping( + SENT_TYPE, + contentBuilder -> { + try { + return contentBuilder + .startObject(SUBSCRIPTION) + .field("type", "keyword") + .field("norms", false) + .endObject() + .startObject(PUBLISH_TIMESTAMP) + .field("type", "date") + .field("index", false) + .endObject() + .startObject(BATCH_ID) + .field("type", "keyword") + .field("norms", false) + .endObject() + .startObject(OFFSET) + .field("type", "long") + .endObject() + .startObject(PARTITION) + .field("type", "integer") + .endObject(); + } catch (IOException e) { + throw new ElasticsearchRepositoryException(e); + } }); + } + + private XContentBuilder prepareMapping( + String indexType, Function additionalMapping) { + try { + XContentBuilder jsonBuilder = + jsonBuilder() + .startObject() + .startObject(indexType) + .field("dynamic", dynamicMappingEnabled) + .startObject("properties") + .startObject(MESSAGE_ID) + .field("type", "keyword") + .field("norms", false) + .endObject() + .startObject(TIMESTAMP) + .field("type", "date") + .field("index", false) + .endObject() + .startObject(TIMESTAMP_SECONDS) + .field("type", "date") + .field("format", "epoch_second") + .endObject() + .startObject(TOPIC_NAME) + .field("type", "keyword") + .field("norms", false) + .endObject() + .startObject(STATUS) + .field("type", "keyword") + .field("norms", false) + .endObject() + .startObject(CLUSTER) + .field("type", "keyword") + .field("norms", false) + .endObject() + .startObject(SOURCE_HOSTNAME) + .field("type", "keyword") + .field("norms", false) + .endObject() + .startObject(REMOTE_HOSTNAME) + .field("type", "keyword") + .field("norms", false) + .endObject() + .startObject(REASON) + .field("type", "text") + .field("norms", false) + .endObject() + .startObject(STORAGE_DATACENTER) + .field("type", "text") + .field("norms", false) + .endObject() + .startObject(EXTRA_REQUEST_HEADERS) + .field("type", "text") + .field("norms", false) + .endObject(); + + return additionalMapping.apply(jsonBuilder).endObject().endObject().endObject(); + } catch (IOException ex) { + throw new ElasticsearchRepositoryException(ex); } - - private XContentBuilder prepareMapping(String indexType, Function additionalMapping) { - try { - XContentBuilder jsonBuilder = jsonBuilder() - .startObject() - .startObject(indexType) - .field("dynamic", dynamicMappingEnabled) - .startObject("properties") - .startObject(MESSAGE_ID).field("type", "keyword").field("norms", false).endObject() - .startObject(TIMESTAMP).field("type", "date").field("index", false).endObject() - .startObject(TIMESTAMP_SECONDS).field("type", "date").field("format", "epoch_second").endObject() - .startObject(TOPIC_NAME).field("type", "keyword").field("norms", false).endObject() - .startObject(STATUS).field("type", "keyword").field("norms", false).endObject() - .startObject(CLUSTER).field("type", "keyword").field("norms", false).endObject() - .startObject(SOURCE_HOSTNAME).field("type", "keyword").field("norms", false).endObject() - .startObject(REMOTE_HOSTNAME).field("type", "keyword").field("norms", false).endObject() - .startObject(REASON).field("type", "text").field("norms", false).endObject() - .startObject(STORAGE_DATACENTER).field("type", "text").field("norms", false).endObject() - .startObject(EXTRA_REQUEST_HEADERS).field("type", "text").field("norms", false).endObject(); - - return additionalMapping.apply(jsonBuilder).endObject().endObject().endObject(); - } catch (IOException ex) { - throw new ElasticsearchRepositoryException(ex); - } - } + } } diff --git a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/consumers/ConsumersDailyIndexFactory.java b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/consumers/ConsumersDailyIndexFactory.java index d5d2d1be4a..aaa8a6459e 100644 --- a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/consumers/ConsumersDailyIndexFactory.java +++ b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/consumers/ConsumersDailyIndexFactory.java @@ -1,17 +1,16 @@ package pl.allegro.tech.hermes.tracker.elasticsearch.consumers; +import java.time.Clock; import pl.allegro.tech.hermes.tracker.elasticsearch.DailyIndexFactory; import pl.allegro.tech.hermes.tracker.elasticsearch.SchemaManager; -import java.time.Clock; - public class ConsumersDailyIndexFactory extends DailyIndexFactory implements ConsumersIndexFactory { - public ConsumersDailyIndexFactory(Clock clock) { - super(SchemaManager.SENT_INDEX, clock); - } + public ConsumersDailyIndexFactory(Clock clock) { + super(SchemaManager.SENT_INDEX, clock); + } - public ConsumersDailyIndexFactory() { - super(SchemaManager.SENT_INDEX); - } + public ConsumersDailyIndexFactory() { + super(SchemaManager.SENT_INDEX); + } } diff --git a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/consumers/ConsumersElasticsearchLogRepository.java b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/consumers/ConsumersElasticsearchLogRepository.java index 14f3384081..b9b1ed45b6 100644 --- a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/consumers/ConsumersElasticsearchLogRepository.java +++ b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/consumers/ConsumersElasticsearchLogRepository.java @@ -1,5 +1,15 @@ package pl.allegro.tech.hermes.tracker.elasticsearch.consumers; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static pl.allegro.tech.hermes.api.SentMessageTraceStatus.DISCARDED; +import static pl.allegro.tech.hermes.api.SentMessageTraceStatus.FAILED; +import static pl.allegro.tech.hermes.api.SentMessageTraceStatus.FILTERED; +import static pl.allegro.tech.hermes.api.SentMessageTraceStatus.INFLIGHT; +import static pl.allegro.tech.hermes.api.SentMessageTraceStatus.SUCCESS; +import static pl.allegro.tech.hermes.tracker.elasticsearch.ElasticsearchDocument.build; + +import java.io.IOException; +import java.util.concurrent.BlockingQueue; import org.elasticsearch.client.Client; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -15,177 +25,185 @@ import pl.allegro.tech.hermes.tracker.elasticsearch.LogSchemaAware; import pl.allegro.tech.hermes.tracker.elasticsearch.SchemaManager; -import java.io.IOException; -import java.util.concurrent.BlockingQueue; - -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static pl.allegro.tech.hermes.api.SentMessageTraceStatus.DISCARDED; -import static pl.allegro.tech.hermes.api.SentMessageTraceStatus.FAILED; -import static pl.allegro.tech.hermes.api.SentMessageTraceStatus.FILTERED; -import static pl.allegro.tech.hermes.api.SentMessageTraceStatus.INFLIGHT; -import static pl.allegro.tech.hermes.api.SentMessageTraceStatus.SUCCESS; -import static pl.allegro.tech.hermes.tracker.elasticsearch.ElasticsearchDocument.build; - public class ConsumersElasticsearchLogRepository - extends BatchingLogRepository - implements LogRepository, LogSchemaAware { - - private static final int DOCUMENT_EXPECTED_SIZE = 1024; - - private final Client elasticClient; - - private ConsumersElasticsearchLogRepository(Client elasticClient, - String clusterName, - String hostname, - int queueSize, - int commitInterval, - IndexFactory indexFactory, - String typeName, - MetricsFacade metricsFacade) { - super(queueSize, clusterName, hostname); - this.elasticClient = elasticClient; - registerMetrics(commitInterval, indexFactory, typeName, metricsFacade.trackerElasticSearch()); - } - - @Override - public void logSuccessful(MessageMetadata message, String hostname, long timestamp) { - queue.offer(build(() -> + extends BatchingLogRepository implements LogRepository, LogSchemaAware { + + private static final int DOCUMENT_EXPECTED_SIZE = 1024; + + private final Client elasticClient; + + private ConsumersElasticsearchLogRepository( + Client elasticClient, + String clusterName, + String hostname, + int queueSize, + int commitInterval, + IndexFactory indexFactory, + String typeName, + MetricsFacade metricsFacade) { + super(queueSize, clusterName, hostname); + this.elasticClient = elasticClient; + registerMetrics(commitInterval, indexFactory, typeName, metricsFacade.trackerElasticSearch()); + } + + @Override + public void logSuccessful(MessageMetadata message, String hostname, long timestamp) { + queue.offer( + build( + () -> notEndedDocument(message, timestamp, SUCCESS.toString()) - .field(REMOTE_HOSTNAME, hostname) - .endObject())); - } - - @Override - public void logFailed(MessageMetadata message, String hostname, long timestamp, String reason) { - queue.offer(build(() -> + .field(REMOTE_HOSTNAME, hostname) + .endObject())); + } + + @Override + public void logFailed(MessageMetadata message, String hostname, long timestamp, String reason) { + queue.offer( + build( + () -> notEndedDocument(message, timestamp, FAILED.toString()) - .field(REASON, reason) - .field(REMOTE_HOSTNAME, hostname) - .endObject())); - } - - @Override - public void logDiscarded(MessageMetadata message, long timestamp, String reason) { - queue.offer(document(message, timestamp, DISCARDED, reason)); + .field(REASON, reason) + .field(REMOTE_HOSTNAME, hostname) + .endObject())); + } + + @Override + public void logDiscarded(MessageMetadata message, long timestamp, String reason) { + queue.offer(document(message, timestamp, DISCARDED, reason)); + } + + @Override + public void logInflight(MessageMetadata message, long timestamp) { + queue.offer(document(message, timestamp, INFLIGHT)); + } + + @Override + public void logFiltered(MessageMetadata message, long timestamp, String reason) { + queue.offer(document(message, timestamp, FILTERED, reason)); + } + + @Override + public void close() { + elasticClient.close(); + } + + private ElasticsearchDocument document( + MessageMetadata message, long createdAt, SentMessageTraceStatus status) { + return build(() -> notEndedDocument(message, createdAt, status.toString()).endObject()); + } + + private ElasticsearchDocument document( + MessageMetadata message, long timestamp, SentMessageTraceStatus status, String reason) { + return build( + () -> + notEndedDocument(message, timestamp, status.toString()) + .field(REASON, reason) + .endObject()); + } + + protected XContentBuilder notEndedDocument(MessageMetadata message, long timestamp, String status) + throws IOException { + return jsonBuilder(new BytesStreamOutput(DOCUMENT_EXPECTED_SIZE)) + .startObject() + .field(MESSAGE_ID, message.getMessageId()) + .field(BATCH_ID, message.getBatchId()) + .field(TIMESTAMP, timestamp) + .field(TIMESTAMP_SECONDS, toSeconds(timestamp)) + .field(PUBLISH_TIMESTAMP, message.getPublishingTimestamp()) + .field(TOPIC_NAME, message.getTopic()) + .field(SUBSCRIPTION, message.getSubscription()) + .field(STATUS, status) + .field(OFFSET, message.getOffset()) + .field(PARTITION, message.getPartition()) + .field(CLUSTER, clusterName) + .field(SOURCE_HOSTNAME, hostname); + } + + private void registerMetrics( + int commitInterval, + IndexFactory indexFactory, + String typeName, + TrackerElasticSearchMetrics trackerMetrics) { + trackerMetrics.registerConsumerTrackerElasticSearchQueueSizeGauge( + this.queue, BlockingQueue::size); + trackerMetrics.registerConsumerTrackerElasticSearchRemainingCapacity( + this.queue, BlockingQueue::size); + + ElasticsearchQueueCommitter.scheduleCommitAtFixedRate( + this.queue, + indexFactory, + typeName, + elasticClient, + trackerMetrics.trackerElasticSearchCommitLatencyTimer(), + commitInterval); + } + + private long toSeconds(long millis) { + return millis / 1000; + } + + public static class Builder { + + private Client elasticClient; + private String clusterName = "primary"; + private String hostName = "unknown"; + private int queueSize = 1000; + private int commitInterval = 100; + private ConsumersIndexFactory indexFactory = new ConsumersDailyIndexFactory(); + private String typeName = SchemaManager.SENT_TYPE; + + private final MetricsFacade metricsFacade; + + public Builder(Client elasticClient, MetricsFacade metricsFacade) { + this.elasticClient = elasticClient; + this.metricsFacade = metricsFacade; } - @Override - public void logInflight(MessageMetadata message, long timestamp) { - queue.offer(document(message, timestamp, INFLIGHT)); + public Builder withElasticClient(Client elasticClient) { + this.elasticClient = elasticClient; + return this; } - @Override - public void logFiltered(MessageMetadata message, long timestamp, String reason) { - queue.offer(document(message, timestamp, FILTERED, reason)); + public Builder withClusterName(String clusterName) { + this.clusterName = clusterName; + return this; } - @Override - public void close() { - elasticClient.close(); + public Builder withHostName(String hostName) { + this.hostName = hostName; + return this; } - private ElasticsearchDocument document(MessageMetadata message, long createdAt, SentMessageTraceStatus status) { - return build(() -> notEndedDocument(message, createdAt, status.toString()).endObject()); + public Builder withQueueSize(int queueSize) { + this.queueSize = queueSize; + return this; } - private ElasticsearchDocument document(MessageMetadata message, long timestamp, SentMessageTraceStatus status, String reason) { - return build(() -> notEndedDocument(message, timestamp, status.toString()).field(REASON, reason).endObject()); + public Builder withCommitInterval(int commitInterval) { + this.commitInterval = commitInterval; + return this; } - protected XContentBuilder notEndedDocument(MessageMetadata message, long timestamp, String status) - throws IOException { - return jsonBuilder(new BytesStreamOutput(DOCUMENT_EXPECTED_SIZE)) - .startObject() - .field(MESSAGE_ID, message.getMessageId()) - .field(BATCH_ID, message.getBatchId()) - .field(TIMESTAMP, timestamp) - .field(TIMESTAMP_SECONDS, toSeconds(timestamp)) - .field(PUBLISH_TIMESTAMP, message.getPublishingTimestamp()) - .field(TOPIC_NAME, message.getTopic()) - .field(SUBSCRIPTION, message.getSubscription()) - .field(STATUS, status) - .field(OFFSET, message.getOffset()) - .field(PARTITION, message.getPartition()) - .field(CLUSTER, clusterName) - .field(SOURCE_HOSTNAME, hostname); - } - - private void registerMetrics(int commitInterval, - IndexFactory indexFactory, - String typeName, - TrackerElasticSearchMetrics trackerMetrics) { - trackerMetrics.registerConsumerTrackerElasticSearchQueueSizeGauge(this.queue, BlockingQueue::size); - trackerMetrics.registerConsumerTrackerElasticSearchRemainingCapacity(this.queue, BlockingQueue::size); - - ElasticsearchQueueCommitter.scheduleCommitAtFixedRate(this.queue, indexFactory, typeName, elasticClient, - trackerMetrics.trackerElasticSearchCommitLatencyTimer(), commitInterval); + public Builder withTypeName(String typeName) { + this.typeName = typeName; + return this; } - private long toSeconds(long millis) { - return millis / 1000; + public Builder withIndexFactory(ConsumersIndexFactory indexFactory) { + this.indexFactory = indexFactory; + return this; } - public static class Builder { - - private Client elasticClient; - private String clusterName = "primary"; - private String hostName = "unknown"; - private int queueSize = 1000; - private int commitInterval = 100; - private ConsumersIndexFactory indexFactory = new ConsumersDailyIndexFactory(); - private String typeName = SchemaManager.SENT_TYPE; - - private final MetricsFacade metricsFacade; - - public Builder(Client elasticClient, MetricsFacade metricsFacade) { - this.elasticClient = elasticClient; - this.metricsFacade = metricsFacade; - } - - public Builder withElasticClient(Client elasticClient) { - this.elasticClient = elasticClient; - return this; - } - - public Builder withClusterName(String clusterName) { - this.clusterName = clusterName; - return this; - } - - public Builder withHostName(String hostName) { - this.hostName = hostName; - return this; - } - - public Builder withQueueSize(int queueSize) { - this.queueSize = queueSize; - return this; - } - - public Builder withCommitInterval(int commitInterval) { - this.commitInterval = commitInterval; - return this; - } - - public Builder withTypeName(String typeName) { - this.typeName = typeName; - return this; - } - - public Builder withIndexFactory(ConsumersIndexFactory indexFactory) { - this.indexFactory = indexFactory; - return this; - } - - public ConsumersElasticsearchLogRepository build() { - return new ConsumersElasticsearchLogRepository(elasticClient, - clusterName, - hostName, - queueSize, - commitInterval, - indexFactory, - typeName, - metricsFacade); - } + public ConsumersElasticsearchLogRepository build() { + return new ConsumersElasticsearchLogRepository( + elasticClient, + clusterName, + hostName, + queueSize, + commitInterval, + indexFactory, + typeName, + metricsFacade); } + } } diff --git a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/consumers/ConsumersIndexFactory.java b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/consumers/ConsumersIndexFactory.java index e00f5e5ac6..1d3b4921f5 100644 --- a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/consumers/ConsumersIndexFactory.java +++ b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/consumers/ConsumersIndexFactory.java @@ -2,5 +2,4 @@ import pl.allegro.tech.hermes.tracker.elasticsearch.IndexFactory; -public interface ConsumersIndexFactory extends IndexFactory { -} +public interface ConsumersIndexFactory extends IndexFactory {} diff --git a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/frontend/FrontendDailyIndexFactory.java b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/frontend/FrontendDailyIndexFactory.java index 74ad4c91ce..63466e3958 100644 --- a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/frontend/FrontendDailyIndexFactory.java +++ b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/frontend/FrontendDailyIndexFactory.java @@ -1,17 +1,16 @@ package pl.allegro.tech.hermes.tracker.elasticsearch.frontend; +import java.time.Clock; import pl.allegro.tech.hermes.tracker.elasticsearch.DailyIndexFactory; import pl.allegro.tech.hermes.tracker.elasticsearch.SchemaManager; -import java.time.Clock; - public class FrontendDailyIndexFactory extends DailyIndexFactory implements FrontendIndexFactory { - public FrontendDailyIndexFactory(Clock clock) { - super(SchemaManager.PUBLISHED_INDEX, clock); - } + public FrontendDailyIndexFactory(Clock clock) { + super(SchemaManager.PUBLISHED_INDEX, clock); + } - public FrontendDailyIndexFactory() { - super(SchemaManager.PUBLISHED_INDEX); - } + public FrontendDailyIndexFactory() { + super(SchemaManager.PUBLISHED_INDEX); + } } diff --git a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/frontend/FrontendElasticsearchLogRepository.java b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/frontend/FrontendElasticsearchLogRepository.java index c8ae1ab94d..6fbcf311e3 100644 --- a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/frontend/FrontendElasticsearchLogRepository.java +++ b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/frontend/FrontendElasticsearchLogRepository.java @@ -1,5 +1,15 @@ package pl.allegro.tech.hermes.tracker.elasticsearch.frontend; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static pl.allegro.tech.hermes.api.PublishedMessageTraceStatus.ERROR; +import static pl.allegro.tech.hermes.api.PublishedMessageTraceStatus.INFLIGHT; +import static pl.allegro.tech.hermes.api.PublishedMessageTraceStatus.SUCCESS; +import static pl.allegro.tech.hermes.common.http.ExtraRequestHeadersCollector.extraRequestHeadersCollector; +import static pl.allegro.tech.hermes.tracker.elasticsearch.ElasticsearchDocument.build; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.BlockingQueue; import org.elasticsearch.client.Client; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -13,204 +23,223 @@ import pl.allegro.tech.hermes.tracker.elasticsearch.SchemaManager; import pl.allegro.tech.hermes.tracker.frontend.LogRepository; -import java.io.IOException; -import java.util.Map; -import java.util.concurrent.BlockingQueue; - -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static pl.allegro.tech.hermes.api.PublishedMessageTraceStatus.ERROR; -import static pl.allegro.tech.hermes.api.PublishedMessageTraceStatus.INFLIGHT; -import static pl.allegro.tech.hermes.api.PublishedMessageTraceStatus.SUCCESS; -import static pl.allegro.tech.hermes.common.http.ExtraRequestHeadersCollector.extraRequestHeadersCollector; -import static pl.allegro.tech.hermes.tracker.elasticsearch.ElasticsearchDocument.build; - -public class FrontendElasticsearchLogRepository - extends BatchingLogRepository - implements LogRepository, LogSchemaAware { - - private static final int DOCUMENT_EXPECTED_SIZE = 1024; - - private final Client elasticClient; - - private FrontendElasticsearchLogRepository(Client elasticClient, - String clusterName, - String hostname, - int queueSize, - int commitInterval, - IndexFactory indexFactory, - String typeName, - MetricsFacade metricsFacade) { - super(queueSize, clusterName, hostname); - this.elasticClient = elasticClient; - registerMetrics(commitInterval, indexFactory, typeName, metricsFacade.trackerElasticSearch()); +public class FrontendElasticsearchLogRepository extends BatchingLogRepository + implements LogRepository, LogSchemaAware { + + private static final int DOCUMENT_EXPECTED_SIZE = 1024; + + private final Client elasticClient; + + private FrontendElasticsearchLogRepository( + Client elasticClient, + String clusterName, + String hostname, + int queueSize, + int commitInterval, + IndexFactory indexFactory, + String typeName, + MetricsFacade metricsFacade) { + super(queueSize, clusterName, hostname); + this.elasticClient = elasticClient; + registerMetrics(commitInterval, indexFactory, typeName, metricsFacade.trackerElasticSearch()); + } + + @Override + public void logPublished( + String messageId, + long timestamp, + String topicName, + String hostname, + String storageDatacenter, + Map extraRequestHeaders) { + queue.offer( + build( + () -> + success( + messageId, + timestamp, + topicName, + hostname, + storageDatacenter, + extraRequestHeaders))); + } + + @Override + public void logError( + String messageId, + long timestamp, + String topicName, + String reason, + String hostname, + Map extraRequestHeaders) { + queue.offer( + build(() -> error(messageId, timestamp, topicName, reason, hostname, extraRequestHeaders))); + } + + @Override + public void logInflight( + String messageId, + long timestamp, + String topicName, + String hostname, + Map extraRequestHeaders) { + queue.offer( + build(() -> inflight(messageId, timestamp, topicName, hostname, extraRequestHeaders))); + } + + @Override + public void close() { + this.elasticClient.close(); + } + + private XContentBuilder success( + String messageId, + long timestamp, + String topicName, + String hostname, + String storageDatacenter, + Map extraRequestHeaders) + throws IOException { + return notEndedDocument( + messageId, timestamp, topicName, SUCCESS.toString(), hostname, extraRequestHeaders) + .field(STORAGE_DATACENTER, storageDatacenter) + .endObject(); + } + + private XContentBuilder inflight( + String messageId, + long timestamp, + String topicName, + String hostname, + Map extraRequestHeaders) + throws IOException { + return notEndedDocument( + messageId, timestamp, topicName, INFLIGHT.name(), hostname, extraRequestHeaders) + .endObject(); + } + + private XContentBuilder error( + String messageId, + long timestamp, + String topicName, + String reason, + String hostname, + Map extraRequestHeaders) + throws IOException { + return notEndedDocument( + messageId, timestamp, topicName, ERROR.toString(), hostname, extraRequestHeaders) + .field(REASON, reason) + .endObject(); + } + + protected XContentBuilder notEndedDocument( + String messageId, + long timestamp, + String topicName, + String status, + String hostname, + Map extraRequestHeaders) + throws IOException { + return jsonBuilder(new BytesStreamOutput(DOCUMENT_EXPECTED_SIZE)) + .startObject() + .field(MESSAGE_ID, messageId) + .field(TIMESTAMP, timestamp) + .field(TIMESTAMP_SECONDS, toSeconds(timestamp)) + .field(TOPIC_NAME, topicName) + .field(STATUS, status) + .field(CLUSTER, clusterName) + .field(SOURCE_HOSTNAME, this.hostname) + .field(REMOTE_HOSTNAME, hostname) + .field( + EXTRA_REQUEST_HEADERS, + extraRequestHeaders.entrySet().stream().collect(extraRequestHeadersCollector())); + } + + private void registerMetrics( + int commitInterval, + IndexFactory indexFactory, + String typeName, + TrackerElasticSearchMetrics trackerMetrics) { + trackerMetrics.registerProducerTrackerElasticSearchQueueSizeGauge( + this.queue, BlockingQueue::size); + trackerMetrics.registerProducerTrackerElasticSearchRemainingCapacity( + this.queue, BlockingQueue::size); + + ElasticsearchQueueCommitter.scheduleCommitAtFixedRate( + this.queue, + indexFactory, + typeName, + elasticClient, + trackerMetrics.trackerElasticSearchCommitLatencyTimer(), + commitInterval); + } + + private long toSeconds(long millis) { + return millis / 1000; + } + + public static class Builder { + + private Client elasticClient; + private String clusterName = "primary"; + private String hostName = "unknown"; + private int queueSize = 1000; + private int commitInterval = 100; + private FrontendIndexFactory indexFactory = new FrontendDailyIndexFactory(); + private String typeName = SchemaManager.PUBLISHED_TYPE; + + private final MetricsFacade metricsFacade; + + public Builder(Client elasticClient, MetricsFacade metricsFacade) { + this.elasticClient = elasticClient; + this.metricsFacade = metricsFacade; } - @Override - public void logPublished(String messageId, - long timestamp, - String topicName, - String hostname, - String storageDatacenter, - Map extraRequestHeaders) { - queue.offer(build(() -> success(messageId, timestamp, topicName, hostname, storageDatacenter, extraRequestHeaders))); + public Builder withElasticClient(Client elasticClient) { + this.elasticClient = elasticClient; + return this; } - @Override - public void logError(String messageId, - long timestamp, - String topicName, - String reason, - String hostname, - Map extraRequestHeaders) { - queue.offer(build(() -> error(messageId, timestamp, topicName, reason, hostname, extraRequestHeaders))); + public Builder withClusterName(String clusterName) { + this.clusterName = clusterName; + return this; } - @Override - public void logInflight(String messageId, - long timestamp, - String topicName, - String hostname, - Map extraRequestHeaders) { - queue.offer(build(() -> inflight(messageId, timestamp, topicName, hostname, extraRequestHeaders))); + public Builder withHostName(String hostName) { + this.hostName = hostName; + return this; } - @Override - public void close() { - this.elasticClient.close(); + public Builder withQueueSize(int queueSize) { + this.queueSize = queueSize; + return this; } - private XContentBuilder success(String messageId, - long timestamp, - String topicName, - String hostname, - String storageDatacenter, - Map extraRequestHeaders) - throws IOException { - return notEndedDocument(messageId, timestamp, topicName, SUCCESS.toString(), hostname, extraRequestHeaders) - .field(STORAGE_DATACENTER, storageDatacenter) - .endObject(); + public Builder withCommitInterval(int commitInterval) { + this.commitInterval = commitInterval; + return this; } - - private XContentBuilder inflight(String messageId, - long timestamp, - String topicName, - String hostname, - Map extraRequestHeaders) - throws IOException { - return notEndedDocument(messageId, timestamp, topicName, INFLIGHT.name(), hostname, extraRequestHeaders).endObject(); - } - - private XContentBuilder error(String messageId, - long timestamp, - String topicName, - String reason, - String hostname, - Map extraRequestHeaders) - throws IOException { - return notEndedDocument(messageId, timestamp, topicName, ERROR.toString(), hostname, extraRequestHeaders) - .field(REASON, reason) - .endObject(); - } - - - protected XContentBuilder notEndedDocument(String messageId, - long timestamp, - String topicName, - String status, - String hostname, - Map extraRequestHeaders) - throws IOException { - return jsonBuilder(new BytesStreamOutput(DOCUMENT_EXPECTED_SIZE)) - .startObject() - .field(MESSAGE_ID, messageId) - .field(TIMESTAMP, timestamp) - .field(TIMESTAMP_SECONDS, toSeconds(timestamp)) - .field(TOPIC_NAME, topicName) - .field(STATUS, status) - .field(CLUSTER, clusterName) - .field(SOURCE_HOSTNAME, this.hostname) - .field(REMOTE_HOSTNAME, hostname) - .field(EXTRA_REQUEST_HEADERS, extraRequestHeaders.entrySet().stream() - .collect(extraRequestHeadersCollector())); - } - - private void registerMetrics(int commitInterval, - IndexFactory indexFactory, - String typeName, - TrackerElasticSearchMetrics trackerMetrics) { - trackerMetrics.registerProducerTrackerElasticSearchQueueSizeGauge(this.queue, BlockingQueue::size); - trackerMetrics.registerProducerTrackerElasticSearchRemainingCapacity(this.queue, BlockingQueue::size); - - ElasticsearchQueueCommitter.scheduleCommitAtFixedRate(this.queue, indexFactory, typeName, elasticClient, - trackerMetrics.trackerElasticSearchCommitLatencyTimer(), commitInterval); + public Builder withTypeName(String typeName) { + this.typeName = typeName; + return this; } - private long toSeconds(long millis) { - return millis / 1000; + public Builder withIndexFactory(FrontendIndexFactory indexFactory) { + this.indexFactory = indexFactory; + return this; } - public static class Builder { - - private Client elasticClient; - private String clusterName = "primary"; - private String hostName = "unknown"; - private int queueSize = 1000; - private int commitInterval = 100; - private FrontendIndexFactory indexFactory = new FrontendDailyIndexFactory(); - private String typeName = SchemaManager.PUBLISHED_TYPE; - - private final MetricsFacade metricsFacade; - - public Builder(Client elasticClient, MetricsFacade metricsFacade) { - this.elasticClient = elasticClient; - this.metricsFacade = metricsFacade; - } - - public Builder withElasticClient(Client elasticClient) { - this.elasticClient = elasticClient; - return this; - } - - public Builder withClusterName(String clusterName) { - this.clusterName = clusterName; - return this; - } - - public Builder withHostName(String hostName) { - this.hostName = hostName; - return this; - } - - public Builder withQueueSize(int queueSize) { - this.queueSize = queueSize; - return this; - } - - public Builder withCommitInterval(int commitInterval) { - this.commitInterval = commitInterval; - return this; - } - - public Builder withTypeName(String typeName) { - this.typeName = typeName; - return this; - } - - public Builder withIndexFactory(FrontendIndexFactory indexFactory) { - this.indexFactory = indexFactory; - return this; - } - - public FrontendElasticsearchLogRepository build() { - return new FrontendElasticsearchLogRepository(elasticClient, - clusterName, - hostName, - queueSize, - commitInterval, - indexFactory, - typeName, - metricsFacade); - } + public FrontendElasticsearchLogRepository build() { + return new FrontendElasticsearchLogRepository( + elasticClient, + clusterName, + hostName, + queueSize, + commitInterval, + indexFactory, + typeName, + metricsFacade); } + } } diff --git a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/frontend/FrontendIndexFactory.java b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/frontend/FrontendIndexFactory.java index a555716fc5..ed94098c3d 100644 --- a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/frontend/FrontendIndexFactory.java +++ b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/frontend/FrontendIndexFactory.java @@ -2,5 +2,4 @@ import pl.allegro.tech.hermes.tracker.elasticsearch.IndexFactory; -public interface FrontendIndexFactory extends IndexFactory { -} +public interface FrontendIndexFactory extends IndexFactory {} diff --git a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/management/ElasticsearchLogRepository.java b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/management/ElasticsearchLogRepository.java index ff514c7c13..4b1fef83fc 100644 --- a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/management/ElasticsearchLogRepository.java +++ b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/management/ElasticsearchLogRepository.java @@ -1,6 +1,16 @@ package pl.allegro.tech.hermes.tracker.elasticsearch.management; +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toList; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static pl.allegro.tech.hermes.tracker.elasticsearch.SchemaManager.schemaManagerWithDailyIndexes; + import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.stream.Stream; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; import org.elasticsearch.index.query.QueryBuilder; @@ -15,99 +25,105 @@ import pl.allegro.tech.hermes.tracker.elasticsearch.SchemaManager; import pl.allegro.tech.hermes.tracker.management.LogRepository; -import java.io.IOException; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.stream.Stream; - -import static java.util.Arrays.stream; -import static java.util.stream.Collectors.toList; -import static org.elasticsearch.index.query.QueryBuilders.boolQuery; -import static org.elasticsearch.index.query.QueryBuilders.termQuery; -import static pl.allegro.tech.hermes.tracker.elasticsearch.SchemaManager.schemaManagerWithDailyIndexes; - public class ElasticsearchLogRepository implements LogRepository, LogSchemaAware { - private static final int LIMIT = 1000; - - private final Client elasticClient; - - private final ObjectMapper objectMapper = new ObjectMapper(); - - public ElasticsearchLogRepository(Client elasticClient) { - this(elasticClient, schemaManagerWithDailyIndexes(elasticClient)); - } - - public ElasticsearchLogRepository(Client elasticClient, SchemaManager schemaManager) { - this.elasticClient = elasticClient; - schemaManager.ensureSchema(); + private static final int LIMIT = 1000; + + private final Client elasticClient; + + private final ObjectMapper objectMapper = new ObjectMapper(); + + public ElasticsearchLogRepository(Client elasticClient) { + this(elasticClient, schemaManagerWithDailyIndexes(elasticClient)); + } + + public ElasticsearchLogRepository(Client elasticClient, SchemaManager schemaManager) { + this.elasticClient = elasticClient; + schemaManager.ensureSchema(); + } + + @Override + public List getLastUndeliveredMessages( + String topicName, String subscriptionName, int limit) { + SearchResponse response = + searchSentMessages( + limit, + SortOrder.DESC, + boolQuery() + .must(termQuery(TOPIC_NAME, topicName)) + .must(termQuery(SUBSCRIPTION, subscriptionName)) + .must(termQuery(STATUS, SentMessageTraceStatus.DISCARDED.name()))); + + return stream(response.getHits().getHits()) + .map(hit -> toMessageTrace(hit, SentMessageTrace.class)) + .collect(toList()); + } + + @Override + public List getMessageStatus( + String topicName, String subscriptionName, String messageId) { + + try { + SearchResponse publishedResponse = + searchPublishedMessages( + LIMIT, + boolQuery() + .must(termQuery(TOPIC_NAME, topicName)) + .must(termQuery(MESSAGE_ID, messageId))); + + SearchResponse sentResponse = + searchSentMessages( + LIMIT, + SortOrder.ASC, + boolQuery() + .must(termQuery(TOPIC_NAME, topicName)) + .must(termQuery(SUBSCRIPTION, subscriptionName)) + .must(termQuery(MESSAGE_ID, messageId))); + + return Stream.concat( + stream(publishedResponse.getHits().getHits()) + .map(hit -> toMessageTrace(hit, PublishedMessageTrace.class)), + stream(sentResponse.getHits().getHits()) + .map(hit -> toMessageTrace(hit, SentMessageTrace.class))) + .collect(toList()); + } catch (InterruptedException | ExecutionException ex) { + throw new ElasticsearchRepositoryException(ex); } - - @Override - public List getLastUndeliveredMessages(String topicName, String subscriptionName, int limit) { - SearchResponse response = searchSentMessages(limit, SortOrder.DESC, - boolQuery() - .must(termQuery(TOPIC_NAME, topicName)) - .must(termQuery(SUBSCRIPTION, subscriptionName)) - .must(termQuery(STATUS, SentMessageTraceStatus.DISCARDED.name()))); - - return stream(response.getHits().getHits()) - .map(hit -> toMessageTrace(hit, SentMessageTrace.class)) - .collect(toList()); - } - - @Override - public List getMessageStatus(String topicName, String subscriptionName, String messageId) { - - try { - SearchResponse publishedResponse = searchPublishedMessages(LIMIT, - boolQuery() - .must(termQuery(TOPIC_NAME, topicName)) - .must(termQuery(MESSAGE_ID, messageId))); - - SearchResponse sentResponse = searchSentMessages(LIMIT, SortOrder.ASC, - boolQuery() - .must(termQuery(TOPIC_NAME, topicName)) - .must(termQuery(SUBSCRIPTION, subscriptionName)) - .must(termQuery(MESSAGE_ID, messageId))); - - return Stream.concat( - stream(publishedResponse.getHits().getHits()).map(hit -> toMessageTrace(hit, PublishedMessageTrace.class)), - stream(sentResponse.getHits().getHits()).map(hit -> toMessageTrace(hit, SentMessageTrace.class))) - .collect(toList()); - } catch (InterruptedException | ExecutionException ex) { - throw new ElasticsearchRepositoryException(ex); - } - } - - private SearchResponse searchSentMessages(int limit, SortOrder sort, QueryBuilder query) { - return elasticClient.prepareSearch(SchemaManager.SENT_ALIAS_NAME) - .setTypes(SchemaManager.SENT_TYPE) - .setTrackScores(true) - .setQuery(query) - .addSort(TIMESTAMP_SECONDS, sort) - .setSize(limit) - .execute() - .actionGet(); - } - - private SearchResponse searchPublishedMessages(int limit, QueryBuilder query) throws InterruptedException, ExecutionException { - return elasticClient.prepareSearch(SchemaManager.PUBLISHED_ALIAS_NAME) - .setTypes(SchemaManager.PUBLISHED_TYPE) - .setTrackScores(true) - .setQuery(query) - .addSort(TIMESTAMP_SECONDS, SortOrder.ASC) - .setSize(limit) - .execute() - .get(); - } - - private T toMessageTrace(SearchHit h, Class messageTraceType) { - try { - return objectMapper.readValue(h.getSourceRef().streamInput(), messageTraceType); - } catch (IOException e) { - throw new RuntimeException( - "Exception during deserialization of message trace class named " + messageTraceType.getCanonicalName(), e); - } + } + + private SearchResponse searchSentMessages(int limit, SortOrder sort, QueryBuilder query) { + return elasticClient + .prepareSearch(SchemaManager.SENT_ALIAS_NAME) + .setTypes(SchemaManager.SENT_TYPE) + .setTrackScores(true) + .setQuery(query) + .addSort(TIMESTAMP_SECONDS, sort) + .setSize(limit) + .execute() + .actionGet(); + } + + private SearchResponse searchPublishedMessages(int limit, QueryBuilder query) + throws InterruptedException, ExecutionException { + return elasticClient + .prepareSearch(SchemaManager.PUBLISHED_ALIAS_NAME) + .setTypes(SchemaManager.PUBLISHED_TYPE) + .setTrackScores(true) + .setQuery(query) + .addSort(TIMESTAMP_SECONDS, SortOrder.ASC) + .setSize(limit) + .execute() + .get(); + } + + private T toMessageTrace(SearchHit h, Class messageTraceType) { + try { + return objectMapper.readValue(h.getSourceRef().streamInput(), messageTraceType); + } catch (IOException e) { + throw new RuntimeException( + "Exception during deserialization of message trace class named " + + messageTraceType.getCanonicalName(), + e); } + } } diff --git a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/management/MultiElasticsearchLogRepository.java b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/management/MultiElasticsearchLogRepository.java index d5aaf7a0ef..54cfc1d472 100644 --- a/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/management/MultiElasticsearchLogRepository.java +++ b/hermes-tracker-elasticsearch/src/main/java/pl/allegro/tech/hermes/tracker/elasticsearch/management/MultiElasticsearchLogRepository.java @@ -1,38 +1,38 @@ package pl.allegro.tech.hermes.tracker.elasticsearch.management; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; import org.elasticsearch.client.Client; import pl.allegro.tech.hermes.api.MessageTrace; import pl.allegro.tech.hermes.api.SentMessageTrace; import pl.allegro.tech.hermes.tracker.elasticsearch.LogSchemaAware; import pl.allegro.tech.hermes.tracker.management.LogRepository; -import java.util.Collection; -import java.util.List; -import java.util.stream.Collectors; - public class MultiElasticsearchLogRepository implements LogRepository, LogSchemaAware { - private final List elasticsearchLogRepositories; + private final List elasticsearchLogRepositories; - public MultiElasticsearchLogRepository(List elasticClients) { - elasticsearchLogRepositories = elasticClients.stream() - .map(ElasticsearchLogRepository::new) - .collect(Collectors.toList()); - } + public MultiElasticsearchLogRepository(List elasticClients) { + elasticsearchLogRepositories = + elasticClients.stream().map(ElasticsearchLogRepository::new).collect(Collectors.toList()); + } - @Override - public List getLastUndeliveredMessages(String topicName, String subscriptionName, int limit) { - return elasticsearchLogRepositories.stream() - .map(repo -> repo.getLastUndeliveredMessages(topicName, subscriptionName, limit)) - .flatMap(Collection::stream) - .collect(Collectors.toList()); - } + @Override + public List getLastUndeliveredMessages( + String topicName, String subscriptionName, int limit) { + return elasticsearchLogRepositories.stream() + .map(repo -> repo.getLastUndeliveredMessages(topicName, subscriptionName, limit)) + .flatMap(Collection::stream) + .collect(Collectors.toList()); + } - @Override - public List getMessageStatus(String qualifiedTopicName, String subscriptionName, String messageId) { - return elasticsearchLogRepositories.stream() - .map(repo -> repo.getMessageStatus(qualifiedTopicName, subscriptionName, messageId)) - .flatMap(Collection::stream) - .collect(Collectors.toList()); - } + @Override + public List getMessageStatus( + String qualifiedTopicName, String subscriptionName, String messageId) { + return elasticsearchLogRepositories.stream() + .map(repo -> repo.getMessageStatus(qualifiedTopicName, subscriptionName, messageId)) + .flatMap(Collection::stream) + .collect(Collectors.toList()); + } } diff --git a/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/ElasticsearchResource.java b/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/ElasticsearchResource.java index 82292ace2b..efc54f195a 100644 --- a/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/ElasticsearchResource.java +++ b/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/ElasticsearchResource.java @@ -1,5 +1,12 @@ package pl.allegro.tech.hermes.tracker.elasticsearch; +import static pl.allegro.tech.embeddedelasticsearch.PopularProperties.CLUSTER_NAME; +import static pl.allegro.tech.embeddedelasticsearch.PopularProperties.HTTP_PORT; +import static pl.allegro.tech.embeddedelasticsearch.PopularProperties.TRANSPORT_TCP_PORT; + +import java.net.InetAddress; +import java.nio.file.Files; +import java.util.concurrent.TimeUnit; import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -11,74 +18,78 @@ import pl.allegro.tech.embeddedelasticsearch.EmbeddedElastic; import pl.allegro.tech.hermes.test.helper.util.Ports; -import java.net.InetAddress; -import java.nio.file.Files; -import java.util.concurrent.TimeUnit; - -import static pl.allegro.tech.embeddedelasticsearch.PopularProperties.CLUSTER_NAME; -import static pl.allegro.tech.embeddedelasticsearch.PopularProperties.HTTP_PORT; -import static pl.allegro.tech.embeddedelasticsearch.PopularProperties.TRANSPORT_TCP_PORT; - public class ElasticsearchResource extends ExternalResource implements LogSchemaAware { - private static final String ELASTIC_VERSION = "6.1.4"; - private static final String CLUSTER_NAME_VALUE = "myTestCluster"; - - private final EmbeddedElastic embeddedElastic; - private Client client; - - public ElasticsearchResource() { - int port = Ports.nextAvailable(); - int httpPort = Ports.nextAvailable(); - - try { - embeddedElastic = EmbeddedElastic.builder() - .withElasticVersion(ELASTIC_VERSION) - .withSetting(TRANSPORT_TCP_PORT, port) - .withSetting(HTTP_PORT, httpPort) - .withSetting(CLUSTER_NAME, CLUSTER_NAME_VALUE) - // embedded elastic search runs with "UseConcMarkSweepGC" which is invalid in Java 17 - .withEsJavaOpts("-Xms128m -Xmx512m -XX:+IgnoreUnrecognizedVMOptions") - .withStartTimeout(1, TimeUnit.MINUTES) - .withCleanInstallationDirectoryOnStop(true) - .withInstallationDirectory(Files.createTempDirectory("elasticsearch-installation-" + port).toFile()) - .build(); - - } catch (Exception e) { - throw new RuntimeException("Unchecked exception", e); - } - } - - @Override - public void before() throws Throwable { - embeddedElastic.start(); - - client = new PreBuiltTransportClient(Settings.builder().put(CLUSTER_NAME, CLUSTER_NAME_VALUE).build()) - .addTransportAddress( - new TransportAddress(InetAddress.getByName("localhost"), embeddedElastic.getTransportTcpPort())); - } - - @Override - public void after() { - embeddedElastic.stop(); - client.close(); - } - - public Client client() { - return client; - } - - public AdminClient adminClient() { - return client.admin(); - } - - public ImmutableOpenMap getIndices() { - return client.admin().cluster().prepareState().execute().actionGet().getState().getMetaData().getIndices(); - } - - public void cleanStructures() { - embeddedElastic.deleteIndices(); - embeddedElastic.deleteTemplates(); + private static final String ELASTIC_VERSION = "6.1.4"; + private static final String CLUSTER_NAME_VALUE = "myTestCluster"; + + private final EmbeddedElastic embeddedElastic; + private Client client; + + public ElasticsearchResource() { + int port = Ports.nextAvailable(); + int httpPort = Ports.nextAvailable(); + + try { + embeddedElastic = + EmbeddedElastic.builder() + .withElasticVersion(ELASTIC_VERSION) + .withSetting(TRANSPORT_TCP_PORT, port) + .withSetting(HTTP_PORT, httpPort) + .withSetting(CLUSTER_NAME, CLUSTER_NAME_VALUE) + // embedded elastic search runs with "UseConcMarkSweepGC" which is invalid in Java 17 + .withEsJavaOpts("-Xms128m -Xmx512m -XX:+IgnoreUnrecognizedVMOptions") + .withStartTimeout(1, TimeUnit.MINUTES) + .withCleanInstallationDirectoryOnStop(true) + .withInstallationDirectory( + Files.createTempDirectory("elasticsearch-installation-" + port).toFile()) + .build(); + + } catch (Exception e) { + throw new RuntimeException("Unchecked exception", e); } - + } + + @Override + public void before() throws Throwable { + embeddedElastic.start(); + + client = + new PreBuiltTransportClient( + Settings.builder().put(CLUSTER_NAME, CLUSTER_NAME_VALUE).build()) + .addTransportAddress( + new TransportAddress( + InetAddress.getByName("localhost"), embeddedElastic.getTransportTcpPort())); + } + + @Override + public void after() { + embeddedElastic.stop(); + client.close(); + } + + public Client client() { + return client; + } + + public AdminClient adminClient() { + return client.admin(); + } + + public ImmutableOpenMap getIndices() { + return client + .admin() + .cluster() + .prepareState() + .execute() + .actionGet() + .getState() + .getMetaData() + .getIndices(); + } + + public void cleanStructures() { + embeddedElastic.deleteIndices(); + embeddedElastic.deleteTemplates(); + } } diff --git a/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/consumers/ConsumersElasticsearchLogRepositoryTest.java b/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/consumers/ConsumersElasticsearchLogRepositoryTest.java index 20d0aa0e6d..4bad544276 100644 --- a/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/consumers/ConsumersElasticsearchLogRepositoryTest.java +++ b/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/consumers/ConsumersElasticsearchLogRepositoryTest.java @@ -1,6 +1,15 @@ package pl.allegro.tech.hermes.tracker.elasticsearch.consumers; +import static org.awaitility.Awaitility.await; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; + import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import java.time.Clock; +import java.time.Duration; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.ZoneOffset; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -16,86 +25,93 @@ import pl.allegro.tech.hermes.tracker.elasticsearch.frontend.FrontendDailyIndexFactory; import pl.allegro.tech.hermes.tracker.elasticsearch.frontend.FrontendIndexFactory; -import java.time.Clock; -import java.time.Duration; -import java.time.LocalDate; -import java.time.ZoneId; -import java.time.ZoneOffset; - -import static org.awaitility.Awaitility.await; -import static org.elasticsearch.index.query.QueryBuilders.boolQuery; -import static org.elasticsearch.index.query.QueryBuilders.termQuery; - -public class ConsumersElasticsearchLogRepositoryTest extends AbstractLogRepositoryTest implements LogSchemaAware { - - private static final String CLUSTER_NAME = "primary"; - - private static final Clock clock = Clock.fixed(LocalDate.now().atStartOfDay().toInstant(ZoneOffset.UTC), ZoneId.systemDefault()); - private static final ConsumersIndexFactory indexFactory = new ConsumersDailyIndexFactory(clock); - private static final FrontendIndexFactory frontendIndexFactory = new FrontendDailyIndexFactory(clock); - private static final MetricsFacade metricsFacade = new MetricsFacade( - new SimpleMeterRegistry() - ); - - - private static final ElasticsearchResource elasticsearch = new ElasticsearchResource(); - private static SchemaManager schemaManager; - - @BeforeClass - public static void beforeAll() throws Throwable { - elasticsearch.before(); - schemaManager = new SchemaManager(elasticsearch.client(), frontendIndexFactory, indexFactory, false); - } - - @AfterClass - public static void afterAll() { - elasticsearch.after(); - } - - @Override - protected LogRepository createLogRepository() { - schemaManager.ensureSchema(); - return new ConsumersElasticsearchLogRepository.Builder(elasticsearch.client(), metricsFacade) - .withIndexFactory(indexFactory) - .build(); - } - - @Override - protected void awaitUntilMessageIsPersisted(String topic, String subscription, String id, - SentMessageTraceStatus status) { - awaitUntilPersisted(getMessageFilter(topic, subscription, id, status)); - } - - @Override - protected void awaitUntilBatchMessageIsPersisted(String topic, String subscription, String messageId, String batchId, - SentMessageTraceStatus status) { - awaitUntilPersisted(getMessageBatchFilter(topic, subscription, messageId, batchId, status)); - } - - private void awaitUntilPersisted(QueryBuilder query) { - await().atMost(Duration.ofMinutes(1)).until(() -> { - SearchResponse response = elasticsearch.client().prepareSearch(indexFactory.createIndex()) - .setTypes(SchemaManager.SENT_TYPE) - .setQuery(query) - .execute().get(); - return response.getHits().getTotalHits() == 1; - }); - } - - private BoolQueryBuilder getMessageFilter(String topic, String subscription, String id, - SentMessageTraceStatus status) { - BoolQueryBuilder queryBuilder = boolQuery() - .must(termQuery(TOPIC_NAME, topic)) - .must(termQuery(SUBSCRIPTION, subscription)) - .must(termQuery(MESSAGE_ID, id)) - .must(termQuery(STATUS, status.toString())) - .must(termQuery(CLUSTER, CLUSTER_NAME)); - return queryBuilder; - } - - private QueryBuilder getMessageBatchFilter(String topic, String subscription, String messageId, String batchId, - SentMessageTraceStatus status) { - return getMessageFilter(topic, subscription, messageId, status) - .must(termQuery(BATCH_ID, batchId)); - } +public class ConsumersElasticsearchLogRepositoryTest extends AbstractLogRepositoryTest + implements LogSchemaAware { + + private static final String CLUSTER_NAME = "primary"; + + private static final Clock clock = + Clock.fixed(LocalDate.now().atStartOfDay().toInstant(ZoneOffset.UTC), ZoneId.systemDefault()); + private static final ConsumersIndexFactory indexFactory = new ConsumersDailyIndexFactory(clock); + private static final FrontendIndexFactory frontendIndexFactory = + new FrontendDailyIndexFactory(clock); + private static final MetricsFacade metricsFacade = new MetricsFacade(new SimpleMeterRegistry()); + + private static final ElasticsearchResource elasticsearch = new ElasticsearchResource(); + private static SchemaManager schemaManager; + + @BeforeClass + public static void beforeAll() throws Throwable { + elasticsearch.before(); + schemaManager = + new SchemaManager(elasticsearch.client(), frontendIndexFactory, indexFactory, false); + } + + @AfterClass + public static void afterAll() { + elasticsearch.after(); + } + + @Override + protected LogRepository createLogRepository() { + schemaManager.ensureSchema(); + return new ConsumersElasticsearchLogRepository.Builder(elasticsearch.client(), metricsFacade) + .withIndexFactory(indexFactory) + .build(); + } + + @Override + protected void awaitUntilMessageIsPersisted( + String topic, String subscription, String id, SentMessageTraceStatus status) { + awaitUntilPersisted(getMessageFilter(topic, subscription, id, status)); + } + + @Override + protected void awaitUntilBatchMessageIsPersisted( + String topic, + String subscription, + String messageId, + String batchId, + SentMessageTraceStatus status) { + awaitUntilPersisted(getMessageBatchFilter(topic, subscription, messageId, batchId, status)); + } + + private void awaitUntilPersisted(QueryBuilder query) { + await() + .atMost(Duration.ofMinutes(1)) + .until( + () -> { + SearchResponse response = + elasticsearch + .client() + .prepareSearch(indexFactory.createIndex()) + .setTypes(SchemaManager.SENT_TYPE) + .setQuery(query) + .execute() + .get(); + return response.getHits().getTotalHits() == 1; + }); + } + + private BoolQueryBuilder getMessageFilter( + String topic, String subscription, String id, SentMessageTraceStatus status) { + BoolQueryBuilder queryBuilder = + boolQuery() + .must(termQuery(TOPIC_NAME, topic)) + .must(termQuery(SUBSCRIPTION, subscription)) + .must(termQuery(MESSAGE_ID, id)) + .must(termQuery(STATUS, status.toString())) + .must(termQuery(CLUSTER, CLUSTER_NAME)); + return queryBuilder; + } + + private QueryBuilder getMessageBatchFilter( + String topic, + String subscription, + String messageId, + String batchId, + SentMessageTraceStatus status) { + return getMessageFilter(topic, subscription, messageId, status) + .must(termQuery(BATCH_ID, batchId)); + } } diff --git a/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/frontend/FrontendElasticsearchLogRepositoryTest.java b/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/frontend/FrontendElasticsearchLogRepositoryTest.java index 329824b414..de7deaa1c4 100644 --- a/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/frontend/FrontendElasticsearchLogRepositoryTest.java +++ b/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/frontend/FrontendElasticsearchLogRepositoryTest.java @@ -1,6 +1,16 @@ package pl.allegro.tech.hermes.tracker.elasticsearch.frontend; +import static org.awaitility.Awaitility.await; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; + import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import java.time.Clock; +import java.time.Duration; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.ZoneOffset; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -16,107 +26,127 @@ import pl.allegro.tech.hermes.tracker.frontend.AbstractLogRepositoryTest; import pl.allegro.tech.hermes.tracker.frontend.LogRepository; -import java.time.Clock; -import java.time.Duration; -import java.time.LocalDate; -import java.time.ZoneId; -import java.time.ZoneOffset; - -import static org.awaitility.Awaitility.await; -import static org.elasticsearch.index.query.QueryBuilders.boolQuery; -import static org.elasticsearch.index.query.QueryBuilders.matchQuery; -import static org.elasticsearch.index.query.QueryBuilders.termQuery; - -public class FrontendElasticsearchLogRepositoryTest extends AbstractLogRepositoryTest implements LogSchemaAware { - - private static final String CLUSTER_NAME = "primary"; - - private static final Clock clock = Clock.fixed(LocalDate.now().atStartOfDay().toInstant(ZoneOffset.UTC), ZoneId.systemDefault()); - private static final FrontendIndexFactory frontendIndexFactory = new FrontendDailyIndexFactory(clock); - private static final ConsumersIndexFactory consumersIndexFactory = new ConsumersDailyIndexFactory(clock); - private static final MetricsFacade metricsFacade = new MetricsFacade( - new SimpleMeterRegistry() - ); - - private static final ElasticsearchResource elasticsearch = new ElasticsearchResource(); - - private static SchemaManager schemaManager; - - @BeforeClass - public static void beforeAll() throws Throwable { - elasticsearch.before(); - schemaManager = new SchemaManager(elasticsearch.client(), frontendIndexFactory, consumersIndexFactory, false); +public class FrontendElasticsearchLogRepositoryTest extends AbstractLogRepositoryTest + implements LogSchemaAware { + + private static final String CLUSTER_NAME = "primary"; + + private static final Clock clock = + Clock.fixed(LocalDate.now().atStartOfDay().toInstant(ZoneOffset.UTC), ZoneId.systemDefault()); + private static final FrontendIndexFactory frontendIndexFactory = + new FrontendDailyIndexFactory(clock); + private static final ConsumersIndexFactory consumersIndexFactory = + new ConsumersDailyIndexFactory(clock); + private static final MetricsFacade metricsFacade = new MetricsFacade(new SimpleMeterRegistry()); + + private static final ElasticsearchResource elasticsearch = new ElasticsearchResource(); + + private static SchemaManager schemaManager; + + @BeforeClass + public static void beforeAll() throws Throwable { + elasticsearch.before(); + schemaManager = + new SchemaManager( + elasticsearch.client(), frontendIndexFactory, consumersIndexFactory, false); + } + + @AfterClass + public static void afterAll() { + elasticsearch.after(); + } + + @Override + protected LogRepository createRepository() { + schemaManager.ensureSchema(); + + return new FrontendElasticsearchLogRepository.Builder(elasticsearch.client(), metricsFacade) + .withIndexFactory(frontendIndexFactory) + .build(); + } + + @Override + protected void awaitUntilSuccessMessageIsPersisted( + String topic, + String id, + String remoteHostname, + String storageDatacenter, + String... extraRequestHeadersKeywords) + throws Exception { + awaitUntilMessageIsIndexed( + getQuery( + topic, + id, + PublishedMessageTraceStatus.SUCCESS, + remoteHostname, + extraRequestHeadersKeywords) + .must(matchQuery(STORAGE_DATACENTER, storageDatacenter))); + } + + @Override + protected void awaitUntilInflightMessageIsPersisted( + String topic, String id, String remoteHostname, String... extraRequestHeadersKeywords) + throws Exception { + awaitUntilMessageIsIndexed( + getQuery( + topic, + id, + PublishedMessageTraceStatus.INFLIGHT, + remoteHostname, + extraRequestHeadersKeywords)); + } + + @Override + protected void awaitUntilErrorMessageIsPersisted( + String topic, + String id, + String reason, + String remoteHostname, + String... extraRequestHeadersKeywords) + throws Exception { + awaitUntilMessageIsIndexed( + getQuery( + topic, + id, + PublishedMessageTraceStatus.ERROR, + remoteHostname, + extraRequestHeadersKeywords) + .must(matchQuery(REASON, reason))); + } + + private BoolQueryBuilder getQuery( + String topic, + String id, + PublishedMessageTraceStatus status, + String remoteHostname, + String... extraRequestHeadersKeywords) { + BoolQueryBuilder queryBuilder = + boolQuery() + .must(termQuery(TOPIC_NAME, topic)) + .must(termQuery(MESSAGE_ID, id)) + .must(termQuery(STATUS, status.toString())) + .must(termQuery(CLUSTER, CLUSTER_NAME)) + .must(termQuery(REMOTE_HOSTNAME, remoteHostname)); + for (String extraRequestHeadersKeyword : extraRequestHeadersKeywords) { + queryBuilder.must(termQuery(EXTRA_REQUEST_HEADERS, extraRequestHeadersKeyword)); } - - @AfterClass - public static void afterAll() { - elasticsearch.after(); - } - - @Override - protected LogRepository createRepository() { - schemaManager.ensureSchema(); - - return new FrontendElasticsearchLogRepository.Builder(elasticsearch.client(), metricsFacade) - .withIndexFactory(frontendIndexFactory) - .build(); - } - - @Override - protected void awaitUntilSuccessMessageIsPersisted(String topic, - String id, - String remoteHostname, - String storageDatacenter, - String... extraRequestHeadersKeywords) throws Exception { - awaitUntilMessageIsIndexed(getQuery(topic, id, PublishedMessageTraceStatus.SUCCESS, remoteHostname, extraRequestHeadersKeywords) - .must(matchQuery(STORAGE_DATACENTER, storageDatacenter)) - ); - } - - @Override - protected void awaitUntilInflightMessageIsPersisted(String topic, - String id, - String remoteHostname, - String... extraRequestHeadersKeywords) throws Exception { - awaitUntilMessageIsIndexed(getQuery(topic, id, PublishedMessageTraceStatus.INFLIGHT, remoteHostname, extraRequestHeadersKeywords)); - } - - @Override - protected void awaitUntilErrorMessageIsPersisted(String topic, - String id, - String reason, - String remoteHostname, - String... extraRequestHeadersKeywords) throws Exception { - awaitUntilMessageIsIndexed( - getQuery(topic, id, PublishedMessageTraceStatus.ERROR, remoteHostname, extraRequestHeadersKeywords) - .must(matchQuery(REASON, reason))); - } - - private BoolQueryBuilder getQuery(String topic, - String id, - PublishedMessageTraceStatus status, - String remoteHostname, - String... extraRequestHeadersKeywords) { - BoolQueryBuilder queryBuilder = boolQuery() - .must(termQuery(TOPIC_NAME, topic)) - .must(termQuery(MESSAGE_ID, id)) - .must(termQuery(STATUS, status.toString())) - .must(termQuery(CLUSTER, CLUSTER_NAME)) - .must(termQuery(REMOTE_HOSTNAME, remoteHostname)); - for (String extraRequestHeadersKeyword : extraRequestHeadersKeywords) { - queryBuilder.must(termQuery(EXTRA_REQUEST_HEADERS, extraRequestHeadersKeyword)); - } - return queryBuilder; - } - - private void awaitUntilMessageIsIndexed(QueryBuilder query) { - await().atMost(Duration.ofMinutes(1)).until(() -> { - SearchResponse response = elasticsearch.client().prepareSearch(frontendIndexFactory.createIndex()) - .setTypes(SchemaManager.PUBLISHED_TYPE) - .setQuery(query) - .execute().get(); - return response.getHits().getTotalHits() == 1; - }); - } - -} \ No newline at end of file + return queryBuilder; + } + + private void awaitUntilMessageIsIndexed(QueryBuilder query) { + await() + .atMost(Duration.ofMinutes(1)) + .until( + () -> { + SearchResponse response = + elasticsearch + .client() + .prepareSearch(frontendIndexFactory.createIndex()) + .setTypes(SchemaManager.PUBLISHED_TYPE) + .setQuery(query) + .execute() + .get(); + return response.getHits().getTotalHits() == 1; + }); + } +} diff --git a/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/management/ElasticsearchLogRepositoryTest.java b/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/management/ElasticsearchLogRepositoryTest.java index 0f36d0a768..b332b8703f 100644 --- a/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/management/ElasticsearchLogRepositoryTest.java +++ b/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/management/ElasticsearchLogRepositoryTest.java @@ -1,7 +1,20 @@ package pl.allegro.tech.hermes.tracker.elasticsearch.management; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static pl.allegro.tech.hermes.api.SentMessageTraceStatus.DISCARDED; +import static pl.allegro.tech.hermes.common.http.ExtraRequestHeadersCollector.extraRequestHeadersCollector; + import com.google.common.collect.ImmutableMap; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import java.time.Clock; +import java.time.Duration; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -26,155 +39,171 @@ import pl.allegro.tech.hermes.tracker.elasticsearch.frontend.FrontendIndexFactory; import pl.allegro.tech.hermes.tracker.management.LogRepository; -import java.time.Clock; -import java.time.Duration; -import java.time.LocalDate; -import java.time.ZoneId; -import java.time.ZoneOffset; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; -import static pl.allegro.tech.hermes.api.SentMessageTraceStatus.DISCARDED; -import static pl.allegro.tech.hermes.common.http.ExtraRequestHeadersCollector.extraRequestHeadersCollector; - public class ElasticsearchLogRepositoryTest implements LogSchemaAware { - private static final String CLUSTER_NAME = "primary"; - private static final String REASON_MESSAGE = "Bad Request"; - - private static final Clock clock = Clock.fixed(LocalDate.now().atStartOfDay().toInstant(ZoneOffset.UTC), ZoneId.systemDefault()); - private static final FrontendIndexFactory frontendIndexFactory = new FrontendDailyIndexFactory(clock); - private static final ConsumersIndexFactory consumersIndexFactory = new ConsumersDailyIndexFactory(clock); - private static final MetricsFacade metricsFacade = new MetricsFacade( - new SimpleMeterRegistry() - ); - - private static final ElasticsearchResource elasticsearch = new ElasticsearchResource(); - - private LogRepository logRepository; - private FrontendElasticsearchLogRepository frontendLogRepository; - private ConsumersElasticsearchLogRepository consumersLogRepository; - - @BeforeClass - public static void beforeAll() throws Throwable { - elasticsearch.before(); - } - - @AfterClass - public static void afterAll() { - elasticsearch.after(); - } - - @Before - public void setUp() { - SchemaManager schemaManager = new SchemaManager(elasticsearch.client(), frontendIndexFactory, consumersIndexFactory, false); - logRepository = new ElasticsearchLogRepository(elasticsearch.client(), schemaManager); - - frontendLogRepository = new FrontendElasticsearchLogRepository.Builder( - elasticsearch.client(), metricsFacade) - .withIndexFactory(frontendIndexFactory) - .build(); - - consumersLogRepository = new ConsumersElasticsearchLogRepository.Builder( - elasticsearch.client(), metricsFacade) - .withIndexFactory(consumersIndexFactory) - .build(); - } - - // TODO: figure out why this test sometimes *consistently* fails on CI - @Ignore - @Test - public void shouldGetLastUndelivered() throws Exception { - //given - String topic = "elasticsearch.lastUndelivered"; - String subscription = "subscription"; - MessageMetadata firstDiscarded = TestMessageMetadata.of("1234", topic, subscription); - long firstTimestamp = System.currentTimeMillis(); - - MessageMetadata secondDiscarded = TestMessageMetadata.of("5678", topic, subscription); - // difference between first and second timestamp must be bigger than 1000ms as query sorts by seconds - long secondTimestamp = firstTimestamp + 2000; - - // when - consumersLogRepository.logDiscarded(firstDiscarded, firstTimestamp, REASON_MESSAGE); - consumersLogRepository.logDiscarded(secondDiscarded, secondTimestamp, REASON_MESSAGE); - - // then - assertThat(fetchLastUndelivered(topic, subscription)) - .containsExactly(sentMessageTrace(secondDiscarded, secondTimestamp, DISCARDED)); - } - - @Test - public void shouldGetMessageStatus() { - //given - String datacenter = "dc1"; - MessageMetadata messageMetadata = TestMessageMetadata.of("1234", "elasticsearch.messageStatus", "subscription"); - long timestamp = System.currentTimeMillis(); - ImmutableMap extraRequestHeaders = ImmutableMap.of("x-header1", "value1", "x-header2", "value2"); - - frontendLogRepository.logPublished("1234", timestamp, "elasticsearch.messageStatus", "localhost", datacenter, extraRequestHeaders); - consumersLogRepository.logSuccessful(messageMetadata, "localhost", timestamp); - - //when - assertThat(fetchMessageStatus(messageMetadata)) - .contains(publishedMessageTrace(messageMetadata, extraRequestHeaders, timestamp, PublishedMessageTraceStatus.SUCCESS, datacenter)) - .contains(sentMessageTrace(messageMetadata, timestamp, SentMessageTraceStatus.SUCCESS)); - } - - private List fetchLastUndelivered(String topic, String subscription) { - final List lastUndelivered = new ArrayList<>(); - - await().atMost(Duration.ofMinutes(1)).until(() -> { - lastUndelivered.clear(); - lastUndelivered.addAll(logRepository.getLastUndeliveredMessages(topic, subscription, 1)); - return lastUndelivered.size() == 1; - }); - return lastUndelivered; - } - - private List fetchMessageStatus(MessageMetadata messageMetadata) { - List status = new ArrayList<>(); - - await().atMost(Duration.ofMinutes(1)).until(() -> { - status.clear(); - status.addAll(logRepository.getMessageStatus(messageMetadata.getTopic(), messageMetadata.getSubscription(), - messageMetadata.getMessageId())); - return status.size() == 2; - }); - - return status; - } - - private SentMessageTrace sentMessageTrace(MessageMetadata messageMetadata, long timestamp, SentMessageTraceStatus status) { - return SentMessageTrace.Builder.sentMessageTrace( - messageMetadata.getMessageId(), - messageMetadata.getBatchId(), - status - ) - .withTimestamp(timestamp) - .withSubscription(messageMetadata.getSubscription()) - .withTopicName(messageMetadata.getTopic()) - .withReason(REASON_MESSAGE) - .withPartition(messageMetadata.getPartition()) - .withOffset(messageMetadata.getOffset()) - .withCluster(CLUSTER_NAME) - .build(); - } - - private PublishedMessageTrace publishedMessageTrace(MessageMetadata messageMetadata, Map extraRequestHeaders, - long timestamp, PublishedMessageTraceStatus status, String storageDatacenter) { - return new PublishedMessageTrace(messageMetadata.getMessageId(), + private static final String CLUSTER_NAME = "primary"; + private static final String REASON_MESSAGE = "Bad Request"; + + private static final Clock clock = + Clock.fixed(LocalDate.now().atStartOfDay().toInstant(ZoneOffset.UTC), ZoneId.systemDefault()); + private static final FrontendIndexFactory frontendIndexFactory = + new FrontendDailyIndexFactory(clock); + private static final ConsumersIndexFactory consumersIndexFactory = + new ConsumersDailyIndexFactory(clock); + private static final MetricsFacade metricsFacade = new MetricsFacade(new SimpleMeterRegistry()); + + private static final ElasticsearchResource elasticsearch = new ElasticsearchResource(); + + private LogRepository logRepository; + private FrontendElasticsearchLogRepository frontendLogRepository; + private ConsumersElasticsearchLogRepository consumersLogRepository; + + @BeforeClass + public static void beforeAll() throws Throwable { + elasticsearch.before(); + } + + @AfterClass + public static void afterAll() { + elasticsearch.after(); + } + + @Before + public void setUp() { + SchemaManager schemaManager = + new SchemaManager( + elasticsearch.client(), frontendIndexFactory, consumersIndexFactory, false); + logRepository = new ElasticsearchLogRepository(elasticsearch.client(), schemaManager); + + frontendLogRepository = + new FrontendElasticsearchLogRepository.Builder(elasticsearch.client(), metricsFacade) + .withIndexFactory(frontendIndexFactory) + .build(); + + consumersLogRepository = + new ConsumersElasticsearchLogRepository.Builder(elasticsearch.client(), metricsFacade) + .withIndexFactory(consumersIndexFactory) + .build(); + } + + // TODO: figure out why this test sometimes *consistently* fails on CI + @Ignore + @Test + public void shouldGetLastUndelivered() throws Exception { + // given + String topic = "elasticsearch.lastUndelivered"; + String subscription = "subscription"; + MessageMetadata firstDiscarded = TestMessageMetadata.of("1234", topic, subscription); + long firstTimestamp = System.currentTimeMillis(); + + MessageMetadata secondDiscarded = TestMessageMetadata.of("5678", topic, subscription); + // difference between first and second timestamp must be bigger than 1000ms as query sorts by + // seconds + long secondTimestamp = firstTimestamp + 2000; + + // when + consumersLogRepository.logDiscarded(firstDiscarded, firstTimestamp, REASON_MESSAGE); + consumersLogRepository.logDiscarded(secondDiscarded, secondTimestamp, REASON_MESSAGE); + + // then + assertThat(fetchLastUndelivered(topic, subscription)) + .containsExactly(sentMessageTrace(secondDiscarded, secondTimestamp, DISCARDED)); + } + + @Test + public void shouldGetMessageStatus() { + // given + String datacenter = "dc1"; + MessageMetadata messageMetadata = + TestMessageMetadata.of("1234", "elasticsearch.messageStatus", "subscription"); + long timestamp = System.currentTimeMillis(); + ImmutableMap extraRequestHeaders = + ImmutableMap.of("x-header1", "value1", "x-header2", "value2"); + + frontendLogRepository.logPublished( + "1234", + timestamp, + "elasticsearch.messageStatus", + "localhost", + datacenter, + extraRequestHeaders); + consumersLogRepository.logSuccessful(messageMetadata, "localhost", timestamp); + + // when + assertThat(fetchMessageStatus(messageMetadata)) + .contains( + publishedMessageTrace( + messageMetadata, + extraRequestHeaders, timestamp, - messageMetadata.getTopic(), - status, - null, - null, - CLUSTER_NAME, - extraRequestHeaders.entrySet().stream() - .collect(extraRequestHeadersCollector()), - storageDatacenter); - } -} \ No newline at end of file + PublishedMessageTraceStatus.SUCCESS, + datacenter)) + .contains(sentMessageTrace(messageMetadata, timestamp, SentMessageTraceStatus.SUCCESS)); + } + + private List fetchLastUndelivered(String topic, String subscription) { + final List lastUndelivered = new ArrayList<>(); + + await() + .atMost(Duration.ofMinutes(1)) + .until( + () -> { + lastUndelivered.clear(); + lastUndelivered.addAll( + logRepository.getLastUndeliveredMessages(topic, subscription, 1)); + return lastUndelivered.size() == 1; + }); + return lastUndelivered; + } + + private List fetchMessageStatus(MessageMetadata messageMetadata) { + List status = new ArrayList<>(); + + await() + .atMost(Duration.ofMinutes(1)) + .until( + () -> { + status.clear(); + status.addAll( + logRepository.getMessageStatus( + messageMetadata.getTopic(), + messageMetadata.getSubscription(), + messageMetadata.getMessageId())); + return status.size() == 2; + }); + + return status; + } + + private SentMessageTrace sentMessageTrace( + MessageMetadata messageMetadata, long timestamp, SentMessageTraceStatus status) { + return SentMessageTrace.Builder.sentMessageTrace( + messageMetadata.getMessageId(), messageMetadata.getBatchId(), status) + .withTimestamp(timestamp) + .withSubscription(messageMetadata.getSubscription()) + .withTopicName(messageMetadata.getTopic()) + .withReason(REASON_MESSAGE) + .withPartition(messageMetadata.getPartition()) + .withOffset(messageMetadata.getOffset()) + .withCluster(CLUSTER_NAME) + .build(); + } + + private PublishedMessageTrace publishedMessageTrace( + MessageMetadata messageMetadata, + Map extraRequestHeaders, + long timestamp, + PublishedMessageTraceStatus status, + String storageDatacenter) { + return new PublishedMessageTrace( + messageMetadata.getMessageId(), + timestamp, + messageMetadata.getTopic(), + status, + null, + null, + CLUSTER_NAME, + extraRequestHeaders.entrySet().stream().collect(extraRequestHeadersCollector()), + storageDatacenter); + } +} diff --git a/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/management/MultiElasticsearchLogRepositoryTest.java b/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/management/MultiElasticsearchLogRepositoryTest.java index 917dbf971c..027152e937 100644 --- a/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/management/MultiElasticsearchLogRepositoryTest.java +++ b/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/management/MultiElasticsearchLogRepositoryTest.java @@ -1,7 +1,20 @@ package pl.allegro.tech.hermes.tracker.elasticsearch.management; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static pl.allegro.tech.hermes.common.http.ExtraRequestHeadersCollector.extraRequestHeadersCollector; + import com.google.common.collect.ImmutableMap; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import java.time.Clock; +import java.time.Duration; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -24,126 +37,135 @@ import pl.allegro.tech.hermes.tracker.elasticsearch.frontend.FrontendIndexFactory; import pl.allegro.tech.hermes.tracker.management.LogRepository; -import java.time.Clock; -import java.time.Duration; -import java.time.LocalDate; -import java.time.ZoneId; -import java.time.ZoneOffset; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; -import static pl.allegro.tech.hermes.common.http.ExtraRequestHeadersCollector.extraRequestHeadersCollector; - - public class MultiElasticsearchLogRepositoryTest implements LogSchemaAware { - private static final String CLUSTER_NAME = "primary"; - private static final String REASON_MESSAGE = "Bad Request"; - - private static final Clock clock = Clock.fixed(LocalDate.now().atStartOfDay().toInstant(ZoneOffset.UTC), ZoneId.systemDefault()); - private static final FrontendIndexFactory frontendIndexFactory = new FrontendDailyIndexFactory(clock); - private static final ConsumersIndexFactory consumersIndexFactory = new ConsumersDailyIndexFactory(clock); - private static final MetricsFacade metricsFacade = new MetricsFacade( - new SimpleMeterRegistry() - ); - - private static final ElasticsearchResource elasticsearch1 = new ElasticsearchResource(); - private static final ElasticsearchResource elasticsearch2 = new ElasticsearchResource(); - - private LogRepository logRepository; - private FrontendElasticsearchLogRepository frontendLogRepository; - private ConsumersElasticsearchLogRepository consumersLogRepository; - - @BeforeClass - public static void beforeAll() throws Throwable { - elasticsearch1.before(); - elasticsearch2.before(); - } - - @AfterClass - public static void afterAll() { - elasticsearch1.after(); - elasticsearch2.after(); - } - - @Before - public void setUp() { - logRepository = new MultiElasticsearchLogRepository(Arrays.asList(elasticsearch1.client(), elasticsearch2.client())); - - frontendLogRepository = new FrontendElasticsearchLogRepository.Builder( - elasticsearch1.client(), metricsFacade) - .withIndexFactory(frontendIndexFactory) - .build(); - - consumersLogRepository = new ConsumersElasticsearchLogRepository.Builder( - elasticsearch2.client(), metricsFacade) - .withIndexFactory(consumersIndexFactory) - .build(); - } - - @Test - public void shouldGetMessageStatus() { - // given - String datacenter = "dc1"; - Map extraRequestHeaders = ImmutableMap.of("x-header1", "value1", "x-header2", "value2"); - MessageMetadata messageMetadata = TestMessageMetadata.of("1234", "elasticsearch1.messageStatus", "subscription"); - long timestamp = System.currentTimeMillis(); - - // when - frontendLogRepository.logPublished("1234", timestamp, "elasticsearch1.messageStatus", "localhost", datacenter, extraRequestHeaders); - consumersLogRepository.logSuccessful(messageMetadata, "localhost", timestamp); - - // then - assertThat(fetchMessageStatus(messageMetadata)) - .contains(publishedMessageTrace(messageMetadata, extraRequestHeaders, timestamp, PublishedMessageTraceStatus.SUCCESS, datacenter)) - .contains(sentMessageTrace(messageMetadata, timestamp, SentMessageTraceStatus.SUCCESS)); - } - - private List fetchMessageStatus(MessageMetadata messageMetadata) { - List status = new ArrayList<>(); - - await().atMost(Duration.ofMinutes(1)).until(() -> { - status.clear(); - status.addAll(logRepository.getMessageStatus(messageMetadata.getTopic(), messageMetadata.getSubscription(), - messageMetadata.getMessageId())); - return status.size() == 2; - }); - - return status; - } - - private SentMessageTrace sentMessageTrace(MessageMetadata messageMetadata, long timestamp, SentMessageTraceStatus status) { - return SentMessageTrace.Builder.sentMessageTrace( - messageMetadata.getMessageId(), - messageMetadata.getBatchId(), - status - ) - .withTimestamp(timestamp) - .withSubscription(messageMetadata.getSubscription()) - .withTopicName(messageMetadata.getTopic()) - .withReason(REASON_MESSAGE) - .withPartition(messageMetadata.getPartition()) - .withOffset(messageMetadata.getOffset()) - .withCluster(CLUSTER_NAME) - .build(); - } - - private PublishedMessageTrace publishedMessageTrace(MessageMetadata messageMetadata, Map extraRequestHeaders, - long timestamp, PublishedMessageTraceStatus status, String storageDatacenter) { - return new PublishedMessageTrace(messageMetadata.getMessageId(), + private static final String CLUSTER_NAME = "primary"; + private static final String REASON_MESSAGE = "Bad Request"; + + private static final Clock clock = + Clock.fixed(LocalDate.now().atStartOfDay().toInstant(ZoneOffset.UTC), ZoneId.systemDefault()); + private static final FrontendIndexFactory frontendIndexFactory = + new FrontendDailyIndexFactory(clock); + private static final ConsumersIndexFactory consumersIndexFactory = + new ConsumersDailyIndexFactory(clock); + private static final MetricsFacade metricsFacade = new MetricsFacade(new SimpleMeterRegistry()); + + private static final ElasticsearchResource elasticsearch1 = new ElasticsearchResource(); + private static final ElasticsearchResource elasticsearch2 = new ElasticsearchResource(); + + private LogRepository logRepository; + private FrontendElasticsearchLogRepository frontendLogRepository; + private ConsumersElasticsearchLogRepository consumersLogRepository; + + @BeforeClass + public static void beforeAll() throws Throwable { + elasticsearch1.before(); + elasticsearch2.before(); + } + + @AfterClass + public static void afterAll() { + elasticsearch1.after(); + elasticsearch2.after(); + } + + @Before + public void setUp() { + logRepository = + new MultiElasticsearchLogRepository( + Arrays.asList(elasticsearch1.client(), elasticsearch2.client())); + + frontendLogRepository = + new FrontendElasticsearchLogRepository.Builder(elasticsearch1.client(), metricsFacade) + .withIndexFactory(frontendIndexFactory) + .build(); + + consumersLogRepository = + new ConsumersElasticsearchLogRepository.Builder(elasticsearch2.client(), metricsFacade) + .withIndexFactory(consumersIndexFactory) + .build(); + } + + @Test + public void shouldGetMessageStatus() { + // given + String datacenter = "dc1"; + Map extraRequestHeaders = + ImmutableMap.of("x-header1", "value1", "x-header2", "value2"); + MessageMetadata messageMetadata = + TestMessageMetadata.of("1234", "elasticsearch1.messageStatus", "subscription"); + long timestamp = System.currentTimeMillis(); + + // when + frontendLogRepository.logPublished( + "1234", + timestamp, + "elasticsearch1.messageStatus", + "localhost", + datacenter, + extraRequestHeaders); + consumersLogRepository.logSuccessful(messageMetadata, "localhost", timestamp); + + // then + assertThat(fetchMessageStatus(messageMetadata)) + .contains( + publishedMessageTrace( + messageMetadata, + extraRequestHeaders, timestamp, - messageMetadata.getTopic(), - status, - null, - null, - CLUSTER_NAME, - extraRequestHeaders.entrySet().stream() - .collect(extraRequestHeadersCollector()), - storageDatacenter - ); - } + PublishedMessageTraceStatus.SUCCESS, + datacenter)) + .contains(sentMessageTrace(messageMetadata, timestamp, SentMessageTraceStatus.SUCCESS)); + } + + private List fetchMessageStatus(MessageMetadata messageMetadata) { + List status = new ArrayList<>(); + + await() + .atMost(Duration.ofMinutes(1)) + .until( + () -> { + status.clear(); + status.addAll( + logRepository.getMessageStatus( + messageMetadata.getTopic(), + messageMetadata.getSubscription(), + messageMetadata.getMessageId())); + return status.size() == 2; + }); + + return status; + } + + private SentMessageTrace sentMessageTrace( + MessageMetadata messageMetadata, long timestamp, SentMessageTraceStatus status) { + return SentMessageTrace.Builder.sentMessageTrace( + messageMetadata.getMessageId(), messageMetadata.getBatchId(), status) + .withTimestamp(timestamp) + .withSubscription(messageMetadata.getSubscription()) + .withTopicName(messageMetadata.getTopic()) + .withReason(REASON_MESSAGE) + .withPartition(messageMetadata.getPartition()) + .withOffset(messageMetadata.getOffset()) + .withCluster(CLUSTER_NAME) + .build(); + } + + private PublishedMessageTrace publishedMessageTrace( + MessageMetadata messageMetadata, + Map extraRequestHeaders, + long timestamp, + PublishedMessageTraceStatus status, + String storageDatacenter) { + return new PublishedMessageTrace( + messageMetadata.getMessageId(), + timestamp, + messageMetadata.getTopic(), + status, + null, + null, + CLUSTER_NAME, + extraRequestHeaders.entrySet().stream().collect(extraRequestHeadersCollector()), + storageDatacenter); + } } diff --git a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/BatchingLogRepository.java b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/BatchingLogRepository.java index 6cfbd51797..b57a5f339a 100644 --- a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/BatchingLogRepository.java +++ b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/BatchingLogRepository.java @@ -5,15 +5,13 @@ public class BatchingLogRepository { - protected final String clusterName; - protected final String hostname; - protected BlockingQueue queue; + protected final String clusterName; + protected final String hostname; + protected BlockingQueue queue; - public BatchingLogRepository(int queueSize, - String clusterName, - String hostname) { - this.queue = new LinkedBlockingQueue<>(queueSize); - this.clusterName = clusterName; - this.hostname = hostname; - } -} \ No newline at end of file + public BatchingLogRepository(int queueSize, String clusterName, String hostname) { + this.queue = new LinkedBlockingQueue<>(queueSize); + this.clusterName = clusterName; + this.hostname = hostname; + } +} diff --git a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/QueueCommitter.java b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/QueueCommitter.java index 51056ba5e4..fb2f98f21c 100644 --- a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/QueueCommitter.java +++ b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/QueueCommitter.java @@ -1,45 +1,44 @@ package pl.allegro.tech.hermes.tracker; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import pl.allegro.tech.hermes.metrics.HermesTimer; -import pl.allegro.tech.hermes.metrics.HermesTimerContext; - import java.util.ArrayList; import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutionException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import pl.allegro.tech.hermes.metrics.HermesTimer; +import pl.allegro.tech.hermes.metrics.HermesTimerContext; public abstract class QueueCommitter implements Runnable { - private static final Logger LOGGER = LoggerFactory.getLogger(QueueCommitter.class); - - private final BlockingQueue queue; - private final HermesTimer timer; - - public QueueCommitter(BlockingQueue queue, HermesTimer timer) { - this.queue = queue; - this.timer = timer; - } - - @Override - public void run() { - try { - if (!queue.isEmpty()) { - HermesTimerContext ctx = timer.time(); - commit(); - ctx.close(); - } - } catch (Exception ex) { - LOGGER.error("Could not commit batch.", ex); - } - } - - private void commit() throws Exception { - List batch = new ArrayList<>(); - queue.drainTo(batch); - processBatch(batch); + private static final Logger LOGGER = LoggerFactory.getLogger(QueueCommitter.class); + + private final BlockingQueue queue; + private final HermesTimer timer; + + public QueueCommitter(BlockingQueue queue, HermesTimer timer) { + this.queue = queue; + this.timer = timer; + } + + @Override + public void run() { + try { + if (!queue.isEmpty()) { + HermesTimerContext ctx = timer.time(); + commit(); + ctx.close(); + } + } catch (Exception ex) { + LOGGER.error("Could not commit batch.", ex); } + } - protected abstract void processBatch(List batch) throws ExecutionException, InterruptedException; + private void commit() throws Exception { + List batch = new ArrayList<>(); + queue.drainTo(batch); + processBatch(batch); + } + protected abstract void processBatch(List batch) + throws ExecutionException, InterruptedException; } diff --git a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/DiscardedSendingTracker.java b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/DiscardedSendingTracker.java index 3c2145185f..ef97d13e21 100644 --- a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/DiscardedSendingTracker.java +++ b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/DiscardedSendingTracker.java @@ -5,37 +5,28 @@ public class DiscardedSendingTracker implements SendingTracker { + private final List repositories; + private final Clock clock; - private final List repositories; - private final Clock clock; + DiscardedSendingTracker(List repositories, Clock clock) { + this.repositories = repositories; + this.clock = clock; + } - DiscardedSendingTracker(List repositories, Clock clock) { - this.repositories = repositories; - this.clock = clock; - } + @Override + public void logSent(MessageMetadata message, String hostname) {} - @Override - public void logSent(MessageMetadata message, String hostname) { + @Override + public void logFailed(MessageMetadata message, String reason, String hostname) {} - } + @Override + public void logDiscarded(MessageMetadata message, String reason) { + repositories.forEach(r -> r.logDiscarded(message, clock.millis(), reason)); + } - @Override - public void logFailed(MessageMetadata message, String reason, String hostname) { - } + @Override + public void logInflight(MessageMetadata message) {} - @Override - public void logDiscarded(MessageMetadata message, String reason) { - repositories.forEach(r -> - r.logDiscarded(message, clock.millis(), reason)); - } - - @Override - public void logInflight(MessageMetadata message) { - - } - - @Override - public void logFiltered(MessageMetadata messageMetadata, String reason) { - - } + @Override + public void logFiltered(MessageMetadata messageMetadata, String reason) {} } diff --git a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/LogRepository.java b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/LogRepository.java index 3a778029b7..227ad0fd04 100644 --- a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/LogRepository.java +++ b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/LogRepository.java @@ -2,15 +2,15 @@ public interface LogRepository { - void logSuccessful(MessageMetadata message, String hostname, long timestamp); + void logSuccessful(MessageMetadata message, String hostname, long timestamp); - void logFailed(MessageMetadata message, String hostname, long timestamp, String reason); + void logFailed(MessageMetadata message, String hostname, long timestamp, String reason); - void logDiscarded(MessageMetadata message, long timestamp, String reason); + void logDiscarded(MessageMetadata message, long timestamp, String reason); - void logInflight(MessageMetadata message, long timestamp); + void logInflight(MessageMetadata message, long timestamp); - void logFiltered(MessageMetadata message, long timestamp, String reason); + void logFiltered(MessageMetadata message, long timestamp, String reason); - void close(); + void close(); } diff --git a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/MessageMetadata.java b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/MessageMetadata.java index 0e5bed04ab..21775d1ad1 100644 --- a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/MessageMetadata.java +++ b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/MessageMetadata.java @@ -2,89 +2,100 @@ public class MessageMetadata { - private final String messageId; - private final String batchId; - private final long offset; - private final int partition; - private final long partitionAssignmentTerm; - private final String topic; - private final String kafkaTopic; - private final String subscription; - private final long publishingTimestamp; - private final long readingTimestamp; + private final String messageId; + private final String batchId; + private final long offset; + private final int partition; + private final long partitionAssignmentTerm; + private final String topic; + private final String kafkaTopic; + private final String subscription; + private final long publishingTimestamp; + private final long readingTimestamp; - public MessageMetadata(String messageId, - long offset, - int partition, - long partitionAssignmentTerm, - String topic, - String subscription, - String kafkaTopic, - long publishingTimestamp, - long readingTimestamp) { - this(messageId, "", offset, partition, partitionAssignmentTerm, topic, subscription, kafkaTopic, publishingTimestamp, - readingTimestamp); - } + public MessageMetadata( + String messageId, + long offset, + int partition, + long partitionAssignmentTerm, + String topic, + String subscription, + String kafkaTopic, + long publishingTimestamp, + long readingTimestamp) { + this( + messageId, + "", + offset, + partition, + partitionAssignmentTerm, + topic, + subscription, + kafkaTopic, + publishingTimestamp, + readingTimestamp); + } - public MessageMetadata(String messageId, - String batchId, - long offset, - int partition, - long partitionAssignmentTerm, - String topic, - String subscription, - String kafkaTopic, - long publishingTimestamp, - long readingTimestamp) { - this.messageId = messageId; - this.batchId = batchId; - this.offset = offset; - this.partition = partition; - this.partitionAssignmentTerm = partitionAssignmentTerm; - this.topic = topic; - this.subscription = subscription; - this.kafkaTopic = kafkaTopic; - this.publishingTimestamp = publishingTimestamp; - this.readingTimestamp = readingTimestamp; - } + public MessageMetadata( + String messageId, + String batchId, + long offset, + int partition, + long partitionAssignmentTerm, + String topic, + String subscription, + String kafkaTopic, + long publishingTimestamp, + long readingTimestamp) { + this.messageId = messageId; + this.batchId = batchId; + this.offset = offset; + this.partition = partition; + this.partitionAssignmentTerm = partitionAssignmentTerm; + this.topic = topic; + this.subscription = subscription; + this.kafkaTopic = kafkaTopic; + this.publishingTimestamp = publishingTimestamp; + this.readingTimestamp = readingTimestamp; + } - public String getMessageId() { - return messageId; - } + public String getMessageId() { + return messageId; + } - public long getOffset() { - return offset; - } + public long getOffset() { + return offset; + } - public int getPartition() { - return partition; - } + public int getPartition() { + return partition; + } - public long getPartitionAssignmentTerm() { - return partitionAssignmentTerm; - } + public long getPartitionAssignmentTerm() { + return partitionAssignmentTerm; + } - public String getTopic() { - return topic; - } + public String getTopic() { + return topic; + } - public String getSubscription() { - return subscription; - } + public String getSubscription() { + return subscription; + } - public long getPublishingTimestamp() { - return publishingTimestamp; - } + public long getPublishingTimestamp() { + return publishingTimestamp; + } - public long getReadingTimestamp() { - return readingTimestamp; - } + public long getReadingTimestamp() { + return readingTimestamp; + } - public String getBatchId() { - return batchId; - } + public String getBatchId() { + return batchId; + } - public String getKafkaTopic() { - return kafkaTopic; - } + public String getKafkaTopic() { + return kafkaTopic; + } } diff --git a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/NoOperationSendingTracker.java b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/NoOperationSendingTracker.java index 9c0a81a756..71e22bb5a2 100644 --- a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/NoOperationSendingTracker.java +++ b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/NoOperationSendingTracker.java @@ -2,28 +2,18 @@ public class NoOperationSendingTracker implements SendingTracker { - @Override - public void logSent(MessageMetadata message, String hostname) { + @Override + public void logSent(MessageMetadata message, String hostname) {} - } + @Override + public void logFailed(MessageMetadata message, String reason, String hostname) {} - @Override - public void logFailed(MessageMetadata message, String reason, String hostname) { + @Override + public void logDiscarded(MessageMetadata message, String reason) {} - } + @Override + public void logInflight(MessageMetadata message) {} - @Override - public void logDiscarded(MessageMetadata message, String reason) { - - } - - @Override - public void logInflight(MessageMetadata message) { - - } - - @Override - public void logFiltered(MessageMetadata messageMetadata, String reason) { - - } + @Override + public void logFiltered(MessageMetadata messageMetadata, String reason) {} } diff --git a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/SendingMessageTracker.java b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/SendingMessageTracker.java index 0b576a2595..8e5fd76a7a 100644 --- a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/SendingMessageTracker.java +++ b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/SendingMessageTracker.java @@ -4,37 +4,36 @@ import java.util.List; class SendingMessageTracker implements SendingTracker { - private final List repositories; - private final Clock clock; - - SendingMessageTracker(List repositories, Clock clock) { - this.repositories = repositories; - this.clock = clock; - } - - @Override - public void logSent(MessageMetadata message, String hostname) { - repositories.forEach(r -> r.logSuccessful(message, hostname, clock.millis())); - } - - @Override - public void logFailed(MessageMetadata message, String reason, String hostname) { - repositories.forEach(r -> r.logFailed(message, hostname, clock.millis(), reason)); - } - - @Override - public void logDiscarded(MessageMetadata message, String reason) { - repositories.forEach(r -> - r.logDiscarded(message, clock.millis(), reason)); - } - - @Override - public void logInflight(MessageMetadata message) { - repositories.forEach(r -> r.logInflight(message, clock.millis())); - } - - @Override - public void logFiltered(MessageMetadata message, String reason) { - repositories.forEach(r -> r.logFiltered(message, clock.millis(), reason)); - } + private final List repositories; + private final Clock clock; + + SendingMessageTracker(List repositories, Clock clock) { + this.repositories = repositories; + this.clock = clock; + } + + @Override + public void logSent(MessageMetadata message, String hostname) { + repositories.forEach(r -> r.logSuccessful(message, hostname, clock.millis())); + } + + @Override + public void logFailed(MessageMetadata message, String reason, String hostname) { + repositories.forEach(r -> r.logFailed(message, hostname, clock.millis(), reason)); + } + + @Override + public void logDiscarded(MessageMetadata message, String reason) { + repositories.forEach(r -> r.logDiscarded(message, clock.millis(), reason)); + } + + @Override + public void logInflight(MessageMetadata message) { + repositories.forEach(r -> r.logInflight(message, clock.millis())); + } + + @Override + public void logFiltered(MessageMetadata message, String reason) { + repositories.forEach(r -> r.logFiltered(message, clock.millis(), reason)); + } } diff --git a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/SendingTracker.java b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/SendingTracker.java index a7f85c9dfa..0b3cb2e861 100644 --- a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/SendingTracker.java +++ b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/SendingTracker.java @@ -1,13 +1,13 @@ package pl.allegro.tech.hermes.tracker.consumers; public interface SendingTracker { - void logSent(MessageMetadata message, String hostname); + void logSent(MessageMetadata message, String hostname); - void logFailed(MessageMetadata message, String reason, String hostname); + void logFailed(MessageMetadata message, String reason, String hostname); - void logDiscarded(MessageMetadata message, String reason); + void logDiscarded(MessageMetadata message, String reason); - void logInflight(MessageMetadata message); + void logInflight(MessageMetadata message); - void logFiltered(MessageMetadata messageMetadata, String reason); + void logFiltered(MessageMetadata messageMetadata, String reason); } diff --git a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/Trackers.java b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/Trackers.java index 127cc6e4d9..dd816ce876 100644 --- a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/Trackers.java +++ b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/consumers/Trackers.java @@ -1,37 +1,35 @@ package pl.allegro.tech.hermes.tracker.consumers; -import pl.allegro.tech.hermes.api.Subscription; - import java.time.Clock; import java.util.List; +import pl.allegro.tech.hermes.api.Subscription; public class Trackers { - private final List repositories; - private final SendingMessageTracker sendingMessageTracker; - private final DiscardedSendingTracker discardedSendingTracker; - private final NoOperationSendingTracker noOperationDeliveryTracker; - - public Trackers(List repositories) { - this.repositories = repositories; - this.sendingMessageTracker = new SendingMessageTracker(repositories, Clock.systemUTC()); - this.discardedSendingTracker = new DiscardedSendingTracker(repositories, Clock.systemUTC()); - this.noOperationDeliveryTracker = new NoOperationSendingTracker(); + private final List repositories; + private final SendingMessageTracker sendingMessageTracker; + private final DiscardedSendingTracker discardedSendingTracker; + private final NoOperationSendingTracker noOperationDeliveryTracker; + + public Trackers(List repositories) { + this.repositories = repositories; + this.sendingMessageTracker = new SendingMessageTracker(repositories, Clock.systemUTC()); + this.discardedSendingTracker = new DiscardedSendingTracker(repositories, Clock.systemUTC()); + this.noOperationDeliveryTracker = new NoOperationSendingTracker(); + } + + public SendingTracker get(Subscription subscription) { + switch (subscription.getTrackingMode()) { + case TRACK_ALL: + return sendingMessageTracker; + case TRACK_DISCARDED_ONLY: + return discardedSendingTracker; + default: + return noOperationDeliveryTracker; } + } - public SendingTracker get(Subscription subscription) { - switch (subscription.getTrackingMode()) { - case TRACK_ALL: - return sendingMessageTracker; - case TRACK_DISCARDED_ONLY: - return discardedSendingTracker; - default: - return noOperationDeliveryTracker; - - } - } - - public void close() { - repositories.forEach(LogRepository::close); - } + public void close() { + repositories.forEach(LogRepository::close); + } } diff --git a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/frontend/LogRepository.java b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/frontend/LogRepository.java index c90756cfc1..604601856d 100644 --- a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/frontend/LogRepository.java +++ b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/frontend/LogRepository.java @@ -4,16 +4,28 @@ public interface LogRepository { - void logPublished(String messageId, long timestamp, String topicName, String hostname, String storageDatacenter, Map extraRequestHeaders); + void logPublished( + String messageId, + long timestamp, + String topicName, + String hostname, + String storageDatacenter, + Map extraRequestHeaders); - void logError(String messageId, - long timestamp, - String topicName, - String reason, - String hostname, - Map extraRequestHeaders); + void logError( + String messageId, + long timestamp, + String topicName, + String reason, + String hostname, + Map extraRequestHeaders); - void logInflight(String messageId, long timestamp, String topicName, String hostname, Map extraRequestHeaders); + void logInflight( + String messageId, + long timestamp, + String topicName, + String hostname, + Map extraRequestHeaders); - void close(); + void close(); } diff --git a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/frontend/NoOperationPublishingTracker.java b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/frontend/NoOperationPublishingTracker.java index 9f3409edc0..477d60efc9 100644 --- a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/frontend/NoOperationPublishingTracker.java +++ b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/frontend/NoOperationPublishingTracker.java @@ -1,20 +1,30 @@ package pl.allegro.tech.hermes.tracker.frontend; -import pl.allegro.tech.hermes.api.TopicName; - import java.util.Map; +import pl.allegro.tech.hermes.api.TopicName; public class NoOperationPublishingTracker implements PublishingTracker { - @Override - public void logInflight(String messageId, TopicName topicName, String hostname, Map extraRequestHeaders) { - } + @Override + public void logInflight( + String messageId, + TopicName topicName, + String hostname, + Map extraRequestHeaders) {} - @Override - public void logPublished(String messageId, TopicName topicName, String hostname, String storageDatacenter, Map extraRequestHeaders) { - } + @Override + public void logPublished( + String messageId, + TopicName topicName, + String hostname, + String storageDatacenter, + Map extraRequestHeaders) {} - @Override - public void logError(String messageId, TopicName topicName, String reason, String hostname, Map extraRequestHeaders) { - } + @Override + public void logError( + String messageId, + TopicName topicName, + String reason, + String hostname, + Map extraRequestHeaders) {} } diff --git a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/frontend/PublishingMessageTracker.java b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/frontend/PublishingMessageTracker.java index f3dd4f6af1..ca8f4652a0 100644 --- a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/frontend/PublishingMessageTracker.java +++ b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/frontend/PublishingMessageTracker.java @@ -1,41 +1,77 @@ package pl.allegro.tech.hermes.tracker.frontend; -import pl.allegro.tech.hermes.api.TopicName; - import java.time.Clock; import java.util.List; import java.util.Map; +import pl.allegro.tech.hermes.api.TopicName; public class PublishingMessageTracker implements PublishingTracker { - private final List repositories; - private final Clock clock; + private final List repositories; + private final Clock clock; - public PublishingMessageTracker(List repositories, Clock clock) { - this.repositories = repositories; - this.clock = clock; - } + public PublishingMessageTracker(List repositories, Clock clock) { + this.repositories = repositories; + this.clock = clock; + } - @Override - public void logInflight(String messageId, TopicName topicName, String hostname, Map extraRequestHeaders) { - repositories.forEach(r -> r.logInflight(messageId, clock.millis(), topicName.qualifiedName(), hostname, extraRequestHeaders)); - } + @Override + public void logInflight( + String messageId, + TopicName topicName, + String hostname, + Map extraRequestHeaders) { + repositories.forEach( + r -> + r.logInflight( + messageId, + clock.millis(), + topicName.qualifiedName(), + hostname, + extraRequestHeaders)); + } - @Override - public void logPublished(String messageId, TopicName topicName, String hostname, String storageDatacenter, Map extraRequestHeaders) { - repositories.forEach(r -> r.logPublished(messageId, clock.millis(), topicName.qualifiedName(), hostname, storageDatacenter, extraRequestHeaders)); - } + @Override + public void logPublished( + String messageId, + TopicName topicName, + String hostname, + String storageDatacenter, + Map extraRequestHeaders) { + repositories.forEach( + r -> + r.logPublished( + messageId, + clock.millis(), + topicName.qualifiedName(), + hostname, + storageDatacenter, + extraRequestHeaders)); + } - @Override - public void logError(String messageId, TopicName topicName, String reason, String hostname, Map extraRequestHeaders) { - repositories.forEach(r -> r.logError(messageId, clock.millis(), topicName.qualifiedName(), reason, hostname, extraRequestHeaders)); - } + @Override + public void logError( + String messageId, + TopicName topicName, + String reason, + String hostname, + Map extraRequestHeaders) { + repositories.forEach( + r -> + r.logError( + messageId, + clock.millis(), + topicName.qualifiedName(), + reason, + hostname, + extraRequestHeaders)); + } - public void add(LogRepository logRepository) { - repositories.add(logRepository); - } + public void add(LogRepository logRepository) { + repositories.add(logRepository); + } - public void close() { - repositories.forEach(LogRepository::close); - } + public void close() { + repositories.forEach(LogRepository::close); + } } diff --git a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/frontend/PublishingTracker.java b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/frontend/PublishingTracker.java index ecd2fa7934..3563dbd9dd 100644 --- a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/frontend/PublishingTracker.java +++ b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/frontend/PublishingTracker.java @@ -1,13 +1,26 @@ package pl.allegro.tech.hermes.tracker.frontend; -import pl.allegro.tech.hermes.api.TopicName; - import java.util.Map; +import pl.allegro.tech.hermes.api.TopicName; public interface PublishingTracker { - void logInflight(String messageId, TopicName topicName, String hostname, Map extraRequestHeaders); + void logInflight( + String messageId, + TopicName topicName, + String hostname, + Map extraRequestHeaders); - void logPublished(String messageId, TopicName topicName, String hostname, String storageDatacenter, Map extraRequestHeaders); + void logPublished( + String messageId, + TopicName topicName, + String hostname, + String storageDatacenter, + Map extraRequestHeaders); - void logError(String messageId, TopicName topicName, String reason, String hostname, Map extraRequestHeaders); + void logError( + String messageId, + TopicName topicName, + String reason, + String hostname, + Map extraRequestHeaders); } diff --git a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/frontend/Trackers.java b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/frontend/Trackers.java index 5b5ac34d09..31d2851f1d 100644 --- a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/frontend/Trackers.java +++ b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/frontend/Trackers.java @@ -1,33 +1,36 @@ package pl.allegro.tech.hermes.tracker.frontend; -import pl.allegro.tech.hermes.api.Topic; - import java.time.Clock; import java.util.List; +import pl.allegro.tech.hermes.api.Topic; public class Trackers { - private final PublishingMessageTracker publishingMessageTracker; - private final NoOperationPublishingTracker noOperationPublishingTracker; - - public Trackers(List logRepositories) { - this(new PublishingMessageTracker(logRepositories, Clock.systemUTC()), new NoOperationPublishingTracker()); - } - - Trackers(PublishingMessageTracker publishingMessageTracker, NoOperationPublishingTracker noOperationPublishingTracker) { - this.publishingMessageTracker = publishingMessageTracker; - this.noOperationPublishingTracker = noOperationPublishingTracker; - } - - public PublishingTracker get(Topic topic) { - return topic.isTrackingEnabled() ? publishingMessageTracker : noOperationPublishingTracker; - } - - public void add(LogRepository logRepository) { - publishingMessageTracker.add(logRepository); - } - - public void close() { - publishingMessageTracker.close(); - } + private final PublishingMessageTracker publishingMessageTracker; + private final NoOperationPublishingTracker noOperationPublishingTracker; + + public Trackers(List logRepositories) { + this( + new PublishingMessageTracker(logRepositories, Clock.systemUTC()), + new NoOperationPublishingTracker()); + } + + Trackers( + PublishingMessageTracker publishingMessageTracker, + NoOperationPublishingTracker noOperationPublishingTracker) { + this.publishingMessageTracker = publishingMessageTracker; + this.noOperationPublishingTracker = noOperationPublishingTracker; + } + + public PublishingTracker get(Topic topic) { + return topic.isTrackingEnabled() ? publishingMessageTracker : noOperationPublishingTracker; + } + + public void add(LogRepository logRepository) { + publishingMessageTracker.add(logRepository); + } + + public void close() { + publishingMessageTracker.close(); + } } diff --git a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/management/LogRepository.java b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/management/LogRepository.java index f3166ab2f8..1c4079ecda 100644 --- a/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/management/LogRepository.java +++ b/hermes-tracker/src/main/java/pl/allegro/tech/hermes/tracker/management/LogRepository.java @@ -1,13 +1,14 @@ package pl.allegro.tech.hermes.tracker.management; +import java.util.List; import pl.allegro.tech.hermes.api.MessageTrace; import pl.allegro.tech.hermes.api.SentMessageTrace; -import java.util.List; - public interface LogRepository { - List getLastUndeliveredMessages(String topicName, String subscriptionName, int limit); + List getLastUndeliveredMessages( + String topicName, String subscriptionName, int limit); - List getMessageStatus(String qualifiedTopicName, String subscriptionName, String messageId); + List getMessageStatus( + String qualifiedTopicName, String subscriptionName, String messageId); } diff --git a/hermes-tracker/src/test/java/pl/allegro/tech/hermes/tracker/consumers/AbstractLogRepositoryTest.java b/hermes-tracker/src/test/java/pl/allegro/tech/hermes/tracker/consumers/AbstractLogRepositoryTest.java index 9144196063..c266a951c1 100644 --- a/hermes-tracker/src/test/java/pl/allegro/tech/hermes/tracker/consumers/AbstractLogRepositoryTest.java +++ b/hermes-tracker/src/test/java/pl/allegro/tech/hermes/tracker/consumers/AbstractLogRepositoryTest.java @@ -1,93 +1,88 @@ package pl.allegro.tech.hermes.tracker.consumers; -import org.junit.Before; -import org.junit.Test; -import pl.allegro.tech.hermes.api.SentMessageTraceStatus; - import static pl.allegro.tech.hermes.api.SentMessageTraceStatus.DISCARDED; import static pl.allegro.tech.hermes.api.SentMessageTraceStatus.INFLIGHT; import static pl.allegro.tech.hermes.api.SentMessageTraceStatus.SUCCESS; +import org.junit.Before; +import org.junit.Test; +import pl.allegro.tech.hermes.api.SentMessageTraceStatus; + public abstract class AbstractLogRepositoryTest { - private static final String SUBSCRIPTION = "subscription"; - - private LogRepository logRepository; - - @Before - public void setUp() { - logRepository = createLogRepository(); - } - - protected abstract LogRepository createLogRepository(); - - @Test - public void shouldLogSentMessage() throws Exception { - // given - String id = "sentMessage"; - String topic = "group.sentMessage"; - - // when - logRepository.logSuccessful(TestMessageMetadata.of(id, topic, SUBSCRIPTION), "host", 1234L); - - // then - awaitUntilMessageIsPersisted(topic, SUBSCRIPTION, id, SUCCESS); - } - - @Test - public void shouldLogInflightMessage() throws Exception { - // given - String id = "inflightMessage"; - String topic = "group.inflightMessage"; - - // when - logRepository.logInflight(TestMessageMetadata.of(id, topic, SUBSCRIPTION), 1234L); - - // then - awaitUntilMessageIsPersisted(topic, SUBSCRIPTION, id, INFLIGHT); - } - - @Test - public void shouldLogUndeliveredMessage() throws Exception { - // given - String id = "undeliveredMessage"; - String topic = "group.undeliveredMessage"; - - // when - logRepository.logDiscarded(TestMessageMetadata.of(id, topic, SUBSCRIPTION), 1234L, "reason"); - - // then - awaitUntilMessageIsPersisted(topic, SUBSCRIPTION, id, DISCARDED); - } - - @Test - public void shouldLogBatchIdInSentMessage() throws Exception { - // given - String messageId = "messageId"; - String batchId = "batchId"; - String topic = "group.sentBatchMessage"; - - // when - logRepository.logSuccessful(TestMessageMetadata.of(messageId, batchId, topic, SUBSCRIPTION), "host", 1234L); - - // then - awaitUntilBatchMessageIsPersisted(topic, SUBSCRIPTION, messageId, batchId, SUCCESS); - } - - protected abstract void awaitUntilMessageIsPersisted( - String topic, - String subscription, - String id, - SentMessageTraceStatus status - ) - throws Exception; - - protected abstract void awaitUntilBatchMessageIsPersisted( - String topic, - String subscription, - String messageId, - String batchId, - SentMessageTraceStatus status - ) - throws Exception; + private static final String SUBSCRIPTION = "subscription"; + + private LogRepository logRepository; + + @Before + public void setUp() { + logRepository = createLogRepository(); + } + + protected abstract LogRepository createLogRepository(); + + @Test + public void shouldLogSentMessage() throws Exception { + // given + String id = "sentMessage"; + String topic = "group.sentMessage"; + + // when + logRepository.logSuccessful(TestMessageMetadata.of(id, topic, SUBSCRIPTION), "host", 1234L); + + // then + awaitUntilMessageIsPersisted(topic, SUBSCRIPTION, id, SUCCESS); + } + + @Test + public void shouldLogInflightMessage() throws Exception { + // given + String id = "inflightMessage"; + String topic = "group.inflightMessage"; + + // when + logRepository.logInflight(TestMessageMetadata.of(id, topic, SUBSCRIPTION), 1234L); + + // then + awaitUntilMessageIsPersisted(topic, SUBSCRIPTION, id, INFLIGHT); + } + + @Test + public void shouldLogUndeliveredMessage() throws Exception { + // given + String id = "undeliveredMessage"; + String topic = "group.undeliveredMessage"; + + // when + logRepository.logDiscarded(TestMessageMetadata.of(id, topic, SUBSCRIPTION), 1234L, "reason"); + + // then + awaitUntilMessageIsPersisted(topic, SUBSCRIPTION, id, DISCARDED); + } + + @Test + public void shouldLogBatchIdInSentMessage() throws Exception { + // given + String messageId = "messageId"; + String batchId = "batchId"; + String topic = "group.sentBatchMessage"; + + // when + logRepository.logSuccessful( + TestMessageMetadata.of(messageId, batchId, topic, SUBSCRIPTION), "host", 1234L); + + // then + awaitUntilBatchMessageIsPersisted(topic, SUBSCRIPTION, messageId, batchId, SUCCESS); + } + + protected abstract void awaitUntilMessageIsPersisted( + String topic, String subscription, String id, SentMessageTraceStatus status) throws Exception; + + protected abstract void awaitUntilBatchMessageIsPersisted( + String topic, + String subscription, + String messageId, + String batchId, + SentMessageTraceStatus status) + throws Exception; } diff --git a/hermes-tracker/src/test/java/pl/allegro/tech/hermes/tracker/consumers/TestMessageMetadata.java b/hermes-tracker/src/test/java/pl/allegro/tech/hermes/tracker/consumers/TestMessageMetadata.java index e4913f3096..fafc491e78 100644 --- a/hermes-tracker/src/test/java/pl/allegro/tech/hermes/tracker/consumers/TestMessageMetadata.java +++ b/hermes-tracker/src/test/java/pl/allegro/tech/hermes/tracker/consumers/TestMessageMetadata.java @@ -2,19 +2,29 @@ public class TestMessageMetadata { - public static MessageMetadata of(String messageId, String topic, String subscription) { - return of(messageId, topic, subscription, 1L, 1); - } + public static MessageMetadata of(String messageId, String topic, String subscription) { + return of(messageId, topic, subscription, 1L, 1); + } - public static MessageMetadata of(String messageId, String topic, String subscription, long offset, int partition) { - return new MessageMetadata(messageId, offset, partition, 123L, topic, subscription, topic, 123456L, 123456L); - } + public static MessageMetadata of( + String messageId, String topic, String subscription, long offset, int partition) { + return new MessageMetadata( + messageId, offset, partition, 123L, topic, subscription, topic, 123456L, 123456L); + } - public static MessageMetadata of(String messageId, String batchId, String topic, String subscription) { - return of(messageId, batchId, topic, subscription, 1L, 1); - } + public static MessageMetadata of( + String messageId, String batchId, String topic, String subscription) { + return of(messageId, batchId, topic, subscription, 1L, 1); + } - public static MessageMetadata of(String messageId, String batchId, String topic, String subscription, long offset, int partition) { - return new MessageMetadata(messageId, batchId, offset, partition, 123L, topic, subscription, topic, 123456L, 123456L); - } + public static MessageMetadata of( + String messageId, + String batchId, + String topic, + String subscription, + long offset, + int partition) { + return new MessageMetadata( + messageId, batchId, offset, partition, 123L, topic, subscription, topic, 123456L, 123456L); + } } diff --git a/hermes-tracker/src/test/java/pl/allegro/tech/hermes/tracker/consumers/TrackersTest.java b/hermes-tracker/src/test/java/pl/allegro/tech/hermes/tracker/consumers/TrackersTest.java index 3d4c79819c..8e8bf30c81 100644 --- a/hermes-tracker/src/test/java/pl/allegro/tech/hermes/tracker/consumers/TrackersTest.java +++ b/hermes-tracker/src/test/java/pl/allegro/tech/hermes/tracker/consumers/TrackersTest.java @@ -1,38 +1,36 @@ package pl.allegro.tech.hermes.tracker.consumers; -import org.junit.Test; -import pl.allegro.tech.hermes.api.Subscription; -import pl.allegro.tech.hermes.api.TrackingMode; - -import java.util.ArrayList; - import static org.assertj.core.api.Assertions.assertThat; import static pl.allegro.tech.hermes.api.TrackingMode.TRACKING_OFF; import static pl.allegro.tech.hermes.api.TrackingMode.TRACK_ALL; import static pl.allegro.tech.hermes.api.TrackingMode.TRACK_DISCARDED_ONLY; import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; -public class TrackersTest { - - @Test - public void shouldDispatchCorrectTracker() { - // given - Trackers trackers = new Trackers(new ArrayList<>()); - - // when - SendingTracker trackingOffTracker = trackers.get(subscriptionWithTrackingMode(TRACKING_OFF)); - SendingTracker trackDiscardedOnlyTracker = trackers.get(subscriptionWithTrackingMode(TRACK_DISCARDED_ONLY)); - SendingTracker trackAllTracker = trackers.get(subscriptionWithTrackingMode(TRACK_ALL)); +import java.util.ArrayList; +import org.junit.Test; +import pl.allegro.tech.hermes.api.Subscription; +import pl.allegro.tech.hermes.api.TrackingMode; - // then - assertThat(trackingOffTracker.getClass()).isEqualTo(NoOperationSendingTracker.class); - assertThat(trackDiscardedOnlyTracker.getClass()).isEqualTo(DiscardedSendingTracker.class); - assertThat(trackAllTracker.getClass()).isEqualTo(SendingMessageTracker.class); - } +public class TrackersTest { - private static Subscription subscriptionWithTrackingMode(TrackingMode trackingMode) { - return subscription("group.topic", "sub") - .withTrackingMode(trackingMode) - .build(); - } -} \ No newline at end of file + @Test + public void shouldDispatchCorrectTracker() { + // given + Trackers trackers = new Trackers(new ArrayList<>()); + + // when + SendingTracker trackingOffTracker = trackers.get(subscriptionWithTrackingMode(TRACKING_OFF)); + SendingTracker trackDiscardedOnlyTracker = + trackers.get(subscriptionWithTrackingMode(TRACK_DISCARDED_ONLY)); + SendingTracker trackAllTracker = trackers.get(subscriptionWithTrackingMode(TRACK_ALL)); + + // then + assertThat(trackingOffTracker.getClass()).isEqualTo(NoOperationSendingTracker.class); + assertThat(trackDiscardedOnlyTracker.getClass()).isEqualTo(DiscardedSendingTracker.class); + assertThat(trackAllTracker.getClass()).isEqualTo(SendingMessageTracker.class); + } + + private static Subscription subscriptionWithTrackingMode(TrackingMode trackingMode) { + return subscription("group.topic", "sub").withTrackingMode(trackingMode).build(); + } +} diff --git a/hermes-tracker/src/test/java/pl/allegro/tech/hermes/tracker/frontend/AbstractLogRepositoryTest.java b/hermes-tracker/src/test/java/pl/allegro/tech/hermes/tracker/frontend/AbstractLogRepositoryTest.java index 8ec5d8ff22..4124863d22 100644 --- a/hermes-tracker/src/test/java/pl/allegro/tech/hermes/tracker/frontend/AbstractLogRepositoryTest.java +++ b/hermes-tracker/src/test/java/pl/allegro/tech/hermes/tracker/frontend/AbstractLogRepositoryTest.java @@ -1,94 +1,93 @@ package pl.allegro.tech.hermes.tracker.frontend; +import static java.lang.System.currentTimeMillis; + import com.google.common.collect.ImmutableMap; +import java.util.Map; import org.junit.Before; import org.junit.Test; -import java.util.Map; - -import static java.lang.System.currentTimeMillis; - public abstract class AbstractLogRepositoryTest { - private LogRepository logRepository; - - @Before - public void setup() { - logRepository = createRepository(); - } - - protected abstract LogRepository createRepository(); - - @Test - public void shouldLogPublished() throws Exception { - // given - String id = "publishedMessage"; - String topic = "group.sentMessage"; - String hostname = "172.16.254.1"; - String datacenter = "dc1"; - Map extraRequestHeaders = ImmutableMap.of("header1", "value1", "header2", "value2"); - - // when - logRepository.logPublished(id, currentTimeMillis(), topic, hostname, datacenter, extraRequestHeaders); - - // then - awaitUntilSuccessMessageIsPersisted(topic, id, hostname, datacenter, "header1", "value1", "header2", "value2"); - } - - @Test - public void shouldLogError() throws Exception { - // given - String id = "errorMessage"; - String topic = "group.sentMessage"; - String hostname = "172.16.254.1"; - Map extraRequestHeaders = ImmutableMap.of("header1", "value1", "header2", "value2"); - - // when - logRepository.logError(id, currentTimeMillis(), topic, "reason", hostname, extraRequestHeaders); - - // then - awaitUntilErrorMessageIsPersisted(topic, id, "reason", hostname, "header1", "value1", "header2", "value2"); - } - - @Test - public void shouldLogInflight() throws Exception { - // given - String id = "inflightMessage"; - String topic = "group.sentMessage"; - String hostname = "172.16.254.1"; - Map extraRequestHeaders = ImmutableMap.of("header1", "value1", "header2", "value2"); - - // when - logRepository.logInflight(id, currentTimeMillis(), topic, hostname, extraRequestHeaders); - - // then - awaitUntilInflightMessageIsPersisted(topic, id, hostname, "header1", "value1", "header2", "value2"); - } - - protected abstract void awaitUntilSuccessMessageIsPersisted( - String topic, - String id, - String remoteHostname, - String storageDatacenter, - String... extraRequestHeadersKeywords - ) - throws Exception; - - protected abstract void awaitUntilInflightMessageIsPersisted( - String topic, - String id, - String remoteHostname, - String... extraRequestHeadersKeywords - ) - throws Exception; - - - protected abstract void awaitUntilErrorMessageIsPersisted( - String topic, - String id, - String reason, - String remoteHostname, - String... extraRequestHeadersKeywords - ) - throws Exception; -} \ No newline at end of file + private LogRepository logRepository; + + @Before + public void setup() { + logRepository = createRepository(); + } + + protected abstract LogRepository createRepository(); + + @Test + public void shouldLogPublished() throws Exception { + // given + String id = "publishedMessage"; + String topic = "group.sentMessage"; + String hostname = "172.16.254.1"; + String datacenter = "dc1"; + Map extraRequestHeaders = + ImmutableMap.of("header1", "value1", "header2", "value2"); + + // when + logRepository.logPublished( + id, currentTimeMillis(), topic, hostname, datacenter, extraRequestHeaders); + + // then + awaitUntilSuccessMessageIsPersisted( + topic, id, hostname, datacenter, "header1", "value1", "header2", "value2"); + } + + @Test + public void shouldLogError() throws Exception { + // given + String id = "errorMessage"; + String topic = "group.sentMessage"; + String hostname = "172.16.254.1"; + Map extraRequestHeaders = + ImmutableMap.of("header1", "value1", "header2", "value2"); + + // when + logRepository.logError(id, currentTimeMillis(), topic, "reason", hostname, extraRequestHeaders); + + // then + awaitUntilErrorMessageIsPersisted( + topic, id, "reason", hostname, "header1", "value1", "header2", "value2"); + } + + @Test + public void shouldLogInflight() throws Exception { + // given + String id = "inflightMessage"; + String topic = "group.sentMessage"; + String hostname = "172.16.254.1"; + Map extraRequestHeaders = + ImmutableMap.of("header1", "value1", "header2", "value2"); + + // when + logRepository.logInflight(id, currentTimeMillis(), topic, hostname, extraRequestHeaders); + + // then + awaitUntilInflightMessageIsPersisted( + topic, id, hostname, "header1", "value1", "header2", "value2"); + } + + protected abstract void awaitUntilSuccessMessageIsPersisted( + String topic, + String id, + String remoteHostname, + String storageDatacenter, + String... extraRequestHeadersKeywords) + throws Exception; + + protected abstract void awaitUntilInflightMessageIsPersisted( + String topic, String id, String remoteHostname, String... extraRequestHeadersKeywords) + throws Exception; + + protected abstract void awaitUntilErrorMessageIsPersisted( + String topic, + String id, + String reason, + String remoteHostname, + String... extraRequestHeadersKeywords) + throws Exception; +} diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/consumers/EndpointAddressResolverConfiguration.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/consumers/EndpointAddressResolverConfiguration.java index 676759d1bd..b5f92f0cd7 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/consumers/EndpointAddressResolverConfiguration.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/consumers/EndpointAddressResolverConfiguration.java @@ -10,10 +10,10 @@ @Configuration public class EndpointAddressResolverConfiguration { - @Bean - @Primary - @Profile("integration") - public EndpointAddressResolver testMultiUrlEndpointAddressResolver() { - return new MultiUrlEndpointAddressResolver(); - } + @Bean + @Primary + @Profile("integration") + public EndpointAddressResolver testMultiUrlEndpointAddressResolver() { + return new MultiUrlEndpointAddressResolver(); + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/consumers/GoogleTransportChannelProviderConfiguration.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/consumers/GoogleTransportChannelProviderConfiguration.java index 52d1e82286..95ae412762 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/consumers/GoogleTransportChannelProviderConfiguration.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/consumers/GoogleTransportChannelProviderConfiguration.java @@ -16,13 +16,16 @@ @EnableConfigurationProperties(GooglePubSubSenderProperties.class) public class GoogleTransportChannelProviderConfiguration { - @Bean - @Primary - @Profile("integration") - public TransportChannelProvider integrationTransportChannelProvider(GooglePubSubSenderProperties googlePubSubSenderProperties) { - final ManagedChannel channel = ManagedChannelBuilder.forTarget( - googlePubSubSenderProperties.getTransportChannelProviderAddress()) - .usePlaintext().build(); - return FixedTransportChannelProvider.create(GrpcTransportChannel.create(channel)); - } + @Bean + @Primary + @Profile("integration") + public TransportChannelProvider integrationTransportChannelProvider( + GooglePubSubSenderProperties googlePubSubSenderProperties) { + final ManagedChannel channel = + ManagedChannelBuilder.forTarget( + googlePubSubSenderProperties.getTransportChannelProviderAddress()) + .usePlaintext() + .build(); + return FixedTransportChannelProvider.create(GrpcTransportChannel.create(channel)); + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/consumers/KafkaNamesMapperConfiguration.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/consumers/KafkaNamesMapperConfiguration.java index a526540e13..e4b01f583c 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/consumers/KafkaNamesMapperConfiguration.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/consumers/KafkaNamesMapperConfiguration.java @@ -13,10 +13,11 @@ @EnableConfigurationProperties(KafkaClustersProperties.class) public class KafkaNamesMapperConfiguration { - @Bean - @Primary - @Profile("integration") - public KafkaNamesMapper testKafkaNamesMapper(KafkaClustersProperties kafkaClustersProperties) { - return new IntegrationTestKafkaNamesMapperFactory(kafkaClustersProperties.getNamespace()).create(); - } + @Bean + @Primary + @Profile("integration") + public KafkaNamesMapper testKafkaNamesMapper(KafkaClustersProperties kafkaClustersProperties) { + return new IntegrationTestKafkaNamesMapperFactory(kafkaClustersProperties.getNamespace()) + .create(); + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/env/BrokerOperations.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/env/BrokerOperations.java index baad64540c..6472e23ae1 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/env/BrokerOperations.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/env/BrokerOperations.java @@ -1,5 +1,22 @@ package pl.allegro.tech.hermes.env; +import static java.util.Collections.singletonList; +import static java.util.concurrent.TimeUnit.MINUTES; +import static java.util.stream.Collectors.toMap; +import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG; +import static org.apache.kafka.clients.CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL; +import static org.apache.kafka.clients.CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG; +import static org.apache.kafka.clients.CommonClientConfigs.SECURITY_PROTOCOL_CONFIG; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topic; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; +import java.util.function.Function; +import java.util.stream.Collectors; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.ListOffsetsResult; import org.apache.kafka.clients.admin.NewTopic; @@ -14,124 +31,111 @@ import pl.allegro.tech.hermes.common.kafka.KafkaTopic; import pl.allegro.tech.hermes.common.kafka.KafkaTopicName; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeoutException; -import java.util.function.Function; -import java.util.stream.Collectors; - -import static java.util.Collections.singletonList; -import static java.util.concurrent.TimeUnit.MINUTES; -import static java.util.stream.Collectors.toMap; -import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG; -import static org.apache.kafka.clients.CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL; -import static org.apache.kafka.clients.CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG; -import static org.apache.kafka.clients.CommonClientConfigs.SECURITY_PROTOCOL_CONFIG; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topic; - public class BrokerOperations { - private static final int DEFAULT_PARTITIONS = 2; - private static final int DEFAULT_REPLICATION_FACTOR = 1; - - private final AdminClient adminClient; - - private final KafkaNamesMapper kafkaNamesMapper; - - public BrokerOperations(String brokerList, String namespace) { - this.adminClient = brokerAdminClient(brokerList); - String namespaceSeparator = "_"; - this.kafkaNamesMapper = new JsonToAvroMigrationKafkaNamesMapper(namespace, namespaceSeparator); - } - - public List getTopicPartitionsOffsets(SubscriptionName subscriptionName) { - ConsumerGroupId consumerGroupId = kafkaNamesMapper.toConsumerGroupId(subscriptionName); - - Map currentOffsets = getTopicCurrentOffsets(consumerGroupId); - Map endOffsets = getEndOffsets(new ArrayList<>(currentOffsets.keySet())); - return currentOffsets.keySet() - .stream() - .map(partition -> new ConsumerGroupOffset( - currentOffsets.get(partition).offset(), - endOffsets.get(partition).offset()) - ).collect(Collectors.toList()); + private static final int DEFAULT_PARTITIONS = 2; + private static final int DEFAULT_REPLICATION_FACTOR = 1; + + private final AdminClient adminClient; + + private final KafkaNamesMapper kafkaNamesMapper; + + public BrokerOperations(String brokerList, String namespace) { + this.adminClient = brokerAdminClient(brokerList); + String namespaceSeparator = "_"; + this.kafkaNamesMapper = new JsonToAvroMigrationKafkaNamesMapper(namespace, namespaceSeparator); + } + + public List getTopicPartitionsOffsets(SubscriptionName subscriptionName) { + ConsumerGroupId consumerGroupId = kafkaNamesMapper.toConsumerGroupId(subscriptionName); + + Map currentOffsets = getTopicCurrentOffsets(consumerGroupId); + Map endOffsets = + getEndOffsets(new ArrayList<>(currentOffsets.keySet())); + return currentOffsets.keySet().stream() + .map( + partition -> + new ConsumerGroupOffset( + currentOffsets.get(partition).offset(), endOffsets.get(partition).offset())) + .collect(Collectors.toList()); + } + + private Map getTopicCurrentOffsets( + ConsumerGroupId consumerGroupId) { + try { + return adminClient + .listConsumerGroupOffsets(consumerGroupId.asString()) + .partitionsToOffsetAndMetadata() + .get(); + } catch (Exception e) { + throw new RuntimeException(e); } - - private Map getTopicCurrentOffsets(ConsumerGroupId consumerGroupId) { - try { - return adminClient.listConsumerGroupOffsets(consumerGroupId.asString()).partitionsToOffsetAndMetadata().get(); - } catch (Exception e) { - throw new RuntimeException(e); - } + } + + private Map getEndOffsets( + List partitions) { + try { + ListOffsetsResult listOffsetsResult = + adminClient.listOffsets( + partitions.stream().collect(toMap(Function.identity(), p -> OffsetSpec.latest()))); + return listOffsetsResult.all().get(); + } catch (Exception e) { + throw new RuntimeException(e); } - - private Map getEndOffsets(List partitions) { - try { - ListOffsetsResult listOffsetsResult = adminClient.listOffsets( - partitions.stream().collect(toMap(Function.identity(), p -> OffsetSpec.latest()))); - return listOffsetsResult.all().get(); - } catch (Exception e) { - throw new RuntimeException(e); - } + } + + public void createTopic(String topicName) { + Topic topic = topic(topicName).build(); + kafkaNamesMapper.toKafkaTopics(topic).forEach(kafkaTopic -> createTopic(kafkaTopic.name())); + } + + private void createTopic(KafkaTopicName topicName) { + try { + NewTopic topic = + new NewTopic( + topicName.asString(), DEFAULT_PARTITIONS, (short) DEFAULT_REPLICATION_FACTOR); + adminClient.createTopics(singletonList(topic)).all().get(1, MINUTES); + } catch (ExecutionException | TimeoutException | InterruptedException e) { + throw new RuntimeException(e); } - - public void createTopic(String topicName) { - Topic topic = topic(topicName).build(); - kafkaNamesMapper.toKafkaTopics(topic) - .forEach(kafkaTopic -> createTopic(kafkaTopic.name())); + } + + public boolean topicExists(String topicName) { + Topic topic = topic(topicName).build(); + return kafkaNamesMapper.toKafkaTopics(topic).allMatch(this::topicExists); + } + + private boolean topicExists(KafkaTopic kafkaTopic) { + try { + return adminClient + .listTopics() + .names() + .get(1, MINUTES) + .contains(kafkaTopic.name().asString()); + } catch (ExecutionException | TimeoutException | InterruptedException e) { + throw new RuntimeException(e); } - - private void createTopic(KafkaTopicName topicName) { - try { - NewTopic topic = new NewTopic(topicName.asString(), DEFAULT_PARTITIONS, (short) DEFAULT_REPLICATION_FACTOR); - adminClient.createTopics(singletonList(topic)) - .all() - .get(1, MINUTES); - } catch (ExecutionException | TimeoutException | InterruptedException e) { - throw new RuntimeException(e); - } - } - - public boolean topicExists(String topicName) { - Topic topic = topic(topicName).build(); - return kafkaNamesMapper.toKafkaTopics(topic) - .allMatch(this::topicExists); + } + + private AdminClient brokerAdminClient(String brokerList) { + Properties props = new Properties(); + props.put(BOOTSTRAP_SERVERS_CONFIG, brokerList); + props.put(SECURITY_PROTOCOL_CONFIG, DEFAULT_SECURITY_PROTOCOL); + props.put(REQUEST_TIMEOUT_MS_CONFIG, 10000); + return AdminClient.create(props); + } + + public static class ConsumerGroupOffset { + private final long currentOffset; + private final long endOffset; + + ConsumerGroupOffset(long currentOffset, long endOffset) { + this.currentOffset = currentOffset; + this.endOffset = endOffset; } - private boolean topicExists(KafkaTopic kafkaTopic) { - try { - return adminClient - .listTopics() - .names() - .get(1, MINUTES) - .contains(kafkaTopic.name().asString()); - } catch (ExecutionException | TimeoutException | InterruptedException e) { - throw new RuntimeException(e); - } - } - - private AdminClient brokerAdminClient(String brokerList) { - Properties props = new Properties(); - props.put(BOOTSTRAP_SERVERS_CONFIG, brokerList); - props.put(SECURITY_PROTOCOL_CONFIG, DEFAULT_SECURITY_PROTOCOL); - props.put(REQUEST_TIMEOUT_MS_CONFIG, 10000); - return AdminClient.create(props); - } - - public static class ConsumerGroupOffset { - private final long currentOffset; - private final long endOffset; - - ConsumerGroupOffset(long currentOffset, long endOffset) { - this.currentOffset = currentOffset; - this.endOffset = endOffset; - } - - public boolean movedToEnd() { - return currentOffset == endOffset; - } + public boolean movedToEnd() { + return currentOffset == endOffset; } + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/env/IntegrationTestKafkaNamesMapperFactory.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/env/IntegrationTestKafkaNamesMapperFactory.java index cc431fd4ee..157a2e6fa2 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/env/IntegrationTestKafkaNamesMapperFactory.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/env/IntegrationTestKafkaNamesMapperFactory.java @@ -5,13 +5,13 @@ public class IntegrationTestKafkaNamesMapperFactory { - private final String namespace; + private final String namespace; - public IntegrationTestKafkaNamesMapperFactory(String namespace) { - this.namespace = namespace; - } + public IntegrationTestKafkaNamesMapperFactory(String namespace) { + this.namespace = namespace; + } - public KafkaNamesMapper create() { - return new JsonToAvroMigrationKafkaNamesMapper(namespace, "_"); - } + public KafkaNamesMapper create() { + return new JsonToAvroMigrationKafkaNamesMapper(namespace, "_"); + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/frontend/AuthConfiguration.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/frontend/AuthConfiguration.java index c278f8a102..b6608d069e 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/frontend/AuthConfiguration.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/frontend/AuthConfiguration.java @@ -11,29 +11,28 @@ @Configuration public class AuthConfiguration { - @Value("${auth.username}") - private String username; + @Value("${auth.username}") + private String username; - @Value("${auth.password}") - private String password; + @Value("${auth.password}") + private String password; - @Bean - @Profile("authRequired") - public AuthenticationConfiguration requiredAuthenticationConfiguration() { - return getAuthConfig(true); - } + @Bean + @Profile("authRequired") + public AuthenticationConfiguration requiredAuthenticationConfiguration() { + return getAuthConfig(true); + } - @Bean - @Profile("authNonRequired") - public AuthenticationConfiguration notRequiredAuthenticationConfiguration() { - return getAuthConfig(false); - } + @Bean + @Profile("authNonRequired") + public AuthenticationConfiguration notRequiredAuthenticationConfiguration() { + return getAuthConfig(false); + } - private AuthenticationConfiguration getAuthConfig(boolean isAuthenticationRequired) { - return new AuthenticationConfiguration( - exchange -> isAuthenticationRequired, - Lists.newArrayList(new BasicAuthenticationMechanism("basicAuthRealm")), - new SingleUserAwareIdentityManager(username, password) - ); - } + private AuthenticationConfiguration getAuthConfig(boolean isAuthenticationRequired) { + return new AuthenticationConfiguration( + exchange -> isAuthenticationRequired, + Lists.newArrayList(new BasicAuthenticationMechanism("basicAuthRealm")), + new SingleUserAwareIdentityManager(username, password)); + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/frontend/FrontendConfigurationProperties.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/frontend/FrontendConfigurationProperties.java index 3f92181942..14f9c1177b 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/frontend/FrontendConfigurationProperties.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/frontend/FrontendConfigurationProperties.java @@ -1,39 +1,50 @@ package pl.allegro.tech.hermes.frontend; public class FrontendConfigurationProperties { - public static String SPRING_PROFILES_ACTIVE = "spring.profiles.active"; - public static String AUTH_USERNAME = "auth.username"; - public static String AUTH_PASSWORD = "auth.password"; - public static String MESSAGES_LOCAL_STORAGE_DIRECTORY = "frontend.messages.local.storage.directory"; - public static String MESSAGES_LOCAL_STORAGE_ENABLED = "frontend.messages.local.storage.enabled"; - public static String FRONTEND_THROUGHPUT_TYPE = "frontend.throughput.type"; - public static String FRONTEND_THROUGHPUT_FIXED_MAX = "frontend.throughput.fixedMax"; - public static String FRONTEND_MESSAGE_PREVIEW_ENABLED = "frontend.message.preview.enabled"; - public static String FRONTEND_MESSAGE_PREVIEW_LOG_PERSIST_PERIOD = "frontend.message.preview.logPersistPeriod"; - public static String FRONTEND_READINESS_CHECK_ENABLED = "frontend.readiness.check.enabled"; - public static String FRONTEND_READINESS_CHECK_KAFKA_CHECK_ENABLED = "frontend.readiness.check.kafkaCheckEnabled"; - public static String FRONTEND_READINESS_CHECK_INTERVAL_SECONDS = "frontend.readiness.check.interval"; - public static String FRONTEND_AUTHENTICATION_MODE = "frontend.handlers.authentication.mode"; - public static String FRONTEND_AUTHENTICATION_ENABLED = "frontend.handlers.authentication.enabled"; - public static String FRONTEND_KEEP_ALIVE_HEADER_ENABLED = "frontend.handlers.keepAliveHeader.enabled"; - public static String FRONTEND_KEEP_ALIVE_HEADER_TIMEOUT = "frontend.handlers.keepAliveHeader.timeout"; - public static String FRONTEND_IDLE_TIMEOUT = "frontend.handlers.idleTimeout"; - public static String METRICS_MICROMETER_REPORT_PERIOD = "frontend.metrics.micrometer.reportPeriod"; - public static String SCHEMA_CACHE_ENABLED = "frontend.schema.cache.enabled"; - public static String SCHEMA_REPOSITORY_SERVER_URL = "frontend.schema.repository.serverUrl"; - public static String FRONTEND_SSL_ENABLED = "frontend.ssl.enabled"; - public static String FRONTEND_SSL_PORT = "frontend.ssl.port"; - public static String FRONTEND_SSL_KEYSTORE_SOURCE = "frontend.ssl.keystoreSource"; - public static String FRONTEND_SSL_TRUSTSTORE_SOURCE = "frontend.ssl.truststoreSource"; - public static String FRONTEND_FORCE_TOPIC_MAX_MESSAGE_SIZE = "frontend.handlers.forceTopicMaxMessageSize"; - public static String FRONTEND_HTTP2_ENABLED = "frontend.server.http2Enabled"; - public static String FRONTEND_GRACEFUL_SHUTDOWN_ENABLED = "frontend.server.gracefulShutdownEnabled"; - public static String FRONTEND_PORT = "frontend.server.port"; - public static String ZOOKEEPER_CONNECTION_STRING = "frontend.zookeeper.clusters.[0].connectionString"; - public static String KAFKA_PRODUCER_METADATA_MAX_AGE = "frontend.kafka.producer.metadataMaxAge"; - public static String KAFKA_BROKER_LIST = "frontend.kafka.clusters.[0].brokerList"; - public static String KAFKA_NAMESPACE = "frontend.kafka.namespace"; - public static String FRONTEND_HEADER_PROPAGATION_ENABLED = "frontend.http.headers.propagationEnabled"; - public static String FRONTEND_HEADER_PROPAGATION_ALLOWED = "frontend.http.headers.allowedSet"; - public static String BROKER_LATENCY_REPORTER_ENABLED = "frontend.brokerLatencyReporter.enabled"; + public static String SPRING_PROFILES_ACTIVE = "spring.profiles.active"; + public static String AUTH_USERNAME = "auth.username"; + public static String AUTH_PASSWORD = "auth.password"; + public static String MESSAGES_LOCAL_STORAGE_DIRECTORY = + "frontend.messages.local.storage.directory"; + public static String MESSAGES_LOCAL_STORAGE_ENABLED = "frontend.messages.local.storage.enabled"; + public static String FRONTEND_THROUGHPUT_TYPE = "frontend.throughput.type"; + public static String FRONTEND_THROUGHPUT_FIXED_MAX = "frontend.throughput.fixedMax"; + public static String FRONTEND_MESSAGE_PREVIEW_ENABLED = "frontend.message.preview.enabled"; + public static String FRONTEND_MESSAGE_PREVIEW_LOG_PERSIST_PERIOD = + "frontend.message.preview.logPersistPeriod"; + public static String FRONTEND_READINESS_CHECK_ENABLED = "frontend.readiness.check.enabled"; + public static String FRONTEND_READINESS_CHECK_KAFKA_CHECK_ENABLED = + "frontend.readiness.check.kafkaCheckEnabled"; + public static String FRONTEND_READINESS_CHECK_INTERVAL_SECONDS = + "frontend.readiness.check.interval"; + public static String FRONTEND_AUTHENTICATION_MODE = "frontend.handlers.authentication.mode"; + public static String FRONTEND_AUTHENTICATION_ENABLED = "frontend.handlers.authentication.enabled"; + public static String FRONTEND_KEEP_ALIVE_HEADER_ENABLED = + "frontend.handlers.keepAliveHeader.enabled"; + public static String FRONTEND_KEEP_ALIVE_HEADER_TIMEOUT = + "frontend.handlers.keepAliveHeader.timeout"; + public static String FRONTEND_IDLE_TIMEOUT = "frontend.handlers.idleTimeout"; + public static String METRICS_MICROMETER_REPORT_PERIOD = + "frontend.metrics.micrometer.reportPeriod"; + public static String SCHEMA_CACHE_ENABLED = "frontend.schema.cache.enabled"; + public static String SCHEMA_REPOSITORY_SERVER_URL = "frontend.schema.repository.serverUrl"; + public static String FRONTEND_SSL_ENABLED = "frontend.ssl.enabled"; + public static String FRONTEND_SSL_PORT = "frontend.ssl.port"; + public static String FRONTEND_SSL_KEYSTORE_SOURCE = "frontend.ssl.keystoreSource"; + public static String FRONTEND_SSL_TRUSTSTORE_SOURCE = "frontend.ssl.truststoreSource"; + public static String FRONTEND_FORCE_TOPIC_MAX_MESSAGE_SIZE = + "frontend.handlers.forceTopicMaxMessageSize"; + public static String FRONTEND_HTTP2_ENABLED = "frontend.server.http2Enabled"; + public static String FRONTEND_GRACEFUL_SHUTDOWN_ENABLED = + "frontend.server.gracefulShutdownEnabled"; + public static String FRONTEND_PORT = "frontend.server.port"; + public static String ZOOKEEPER_CONNECTION_STRING = + "frontend.zookeeper.clusters.[0].connectionString"; + public static String KAFKA_PRODUCER_METADATA_MAX_AGE = "frontend.kafka.producer.metadataMaxAge"; + public static String KAFKA_BROKER_LIST = "frontend.kafka.clusters.[0].brokerList"; + public static String KAFKA_NAMESPACE = "frontend.kafka.namespace"; + public static String FRONTEND_HEADER_PROPAGATION_ENABLED = + "frontend.http.headers.propagationEnabled"; + public static String FRONTEND_HEADER_PROPAGATION_ALLOWED = "frontend.http.headers.allowedSet"; + public static String BROKER_LATENCY_REPORTER_ENABLED = "frontend.brokerLatencyReporter.enabled"; } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/frontend/KafkaNamesMapperConfiguration.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/frontend/KafkaNamesMapperConfiguration.java index 822cbe6c67..344040b180 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/frontend/KafkaNamesMapperConfiguration.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/frontend/KafkaNamesMapperConfiguration.java @@ -13,10 +13,11 @@ @EnableConfigurationProperties(KafkaClustersProperties.class) public class KafkaNamesMapperConfiguration { - @Bean - @Primary - @Profile("integration") - public KafkaNamesMapper testKafkaNamesMapper(KafkaClustersProperties kafkaClustersProperties) { - return new IntegrationTestKafkaNamesMapperFactory(kafkaClustersProperties.getNamespace()).create(); - } + @Bean + @Primary + @Profile("integration") + public KafkaNamesMapper testKafkaNamesMapper(KafkaClustersProperties kafkaClustersProperties) { + return new IntegrationTestKafkaNamesMapperFactory(kafkaClustersProperties.getNamespace()) + .create(); + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/frontend/SingleUserAwareIdentityManager.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/frontend/SingleUserAwareIdentityManager.java index 72474bc3f7..608785cd8b 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/frontend/SingleUserAwareIdentityManager.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/frontend/SingleUserAwareIdentityManager.java @@ -4,65 +4,64 @@ import io.undertow.security.idm.Credential; import io.undertow.security.idm.IdentityManager; import io.undertow.security.idm.PasswordCredential; -import pl.allegro.tech.hermes.frontend.server.auth.Roles; - import java.security.Principal; import java.util.Collections; import java.util.Set; +import pl.allegro.tech.hermes.frontend.server.auth.Roles; public class SingleUserAwareIdentityManager implements IdentityManager { - private final String username; - private final String password; + private final String username; + private final String password; - public SingleUserAwareIdentityManager(String username, String password) { - this.username = username; - this.password = password; - } + public SingleUserAwareIdentityManager(String username, String password) { + this.username = username; + this.password = password; + } - @Override - public Account verify(Account account) { - return null; - } + @Override + public Account verify(Account account) { + return null; + } - @Override - public Account verify(String username, Credential credential) { - String password = new String(((PasswordCredential) credential).getPassword()); - if (this.username.equals(username) && this.password.equals(password)) { - return new SomeUserAccount(username); - } - return null; + @Override + public Account verify(String username, Credential credential) { + String password = new String(((PasswordCredential) credential).getPassword()); + if (this.username.equals(username) && this.password.equals(password)) { + return new SomeUserAccount(username); } + return null; + } - @Override - public Account verify(Credential credential) { - return null; - } + @Override + public Account verify(Credential credential) { + return null; + } - private static final class SomeUserAccount implements Account { + private static final class SomeUserAccount implements Account { - private final Principal principal; + private final Principal principal; - private SomeUserAccount(String username) { - this.principal = new SomeUserPrincipal(username); - } + private SomeUserAccount(String username) { + this.principal = new SomeUserPrincipal(username); + } - @Override - public Principal getPrincipal() { - return principal; - } + @Override + public Principal getPrincipal() { + return principal; + } - @Override - public Set getRoles() { - return Collections.singleton(Roles.PUBLISHER); - } + @Override + public Set getRoles() { + return Collections.singleton(Roles.PUBLISHER); } + } - private record SomeUserPrincipal(String username) implements Principal { + private record SomeUserPrincipal(String username) implements Principal { - @Override - public String getName() { - return username; - } - } -} \ No newline at end of file + @Override + public String getName() { + return username; + } + } +} diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/assertions/GooglePubSubAssertion.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/assertions/GooglePubSubAssertion.java index 76783fc483..c494213ea2 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/assertions/GooglePubSubAssertion.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/assertions/GooglePubSubAssertion.java @@ -1,23 +1,23 @@ package pl.allegro.tech.hermes.integrationtests.assertions; +import static org.assertj.core.api.Assertions.assertThat; + import com.google.pubsub.v1.PubsubMessage; import org.assertj.core.api.AbstractAssert; -import static org.assertj.core.api.Assertions.assertThat; - public class GooglePubSubAssertion extends AbstractAssert { - GooglePubSubAssertion(PubsubMessage actual) { - super(actual, GooglePubSubAssertion.class); - } + GooglePubSubAssertion(PubsubMessage actual) { + super(actual, GooglePubSubAssertion.class); + } - public GooglePubSubAssertion hasAttribute(String name) { - assertThat(actual.getAttributesMap()).containsKey(name); - return this; - } + public GooglePubSubAssertion hasAttribute(String name) { + assertThat(actual.getAttributesMap()).containsKey(name); + return this; + } - public GooglePubSubAssertion hasBody(String body) { - assertThat(actual.getData().toStringUtf8()).isEqualTo(body); - return this; - } + public GooglePubSubAssertion hasBody(String body) { + assertThat(actual.getData().toStringUtf8()).isEqualTo(body); + return this; + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/assertions/HermesAssertions.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/assertions/HermesAssertions.java index 53f03e3eab..a60cfdd0ef 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/assertions/HermesAssertions.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/assertions/HermesAssertions.java @@ -7,23 +7,21 @@ public final class HermesAssertions extends Assertions { - private HermesAssertions() { - } + private HermesAssertions() {} - public static HttpResponseAssertion assertThat(Response response) { - return new HttpResponseAssertion(response); - } + public static HttpResponseAssertion assertThat(Response response) { + return new HttpResponseAssertion(response); + } + public static WiremockRequestAssertion assertThat(LoggedRequest request) { + return new WiremockRequestAssertion(request); + } - public static WiremockRequestAssertion assertThat(LoggedRequest request) { - return new WiremockRequestAssertion(request); - } + public static PrometheusMetricsAssertion assertThatMetrics(String body) { + return new PrometheusMetricsAssertion(body); + } - public static PrometheusMetricsAssertion assertThatMetrics(String body) { - return new PrometheusMetricsAssertion(body); - } - - public static GooglePubSubAssertion assertThat(PubsubMessage message) { - return new GooglePubSubAssertion(message); - } + public static GooglePubSubAssertion assertThat(PubsubMessage message) { + return new GooglePubSubAssertion(message); + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/assertions/HttpResponseAssertion.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/assertions/HttpResponseAssertion.java index ddc5f7b171..9bba9f88dc 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/assertions/HttpResponseAssertion.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/assertions/HttpResponseAssertion.java @@ -1,18 +1,18 @@ package pl.allegro.tech.hermes.integrationtests.assertions; +import static org.assertj.core.api.Assertions.assertThat; + import jakarta.ws.rs.core.Response; import org.assertj.core.api.AbstractAssert; -import static org.assertj.core.api.Assertions.assertThat; - public class HttpResponseAssertion extends AbstractAssert { - HttpResponseAssertion(Response actual) { - super(actual, HttpResponseAssertion.class); - } + HttpResponseAssertion(Response actual) { + super(actual, HttpResponseAssertion.class); + } - public HttpResponseAssertion hasStatus(Response.Status status) { - assertThat(actual.getStatus()).isEqualTo(status.getStatusCode()); - return this; - } + public HttpResponseAssertion hasStatus(Response.Status status) { + assertThat(actual.getStatus()).isEqualTo(status.getStatusCode()); + return this; + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/assertions/PrometheusMetricsAssertion.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/assertions/PrometheusMetricsAssertion.java index 5e705533dc..daec625d66 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/assertions/PrometheusMetricsAssertion.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/assertions/PrometheusMetricsAssertion.java @@ -1,118 +1,122 @@ package pl.allegro.tech.hermes.integrationtests.assertions; +import static org.assertj.core.api.Assertions.assertThat; + import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static org.assertj.core.api.Assertions.assertThat; - public class PrometheusMetricsAssertion { - private static final Pattern METRIC_LINE_PATTERN = Pattern.compile("^[a-z_]+\\{(.*)\\} (\\d+\\.\\d+)$"); + private static final Pattern METRIC_LINE_PATTERN = + Pattern.compile("^[a-z_]+\\{(.*)\\} (\\d+\\.\\d+)$"); + + private final String actualBody; + + PrometheusMetricsAssertion(String actualBody) { + this.actualBody = actualBody; + } + + public PrometheusMetricWithNameAssertion contains(String metricName) { + List matchedLines = + actualBody.lines().filter(line -> line.startsWith(metricName + "{")).toList(); + assertThat(matchedLines).overridingErrorMessage("Metric %s doesn't exist").isNotEmpty(); + return new PrometheusMetricWithNameAssertion(matchedLines); + } + + public static class PrometheusMetricWithNameAssertion { + + private final List actualMetrics; + + PrometheusMetricWithNameAssertion(List actualMetrics) { + this.actualMetrics = actualMetrics; + } + + public PrometheusMetricAssertion withLabels( + String label0, String value0, String label1, String value1) { + return withLabels(new String[] {label0, label1}, new String[] {value0, value1}); + } + + public PrometheusMetricAssertion withLabels( + String label0, String value0, String label1, String value1, String label2, String value2) { + return withLabels( + new String[] {label0, label1, label2}, new String[] {value0, value1, value2}); + } + + public PrometheusMetricAssertion withLabels( + String label0, + String value0, + String label1, + String value1, + String label2, + String value2, + String label3, + String value3) { + return withLabels( + new String[] {label0, label1, label2, label3}, + new String[] {value0, value1, value2, value3}); + } - private final String actualBody; + private PrometheusMetricAssertion withLabels(String[] names, String[] values) { + List matchedLines = + actualMetrics.stream() + .filter( + line -> { + Matcher matcher = METRIC_LINE_PATTERN.matcher(line); + String labels; + if (matcher.matches()) { + labels = matcher.group(1); + } else { + throw new IllegalStateException("Unexpected line: " + line); + } + for (int i = 0; i < names.length; i++) { + if (!labels.contains(names[i] + "=\"" + values[i] + "\"")) { + return false; + } + } + return true; + }) + .toList(); + assertThat(matchedLines) + .overridingErrorMessage("There is no metric with provided labels") + .isNotEmpty(); + assertThat(matchedLines) + .overridingErrorMessage("Found more than one metric with provided labels") + .hasSize(1); + return new PrometheusMetricAssertion(matchedLines.get(0)); + } + } + + public static class PrometheusMetricAssertion { + + private final String actualLine; + + PrometheusMetricAssertion(String actualLine) { + this.actualLine = actualLine; + } - PrometheusMetricsAssertion(String actualBody) { - this.actualBody = actualBody; + public void withValue(double expectedValue) { + double actualValue = extractValue(); + assertThat(actualValue).isEqualTo(expectedValue); } - public PrometheusMetricWithNameAssertion contains(String metricName) { - List matchedLines = actualBody.lines() - .filter(line -> line.startsWith(metricName + "{")) - .toList(); - assertThat(matchedLines).overridingErrorMessage("Metric %s doesn't exist").isNotEmpty(); - return new PrometheusMetricWithNameAssertion(matchedLines); + public double withInitialValue() { + return extractValue(); } - public static class PrometheusMetricWithNameAssertion { - - private final List actualMetrics; - - PrometheusMetricWithNameAssertion(List actualMetrics) { - this.actualMetrics = actualMetrics; - } - - public PrometheusMetricAssertion withLabels(String label0, String value0, - String label1, String value1) { - return withLabels( - new String[]{label0, label1}, - new String[]{value0, value1} - ); - } - - public PrometheusMetricAssertion withLabels(String label0, String value0, - String label1, String value1, - String label2, String value2) { - return withLabels( - new String[]{label0, label1, label2}, - new String[]{value0, value1, value2} - ); - } - - public PrometheusMetricAssertion withLabels(String label0, String value0, - String label1, String value1, - String label2, String value2, - String label3, String value3) { - return withLabels( - new String[]{label0, label1, label2, label3}, - new String[]{value0, value1, value2, value3} - ); - } - - private PrometheusMetricAssertion withLabels(String[] names, String[] values) { - List matchedLines = actualMetrics.stream() - .filter(line -> { - Matcher matcher = METRIC_LINE_PATTERN.matcher(line); - String labels; - if (matcher.matches()) { - labels = matcher.group(1); - } else { - throw new IllegalStateException("Unexpected line: " + line); - } - for (int i = 0; i < names.length; i++) { - if (!labels.contains(names[i] + "=\"" + values[i] + "\"")) { - return false; - } - } - return true; - }) - .toList(); - assertThat(matchedLines).overridingErrorMessage("There is no metric with provided labels").isNotEmpty(); - assertThat(matchedLines).overridingErrorMessage("Found more than one metric with provided labels").hasSize(1); - return new PrometheusMetricAssertion(matchedLines.get(0)); - } + public void withValueGreaterThan(double expectedValue) { + double actualValue = extractValue(); + assertThat(actualValue).isGreaterThan(expectedValue); } - public static class PrometheusMetricAssertion { - - private final String actualLine; - - PrometheusMetricAssertion(String actualLine) { - this.actualLine = actualLine; - } - - public void withValue(double expectedValue) { - double actualValue = extractValue(); - assertThat(actualValue).isEqualTo(expectedValue); - } - - public double withInitialValue() { - return extractValue(); - } - - public void withValueGreaterThan(double expectedValue) { - double actualValue = extractValue(); - assertThat(actualValue).isGreaterThan(expectedValue); - } - - private double extractValue() { - Matcher matcher = METRIC_LINE_PATTERN.matcher(actualLine); - if (matcher.matches()) { - String valueStr = matcher.group(2); - return Double.parseDouble(valueStr); - } else { - throw new IllegalStateException("Unexpected line: " + actualLine); - } - } + private double extractValue() { + Matcher matcher = METRIC_LINE_PATTERN.matcher(actualLine); + if (matcher.matches()) { + String valueStr = matcher.group(2); + return Double.parseDouble(valueStr); + } else { + throw new IllegalStateException("Unexpected line: " + actualLine); + } } + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/assertions/WiremockRequestAssertion.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/assertions/WiremockRequestAssertion.java index ca59f4a69e..c6cc0e87eb 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/assertions/WiremockRequestAssertion.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/assertions/WiremockRequestAssertion.java @@ -1,25 +1,25 @@ package pl.allegro.tech.hermes.integrationtests.assertions; -import com.github.tomakehurst.wiremock.verification.LoggedRequest; -import org.assertj.core.api.AbstractAssert; +import static org.assertj.core.api.Assertions.assertThat; +import com.github.tomakehurst.wiremock.verification.LoggedRequest; import java.util.Map; +import org.assertj.core.api.AbstractAssert; -import static org.assertj.core.api.Assertions.assertThat; - -public class WiremockRequestAssertion extends AbstractAssert { +public class WiremockRequestAssertion + extends AbstractAssert { - protected WiremockRequestAssertion(LoggedRequest actual) { - super(actual, WiremockRequestAssertion.class); - } + protected WiremockRequestAssertion(LoggedRequest actual) { + super(actual, WiremockRequestAssertion.class); + } - public WiremockRequestAssertion hasHeaderValue(String header, String value) { - assertThat(actual.getHeader(header)).isEqualTo(value); - return this; - } + public WiremockRequestAssertion hasHeaderValue(String header, String value) { + assertThat(actual.getHeader(header)).isEqualTo(value); + return this; + } - public WiremockRequestAssertion containsAllHeaders(Map headers) { - headers.forEach((header, value) -> assertThat(actual.getHeader(header)).isEqualTo(value)); - return this; - } + public WiremockRequestAssertion containsAllHeaders(Map headers) { + headers.forEach((header, value) -> assertThat(actual.getHeader(header)).isEqualTo(value)); + return this; + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/helpers/TraceHeaders.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/helpers/TraceHeaders.java index e8d438dba0..68ce7649e1 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/helpers/TraceHeaders.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/helpers/TraceHeaders.java @@ -1,17 +1,15 @@ package pl.allegro.tech.hermes.integrationtests.helpers; -import pl.allegro.tech.hermes.integrationtests.metadata.TraceContext; - import java.util.Map; +import pl.allegro.tech.hermes.integrationtests.metadata.TraceContext; public class TraceHeaders { - public static Map fromTraceContext(TraceContext traceContext) { - return Map.of( - "Trace-Id", traceContext.traceId(), - "Span-Id", traceContext.spanId(), - "Parent-Span-Id", traceContext.parentSpanId(), - "Trace-Sampled", traceContext.traceSampled(), - "Trace-Reported", traceContext.traceReported() - ); - } + public static Map fromTraceContext(TraceContext traceContext) { + return Map.of( + "Trace-Id", traceContext.traceId(), + "Span-Id", traceContext.spanId(), + "Parent-Span-Id", traceContext.parentSpanId(), + "Trace-Sampled", traceContext.traceSampled(), + "Trace-Reported", traceContext.traceReported()); + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/metadata/TraceContext.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/metadata/TraceContext.java index c12dbb3e0c..df2de78570 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/metadata/TraceContext.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/metadata/TraceContext.java @@ -1,70 +1,69 @@ package pl.allegro.tech.hermes.integrationtests.metadata; import com.google.common.collect.ImmutableMap; - import java.util.Map; import java.util.UUID; -public record TraceContext(String traceId, String spanId, String parentSpanId, String traceSampled, - String traceReported) { +public record TraceContext( + String traceId, String spanId, String parentSpanId, String traceSampled, String traceReported) { - public Map asMap() { + public Map asMap() { - return ImmutableMap.builder() - .put("Trace-Id", traceId) - .put("Span-Id", spanId) - .put("Parent-Span-Id", parentSpanId) - .put("Trace-Sampled", traceSampled) - .put("Trace-Reported", traceReported) - .build(); - } + return ImmutableMap.builder() + .put("Trace-Id", traceId) + .put("Span-Id", spanId) + .put("Parent-Span-Id", parentSpanId) + .put("Trace-Sampled", traceSampled) + .put("Trace-Reported", traceReported) + .build(); + } - public static TraceContext random() { + public static TraceContext random() { - return new Builder() - .withTraceId(UUID.randomUUID().toString()) - .withSpanId(UUID.randomUUID().toString()) - .withParentSpanId(UUID.randomUUID().toString()) - .withTraceSampled("1") - .withTraceReported("0") - .build(); - } + return new Builder() + .withTraceId(UUID.randomUUID().toString()) + .withSpanId(UUID.randomUUID().toString()) + .withParentSpanId(UUID.randomUUID().toString()) + .withTraceSampled("1") + .withTraceReported("0") + .build(); + } - public static class Builder { + public static class Builder { - private String traceId; - private String spanId; - private String parentSpanId; - private String traceSampled; - private String traceReported; + private String traceId; + private String spanId; + private String parentSpanId; + private String traceSampled; + private String traceReported; - public Builder withTraceId(String traceId) { - this.traceId = traceId; - return this; - } + public Builder withTraceId(String traceId) { + this.traceId = traceId; + return this; + } - public Builder withSpanId(String spanId) { - this.spanId = spanId; - return this; - } + public Builder withSpanId(String spanId) { + this.spanId = spanId; + return this; + } - public Builder withParentSpanId(String parentSpanId) { - this.parentSpanId = parentSpanId; - return this; - } + public Builder withParentSpanId(String parentSpanId) { + this.parentSpanId = parentSpanId; + return this; + } - public Builder withTraceSampled(String traceSampled) { - this.traceSampled = traceSampled; - return this; - } + public Builder withTraceSampled(String traceSampled) { + this.traceSampled = traceSampled; + return this; + } - public Builder withTraceReported(String traceReported) { - this.traceReported = traceReported; - return this; - } + public Builder withTraceReported(String traceReported) { + this.traceReported = traceReported; + return this; + } - public TraceContext build() { - return new TraceContext(traceId, spanId, parentSpanId, traceSampled, traceReported); - } + public TraceContext build() { + return new TraceContext(traceId, spanId, parentSpanId, traceSampled, traceReported); } + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/prometheus/PrometheusExtension.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/prometheus/PrometheusExtension.java index b39b5c3f48..77be5dd796 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/prometheus/PrometheusExtension.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/prometheus/PrometheusExtension.java @@ -1,137 +1,143 @@ package pl.allegro.tech.hermes.integrationtests.prometheus; +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; +import static pl.allegro.tech.hermes.management.infrastructure.prometheus.PrometheusClient.forSubscription; +import static pl.allegro.tech.hermes.management.infrastructure.prometheus.PrometheusClient.forSubscriptionStatusCode; +import static pl.allegro.tech.hermes.management.infrastructure.prometheus.PrometheusClient.forTopic; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.github.tomakehurst.wiremock.WireMockServer; import com.github.tomakehurst.wiremock.matching.AnythingPattern; +import java.time.Duration; +import java.util.List; import org.junit.jupiter.api.extension.AfterEachCallback; import org.junit.jupiter.api.extension.BeforeAllCallback; import org.junit.jupiter.api.extension.ExtensionContext; import pl.allegro.tech.hermes.api.SubscriptionName; import pl.allegro.tech.hermes.api.TopicName; -import java.time.Duration; -import java.util.List; - -import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; -import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; -import static com.github.tomakehurst.wiremock.client.WireMock.get; -import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; -import static pl.allegro.tech.hermes.management.infrastructure.prometheus.PrometheusClient.forSubscription; -import static pl.allegro.tech.hermes.management.infrastructure.prometheus.PrometheusClient.forSubscriptionStatusCode; -import static pl.allegro.tech.hermes.management.infrastructure.prometheus.PrometheusClient.forTopic; +public class PrometheusExtension + implements AfterEachCallback, BeforeAllCallback, ExtensionContext.Store.CloseableResource { -public class PrometheusExtension implements AfterEachCallback, BeforeAllCallback, ExtensionContext.Store.CloseableResource { + private static final WireMockServer wiremock = new WireMockServer(0); + private static boolean started = false; + private final ObjectMapper objectMapper = new ObjectMapper(); - private static final WireMockServer wiremock = new WireMockServer(0); - private static boolean started = false; - private final ObjectMapper objectMapper = new ObjectMapper(); - - @Override - public void beforeAll(ExtensionContext context) { - if (!started) { - wiremock.start(); - started = true; - } - } - - @Override - public void afterEach(ExtensionContext context) { - wiremock.resetAll(); - } - - @Override - public void close() { - wiremock.shutdown(); + @Override + public void beforeAll(ExtensionContext context) { + if (!started) { + wiremock.start(); + started = true; } - - public String getEndpoint() { - return "http://localhost:" + wiremock.port(); - } - - @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") - public void stubSubscriptionMetrics(SubscriptionMetrics metrics) { - SubscriptionName subName = metrics.name(); - String deliveredQuery = forSubscription("hermes_consumers_subscription_delivered_total", subName, ""); - String timeoutsQuery = forSubscription("hermes_consumers_subscription_timeouts_total", subName, ""); - String retriesQuery = forSubscription("hermes_consumers_subscription_retries_total", subName, ""); - String throughputQuery = forSubscription("hermes_consumers_subscription_throughput_bytes_total", subName, ""); - String errorsQuery = forSubscription("hermes_consumers_subscription_other_errors_total", subName, ""); - String batchesQuery = forSubscription("hermes_consumers_subscription_batches_total", subName, ""); - String statusCodes2xxQuery = forSubscriptionStatusCode("hermes_consumers_subscription_http_status_codes_total", subName, "2.*", ""); - String statusCodes4xxQuery = forSubscriptionStatusCode("hermes_consumers_subscription_http_status_codes_total", subName, "4.*", ""); - String statusCodes5xxQuery = forSubscriptionStatusCode("hermes_consumers_subscription_http_status_codes_total", subName, "5.*", ""); - - stub(deliveredQuery, metrics.toPrometheusRateResponse()); - stub(timeoutsQuery, metrics.toPrometheusDefaultResponse()); - stub(retriesQuery, metrics.toPrometheusDefaultResponse()); - stub(throughputQuery, metrics.toPrometheusThroughputResponse()); - stub(errorsQuery, metrics.toPrometheusDefaultResponse()); - stub(batchesQuery, metrics.toPrometheusDefaultResponse()); - stub(statusCodes2xxQuery, metrics.toPrometheusStatusCodesResponse()); - stub(statusCodes4xxQuery, metrics.toPrometheusStatusCodesResponse()); - stub(statusCodes5xxQuery, metrics.toPrometheusStatusCodesResponse()); - } - - private void stub(String query, PrometheusResponse response) { - wiremock.addStubMapping( - get(urlPathEqualTo("/api/v1/query")) - .withQueryParam("query", equalTo(query)) - .willReturn( - aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withBody(writeValueAsString(response)) - ) - .build() - ); - } - - public void stubTopicMetrics(TopicMetrics metrics) { - TopicName topicName = metrics.name(); - String requestsQuery = forTopic("hermes_frontend_topic_requests_total", topicName, ""); - String deliveredQuery = forTopic("hermes_consumers_subscription_delivered_total", topicName, ""); - String throughputQuery = forTopic("hermes_frontend_topic_throughput_bytes_total", topicName, ""); - - stub(requestsQuery, metrics.toPrometheusRequestsResponse()); - stub(deliveredQuery, metrics.toDeliveredResponse()); - stub(throughputQuery, metrics.toPrometheusThroughputResponse()); - } - - public void stubDelay(Duration duration) { - var response = new PrometheusResponse("success", new PrometheusResponse.Data("vector", List.of())); - wiremock.addStubMapping( - get(urlPathEqualTo("/api/v1/query")) - .withQueryParam("query", new AnythingPattern()) - .willReturn( - aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withBody(writeValueAsString(response)) - .withFixedDelay((int) duration.toMillis()) - ) - .build() - ); - } - - public void stub500Error() { - wiremock.addStubMapping( - get(urlPathEqualTo("/api/v1/query")) - .withQueryParam("query", new AnythingPattern()) - .willReturn( - aResponse() - .withStatus(500) - .withHeader("Content-Type", "application/json") - ) - .build() - ); - } - - private String writeValueAsString(Object o) { - try { - return objectMapper.writeValueAsString(o); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } + } + + @Override + public void afterEach(ExtensionContext context) { + wiremock.resetAll(); + } + + @Override + public void close() { + wiremock.shutdown(); + } + + public String getEndpoint() { + return "http://localhost:" + wiremock.port(); + } + + @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") + public void stubSubscriptionMetrics(SubscriptionMetrics metrics) { + SubscriptionName subName = metrics.name(); + String deliveredQuery = + forSubscription("hermes_consumers_subscription_delivered_total", subName, ""); + String timeoutsQuery = + forSubscription("hermes_consumers_subscription_timeouts_total", subName, ""); + String retriesQuery = + forSubscription("hermes_consumers_subscription_retries_total", subName, ""); + String throughputQuery = + forSubscription("hermes_consumers_subscription_throughput_bytes_total", subName, ""); + String errorsQuery = + forSubscription("hermes_consumers_subscription_other_errors_total", subName, ""); + String batchesQuery = + forSubscription("hermes_consumers_subscription_batches_total", subName, ""); + String statusCodes2xxQuery = + forSubscriptionStatusCode( + "hermes_consumers_subscription_http_status_codes_total", subName, "2.*", ""); + String statusCodes4xxQuery = + forSubscriptionStatusCode( + "hermes_consumers_subscription_http_status_codes_total", subName, "4.*", ""); + String statusCodes5xxQuery = + forSubscriptionStatusCode( + "hermes_consumers_subscription_http_status_codes_total", subName, "5.*", ""); + + stub(deliveredQuery, metrics.toPrometheusRateResponse()); + stub(timeoutsQuery, metrics.toPrometheusDefaultResponse()); + stub(retriesQuery, metrics.toPrometheusDefaultResponse()); + stub(throughputQuery, metrics.toPrometheusThroughputResponse()); + stub(errorsQuery, metrics.toPrometheusDefaultResponse()); + stub(batchesQuery, metrics.toPrometheusDefaultResponse()); + stub(statusCodes2xxQuery, metrics.toPrometheusStatusCodesResponse()); + stub(statusCodes4xxQuery, metrics.toPrometheusStatusCodesResponse()); + stub(statusCodes5xxQuery, metrics.toPrometheusStatusCodesResponse()); + } + + private void stub(String query, PrometheusResponse response) { + wiremock.addStubMapping( + get(urlPathEqualTo("/api/v1/query")) + .withQueryParam("query", equalTo(query)) + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody(writeValueAsString(response))) + .build()); + } + + public void stubTopicMetrics(TopicMetrics metrics) { + TopicName topicName = metrics.name(); + String requestsQuery = forTopic("hermes_frontend_topic_requests_total", topicName, ""); + String deliveredQuery = + forTopic("hermes_consumers_subscription_delivered_total", topicName, ""); + String throughputQuery = + forTopic("hermes_frontend_topic_throughput_bytes_total", topicName, ""); + + stub(requestsQuery, metrics.toPrometheusRequestsResponse()); + stub(deliveredQuery, metrics.toDeliveredResponse()); + stub(throughputQuery, metrics.toPrometheusThroughputResponse()); + } + + public void stubDelay(Duration duration) { + var response = + new PrometheusResponse("success", new PrometheusResponse.Data("vector", List.of())); + wiremock.addStubMapping( + get(urlPathEqualTo("/api/v1/query")) + .withQueryParam("query", new AnythingPattern()) + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody(writeValueAsString(response)) + .withFixedDelay((int) duration.toMillis())) + .build()); + } + + public void stub500Error() { + wiremock.addStubMapping( + get(urlPathEqualTo("/api/v1/query")) + .withQueryParam("query", new AnythingPattern()) + .willReturn(aResponse().withStatus(500).withHeader("Content-Type", "application/json")) + .build()); + } + + private String writeValueAsString(Object o) { + try { + return objectMapper.writeValueAsString(o); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); } + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/prometheus/PrometheusResponse.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/prometheus/PrometheusResponse.java index df4bd8757d..3e92bbf33e 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/prometheus/PrometheusResponse.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/prometheus/PrometheusResponse.java @@ -1,16 +1,13 @@ package pl.allegro.tech.hermes.integrationtests.prometheus; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.List; -record PrometheusResponse(@JsonProperty("status") String status, - @JsonProperty("data") Data data) { +record PrometheusResponse(@JsonProperty("status") String status, @JsonProperty("data") Data data) { - record Data(@JsonProperty("resultType") String resultType, - @JsonProperty("result") List results) { - } + record Data( + @JsonProperty("resultType") String resultType, + @JsonProperty("result") List results) {} - record Result(@JsonProperty("value") List values) { - } + record Result(@JsonProperty("value") List values) {} } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/prometheus/SubscriptionMetrics.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/prometheus/SubscriptionMetrics.java index 438def3702..757b400385 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/prometheus/SubscriptionMetrics.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/prometheus/SubscriptionMetrics.java @@ -1,89 +1,74 @@ package pl.allegro.tech.hermes.integrationtests.prometheus; -import pl.allegro.tech.hermes.api.SubscriptionName; - import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import pl.allegro.tech.hermes.api.SubscriptionName; -public record SubscriptionMetrics(SubscriptionName name, int rate, int throughput, - Map ratePerStatusCode) { +public record SubscriptionMetrics( + SubscriptionName name, int rate, int throughput, Map ratePerStatusCode) { - private static final String TIMESTAMP = "1396860420"; + private static final String TIMESTAMP = "1396860420"; - public static SubscriptionMetricsBuilder subscriptionMetrics(SubscriptionName name) { - return new SubscriptionMetricsBuilder(name); - } + public static SubscriptionMetricsBuilder subscriptionMetrics(SubscriptionName name) { + return new SubscriptionMetricsBuilder(name); + } - PrometheusResponse toPrometheusRateResponse() { - List results = new ArrayList<>(); - results.add( - new PrometheusResponse.Result( - List.of(TIMESTAMP, String.valueOf(rate))) - ); - return new PrometheusResponse("success", new PrometheusResponse.Data("vector", results)); - } + PrometheusResponse toPrometheusRateResponse() { + List results = new ArrayList<>(); + results.add(new PrometheusResponse.Result(List.of(TIMESTAMP, String.valueOf(rate)))); + return new PrometheusResponse("success", new PrometheusResponse.Data("vector", results)); + } - PrometheusResponse toPrometheusThroughputResponse() { - List results = new ArrayList<>(); - results.add( - new PrometheusResponse.Result( - List.of(TIMESTAMP, String.valueOf(throughput)) - ) - ); - return new PrometheusResponse("success", new PrometheusResponse.Data("vector", results)); - } + PrometheusResponse toPrometheusThroughputResponse() { + List results = new ArrayList<>(); + results.add(new PrometheusResponse.Result(List.of(TIMESTAMP, String.valueOf(throughput)))); + return new PrometheusResponse("success", new PrometheusResponse.Data("vector", results)); + } - PrometheusResponse toPrometheusStatusCodesResponse() { - List results = new ArrayList<>(); - ratePerStatusCode.forEach((code, rate) -> results.add( - new PrometheusResponse.Result( - List.of(TIMESTAMP, String.valueOf(rate)) - ) - ) - ); - return new PrometheusResponse("success", new PrometheusResponse.Data("vector", results)); - } + PrometheusResponse toPrometheusStatusCodesResponse() { + List results = new ArrayList<>(); + ratePerStatusCode.forEach( + (code, rate) -> + results.add(new PrometheusResponse.Result(List.of(TIMESTAMP, String.valueOf(rate))))); + return new PrometheusResponse("success", new PrometheusResponse.Data("vector", results)); + } - PrometheusResponse toPrometheusDefaultResponse() { - List results = new ArrayList<>(); - ratePerStatusCode.forEach((code, rate) -> results.add( - new PrometheusResponse.Result( - List.of(TIMESTAMP, "0.0") - ) - ) - ); - return new PrometheusResponse("success", new PrometheusResponse.Data("vector", results)); - } + PrometheusResponse toPrometheusDefaultResponse() { + List results = new ArrayList<>(); + ratePerStatusCode.forEach( + (code, rate) -> results.add(new PrometheusResponse.Result(List.of(TIMESTAMP, "0.0")))); + return new PrometheusResponse("success", new PrometheusResponse.Data("vector", results)); + } - public static class SubscriptionMetricsBuilder { - private final SubscriptionName name; - private int rate = 0; - private int throughput = 0; - private final Map ratePerStatusCode = new HashMap<>(); + public static class SubscriptionMetricsBuilder { + private final SubscriptionName name; + private int rate = 0; + private int throughput = 0; + private final Map ratePerStatusCode = new HashMap<>(); - private SubscriptionMetricsBuilder(SubscriptionName name) { - this.name = name; - } + private SubscriptionMetricsBuilder(SubscriptionName name) { + this.name = name; + } - public SubscriptionMetricsBuilder withRate(int rate) { - this.rate = rate; - return this; - } + public SubscriptionMetricsBuilder withRate(int rate) { + this.rate = rate; + return this; + } - public SubscriptionMetricsBuilder with500Rate(int rate) { - ratePerStatusCode.put("500", rate); - return this; - } + public SubscriptionMetricsBuilder with500Rate(int rate) { + ratePerStatusCode.put("500", rate); + return this; + } - public SubscriptionMetricsBuilder withThroughput(int throughput) { - this.throughput = throughput; - return this; - } + public SubscriptionMetricsBuilder withThroughput(int throughput) { + this.throughput = throughput; + return this; + } - public SubscriptionMetrics build() { - return new SubscriptionMetrics(name, rate, throughput, ratePerStatusCode); - } + public SubscriptionMetrics build() { + return new SubscriptionMetrics(name, rate, throughput, ratePerStatusCode); } + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/prometheus/TopicMetrics.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/prometheus/TopicMetrics.java index 3481e953ea..321292f5e7 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/prometheus/TopicMetrics.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/prometheus/TopicMetrics.java @@ -1,86 +1,69 @@ package pl.allegro.tech.hermes.integrationtests.prometheus; -import pl.allegro.tech.hermes.api.TopicName; - import java.util.List; +import pl.allegro.tech.hermes.api.TopicName; public record TopicMetrics(TopicName name, int rate, int deliveryRate, int throughput) { - private static final String TIMESTAMP = "1396860420"; + private static final String TIMESTAMP = "1396860420"; - public static TopicMetricsBuilder topicMetrics(TopicName name) { - return new TopicMetricsBuilder(name); - } + public static TopicMetricsBuilder topicMetrics(TopicName name) { + return new TopicMetricsBuilder(name); + } - PrometheusResponse toPrometheusRequestsResponse() { - return new PrometheusResponse( - "success", - new PrometheusResponse.Data( - "vector", - List.of( - new PrometheusResponse.Result( - List.of(TIMESTAMP, String.valueOf(rate)) - ) - ) - ) - ); - } + PrometheusResponse toPrometheusRequestsResponse() { + return new PrometheusResponse( + "success", + new PrometheusResponse.Data( + "vector", + List.of(new PrometheusResponse.Result(List.of(TIMESTAMP, String.valueOf(rate)))))); + } - PrometheusResponse toDeliveredResponse() { - return new PrometheusResponse( - "success", - new PrometheusResponse.Data( - "vector", - List.of( - new PrometheusResponse.Result( - List.of(TIMESTAMP, String.valueOf(deliveryRate)) - ) - ) - ) - ); - } + PrometheusResponse toDeliveredResponse() { + return new PrometheusResponse( + "success", + new PrometheusResponse.Data( + "vector", + List.of( + new PrometheusResponse.Result(List.of(TIMESTAMP, String.valueOf(deliveryRate)))))); + } - PrometheusResponse toPrometheusThroughputResponse() { - return new PrometheusResponse( - "success", - new PrometheusResponse.Data( - "vector", - List.of( - new PrometheusResponse.Result( - List.of(TIMESTAMP, String.valueOf(throughput)) - ) - ) - ) - ); - } + PrometheusResponse toPrometheusThroughputResponse() { + return new PrometheusResponse( + "success", + new PrometheusResponse.Data( + "vector", + List.of( + new PrometheusResponse.Result(List.of(TIMESTAMP, String.valueOf(throughput)))))); + } - public static class TopicMetricsBuilder { - private final TopicName name; - private int rate = 0; - private int deliveryRate = 0; - private int throughput = 0; + public static class TopicMetricsBuilder { + private final TopicName name; + private int rate = 0; + private int deliveryRate = 0; + private int throughput = 0; - private TopicMetricsBuilder(TopicName name) { - this.name = name; - } + private TopicMetricsBuilder(TopicName name) { + this.name = name; + } - public TopicMetricsBuilder withRate(int rate) { - this.rate = rate; - return this; - } + public TopicMetricsBuilder withRate(int rate) { + this.rate = rate; + return this; + } - public TopicMetricsBuilder withDeliveryRate(int deliveryRate) { - this.deliveryRate = deliveryRate; - return this; - } + public TopicMetricsBuilder withDeliveryRate(int deliveryRate) { + this.deliveryRate = deliveryRate; + return this; + } - public TopicMetricsBuilder withThroughput(int throughput) { - this.throughput = throughput; - return this; - } + public TopicMetricsBuilder withThroughput(int throughput) { + this.throughput = throughput; + return this; + } - public TopicMetrics build() { - return new TopicMetrics(name, rate, deliveryRate, throughput); - } + public TopicMetrics build() { + return new TopicMetrics(name, rate, deliveryRate, throughput); } + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/GooglePubSubExtension.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/GooglePubSubExtension.java index 2943486e47..6b7d3b136d 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/GooglePubSubExtension.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/GooglePubSubExtension.java @@ -1,40 +1,39 @@ package pl.allegro.tech.hermes.integrationtests.setup; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import org.junit.jupiter.api.extension.AfterAllCallback; import org.junit.jupiter.api.extension.BeforeAllCallback; import org.junit.jupiter.api.extension.ExtensionContext; import pl.allegro.tech.hermes.integrationtests.subscriber.TestGooglePubSubSubscriber; import pl.allegro.tech.hermes.test.helper.containers.GooglePubSubContainer; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - public class GooglePubSubExtension implements BeforeAllCallback, AfterAllCallback { - private final GooglePubSubContainer googlePubSubContainer = new GooglePubSubContainer(); - private final List subscribers = new ArrayList<>(); - - @Override - public void beforeAll(ExtensionContext context) { - googlePubSubContainer.start(); - } - - @Override - public void afterAll(ExtensionContext context) { - for (TestGooglePubSubSubscriber subscriber : subscribers) { - subscriber.stop(); - } - googlePubSubContainer.stop(); - } + private final GooglePubSubContainer googlePubSubContainer = new GooglePubSubContainer(); + private final List subscribers = new ArrayList<>(); - public TestGooglePubSubSubscriber subscriber() throws IOException { - TestGooglePubSubSubscriber subscriber = new TestGooglePubSubSubscriber(getEmulatorEndpoint()); - subscribers.add(subscriber); - return subscriber; - } + @Override + public void beforeAll(ExtensionContext context) { + googlePubSubContainer.start(); + } - public String getEmulatorEndpoint() { - return googlePubSubContainer.getEmulatorEndpoint(); + @Override + public void afterAll(ExtensionContext context) { + for (TestGooglePubSubSubscriber subscriber : subscribers) { + subscriber.stop(); } + googlePubSubContainer.stop(); + } + + public TestGooglePubSubSubscriber subscriber() throws IOException { + TestGooglePubSubSubscriber subscriber = new TestGooglePubSubSubscriber(getEmulatorEndpoint()); + subscribers.add(subscriber); + return subscriber; + } + + public String getEmulatorEndpoint() { + return googlePubSubContainer.getEmulatorEndpoint(); + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/HermesConsumersTestApp.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/HermesConsumersTestApp.java index f6151023e0..1fb27df659 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/HermesConsumersTestApp.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/HermesConsumersTestApp.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.integrationtests.setup; +import java.time.Duration; +import java.util.List; import org.springframework.boot.WebApplicationType; import org.springframework.boot.builder.SpringApplicationBuilder; import pl.allegro.tech.hermes.consumers.HermesConsumers; @@ -9,91 +11,89 @@ import pl.allegro.tech.hermes.test.helper.containers.ZookeeperContainer; import pl.allegro.tech.hermes.test.helper.environment.HermesTestApp; -import java.time.Duration; -import java.util.List; - public class HermesConsumersTestApp implements HermesTestApp { - private final ZookeeperContainer hermesZookeeper; - private final KafkaContainerCluster kafka; - private final ConfluentSchemaRegistryContainer schemaRegistry; + private final ZookeeperContainer hermesZookeeper; + private final KafkaContainerCluster kafka; + private final ConfluentSchemaRegistryContainer schemaRegistry; - private int port = -1; - private SpringApplicationBuilder app = null; - private List currentArgs = List.of(); - private GooglePubSubExtension googlePubSub = null; + private int port = -1; + private SpringApplicationBuilder app = null; + private List currentArgs = List.of(); + private GooglePubSubExtension googlePubSub = null; - public HermesConsumersTestApp(ZookeeperContainer hermesZookeeper, - KafkaContainerCluster kafka, - ConfluentSchemaRegistryContainer schemaRegistry) { - this.hermesZookeeper = hermesZookeeper; - this.kafka = kafka; - this.schemaRegistry = schemaRegistry; - } + public HermesConsumersTestApp( + ZookeeperContainer hermesZookeeper, + KafkaContainerCluster kafka, + ConfluentSchemaRegistryContainer schemaRegistry) { + this.hermesZookeeper = hermesZookeeper; + this.kafka = kafka; + this.schemaRegistry = schemaRegistry; + } - @Override - public HermesTestApp start() { - app = new SpringApplicationBuilder(HermesConsumers.class) - .web(WebApplicationType.NONE); - currentArgs = createArgs(); - app.run(currentArgs.toArray(new String[0])); - port = app.context().getBean(ConsumerHttpServer.class).getPort(); - return this; - } + @Override + public HermesTestApp start() { + app = new SpringApplicationBuilder(HermesConsumers.class).web(WebApplicationType.NONE); + currentArgs = createArgs(); + app.run(currentArgs.toArray(new String[0])); + port = app.context().getBean(ConsumerHttpServer.class).getPort(); + return this; + } - @Override - public void stop() { - if (app != null) { - app.context().close(); - app = null; - } + @Override + public void stop() { + if (app != null) { + app.context().close(); + app = null; } + } - @Override - public int getPort() { - if (port == -1) { - throw new IllegalStateException("hermes-consumers port hasn't been initialized"); - } - return port; + @Override + public int getPort() { + if (port == -1) { + throw new IllegalStateException("hermes-consumers port hasn't been initialized"); } + return port; + } - @Override - public boolean shouldBeRestarted() { - List args = createArgs(); - return !args.equals(currentArgs); - } + @Override + public boolean shouldBeRestarted() { + List args = createArgs(); + return !args.equals(currentArgs); + } - private List createArgs() { - return List.of( - "--spring.profiles.active=integration", - "--consumer.healthCheckPort=0", - "--consumer.kafka.namespace=itTest", - "--consumer.kafka.clusters.[0].brokerList=" + kafka.getBootstrapServersForExternalClients(), - "--consumer.kafka.clusters.[0].clusterName=" + "primary-dc", - "--consumer.zookeeper.clusters.[0].connectionString=" + hermesZookeeper.getConnectionString(), - "--consumer.schema.repository.serverUrl=" + schemaRegistry.getUrl(), - "--consumer.backgroundSupervisor.interval=" + Duration.ofMillis(100), - "--consumer.workload.rebalanceInterval=" + Duration.ofSeconds(1), - "--consumer.commit.offset.period=" + Duration.ofSeconds(1), - "--consumer.metrics.micrometer.reportPeriod=" + Duration.ofSeconds(5), - "--consumer.schema.cache.enabled=true", - "--consumer.google.pubsub.sender.transportChannelProviderAddress=" + getGooglePubSubEndpoint() - ); - } + private List createArgs() { + return List.of( + "--spring.profiles.active=integration", + "--consumer.healthCheckPort=0", + "--consumer.kafka.namespace=itTest", + "--consumer.kafka.clusters.[0].brokerList=" + kafka.getBootstrapServersForExternalClients(), + "--consumer.kafka.clusters.[0].clusterName=" + "primary-dc", + "--consumer.zookeeper.clusters.[0].connectionString=" + + hermesZookeeper.getConnectionString(), + "--consumer.schema.repository.serverUrl=" + schemaRegistry.getUrl(), + "--consumer.backgroundSupervisor.interval=" + Duration.ofMillis(100), + "--consumer.workload.rebalanceInterval=" + Duration.ofSeconds(1), + "--consumer.commit.offset.period=" + Duration.ofSeconds(1), + "--consumer.metrics.micrometer.reportPeriod=" + Duration.ofSeconds(5), + "--consumer.schema.cache.enabled=true", + "--consumer.google.pubsub.sender.transportChannelProviderAddress=" + + getGooglePubSubEndpoint()); + } - private String getGooglePubSubEndpoint() { - if (googlePubSub == null) { - return "integration"; - } - return googlePubSub.getEmulatorEndpoint(); + private String getGooglePubSubEndpoint() { + if (googlePubSub == null) { + return "integration"; } + return googlePubSub.getEmulatorEndpoint(); + } - void withGooglePubSubEndpoint(GooglePubSubExtension googlePubSub) { - this.googlePubSub = googlePubSub; - } + void withGooglePubSubEndpoint(GooglePubSubExtension googlePubSub) { + this.googlePubSub = googlePubSub; + } - @Override - public void restoreDefaultSettings() { - googlePubSub = null; - } + @Override + public void restoreDefaultSettings() { + googlePubSub = null; + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/HermesExtension.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/HermesExtension.java index ff1e4bb4ed..6f008480bd 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/HermesExtension.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/HermesExtension.java @@ -1,5 +1,12 @@ package pl.allegro.tech.hermes.integrationtests.setup; +import static org.awaitility.Awaitility.waitAtMost; +import static pl.allegro.tech.hermes.integrationtests.setup.HermesManagementTestApp.AUDIT_EVENT_PATH; +import static pl.allegro.tech.hermes.test.helper.endpoint.TimeoutAdjuster.adjust; + +import java.time.Duration; +import java.util.List; +import java.util.stream.Stream; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.extension.AfterAllCallback; import org.junit.jupiter.api.extension.BeforeAllCallback; @@ -26,150 +33,150 @@ import pl.allegro.tech.hermes.test.helper.containers.ZookeeperContainer; import pl.allegro.tech.hermes.test.helper.environment.HermesTestApp; -import java.time.Duration; -import java.util.List; -import java.util.stream.Stream; - -import static org.awaitility.Awaitility.waitAtMost; -import static pl.allegro.tech.hermes.integrationtests.setup.HermesManagementTestApp.AUDIT_EVENT_PATH; -import static pl.allegro.tech.hermes.test.helper.endpoint.TimeoutAdjuster.adjust; - -public class HermesExtension implements BeforeAllCallback, AfterAllCallback, ExtensionContext.Store.CloseableResource { - - private static final Logger logger = LoggerFactory.getLogger(HermesExtension.class); - - public static final TestSubscribersExtension auditEventsReceiver = new TestSubscribersExtension(); - - private static final ZookeeperContainer hermesZookeeper = new ZookeeperContainer("HermesZookeeper"); - private static final KafkaContainerCluster kafka = new KafkaContainerCluster(1); - public static final ConfluentSchemaRegistryContainer schemaRegistry = new ConfluentSchemaRegistryContainer() - .withKafkaCluster(kafka); - private static final HermesConsumersTestApp consumers = new HermesConsumersTestApp(hermesZookeeper, kafka, schemaRegistry); - private static final HermesManagementTestApp management = new HermesManagementTestApp(hermesZookeeper, kafka, schemaRegistry); - private static final HermesFrontendTestApp frontend = new HermesFrontendTestApp(hermesZookeeper, kafka, schemaRegistry); - private HermesTestClient hermesTestClient; - private HermesInitHelper hermesInitHelper; - private static final RequestUser testUser = new TestUser(); - - private static boolean started = false; - - public static TestSubscriber auditEvents; - - public static BrokerOperations brokerOperations; - - @Override - public void beforeAll(ExtensionContext context) { - if (!started) { - Stream.of(hermesZookeeper, kafka).parallel().forEach(Startable::start); - schemaRegistry.start(); - management.addEventAuditorListener(auditEventsReceiver.getPort()); - management.start(); - Stream.of(consumers, frontend).forEach(HermesTestApp::start); - started = true; - } - Stream.of(management, consumers, frontend).forEach(app -> { - if (app.shouldBeRestarted()) { +public class HermesExtension + implements BeforeAllCallback, AfterAllCallback, ExtensionContext.Store.CloseableResource { + + private static final Logger logger = LoggerFactory.getLogger(HermesExtension.class); + + public static final TestSubscribersExtension auditEventsReceiver = new TestSubscribersExtension(); + + private static final ZookeeperContainer hermesZookeeper = + new ZookeeperContainer("HermesZookeeper"); + private static final KafkaContainerCluster kafka = new KafkaContainerCluster(1); + public static final ConfluentSchemaRegistryContainer schemaRegistry = + new ConfluentSchemaRegistryContainer().withKafkaCluster(kafka); + private static final HermesConsumersTestApp consumers = + new HermesConsumersTestApp(hermesZookeeper, kafka, schemaRegistry); + private static final HermesManagementTestApp management = + new HermesManagementTestApp(hermesZookeeper, kafka, schemaRegistry); + private static final HermesFrontendTestApp frontend = + new HermesFrontendTestApp(hermesZookeeper, kafka, schemaRegistry); + private HermesTestClient hermesTestClient; + private HermesInitHelper hermesInitHelper; + private static final RequestUser testUser = new TestUser(); + + private static boolean started = false; + + public static TestSubscriber auditEvents; + + public static BrokerOperations brokerOperations; + + @Override + public void beforeAll(ExtensionContext context) { + if (!started) { + Stream.of(hermesZookeeper, kafka).parallel().forEach(Startable::start); + schemaRegistry.start(); + management.addEventAuditorListener(auditEventsReceiver.getPort()); + management.start(); + Stream.of(consumers, frontend).forEach(HermesTestApp::start); + started = true; + } + Stream.of(management, consumers, frontend) + .forEach( + app -> { + if (app.shouldBeRestarted()) { app.stop(); app.start(); - } - }); - hermesTestClient = new HermesTestClient(management.getPort(), frontend.getPort(), consumers.getPort()); - hermesInitHelper = new HermesInitHelper(management.getPort()); - auditEvents = auditEventsReceiver.createSubscriberWithStrictPath(200, AUDIT_EVENT_PATH); - brokerOperations = new BrokerOperations(kafka.getBootstrapServersForExternalClients(), "itTest"); - } - - @Override - public void close() { - Stream.of(management, consumers, frontend).parallel().forEach(HermesTestApp::stop); - Stream.of(hermesZookeeper, kafka, schemaRegistry).parallel().forEach(Startable::stop); - started = false; - } - - public int getFrontendPort() { - return frontend.getPort(); - } - - public HermesTestClient api() { - return hermesTestClient; - } - - public HermesInitHelper initHelper() { - return hermesInitHelper; - } - - public void cutOffConnectionsBetweenBrokersAndClients() { - kafka.cutOffConnectionsBetweenBrokersAndClients(); - } - - public void restoreConnectionsBetweenBrokersAndClients() { - kafka.restoreConnectionsBetweenBrokersAndClients(); - } - - private void removeSubscriptions() { - SubscriptionService service = management.subscriptionService(); - List subscriptions = service.getAllSubscriptions(); - for (Subscription subscription : subscriptions) { - service.removeSubscription(subscription.getTopicName(), subscription.getName(), testUser); - } - - waitAtMost(adjust(Duration.ofMinutes(1))).untilAsserted(() -> - Assertions.assertThat(service.getAllSubscriptions().size()).isEqualTo(0) - ); - } - - private void removeTopics() { - TopicService service = management.topicService(); - List topics = service.getAllTopics(); - for (Topic topic : topics) { - service.removeTopicWithSchema(topic, testUser); - } - - waitAtMost(adjust(Duration.ofMinutes(1))).untilAsserted(() -> - Assertions.assertThat(service.getAllTopics().size()).isEqualTo(0) - ); - } - - private void removeGroups() { - GroupService service = management.groupService(); - List groups = service.listGroups(); - for (Group group : groups) { - try { - service.removeGroup(group.getGroupName(), testUser); - } catch (GroupNotEmptyException e) { - logger.warn("Error during removing group: {}", group, e); - } - } - } - - public void clearManagementData() { - removeSubscriptions(); - removeTopics(); - removeGroups(); - } - - public HermesExtension withPrometheus(PrometheusExtension prometheus) { - management.withPrometheus(prometheus); - return this; - } - - public HermesExtension withGooglePubSub(GooglePubSubExtension googlePubSub) { - consumers.withGooglePubSubEndpoint(googlePubSub); - return this; - } - - public HermesExtension withFrontendProperty(String name, Object value) { - frontend.withProperty(name, value); - return this; - } - - public HermesExtension withFrontendProfile(String profile) { - frontend.withSpringProfile(profile); - return this; - } - - @Override - public void afterAll(ExtensionContext context) { - Stream.of(management, consumers, frontend).forEach(HermesTestApp::restoreDefaultSettings); - } + } + }); + hermesTestClient = + new HermesTestClient(management.getPort(), frontend.getPort(), consumers.getPort()); + hermesInitHelper = new HermesInitHelper(management.getPort()); + auditEvents = auditEventsReceiver.createSubscriberWithStrictPath(200, AUDIT_EVENT_PATH); + brokerOperations = + new BrokerOperations(kafka.getBootstrapServersForExternalClients(), "itTest"); + } + + @Override + public void close() { + Stream.of(management, consumers, frontend).parallel().forEach(HermesTestApp::stop); + Stream.of(hermesZookeeper, kafka, schemaRegistry).parallel().forEach(Startable::stop); + started = false; + } + + public int getFrontendPort() { + return frontend.getPort(); + } + + public HermesTestClient api() { + return hermesTestClient; + } + + public HermesInitHelper initHelper() { + return hermesInitHelper; + } + + public void cutOffConnectionsBetweenBrokersAndClients() { + kafka.cutOffConnectionsBetweenBrokersAndClients(); + } + + public void restoreConnectionsBetweenBrokersAndClients() { + kafka.restoreConnectionsBetweenBrokersAndClients(); + } + + private void removeSubscriptions() { + SubscriptionService service = management.subscriptionService(); + List subscriptions = service.getAllSubscriptions(); + for (Subscription subscription : subscriptions) { + service.removeSubscription(subscription.getTopicName(), subscription.getName(), testUser); + } + + waitAtMost(adjust(Duration.ofMinutes(1))) + .untilAsserted( + () -> Assertions.assertThat(service.getAllSubscriptions().size()).isEqualTo(0)); + } + + private void removeTopics() { + TopicService service = management.topicService(); + List topics = service.getAllTopics(); + for (Topic topic : topics) { + service.removeTopicWithSchema(topic, testUser); + } + + waitAtMost(adjust(Duration.ofMinutes(1))) + .untilAsserted(() -> Assertions.assertThat(service.getAllTopics().size()).isEqualTo(0)); + } + + private void removeGroups() { + GroupService service = management.groupService(); + List groups = service.listGroups(); + for (Group group : groups) { + try { + service.removeGroup(group.getGroupName(), testUser); + } catch (GroupNotEmptyException e) { + logger.warn("Error during removing group: {}", group, e); + } + } + } + + public void clearManagementData() { + removeSubscriptions(); + removeTopics(); + removeGroups(); + } + + public HermesExtension withPrometheus(PrometheusExtension prometheus) { + management.withPrometheus(prometheus); + return this; + } + + public HermesExtension withGooglePubSub(GooglePubSubExtension googlePubSub) { + consumers.withGooglePubSubEndpoint(googlePubSub); + return this; + } + + public HermesExtension withFrontendProperty(String name, Object value) { + frontend.withProperty(name, value); + return this; + } + + public HermesExtension withFrontendProfile(String profile) { + frontend.withSpringProfile(profile); + return this; + } + + @Override + public void afterAll(ExtensionContext context) { + Stream.of(management, consumers, frontend).forEach(HermesTestApp::restoreDefaultSettings); + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/HermesFrontendTestApp.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/HermesFrontendTestApp.java index 2a13efbaa9..bac6553a69 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/HermesFrontendTestApp.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/HermesFrontendTestApp.java @@ -1,20 +1,5 @@ package pl.allegro.tech.hermes.integrationtests.setup; -import org.springframework.boot.WebApplicationType; -import org.springframework.boot.builder.SpringApplicationBuilder; -import pl.allegro.tech.hermes.frontend.HermesFrontend; -import pl.allegro.tech.hermes.frontend.server.HermesServer; -import pl.allegro.tech.hermes.test.helper.containers.ConfluentSchemaRegistryContainer; -import pl.allegro.tech.hermes.test.helper.containers.KafkaContainerCluster; -import pl.allegro.tech.hermes.test.helper.containers.ZookeeperContainer; -import pl.allegro.tech.hermes.test.helper.environment.HermesTestApp; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.AUTH_PASSWORD; import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.AUTH_USERNAME; import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_FORCE_TOPIC_MAX_MESSAGE_SIZE; @@ -38,168 +23,187 @@ import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.SPRING_PROFILES_ACTIVE; import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.ZOOKEEPER_CONNECTION_STRING; -public class HermesFrontendTestApp implements HermesTestApp { - - private final ZookeeperContainer hermesZookeeper; - private final Map kafkaClusters; - private final ConfluentSchemaRegistryContainer schemaRegistry; - private SpringApplicationBuilder app; - - private int port = -1; - private boolean kafkaCheckEnabled = false; - private Duration metadataMaxAge = Duration.ofMinutes(5); - private Duration readinessCheckInterval = Duration.ofSeconds(1); - private final Map extraArgs = new HashMap<>(); - private final List profiles = new ArrayList<>(List.of("integration")); - private List currentArgs = List.of(); - - public HermesFrontendTestApp(ZookeeperContainer hermesZookeeper, - KafkaContainerCluster kafka, - ConfluentSchemaRegistryContainer schemaRegistry) { - this.hermesZookeeper = hermesZookeeper; - this.schemaRegistry = schemaRegistry; - this.kafkaClusters = Map.of("dc", kafka); - - } - public HermesFrontendTestApp(ZookeeperContainer hermesZookeeper, - Map kafkaClusters, - ConfluentSchemaRegistryContainer schemaRegistry) { - this.hermesZookeeper = hermesZookeeper; - this.schemaRegistry = schemaRegistry; - this.kafkaClusters = kafkaClusters; - } - - private String kafkaClusterProperty(int index, String name) { - return String.format("frontend.kafka.clusters[%d].%s", index, name); - } - - public HermesFrontendTestApp withProperty(String name, Object value) { - this.extraArgs.put(name, value); - return this; - } - - public HermesFrontendTestApp withSpringProfile(String profile) { - profiles.add(profile); - return this; - } - - private List createArgs() { - Map args = new HashMap<>(); - args.put(SPRING_PROFILES_ACTIVE, String.join(",", profiles)); - args.put(FRONTEND_PORT, 0); - - args.put(KAFKA_NAMESPACE, "itTest"); - - var i = 0; - for (var entry : kafkaClusters.entrySet()) { - args.put(kafkaClusterProperty(i, "datacenter"), entry.getKey()); - args.put(kafkaClusterProperty(i, "brokerList"), entry.getValue().getBootstrapServersForExternalClients()); - i++; - } - - args.put(ZOOKEEPER_CONNECTION_STRING, hermesZookeeper.getConnectionString()); - - args.put(SCHEMA_CACHE_ENABLED, true); - args.put(SCHEMA_REPOSITORY_SERVER_URL, schemaRegistry.getUrl()); - - args.put(FRONTEND_READINESS_CHECK_KAFKA_CHECK_ENABLED, kafkaCheckEnabled); - args.put(FRONTEND_READINESS_CHECK_ENABLED, true); - args.put(FRONTEND_READINESS_CHECK_INTERVAL_SECONDS, readinessCheckInterval); - - args.put(FRONTEND_HEADER_PROPAGATION_ENABLED, true); - args.put(FRONTEND_HEADER_PROPAGATION_ALLOWED, "trace-id, span-id, parent-span-id, trace-sampled, trace-reported"); - - args.put(KAFKA_PRODUCER_METADATA_MAX_AGE, metadataMaxAge); - - args.put(FRONTEND_FORCE_TOPIC_MAX_MESSAGE_SIZE, true); - args.put(FRONTEND_IDLE_TIMEOUT, Duration.ofSeconds(2)); - - args.put(FRONTEND_THROUGHPUT_TYPE, "fixed"); - args.put(FRONTEND_THROUGHPUT_FIXED_MAX, 50 * 1024L); - - args.put(FRONTEND_GRACEFUL_SHUTDOWN_ENABLED, false); - - args.put(METRICS_MICROMETER_REPORT_PERIOD, Duration.ofSeconds(1)); - - args.put(FRONTEND_MESSAGE_PREVIEW_ENABLED, true); - args.put(FRONTEND_MESSAGE_PREVIEW_LOG_PERSIST_PERIOD, Duration.ofSeconds(1)); - - args.put(AUTH_USERNAME, "username"); - args.put(AUTH_PASSWORD, "password"); - - args.putAll(extraArgs); - - return args.entrySet().stream().map(e -> getArgument(e.getKey(), e.getValue())).toList(); - } - - @Override - public HermesTestApp start() { - app = new SpringApplicationBuilder(HermesFrontend.class) - .web(WebApplicationType.NONE); - currentArgs = createArgs(); - app.run(currentArgs.toArray(new String[0])); - port = app.context().getBean(HermesServer.class).getPort(); - return this; - } - - @Override - public void stop() { - if (app != null) { - app.context().close(); - app = null; - } - } - - @Override - public int getPort() { - if (port == -1) { - throw new IllegalStateException("hermes-frontend port hasn't been initialized"); - } - return port; - } - - public int getSSLPort() { - return app.context().getBean(HermesServer.class).getSSLPort(); - } +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.builder.SpringApplicationBuilder; +import pl.allegro.tech.hermes.frontend.HermesFrontend; +import pl.allegro.tech.hermes.frontend.server.HermesServer; +import pl.allegro.tech.hermes.test.helper.containers.ConfluentSchemaRegistryContainer; +import pl.allegro.tech.hermes.test.helper.containers.KafkaContainerCluster; +import pl.allegro.tech.hermes.test.helper.containers.ZookeeperContainer; +import pl.allegro.tech.hermes.test.helper.environment.HermesTestApp; - public T getBean(Class clazz) { - return app.context().getBean(clazz); - } +public class HermesFrontendTestApp implements HermesTestApp { - public HermesFrontendTestApp metadataMaxAgeInSeconds(int value) { - metadataMaxAge = Duration.ofSeconds(value); - return this; - } + private final ZookeeperContainer hermesZookeeper; + private final Map kafkaClusters; + private final ConfluentSchemaRegistryContainer schemaRegistry; + private SpringApplicationBuilder app; + + private int port = -1; + private boolean kafkaCheckEnabled = false; + private Duration metadataMaxAge = Duration.ofMinutes(5); + private Duration readinessCheckInterval = Duration.ofSeconds(1); + private final Map extraArgs = new HashMap<>(); + private final List profiles = new ArrayList<>(List.of("integration")); + private List currentArgs = List.of(); + + public HermesFrontendTestApp( + ZookeeperContainer hermesZookeeper, + KafkaContainerCluster kafka, + ConfluentSchemaRegistryContainer schemaRegistry) { + this.hermesZookeeper = hermesZookeeper; + this.schemaRegistry = schemaRegistry; + this.kafkaClusters = Map.of("dc", kafka); + } + + public HermesFrontendTestApp( + ZookeeperContainer hermesZookeeper, + Map kafkaClusters, + ConfluentSchemaRegistryContainer schemaRegistry) { + this.hermesZookeeper = hermesZookeeper; + this.schemaRegistry = schemaRegistry; + this.kafkaClusters = kafkaClusters; + } + + private String kafkaClusterProperty(int index, String name) { + return String.format("frontend.kafka.clusters[%d].%s", index, name); + } + + public HermesFrontendTestApp withProperty(String name, Object value) { + this.extraArgs.put(name, value); + return this; + } + + public HermesFrontendTestApp withSpringProfile(String profile) { + profiles.add(profile); + return this; + } + + private List createArgs() { + Map args = new HashMap<>(); + args.put(SPRING_PROFILES_ACTIVE, String.join(",", profiles)); + args.put(FRONTEND_PORT, 0); + + args.put(KAFKA_NAMESPACE, "itTest"); + + var i = 0; + for (var entry : kafkaClusters.entrySet()) { + args.put(kafkaClusterProperty(i, "datacenter"), entry.getKey()); + args.put( + kafkaClusterProperty(i, "brokerList"), + entry.getValue().getBootstrapServersForExternalClients()); + i++; + } + + args.put(ZOOKEEPER_CONNECTION_STRING, hermesZookeeper.getConnectionString()); + + args.put(SCHEMA_CACHE_ENABLED, true); + args.put(SCHEMA_REPOSITORY_SERVER_URL, schemaRegistry.getUrl()); + + args.put(FRONTEND_READINESS_CHECK_KAFKA_CHECK_ENABLED, kafkaCheckEnabled); + args.put(FRONTEND_READINESS_CHECK_ENABLED, true); + args.put(FRONTEND_READINESS_CHECK_INTERVAL_SECONDS, readinessCheckInterval); + + args.put(FRONTEND_HEADER_PROPAGATION_ENABLED, true); + args.put( + FRONTEND_HEADER_PROPAGATION_ALLOWED, + "trace-id, span-id, parent-span-id, trace-sampled, trace-reported"); + + args.put(KAFKA_PRODUCER_METADATA_MAX_AGE, metadataMaxAge); + + args.put(FRONTEND_FORCE_TOPIC_MAX_MESSAGE_SIZE, true); + args.put(FRONTEND_IDLE_TIMEOUT, Duration.ofSeconds(2)); + + args.put(FRONTEND_THROUGHPUT_TYPE, "fixed"); + args.put(FRONTEND_THROUGHPUT_FIXED_MAX, 50 * 1024L); + + args.put(FRONTEND_GRACEFUL_SHUTDOWN_ENABLED, false); + + args.put(METRICS_MICROMETER_REPORT_PERIOD, Duration.ofSeconds(1)); + + args.put(FRONTEND_MESSAGE_PREVIEW_ENABLED, true); + args.put(FRONTEND_MESSAGE_PREVIEW_LOG_PERSIST_PERIOD, Duration.ofSeconds(1)); + + args.put(AUTH_USERNAME, "username"); + args.put(AUTH_PASSWORD, "password"); + + args.putAll(extraArgs); + + return args.entrySet().stream().map(e -> getArgument(e.getKey(), e.getValue())).toList(); + } + + @Override + public HermesTestApp start() { + app = new SpringApplicationBuilder(HermesFrontend.class).web(WebApplicationType.NONE); + currentArgs = createArgs(); + app.run(currentArgs.toArray(new String[0])); + port = app.context().getBean(HermesServer.class).getPort(); + return this; + } + + @Override + public void stop() { + if (app != null) { + app.context().close(); + app = null; + } + } + + @Override + public int getPort() { + if (port == -1) { + throw new IllegalStateException("hermes-frontend port hasn't been initialized"); + } + return port; + } + + public int getSSLPort() { + return app.context().getBean(HermesServer.class).getSSLPort(); + } + + public T getBean(Class clazz) { + return app.context().getBean(clazz); + } - public HermesFrontendTestApp readinessCheckIntervalInSeconds(int value) { - readinessCheckInterval = Duration.ofSeconds(value); - return this; - } + public HermesFrontendTestApp metadataMaxAgeInSeconds(int value) { + metadataMaxAge = Duration.ofSeconds(value); + return this; + } + + public HermesFrontendTestApp readinessCheckIntervalInSeconds(int value) { + readinessCheckInterval = Duration.ofSeconds(value); + return this; + } - public HermesFrontendTestApp kafkaCheckEnabled() { - kafkaCheckEnabled = true; - return this; - } + public HermesFrontendTestApp kafkaCheckEnabled() { + kafkaCheckEnabled = true; + return this; + } - public HermesFrontendTestApp kafkaCheckDisabled() { - kafkaCheckEnabled = false; - return this; - } + public HermesFrontendTestApp kafkaCheckDisabled() { + kafkaCheckEnabled = false; + return this; + } - private static String getArgument(String config, Object value) { - return "--" + config + "=" + value; - } + private static String getArgument(String config, Object value) { + return "--" + config + "=" + value; + } - @Override - public boolean shouldBeRestarted() { - List args = createArgs(); - return !args.equals(currentArgs); - } + @Override + public boolean shouldBeRestarted() { + List args = createArgs(); + return !args.equals(currentArgs); + } - @Override - public void restoreDefaultSettings() { - extraArgs.clear(); - profiles.clear(); - profiles.add("integration"); - } + @Override + public void restoreDefaultSettings() { + extraArgs.clear(); + profiles.clear(); + profiles.add("integration"); + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/HermesManagementExtension.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/HermesManagementExtension.java index 41fc331b1a..260a4b2788 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/HermesManagementExtension.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/HermesManagementExtension.java @@ -7,25 +7,26 @@ public class HermesManagementExtension implements BeforeAllCallback, AfterAllCallback { - private final HermesManagementTestApp management; - private HermesInitHelper initHelper; + private final HermesManagementTestApp management; + private HermesInitHelper initHelper; - public HermesManagementExtension(InfrastructureExtension infra) { - management = new HermesManagementTestApp(infra.hermesZookeeper(), infra.kafka(), infra.schemaRegistry()); - } + public HermesManagementExtension(InfrastructureExtension infra) { + management = + new HermesManagementTestApp(infra.hermesZookeeper(), infra.kafka(), infra.schemaRegistry()); + } - @Override - public void beforeAll(ExtensionContext context) { - management.start(); - initHelper = new HermesInitHelper(management.getPort()); - } + @Override + public void beforeAll(ExtensionContext context) { + management.start(); + initHelper = new HermesInitHelper(management.getPort()); + } - @Override - public void afterAll(ExtensionContext context) { - management.stop(); - } + @Override + public void afterAll(ExtensionContext context) { + management.stop(); + } - public HermesInitHelper initHelper() { - return initHelper; - } + public HermesInitHelper initHelper() { + return initHelper; + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/HermesManagementTestApp.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/HermesManagementTestApp.java index 16ebf4bbe5..4031abb32e 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/HermesManagementTestApp.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/HermesManagementTestApp.java @@ -1,16 +1,9 @@ package pl.allegro.tech.hermes.integrationtests.setup; -import org.springframework.boot.builder.SpringApplicationBuilder; -import org.springframework.core.env.Environment; -import pl.allegro.tech.hermes.integrationtests.prometheus.PrometheusExtension; -import pl.allegro.tech.hermes.management.HermesManagement; -import pl.allegro.tech.hermes.management.domain.group.GroupService; -import pl.allegro.tech.hermes.management.domain.subscription.SubscriptionService; -import pl.allegro.tech.hermes.management.domain.topic.TopicService; -import pl.allegro.tech.hermes.test.helper.containers.ConfluentSchemaRegistryContainer; -import pl.allegro.tech.hermes.test.helper.containers.KafkaContainerCluster; -import pl.allegro.tech.hermes.test.helper.containers.ZookeeperContainer; -import pl.allegro.tech.hermes.test.helper.environment.HermesTestApp; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.waitAtMost; +import static pl.allegro.tech.hermes.infrastructure.dc.DefaultDatacenterNameProvider.DEFAULT_DC_NAME; +import static pl.allegro.tech.hermes.test.helper.endpoint.TimeoutAdjuster.adjust; import java.io.IOException; import java.net.URI; @@ -22,175 +15,199 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.waitAtMost; -import static pl.allegro.tech.hermes.infrastructure.dc.DefaultDatacenterNameProvider.DEFAULT_DC_NAME; -import static pl.allegro.tech.hermes.test.helper.endpoint.TimeoutAdjuster.adjust; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.core.env.Environment; +import pl.allegro.tech.hermes.integrationtests.prometheus.PrometheusExtension; +import pl.allegro.tech.hermes.management.HermesManagement; +import pl.allegro.tech.hermes.management.domain.group.GroupService; +import pl.allegro.tech.hermes.management.domain.subscription.SubscriptionService; +import pl.allegro.tech.hermes.management.domain.topic.TopicService; +import pl.allegro.tech.hermes.test.helper.containers.ConfluentSchemaRegistryContainer; +import pl.allegro.tech.hermes.test.helper.containers.KafkaContainerCluster; +import pl.allegro.tech.hermes.test.helper.containers.ZookeeperContainer; +import pl.allegro.tech.hermes.test.helper.environment.HermesTestApp; public class HermesManagementTestApp implements HermesTestApp { - private int port = -1; - - private int auditEventPort = -1; - - public static String AUDIT_EVENT_PATH = "/audit-events"; - - private final Map hermesZookeepers; - private final Map kafkaClusters; - private final ConfluentSchemaRegistryContainer schemaRegistry; - private SpringApplicationBuilder app = null; - private List currentArgs = List.of(); - private PrometheusExtension prometheus = null; - - public HermesManagementTestApp(ZookeeperContainer hermesZookeeper, - KafkaContainerCluster kafka, - ConfluentSchemaRegistryContainer schemaRegistry) { - this(Map.of(DEFAULT_DC_NAME, hermesZookeeper), Map.of(DEFAULT_DC_NAME, kafka), schemaRegistry); - } - - public HermesManagementTestApp(Map hermesZookeepers, - Map kafkaClusters, - ConfluentSchemaRegistryContainer schemaRegistry) { - this.hermesZookeepers = hermesZookeepers; - this.kafkaClusters = kafkaClusters; - this.schemaRegistry = schemaRegistry; - } - - @Override - public HermesTestApp start() { - currentArgs = createArgs(); - app = new SpringApplicationBuilder(HermesManagement.class); - app.run(currentArgs.toArray(new String[0])); - String localServerPort = app.context().getBean(Environment.class).getProperty("local.server.port"); - if (localServerPort == null) { - throw new IllegalStateException("Cannot get hermes-management port"); - } - port = Integer.parseInt(localServerPort); - waitUntilReady(); - return this; + private int port = -1; + + private int auditEventPort = -1; + + public static String AUDIT_EVENT_PATH = "/audit-events"; + + private final Map hermesZookeepers; + private final Map kafkaClusters; + private final ConfluentSchemaRegistryContainer schemaRegistry; + private SpringApplicationBuilder app = null; + private List currentArgs = List.of(); + private PrometheusExtension prometheus = null; + + public HermesManagementTestApp( + ZookeeperContainer hermesZookeeper, + KafkaContainerCluster kafka, + ConfluentSchemaRegistryContainer schemaRegistry) { + this(Map.of(DEFAULT_DC_NAME, hermesZookeeper), Map.of(DEFAULT_DC_NAME, kafka), schemaRegistry); + } + + public HermesManagementTestApp( + Map hermesZookeepers, + Map kafkaClusters, + ConfluentSchemaRegistryContainer schemaRegistry) { + this.hermesZookeepers = hermesZookeepers; + this.kafkaClusters = kafkaClusters; + this.schemaRegistry = schemaRegistry; + } + + @Override + public HermesTestApp start() { + currentArgs = createArgs(); + app = new SpringApplicationBuilder(HermesManagement.class); + app.run(currentArgs.toArray(new String[0])); + String localServerPort = + app.context().getBean(Environment.class).getProperty("local.server.port"); + if (localServerPort == null) { + throw new IllegalStateException("Cannot get hermes-management port"); } - - private void waitUntilReady() { - try { - HttpRequest request = HttpRequest.newBuilder() - .uri(new URI("http://localhost:" + getPort() + "/mode")) - .GET() - .build(); - HttpClient httpClient = HttpClient.newHttpClient(); - - waitAtMost(adjust(240), TimeUnit.SECONDS).untilAsserted(() -> { + port = Integer.parseInt(localServerPort); + waitUntilReady(); + return this; + } + + private void waitUntilReady() { + try { + HttpRequest request = + HttpRequest.newBuilder() + .uri(new URI("http://localhost:" + getPort() + "/mode")) + .GET() + .build(); + HttpClient httpClient = HttpClient.newHttpClient(); + + waitAtMost(adjust(240), TimeUnit.SECONDS) + .untilAsserted( + () -> { try { - HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); - assertThat(response.body()).isEqualTo("readWrite"); + HttpResponse response = + httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + assertThat(response.body()).isEqualTo("readWrite"); } catch (IOException | InterruptedException e) { - throw new AssertionError("Reading management mode failed", e); + throw new AssertionError("Reading management mode failed", e); } - }); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } + }); + } catch (URISyntaxException e) { + throw new RuntimeException(e); } + } - @Override - public int getPort() { - if (port == -1) { - throw new IllegalStateException("hermes-management port hasn't been initialized"); - } - return port; + @Override + public int getPort() { + if (port == -1) { + throw new IllegalStateException("hermes-management port hasn't been initialized"); } - - @Override - public void stop() { - if (app != null) { - app.context().close(); - app = null; - } + return port; + } + + @Override + public void stop() { + if (app != null) { + app.context().close(); + app = null; } - - public void addEventAuditorListener(int port) { - auditEventPort = port; + } + + public void addEventAuditorListener(int port) { + auditEventPort = port; + } + + void withPrometheus(PrometheusExtension prometheus) { + this.prometheus = prometheus; + } + + @Override + public void restoreDefaultSettings() { + prometheus = null; + } + + @Override + public boolean shouldBeRestarted() { + List args = createArgs(); + return !args.equals(currentArgs); + } + + private List createArgs() { + List args = new ArrayList<>(); + args.add("--spring.profiles.active=integration"); + args.add("--server.port=0"); + args.add("--prometheus.client.enabled=true"); + args.add("--prometheus.client.socketTimeoutMillis=500"); + if (prometheus != null) { + args.add("--prometheus.client.externalMonitoringUrl=" + prometheus.getEndpoint()); + args.add("--prometheus.client.cacheTtlSeconds=0"); } - - void withPrometheus(PrometheusExtension prometheus) { - this.prometheus = prometheus; + args.add("--topic.partitions=2"); + args.add("--topic.uncleanLeaderElectionEnabled=false"); + int smallestClusterSize = + kafkaClusters.values().stream() + .map(cluster -> cluster.getAllBrokers().size()) + .min(Integer::compareTo) + .orElse(1); + args.add("--topic.replicationFactor=" + smallestClusterSize); + int idx = 0; + for (Map.Entry zk : hermesZookeepers.entrySet()) { + args.add("--storage.clusters[" + idx + "].datacenter=" + zk.getKey()); + args.add("--storage.clusters[" + idx + "].clusterName=zk"); + args.add( + "--storage.clusters[" + + idx + + "].connectionString=" + + zk.getValue().getConnectionString()); + idx++; } - - @Override - public void restoreDefaultSettings() { - prometheus = null; + idx = 0; + for (Map.Entry kafka : kafkaClusters.entrySet()) { + args.add("--kafka.clusters[" + idx + "].datacenter=" + kafka.getKey()); + args.add("--kafka.clusters[" + idx + "].clusterName=primary"); + args.add( + "--kafka.clusters[" + + idx + + "].bootstrapKafkaServer=" + + kafka.getValue().getBootstrapServersForExternalClients()); + args.add("--kafka.clusters[" + idx + "].namespace=itTest"); + idx++; } - @Override - public boolean shouldBeRestarted() { - List args = createArgs(); - return !args.equals(currentArgs); + args.add("--schema.repository.serverUrl=" + schemaRegistry.getUrl()); + args.add("--topic.touchSchedulerEnabled=" + false); + args.add("--topic.allowRemoval=" + true); + args.add("--topic.allowedTopicLabels=" + "label-1, label-2, label-3"); + if (auditEventPort != -1) { + args.add("--audit.isEventAuditEnabled=" + true); + args.add("--audit.eventUrl=" + "http://localhost:" + auditEventPort + AUDIT_EVENT_PATH); } - private List createArgs() { - List args = new ArrayList<>(); - args.add("--spring.profiles.active=integration"); - args.add("--server.port=0"); - args.add("--prometheus.client.enabled=true"); - args.add("--prometheus.client.socketTimeoutMillis=500"); - if (prometheus != null) { - args.add("--prometheus.client.externalMonitoringUrl=" + prometheus.getEndpoint()); - args.add("--prometheus.client.cacheTtlSeconds=0"); - } - args.add("--topic.partitions=2"); - args.add("--topic.uncleanLeaderElectionEnabled=false"); - int smallestClusterSize = kafkaClusters.values().stream() - .map(cluster -> cluster.getAllBrokers().size()) - .min(Integer::compareTo) - .orElse(1); - args.add("--topic.replicationFactor=" + smallestClusterSize); - int idx = 0; - for (Map.Entry zk : hermesZookeepers.entrySet()) { - args.add("--storage.clusters[" + idx + "].datacenter=" + zk.getKey()); - args.add("--storage.clusters[" + idx + "].clusterName=zk"); - args.add("--storage.clusters[" + idx + "].connectionString=" + zk.getValue().getConnectionString()); - idx++; - } - idx = 0; - for (Map.Entry kafka : kafkaClusters.entrySet()) { - args.add("--kafka.clusters[" + idx + "].datacenter=" + kafka.getKey()); - args.add("--kafka.clusters[" + idx + "].clusterName=primary"); - args.add("--kafka.clusters[" + idx + "].bootstrapKafkaServer=" + kafka.getValue().getBootstrapServersForExternalClients()); - args.add("--kafka.clusters[" + idx + "].namespace=itTest"); - idx++; - } - - args.add("--schema.repository.serverUrl=" + schemaRegistry.getUrl()); - args.add("--topic.touchSchedulerEnabled=" + false); - args.add("--topic.allowRemoval=" + true); - args.add("--topic.allowedTopicLabels=" + "label-1, label-2, label-3"); - if (auditEventPort != -1) { - args.add("--audit.isEventAuditEnabled=" + true); - args.add("--audit.eventUrl=" + "http://localhost:" + auditEventPort + AUDIT_EVENT_PATH); - } - - args.add("--topic.removeSchema=" + true); - args.add("--storage.pathPrefix=" + "/hermes"); - args.add("--subscription.subscribersWithAccessToAnyTopic[0].ownerSource=" + "Plaintext"); - args.add("--subscription.subscribersWithAccessToAnyTopic[0].ownerId=" + "subscriberAllowedToAccessAnyTopic"); - args.add("--subscription.subscribersWithAccessToAnyTopic[0].protocols=" + "http, https"); - args.add("--group.allowedGroupNameRegex=" + "[a-zA-Z0-9_.-]+"); - args.add("--group.nonAdminCreationEnabled=" + true); - args.add("--schema.repository.type=schema_registry"); - args.add("--schema.repository.deleteSchemaPathSuffix="); - - return args; - } - - public SubscriptionService subscriptionService() { - return app.context().getBean(SubscriptionService.class); - } - - public TopicService topicService() { - return app.context().getBean(TopicService.class); - } - - public GroupService groupService() { - return app.context().getBean(GroupService.class); - } + args.add("--topic.removeSchema=" + true); + args.add("--storage.pathPrefix=" + "/hermes"); + args.add("--subscription.subscribersWithAccessToAnyTopic[0].ownerSource=" + "Plaintext"); + args.add( + "--subscription.subscribersWithAccessToAnyTopic[0].ownerId=" + + "subscriberAllowedToAccessAnyTopic"); + args.add("--subscription.subscribersWithAccessToAnyTopic[0].protocols=" + "http, https"); + args.add("--group.allowedGroupNameRegex=" + "[a-zA-Z0-9_.-]+"); + args.add("--group.nonAdminCreationEnabled=" + true); + args.add("--schema.repository.type=schema_registry"); + args.add("--schema.repository.deleteSchemaPathSuffix="); + + return args; + } + + public SubscriptionService subscriptionService() { + return app.context().getBean(SubscriptionService.class); + } + + public TopicService topicService() { + return app.context().getBean(TopicService.class); + } + + public GroupService groupService() { + return app.context().getBean(GroupService.class); + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/InfrastructureExtension.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/InfrastructureExtension.java index fbddd75730..a8088266b2 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/InfrastructureExtension.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/InfrastructureExtension.java @@ -1,5 +1,6 @@ package pl.allegro.tech.hermes.integrationtests.setup; +import java.util.stream.Stream; import org.junit.jupiter.api.extension.BeforeAllCallback; import org.junit.jupiter.api.extension.ExtensionContext; import org.testcontainers.lifecycle.Startable; @@ -7,40 +8,40 @@ import pl.allegro.tech.hermes.test.helper.containers.KafkaContainerCluster; import pl.allegro.tech.hermes.test.helper.containers.ZookeeperContainer; -import java.util.stream.Stream; - -public class InfrastructureExtension implements BeforeAllCallback, ExtensionContext.Store.CloseableResource { - private static final KafkaContainerCluster kafka = new KafkaContainerCluster(1); - private static final ZookeeperContainer hermesZookeeper = new ZookeeperContainer("HermesZookeeper"); - private static final ConfluentSchemaRegistryContainer schemaRegistry = new ConfluentSchemaRegistryContainer() - .withKafkaCluster(kafka); - - private static boolean started = false; - - public KafkaContainerCluster kafka() { - return kafka; - } - - public ZookeeperContainer hermesZookeeper() { - return hermesZookeeper; - } - - public ConfluentSchemaRegistryContainer schemaRegistry() { - return schemaRegistry; +public class InfrastructureExtension + implements BeforeAllCallback, ExtensionContext.Store.CloseableResource { + private static final KafkaContainerCluster kafka = new KafkaContainerCluster(1); + private static final ZookeeperContainer hermesZookeeper = + new ZookeeperContainer("HermesZookeeper"); + private static final ConfluentSchemaRegistryContainer schemaRegistry = + new ConfluentSchemaRegistryContainer().withKafkaCluster(kafka); + + private static boolean started = false; + + public KafkaContainerCluster kafka() { + return kafka; + } + + public ZookeeperContainer hermesZookeeper() { + return hermesZookeeper; + } + + public ConfluentSchemaRegistryContainer schemaRegistry() { + return schemaRegistry; + } + + @Override + public void beforeAll(ExtensionContext context) { + if (!started) { + Stream.of(hermesZookeeper, kafka).parallel().forEach(Startable::start); + schemaRegistry.start(); + started = true; } + } - @Override - public void beforeAll(ExtensionContext context) { - if (!started) { - Stream.of(hermesZookeeper, kafka).parallel().forEach(Startable::start); - schemaRegistry.start(); - started = true; - } - } - - @Override - public void close() { - Stream.of(hermesZookeeper, kafka, schemaRegistry).parallel().forEach(Startable::stop); - started = false; - } + @Override + public void close() { + Stream.of(hermesZookeeper, kafka, schemaRegistry).parallel().forEach(Startable::stop); + started = false; + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/JmsStarter.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/JmsStarter.java index d735c5b9e4..70197cf3b0 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/JmsStarter.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/JmsStarter.java @@ -5,21 +5,21 @@ public class JmsStarter implements Starter { - private EmbeddedJMS jmsServer; + private EmbeddedJMS jmsServer; - @Override - public void start() throws Exception { - jmsServer = new EmbeddedJMS(); - jmsServer.start(); - } + @Override + public void start() throws Exception { + jmsServer = new EmbeddedJMS(); + jmsServer.start(); + } - @Override - public void stop() throws Exception { - jmsServer.stop(); - } + @Override + public void stop() throws Exception { + jmsServer.stop(); + } - @Override - public EmbeddedJMS instance() { - return jmsServer; - } + @Override + public EmbeddedJMS instance() { + return jmsServer; + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/TestUser.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/TestUser.java index 0cb65a3fc8..dac3b7a817 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/TestUser.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/TestUser.java @@ -4,18 +4,18 @@ import pl.allegro.tech.hermes.management.domain.auth.RequestUser; class TestUser implements RequestUser { - @Override - public String getUsername() { - return "test_user"; - } + @Override + public String getUsername() { + return "test_user"; + } - @Override - public boolean isAdmin() { - return true; - } + @Override + public boolean isAdmin() { + return true; + } - @Override - public boolean isOwner(OwnerId ownerId) { - return true; - } + @Override + public boolean isOwner(OwnerId ownerId) { + return true; + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/TraceContext.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/TraceContext.java index a6519ddbc6..ae766cb863 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/TraceContext.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/setup/TraceContext.java @@ -2,55 +2,55 @@ import java.util.UUID; -public record TraceContext(String traceId, String spanId, String parentSpanId, String traceSampled, - String traceReported) { - - public static TraceContext random() { - - return new Builder() - .withTraceId(UUID.randomUUID().toString()) - .withSpanId(UUID.randomUUID().toString()) - .withParentSpanId(UUID.randomUUID().toString()) - .withTraceSampled("1") - .withTraceReported("0") - .build(); +public record TraceContext( + String traceId, String spanId, String parentSpanId, String traceSampled, String traceReported) { + + public static TraceContext random() { + + return new Builder() + .withTraceId(UUID.randomUUID().toString()) + .withSpanId(UUID.randomUUID().toString()) + .withParentSpanId(UUID.randomUUID().toString()) + .withTraceSampled("1") + .withTraceReported("0") + .build(); + } + + public static class Builder { + + private String traceId; + private String spanId; + private String parentSpanId; + private String traceSampled; + private String traceReported; + + public Builder withTraceId(String traceId) { + this.traceId = traceId; + return this; } - public static class Builder { - - private String traceId; - private String spanId; - private String parentSpanId; - private String traceSampled; - private String traceReported; - - public Builder withTraceId(String traceId) { - this.traceId = traceId; - return this; - } - - public Builder withSpanId(String spanId) { - this.spanId = spanId; - return this; - } - - public Builder withParentSpanId(String parentSpanId) { - this.parentSpanId = parentSpanId; - return this; - } - - public Builder withTraceSampled(String traceSampled) { - this.traceSampled = traceSampled; - return this; - } - - public Builder withTraceReported(String traceReported) { - this.traceReported = traceReported; - return this; - } - - public TraceContext build() { - return new TraceContext(traceId, spanId, parentSpanId, traceSampled, traceReported); - } + public Builder withSpanId(String spanId) { + this.spanId = spanId; + return this; } + + public Builder withParentSpanId(String parentSpanId) { + this.parentSpanId = parentSpanId; + return this; + } + + public Builder withTraceSampled(String traceSampled) { + this.traceSampled = traceSampled; + return this; + } + + public Builder withTraceReported(String traceReported) { + this.traceReported = traceReported; + return this; + } + + public TraceContext build() { + return new TraceContext(traceId, spanId, parentSpanId, traceSampled, traceReported); + } + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/subscriber/TestGooglePubSubSubscriber.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/subscriber/TestGooglePubSubSubscriber.java index 85616b5446..f7b576570c 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/subscriber/TestGooglePubSubSubscriber.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/subscriber/TestGooglePubSubSubscriber.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.integrationtests.subscriber; +import static org.assertj.core.api.Assertions.assertThat; + import com.google.api.gax.core.CredentialsProvider; import com.google.api.gax.core.NoCredentialsProvider; import com.google.api.gax.grpc.GrpcTransportChannel; @@ -20,86 +22,91 @@ import com.google.pubsub.v1.TopicName; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; - import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; -import static org.assertj.core.api.Assertions.assertThat; - public class TestGooglePubSubSubscriber { - private static final String GOOGLE_PUBSUB_PROJECT_ID = "test-project"; - private static final AtomicInteger sequence = new AtomicInteger(); + private static final String GOOGLE_PUBSUB_PROJECT_ID = "test-project"; + private static final AtomicInteger sequence = new AtomicInteger(); - private final CredentialsProvider credentialsProvider; - private final TransportChannelProvider transportChannelProvider; - private final ManagedChannel channel; - private final String endpoint; - private final String subscription; - private final List receivedMessages = new ArrayList<>(); + private final CredentialsProvider credentialsProvider; + private final TransportChannelProvider transportChannelProvider; + private final ManagedChannel channel; + private final String endpoint; + private final String subscription; + private final List receivedMessages = new ArrayList<>(); - public TestGooglePubSubSubscriber(String emulatorEndpoint) { - this.credentialsProvider = NoCredentialsProvider.create(); - this.channel = ManagedChannelBuilder.forTarget(emulatorEndpoint).usePlaintext().build(); - this.transportChannelProvider = FixedTransportChannelProvider.create(GrpcTransportChannel.create(channel)); - String topicId = "test-topic" + sequence.incrementAndGet(); - String topic = TopicName.format(GOOGLE_PUBSUB_PROJECT_ID, topicId); - subscription = ProjectSubscriptionName.format(GOOGLE_PUBSUB_PROJECT_ID, "test-subscription" + sequence.incrementAndGet()); - try { - createTopic(topic); - createSubscription(subscription, topic); - } catch (IOException e) { - throw new RuntimeException(e); - } - endpoint = "googlepubsub://pubsub.googleapis.com:443/projects/%s/topics/%s".formatted(GOOGLE_PUBSUB_PROJECT_ID, topicId); + public TestGooglePubSubSubscriber(String emulatorEndpoint) { + this.credentialsProvider = NoCredentialsProvider.create(); + this.channel = ManagedChannelBuilder.forTarget(emulatorEndpoint).usePlaintext().build(); + this.transportChannelProvider = + FixedTransportChannelProvider.create(GrpcTransportChannel.create(channel)); + String topicId = "test-topic" + sequence.incrementAndGet(); + String topic = TopicName.format(GOOGLE_PUBSUB_PROJECT_ID, topicId); + subscription = + ProjectSubscriptionName.format( + GOOGLE_PUBSUB_PROJECT_ID, "test-subscription" + sequence.incrementAndGet()); + try { + createTopic(topic); + createSubscription(subscription, topic); + } catch (IOException e) { + throw new RuntimeException(e); } + endpoint = + "googlepubsub://pubsub.googleapis.com:443/projects/%s/topics/%s" + .formatted(GOOGLE_PUBSUB_PROJECT_ID, topicId); + } - private void createTopic(String topicName) throws IOException { - TopicAdminSettings topicAdminSettings = TopicAdminSettings.newBuilder() - .setTransportChannelProvider(transportChannelProvider) - .setCredentialsProvider(credentialsProvider) - .build(); - try (TopicAdminClient topicAdminClient = TopicAdminClient.create(topicAdminSettings)) { - topicAdminClient.createTopic(topicName); - } + private void createTopic(String topicName) throws IOException { + TopicAdminSettings topicAdminSettings = + TopicAdminSettings.newBuilder() + .setTransportChannelProvider(transportChannelProvider) + .setCredentialsProvider(credentialsProvider) + .build(); + try (TopicAdminClient topicAdminClient = TopicAdminClient.create(topicAdminSettings)) { + topicAdminClient.createTopic(topicName); } + } - private void createSubscription(String subscriptionName, String topicName) throws IOException { - SubscriptionAdminSettings subscriptionAdminSettings = SubscriptionAdminSettings.newBuilder() - .setTransportChannelProvider(transportChannelProvider) - .setCredentialsProvider(credentialsProvider) - .build(); - SubscriptionAdminClient subscriptionAdminClient = SubscriptionAdminClient.create(subscriptionAdminSettings); - subscriptionAdminClient.createSubscription(subscriptionName, topicName, PushConfig.getDefaultInstance(), 10); - } + private void createSubscription(String subscriptionName, String topicName) throws IOException { + SubscriptionAdminSettings subscriptionAdminSettings = + SubscriptionAdminSettings.newBuilder() + .setTransportChannelProvider(transportChannelProvider) + .setCredentialsProvider(credentialsProvider) + .build(); + SubscriptionAdminClient subscriptionAdminClient = + SubscriptionAdminClient.create(subscriptionAdminSettings); + subscriptionAdminClient.createSubscription( + subscriptionName, topicName, PushConfig.getDefaultInstance(), 10); + } - public String getEndpoint() { - return endpoint; - } + public String getEndpoint() { + return endpoint; + } - public void waitUntilAnyMessageReceived() throws IOException { - SubscriberStubSettings subscriberStubSettings = SubscriberStubSettings.newBuilder() - .setTransportChannelProvider(transportChannelProvider) - .setCredentialsProvider(credentialsProvider) - .build(); - try (SubscriberStub subscriber = GrpcSubscriberStub.create(subscriberStubSettings)) { - PullRequest pullRequest = PullRequest.newBuilder() - .setMaxMessages(1) - .setSubscription(subscription) - .build(); - PullResponse pullResponse = subscriber.pullCallable().call(pullRequest); - assertThat(pullResponse.getReceivedMessagesList()).isNotEmpty(); - receivedMessages.addAll(pullResponse.getReceivedMessagesList()); - } + public void waitUntilAnyMessageReceived() throws IOException { + SubscriberStubSettings subscriberStubSettings = + SubscriberStubSettings.newBuilder() + .setTransportChannelProvider(transportChannelProvider) + .setCredentialsProvider(credentialsProvider) + .build(); + try (SubscriberStub subscriber = GrpcSubscriberStub.create(subscriberStubSettings)) { + PullRequest pullRequest = + PullRequest.newBuilder().setMaxMessages(1).setSubscription(subscription).build(); + PullResponse pullResponse = subscriber.pullCallable().call(pullRequest); + assertThat(pullResponse.getReceivedMessagesList()).isNotEmpty(); + receivedMessages.addAll(pullResponse.getReceivedMessagesList()); } + } - public List getAllReceivedMessages() { - return receivedMessages; - } + public List getAllReceivedMessages() { + return receivedMessages; + } - public void stop() { - channel.shutdown(); - } + public void stop() { + channel.shutdown(); + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/subscriber/TestJmsSubscriber.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/subscriber/TestJmsSubscriber.java index dabfedda37..fff3843b3d 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/subscriber/TestJmsSubscriber.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/subscriber/TestJmsSubscriber.java @@ -1,9 +1,8 @@ package pl.allegro.tech.hermes.integrationtests.subscriber; -import org.hornetq.api.core.TransportConfiguration; -import org.hornetq.api.jms.HornetQJMSClient; -import org.hornetq.api.jms.JMSFactoryType; -import org.hornetq.core.remoting.impl.netty.NettyConnectorFactory; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static pl.allegro.tech.hermes.test.helper.endpoint.TimeoutAdjuster.adjust; import java.nio.charset.StandardCharsets; import java.time.Duration; @@ -15,74 +14,82 @@ import javax.jms.JMSException; import javax.jms.Message; import javax.jms.Topic; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; -import static pl.allegro.tech.hermes.test.helper.endpoint.TimeoutAdjuster.adjust; +import org.hornetq.api.core.TransportConfiguration; +import org.hornetq.api.jms.HornetQJMSClient; +import org.hornetq.api.jms.JMSFactoryType; +import org.hornetq.core.remoting.impl.netty.NettyConnectorFactory; public class TestJmsSubscriber { - private static final int DEFAULT_WAIT_TIME_IN_SEC = 10; - - private final List receivedRequests = Collections.synchronizedList(new ArrayList<>()); + private static final int DEFAULT_WAIT_TIME_IN_SEC = 10; - public TestJmsSubscriber(String topicName) { - initializeContext(topicName); - } + private final List receivedRequests = Collections.synchronizedList(new ArrayList<>()); - private void initializeContext(String topicName) { - TransportConfiguration transportConfiguration = new TransportConfiguration(NettyConnectorFactory.class.getName()); - ConnectionFactory connectionFactory = HornetQJMSClient.createConnectionFactoryWithHA( - JMSFactoryType.TOPIC_CF, - transportConfiguration); + public TestJmsSubscriber(String topicName) { + initializeContext(topicName); + } - JMSContext jmsContext = connectionFactory.createContext(); + private void initializeContext(String topicName) { + TransportConfiguration transportConfiguration = + new TransportConfiguration(NettyConnectorFactory.class.getName()); + ConnectionFactory connectionFactory = + HornetQJMSClient.createConnectionFactoryWithHA( + JMSFactoryType.TOPIC_CF, transportConfiguration); - Topic topic = jmsContext.createTopic(topicName); - jmsContext.createConsumer(topic).setMessageListener(this::onRequestReceived); + JMSContext jmsContext = connectionFactory.createContext(); - } + Topic topic = jmsContext.createTopic(topicName); + jmsContext.createConsumer(topic).setMessageListener(this::onRequestReceived); + } - void onRequestReceived(Message request) { - receivedRequests.add(request); - } + void onRequestReceived(Message request) { + receivedRequests.add(request); + } - public void waitUntilReceived(String body) { - awaitWithSyncRequests(() -> - assertThat( - receivedRequests.stream() - .filter(r -> { - try { - String s = new String(r.getBody(byte[].class), StandardCharsets.UTF_8); - return s.equals(body); - } catch (JMSException e) { - throw new RuntimeException(e); - } - }) - .findFirst() - ).isNotEmpty()); - } + public void waitUntilReceived(String body) { + awaitWithSyncRequests( + () -> + assertThat( + receivedRequests.stream() + .filter( + r -> { + try { + String s = + new String(r.getBody(byte[].class), StandardCharsets.UTF_8); + return s.equals(body); + } catch (JMSException e) { + throw new RuntimeException(e); + } + }) + .findFirst()) + .isNotEmpty()); + } - public void waitUntilMessageWithHeaderReceived(String headerName, String headerValue) { - awaitWithSyncRequests(() -> - assertThat( - receivedRequests.stream() - .filter(r -> { - try { - return r.getStringProperty(headerName).equals(headerValue); - } catch (JMSException e) { - throw new RuntimeException(e); - } - }) - .findFirst()) - .isNotEmpty()); - } + public void waitUntilMessageWithHeaderReceived(String headerName, String headerValue) { + awaitWithSyncRequests( + () -> + assertThat( + receivedRequests.stream() + .filter( + r -> { + try { + return r.getStringProperty(headerName).equals(headerValue); + } catch (JMSException e) { + throw new RuntimeException(e); + } + }) + .findFirst()) + .isNotEmpty()); + } - private void awaitWithSyncRequests(Runnable runnable) { - await().atMost(adjust(Duration.ofSeconds(DEFAULT_WAIT_TIME_IN_SEC))).untilAsserted(() -> { - synchronized (receivedRequests) { + private void awaitWithSyncRequests(Runnable runnable) { + await() + .atMost(adjust(Duration.ofSeconds(DEFAULT_WAIT_TIME_IN_SEC))) + .untilAsserted( + () -> { + synchronized (receivedRequests) { runnable.run(); - } - }); - } -} \ No newline at end of file + } + }); + } +} diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/subscriber/TestSubscriber.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/subscriber/TestSubscriber.java index 59d2ace7fa..d20a33f6e6 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/subscriber/TestSubscriber.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/subscriber/TestSubscriber.java @@ -1,8 +1,11 @@ package pl.allegro.tech.hermes.integrationtests.subscriber; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static pl.allegro.tech.hermes.test.helper.endpoint.TimeoutAdjuster.adjust; + import com.github.tomakehurst.wiremock.verification.LoggedRequest; import com.google.common.collect.Streams; - import java.net.URI; import java.time.Duration; import java.util.ArrayList; @@ -13,122 +16,135 @@ import java.util.function.Consumer; import java.util.stream.Collectors; -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; -import static pl.allegro.tech.hermes.test.helper.endpoint.TimeoutAdjuster.adjust; - public class TestSubscriber { - private static final int DEFAULT_WAIT_TIME_IN_SEC = 10; - private final URI subscriberUrl; - private final List receivedRequests = Collections.synchronizedList(new ArrayList<>()); + private static final int DEFAULT_WAIT_TIME_IN_SEC = 10; + private final URI subscriberUrl; + private final List receivedRequests = + Collections.synchronizedList(new ArrayList<>()); - TestSubscriber(URI subscriberUrl) { - this.subscriberUrl = subscriberUrl; - } + TestSubscriber(URI subscriberUrl) { + this.subscriberUrl = subscriberUrl; + } - void onRequestReceived(LoggedRequest request) { - receivedRequests.add(request); - } + void onRequestReceived(LoggedRequest request) { + receivedRequests.add(request); + } - public String getEndpoint() { - return subscriberUrl.toString(); - } + public String getEndpoint() { + return subscriberUrl.toString(); + } - public int getPort() { - return subscriberUrl.getPort(); - } + public int getPort() { + return subscriberUrl.getPort(); + } - public String getPath() { - return subscriberUrl.getPath(); - } + public String getPath() { + return subscriberUrl.getPath(); + } - public void waitUntilReceived(String body) { - awaitWithSyncRequests(() -> + public void waitUntilReceived(String body) { + awaitWithSyncRequests( + () -> assertThat( - receivedRequests.stream() - .filter(r -> r.getBodyAsString().equals(body)) - .findFirst() - ).isNotEmpty()); - } - - public void waitUntilReceived(Duration duration, int numberOfExpectedMessages) { - await().atMost(adjust(duration)).untilAsserted(() -> - assertThat(receivedRequests.size()).isEqualTo(numberOfExpectedMessages)); - } - - public void waitUntilAnyMessageReceived() { - await().atMost(adjust(Duration.ofSeconds(DEFAULT_WAIT_TIME_IN_SEC))).untilAsserted(() -> - assertThat(receivedRequests.size()).isPositive()); - } - - public void waitUntilRequestReceived(Consumer requestConsumer) { - waitUntilAnyMessageReceived(); - - synchronized (receivedRequests) { - receivedRequests.forEach(requestConsumer); - } - } - - public void waitUntilRequestsReceived(Consumer> requestsConsumer) { - await().atMost(adjust(Duration.ofSeconds(DEFAULT_WAIT_TIME_IN_SEC))).untilAsserted( - () -> { - synchronized (receivedRequests) { - requestsConsumer.accept(receivedRequests); - } - } - ); - } - - public void noMessagesReceived() { - assertThat(receivedRequests).isEmpty(); - } - - public void waitUntilMessageWithHeaderReceived(String headerName, String headerValue) { - awaitWithSyncRequests(() -> + receivedRequests.stream() + .filter(r -> r.getBodyAsString().equals(body)) + .findFirst()) + .isNotEmpty()); + } + + public void waitUntilReceived(Duration duration, int numberOfExpectedMessages) { + await() + .atMost(adjust(duration)) + .untilAsserted( + () -> assertThat(receivedRequests.size()).isEqualTo(numberOfExpectedMessages)); + } + + public void waitUntilAnyMessageReceived() { + await() + .atMost(adjust(Duration.ofSeconds(DEFAULT_WAIT_TIME_IN_SEC))) + .untilAsserted(() -> assertThat(receivedRequests.size()).isPositive()); + } + + public void waitUntilRequestReceived(Consumer requestConsumer) { + waitUntilAnyMessageReceived(); + + synchronized (receivedRequests) { + receivedRequests.forEach(requestConsumer); + } + } + + public void waitUntilRequestsReceived(Consumer> requestsConsumer) { + await() + .atMost(adjust(Duration.ofSeconds(DEFAULT_WAIT_TIME_IN_SEC))) + .untilAsserted( + () -> { + synchronized (receivedRequests) { + requestsConsumer.accept(receivedRequests); + } + }); + } + + public void noMessagesReceived() { + assertThat(receivedRequests).isEmpty(); + } + + public void waitUntilMessageWithHeaderReceived(String headerName, String headerValue) { + awaitWithSyncRequests( + () -> assertThat( - receivedRequests.stream() - .filter(r -> r.containsHeader(headerName) && r.getHeader(headerName).equals(headerValue)) - .findFirst()) - .isNotEmpty()); - } - - public java.time.Duration durationBetweenFirstAndLastRequest() { - return java.time.Duration.between( - getFirstReceivedRequest().getLoggedDate().toInstant(), - getLastReceivedRequest().getLoggedDate().toInstant()); - } - - private void awaitWithSyncRequests(Runnable runnable) { - await().atMost(adjust(Duration.ofSeconds(DEFAULT_WAIT_TIME_IN_SEC))).untilAsserted(() -> { - synchronized (receivedRequests) { + receivedRequests.stream() + .filter( + r -> + r.containsHeader(headerName) + && r.getHeader(headerName).equals(headerValue)) + .findFirst()) + .isNotEmpty()); + } + + public java.time.Duration durationBetweenFirstAndLastRequest() { + return java.time.Duration.between( + getFirstReceivedRequest().getLoggedDate().toInstant(), + getLastReceivedRequest().getLoggedDate().toInstant()); + } + + private void awaitWithSyncRequests(Runnable runnable) { + await() + .atMost(adjust(Duration.ofSeconds(DEFAULT_WAIT_TIME_IN_SEC))) + .untilAsserted( + () -> { + synchronized (receivedRequests) { runnable.run(); - } - }); - } - - private LoggedRequest getFirstReceivedRequest() { - synchronized (receivedRequests) { - return receivedRequests.stream().findFirst().orElseThrow(NoSuchElementException::new); - } - } - - public LoggedRequest getLastReceivedRequest() { - synchronized (receivedRequests) { - return Streams.findLast(receivedRequests.stream()).orElseThrow(NoSuchElementException::new); - } - } - - public void waitUntilAllReceivedStrict(Set expectedBodies) { - awaitWithSyncRequests(() -> { - int receivedCount = receivedRequests.size(); - assertThat(receivedCount).isEqualTo(expectedBodies.size()); - Set actual = receivedRequests.stream().map(LoggedRequest::getBodyAsString).collect(Collectors.toSet()); - assertThat(actual).isEqualTo(expectedBodies); + } + }); + } + + private LoggedRequest getFirstReceivedRequest() { + synchronized (receivedRequests) { + return receivedRequests.stream().findFirst().orElseThrow(NoSuchElementException::new); + } + } + + public LoggedRequest getLastReceivedRequest() { + synchronized (receivedRequests) { + return Streams.findLast(receivedRequests.stream()).orElseThrow(NoSuchElementException::new); + } + } + + public void waitUntilAllReceivedStrict(Set expectedBodies) { + awaitWithSyncRequests( + () -> { + int receivedCount = receivedRequests.size(); + assertThat(receivedCount).isEqualTo(expectedBodies.size()); + Set actual = + receivedRequests.stream() + .map(LoggedRequest::getBodyAsString) + .collect(Collectors.toSet()); + assertThat(actual).isEqualTo(expectedBodies); }); - } + } - public void reset() { - this.receivedRequests.clear(); - } + public void reset() { + this.receivedRequests.clear(); + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/subscriber/TestSubscribersExtension.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/subscriber/TestSubscribersExtension.java index c1a7d1e211..99aa994ff0 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/subscriber/TestSubscribersExtension.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/integrationtests/subscriber/TestSubscribersExtension.java @@ -1,135 +1,136 @@ package pl.allegro.tech.hermes.integrationtests.subscriber; +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; +import static com.github.tomakehurst.wiremock.stubbing.Scenario.STARTED; +import static jakarta.ws.rs.core.Response.Status.OK; + import com.github.tomakehurst.wiremock.WireMockServer; import com.github.tomakehurst.wiremock.matching.EqualToPattern; import com.github.tomakehurst.wiremock.verification.LoggedRequest; -import org.junit.jupiter.api.extension.AfterAllCallback; -import org.junit.jupiter.api.extension.AfterEachCallback; -import org.junit.jupiter.api.extension.ExtensionContext; - import java.net.URI; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; - -import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; -import static com.github.tomakehurst.wiremock.client.WireMock.post; -import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; -import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; -import static com.github.tomakehurst.wiremock.stubbing.Scenario.STARTED; -import static jakarta.ws.rs.core.Response.Status.OK; +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.ExtensionContext; public class TestSubscribersExtension implements AfterEachCallback, AfterAllCallback { - private final WireMockServer service; - private final URI serviceUrl; - private final AtomicInteger subscriberIndex = new AtomicInteger(); - private final Map subscribersPerPath = new ConcurrentHashMap<>(); - - public TestSubscribersExtension() { - this(0); - } - - public TestSubscribersExtension(int port) { - service = new WireMockServer(port); - service.start(); - serviceUrl = URI.create("http://localhost:" + service.port()); - service.addMockServiceRequestListener((request, response) -> { - TestSubscriber subscriber = subscribersPerPath.get(request.getUrl()); // getUrl() returns path here - if (subscriber != null) { - subscriber.onRequestReceived(LoggedRequest.createFrom(request)); - } + private final WireMockServer service; + private final URI serviceUrl; + private final AtomicInteger subscriberIndex = new AtomicInteger(); + private final Map subscribersPerPath = new ConcurrentHashMap<>(); + + public TestSubscribersExtension() { + this(0); + } + + public TestSubscribersExtension(int port) { + service = new WireMockServer(port); + service.start(); + serviceUrl = URI.create("http://localhost:" + service.port()); + service.addMockServiceRequestListener( + (request, response) -> { + TestSubscriber subscriber = + subscribersPerPath.get(request.getUrl()); // getUrl() returns path here + if (subscriber != null) { + subscriber.onRequestReceived(LoggedRequest.createFrom(request)); + } }); - } - - public TestSubscriber createSubscriber(String endpointPathSuffix) { - return createSubscriber(OK.getStatusCode(), endpointPathSuffix); - } - - public TestSubscriber createSubscriber() { - return createSubscriber(""); - } - - public TestSubscriber createSubscriber(int statusCode) { - return createSubscriber(statusCode, ""); - } - - public TestSubscriber createSubscriber(int statusCode, String endpointPathSuffix) { - String path = createPath(endpointPathSuffix); - return createSubscriberWithStrictPath(statusCode, path); - } - - public TestSubscriber createSubscriberWithStrictPath(int statusCode, String path) { - service.addStubMapping(post(urlPathEqualTo(path)).willReturn(aResponse().withStatus(statusCode)).build()); - TestSubscriber subscriber = new TestSubscriber(createSubscriberURI(path)); - subscribersPerPath.put(path, subscriber); - return subscriber; - } - - public TestSubscriber createSubscriberWithRetry(String message, int delay) { - String path = createPath(""); - int firstStatusCode = 503; - int secondStatusCode = 200; - String scenarioName = "Retrying"; - String secondScenarioState = "Retried"; - service.addStubMapping( - post(urlEqualTo(path)) + } + + public TestSubscriber createSubscriber(String endpointPathSuffix) { + return createSubscriber(OK.getStatusCode(), endpointPathSuffix); + } + + public TestSubscriber createSubscriber() { + return createSubscriber(""); + } + + public TestSubscriber createSubscriber(int statusCode) { + return createSubscriber(statusCode, ""); + } + + public TestSubscriber createSubscriber(int statusCode, String endpointPathSuffix) { + String path = createPath(endpointPathSuffix); + return createSubscriberWithStrictPath(statusCode, path); + } + + public TestSubscriber createSubscriberWithStrictPath(int statusCode, String path) { + service.addStubMapping( + post(urlPathEqualTo(path)).willReturn(aResponse().withStatus(statusCode)).build()); + TestSubscriber subscriber = new TestSubscriber(createSubscriberURI(path)); + subscribersPerPath.put(path, subscriber); + return subscriber; + } + + public TestSubscriber createSubscriberWithRetry(String message, int delay) { + String path = createPath(""); + int firstStatusCode = 503; + int secondStatusCode = 200; + String scenarioName = "Retrying"; + String secondScenarioState = "Retried"; + service.addStubMapping( + post(urlEqualTo(path)) .withRequestBody(new EqualToPattern(message)) .inScenario(scenarioName) .whenScenarioStateIs(STARTED) .willSetStateTo(secondScenarioState) - .willReturn(aResponse() - .withStatus(firstStatusCode) - .withHeader("Retry-After", Integer.toString(delay)) - .withFixedDelay(delay)) + .willReturn( + aResponse() + .withStatus(firstStatusCode) + .withHeader("Retry-After", Integer.toString(delay)) + .withFixedDelay(delay)) .build()); - service.addStubMapping( - post(urlEqualTo(path)) + service.addStubMapping( + post(urlEqualTo(path)) .withRequestBody(new EqualToPattern(message)) .inScenario(scenarioName) .whenScenarioStateIs(secondScenarioState) - .willReturn(aResponse() - .withStatus(secondStatusCode) - .withFixedDelay(delay)) + .willReturn(aResponse().withStatus(secondStatusCode).withFixedDelay(delay)) .build()); - TestSubscriber subscriber = new TestSubscriber(createSubscriberURI(path)); - subscribersPerPath.put(path, subscriber); - return subscriber; - } - - private String createPath(String pathSuffix) { - return "/subscriber-" + subscriberIndex.incrementAndGet() + pathSuffix; - } - - public interface SubscriberScenario { - void apply(WireMockServer subscriber, String endpoint); - } - - public TestSubscriber createSubscriber(SubscriberScenario scenario) { - String path = createPath(""); - scenario.apply(service, path); - TestSubscriber subscriber = new TestSubscriber(createSubscriberURI(path)); - subscribersPerPath.put(path, subscriber); - return subscriber; - } - - private URI createSubscriberURI(String path) { - return serviceUrl.resolve(path); - } - - @Override - public void afterEach(ExtensionContext context) { - service.resetRequests(); - subscribersPerPath.clear(); - } - - @Override - public void afterAll(ExtensionContext context) { - service.stop(); - } - - public int getPort() { - return service.port(); - } + TestSubscriber subscriber = new TestSubscriber(createSubscriberURI(path)); + subscribersPerPath.put(path, subscriber); + return subscriber; + } + + private String createPath(String pathSuffix) { + return "/subscriber-" + subscriberIndex.incrementAndGet() + pathSuffix; + } + + public interface SubscriberScenario { + void apply(WireMockServer subscriber, String endpoint); + } + + public TestSubscriber createSubscriber(SubscriberScenario scenario) { + String path = createPath(""); + scenario.apply(service, path); + TestSubscriber subscriber = new TestSubscriber(createSubscriberURI(path)); + subscribersPerPath.put(path, subscriber); + return subscriber; + } + + private URI createSubscriberURI(String path) { + return serviceUrl.resolve(path); + } + + @Override + public void afterEach(ExtensionContext context) { + service.resetRequests(); + subscribersPerPath.clear(); + } + + @Override + public void afterAll(ExtensionContext context) { + service.stop(); + } + + public int getPort() { + return service.port(); + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/management/JsonToAvroKafkaNamesMappersConfiguration.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/management/JsonToAvroKafkaNamesMappersConfiguration.java index 7c08d2b1ca..1239a13d5c 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/management/JsonToAvroKafkaNamesMappersConfiguration.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/management/JsonToAvroKafkaNamesMappersConfiguration.java @@ -12,13 +12,15 @@ @Configuration @EnableConfigurationProperties(KafkaClustersProperties.class) -public class JsonToAvroKafkaNamesMappersConfiguration implements MultipleDcKafkaNamesMappersFactory { +public class JsonToAvroKafkaNamesMappersConfiguration + implements MultipleDcKafkaNamesMappersFactory { - @Bean - @Primary - @Profile("integration") - KafkaNamesMappers testKafkaNameMappers(KafkaClustersProperties kafkaClustersProperties) { - return createKafkaNamesMapper(kafkaClustersProperties, namespace -> - new IntegrationTestKafkaNamesMapperFactory(namespace).create()); - } + @Bean + @Primary + @Profile("integration") + KafkaNamesMappers testKafkaNameMappers(KafkaClustersProperties kafkaClustersProperties) { + return createKafkaNamesMapper( + kafkaClustersProperties, + namespace -> new IntegrationTestKafkaNamesMapperFactory(namespace).create()); + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/management/TestSecurityProvider.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/management/TestSecurityProvider.java index afcc375b62..b521cb1e06 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/management/TestSecurityProvider.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/management/TestSecurityProvider.java @@ -1,79 +1,78 @@ package pl.allegro.tech.hermes.management; +import static java.util.stream.Collectors.toSet; + import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.SecurityContext; -import org.apache.commons.lang3.NotImplementedException; -import pl.allegro.tech.hermes.api.OwnerId; -import pl.allegro.tech.hermes.management.api.auth.SecurityProvider; - import java.security.Principal; import java.util.Arrays; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; - -import static java.util.stream.Collectors.toSet; +import org.apache.commons.lang3.NotImplementedException; +import pl.allegro.tech.hermes.api.OwnerId; +import pl.allegro.tech.hermes.management.api.auth.SecurityProvider; public class TestSecurityProvider implements SecurityProvider { - private static volatile boolean userIsAdmin = true; - private static final Set ownerIds = ConcurrentHashMap.newKeySet(); - - public static void setUserIsAdmin(boolean userIsAdmin) { - TestSecurityProvider.userIsAdmin = userIsAdmin; - } - - public static void setUserAsOwner(OwnerId... ownerIds) { - TestSecurityProvider.ownerIds.addAll(Arrays.stream(ownerIds).collect(toSet())); + private static volatile boolean userIsAdmin = true; + private static final Set ownerIds = ConcurrentHashMap.newKeySet(); + + public static void setUserIsAdmin(boolean userIsAdmin) { + TestSecurityProvider.userIsAdmin = userIsAdmin; + } + + public static void setUserAsOwner(OwnerId... ownerIds) { + TestSecurityProvider.ownerIds.addAll(Arrays.stream(ownerIds).collect(toSet())); + } + + public static void reset() { + userIsAdmin = true; + ownerIds.clear(); + } + + @Override + public HermesSecurity security(ContainerRequestContext requestContext) { + return new HermesSecurity(securityContext(), ownerIds::contains); + } + + private SecurityContext securityContext() { + + return new SecurityContext() { + @Override + public Principal getUserPrincipal() { + return new TestUserPrincipal(); + } + + @Override + public boolean isUserInRole(String role) { + return checkRoles(role); + } + + @Override + public boolean isSecure() { + throw new NotImplementedException(); + } + + @Override + public String getAuthenticationScheme() { + throw new NotImplementedException(); + } + }; + } + + private boolean checkRoles(String role) { + if (role.equalsIgnoreCase("admin")) { + return userIsAdmin; + } else { + return true; } + } - public static void reset() { - userIsAdmin = true; - ownerIds.clear(); - } + private static class TestUserPrincipal implements Principal { @Override - public HermesSecurity security(ContainerRequestContext requestContext) { - return new HermesSecurity(securityContext(), ownerIds::contains); - } - - private SecurityContext securityContext() { - - return new SecurityContext() { - @Override - public Principal getUserPrincipal() { - return new TestUserPrincipal(); - } - - @Override - public boolean isUserInRole(String role) { - return checkRoles(role); - } - - @Override - public boolean isSecure() { - throw new NotImplementedException(); - } - - @Override - public String getAuthenticationScheme() { - throw new NotImplementedException(); - } - }; - } - - private boolean checkRoles(String role) { - if (role.equalsIgnoreCase("admin")) { - return userIsAdmin; - } else { - return true; - } - } - - private static class TestUserPrincipal implements Principal { - - @Override - public String getName() { - return "test-user"; - } + public String getName() { + return "test-user"; } + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/management/TestSecurityProviderConfiguration.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/management/TestSecurityProviderConfiguration.java index 223c4f91df..ec21e99d78 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/management/TestSecurityProviderConfiguration.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/management/TestSecurityProviderConfiguration.java @@ -9,10 +9,10 @@ @Configuration public class TestSecurityProviderConfiguration { - @Bean - @Primary - @Profile("integration") - SecurityProvider testAuthorization() { - return new TestSecurityProvider(); - } + @Bean + @Primary + @Profile("integration") + SecurityProvider testAuthorization() { + return new TestSecurityProvider(); + } } diff --git a/integration-tests/src/common/java/pl/allegro/tech/hermes/utils/Headers.java b/integration-tests/src/common/java/pl/allegro/tech/hermes/utils/Headers.java index 75911b9590..75225335d7 100644 --- a/integration-tests/src/common/java/pl/allegro/tech/hermes/utils/Headers.java +++ b/integration-tests/src/common/java/pl/allegro/tech/hermes/utils/Headers.java @@ -1,13 +1,12 @@ package pl.allegro.tech.hermes.utils; -import org.springframework.http.HttpHeaders; - import java.util.Map; +import org.springframework.http.HttpHeaders; public class Headers { - public static HttpHeaders createHeaders(Map map) { - HttpHeaders headers = new HttpHeaders(); - map.forEach(headers::add); - return headers; - } + public static HttpHeaders createHeaders(Map map) { + HttpHeaders headers = new HttpHeaders(); + map.forEach(headers::add); + return headers; + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/BasicAuthSubscribingTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/BasicAuthSubscribingTest.java index 3c5dea2995..ab653658b6 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/BasicAuthSubscribingTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/BasicAuthSubscribingTest.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.integrationtests; +import static org.assertj.core.api.Assertions.assertThat; +import static pl.allegro.tech.hermes.api.PatchData.patchData; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import pl.allegro.tech.hermes.api.Subscription; @@ -9,52 +14,59 @@ import pl.allegro.tech.hermes.integrationtests.subscriber.TestSubscribersExtension; import pl.allegro.tech.hermes.test.helper.message.TestMessage; -import static org.assertj.core.api.Assertions.assertThat; -import static pl.allegro.tech.hermes.api.PatchData.patchData; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class BasicAuthSubscribingTest { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); - @RegisterExtension - public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); + @RegisterExtension + public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); - @Test - public void shouldAuthorizeUsingBasicAuthWhenSubscriptionHasCredentials() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); + @Test + public void shouldAuthorizeUsingBasicAuthWhenSubscriptionHasCredentials() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - hermes.initHelper().createSubscription(subscription( - topic.getQualifiedName(), - "subscription", - "http://user:password@localhost:" + subscriber.getPort() + subscriber.getPath()).build()); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + hermes + .initHelper() + .createSubscription( + subscription( + topic.getQualifiedName(), + "subscription", + "http://user:password@localhost:" + subscriber.getPort() + subscriber.getPath()) + .build()); - TestMessage message = TestMessage.of("hello", "world"); + TestMessage message = TestMessage.of("hello", "world"); - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - // then - subscriber.waitUntilMessageWithHeaderReceived("Authorization", "Basic dXNlcjpwYXNzd29yZA=="); - } + // then + subscriber.waitUntilMessageWithHeaderReceived("Authorization", "Basic dXNlcjpwYXNzd29yZA=="); + } - @Test - public void shouldUpdateSubscriptionUsernameAndPassword() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - hermes.initHelper().createSubscription(subscription( - topic.getQualifiedName(), "subscription", "http://user:password@localhost:1234").build()); + @Test + public void shouldUpdateSubscriptionUsernameAndPassword() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + hermes + .initHelper() + .createSubscription( + subscription( + topic.getQualifiedName(), "subscription", "http://user:password@localhost:1234") + .build()); - // when - hermes.api().updateSubscription(topic, "subscription", + // when + hermes + .api() + .updateSubscription( + topic, + "subscription", patchData().set("endpoint", "http://newuser:newpassword@localhost:1234").build()); - // then - Subscription subscription = hermes.api().getSubscription(topic.getQualifiedName(), "subscription"); - assertThat(subscription.getEndpoint().getUsername()).isEqualTo("newuser"); - } + // then + Subscription subscription = + hermes.api().getSubscription(topic.getQualifiedName(), "subscription"); + assertThat(subscription.getEndpoint().getUsername()).isEqualTo("newuser"); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/BatchDeliveryTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/BatchDeliveryTest.java index e994641533..560c358bac 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/BatchDeliveryTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/BatchDeliveryTest.java @@ -1,6 +1,24 @@ package pl.allegro.tech.hermes.integrationtests; +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.stubbing.Scenario.STARTED; +import static com.google.common.collect.ImmutableMap.of; +import static java.util.Arrays.stream; +import static org.awaitility.Awaitility.waitAtMost; +import static pl.allegro.tech.hermes.api.BatchSubscriptionPolicy.Builder.batchSubscriptionPolicy; +import static pl.allegro.tech.hermes.api.TopicWithSchema.topicWithSchema; +import static pl.allegro.tech.hermes.integrationtests.assertions.HermesAssertions.assertThatMetrics; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.time.Duration; +import java.util.List; +import java.util.Map; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -15,410 +33,415 @@ import pl.allegro.tech.hermes.test.helper.avro.AvroUser; import pl.allegro.tech.hermes.test.helper.message.TestMessage; -import java.io.IOException; -import java.time.Duration; -import java.util.List; -import java.util.Map; - -import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; -import static com.github.tomakehurst.wiremock.client.WireMock.post; -import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; -import static com.github.tomakehurst.wiremock.stubbing.Scenario.STARTED; -import static com.google.common.collect.ImmutableMap.of; -import static java.util.Arrays.stream; -import static org.awaitility.Awaitility.waitAtMost; -import static pl.allegro.tech.hermes.api.BatchSubscriptionPolicy.Builder.batchSubscriptionPolicy; -import static pl.allegro.tech.hermes.api.TopicWithSchema.topicWithSchema; -import static pl.allegro.tech.hermes.integrationtests.assertions.HermesAssertions.assertThatMetrics; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class BatchDeliveryTest { - private final ObjectMapper mapper = new ObjectMapper(); - - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - @RegisterExtension - public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); - - static final AvroUser BOB = new AvroUser("Bob", 50, "blue"); - - static final AvroUser ALICE = new AvroUser("Alice", 20, "magenta"); - - private static final TestMessage[] SMALL_BATCH = TestMessage.simpleMessages(2); - - private static final TestMessage SINGLE_MESSAGE = TestMessage.simple(); - - private static final TestMessage SINGLE_MESSAGE_FILTERED = BOB.asTestMessage(); - - private static final MessageFilterSpecification MESSAGE_NAME_FILTER = - new MessageFilterSpecification(of("type", "jsonpath", "path", ".name", "matcher", "^Bob.*")); - - @Test - public void shouldDeliverMessagesInBatch() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestSubscriber subscriber = subscribers.createSubscriber(); - - Subscription subscription = subscription(topic.getQualifiedName(), - "subscription", subscriber.getEndpoint()) - .withSubscriptionPolicy(buildBatchPolicy() - .withBatchSize(2) - .withBatchTime(Integer.MAX_VALUE) - .withBatchVolume(1024) - .build() - ).build(); - - hermes.initHelper().createSubscription(subscription); - - // when - publishAll(topic.getQualifiedName(), SMALL_BATCH); - - // then - expectSingleBatch(subscriber, SMALL_BATCH); - } - - @Test - public void shouldFilterIncomingEventsForBatch() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - final Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()) - .withSubscriptionPolicy(buildBatchPolicy() - .withBatchSize(2) - .withBatchTime(3) - .withBatchVolume(1024) - .build()) - .withFilter(MESSAGE_NAME_FILTER) - .build()); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), BOB.asJson()); - hermes.api().publishUntilSuccess(topic.getQualifiedName(), ALICE.asJson()); - - // then - expectSingleBatch(subscriber, SINGLE_MESSAGE_FILTERED); - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> - hermes.api().getConsumersMetrics() - .expectStatus() - .isOk() - .expectBody(String.class) - .value((body) -> assertThatMetrics(body) + private final ObjectMapper mapper = new ObjectMapper(); + + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); + + @RegisterExtension + public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); + + static final AvroUser BOB = new AvroUser("Bob", 50, "blue"); + + static final AvroUser ALICE = new AvroUser("Alice", 20, "magenta"); + + private static final TestMessage[] SMALL_BATCH = TestMessage.simpleMessages(2); + + private static final TestMessage SINGLE_MESSAGE = TestMessage.simple(); + + private static final TestMessage SINGLE_MESSAGE_FILTERED = BOB.asTestMessage(); + + private static final MessageFilterSpecification MESSAGE_NAME_FILTER = + new MessageFilterSpecification(of("type", "jsonpath", "path", ".name", "matcher", "^Bob.*")); + + @Test + public void shouldDeliverMessagesInBatch() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestSubscriber subscriber = subscribers.createSubscriber(); + + Subscription subscription = + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) + .withSubscriptionPolicy( + buildBatchPolicy() + .withBatchSize(2) + .withBatchTime(Integer.MAX_VALUE) + .withBatchVolume(1024) + .build()) + .build(); + + hermes.initHelper().createSubscription(subscription); + + // when + publishAll(topic.getQualifiedName(), SMALL_BATCH); + + // then + expectSingleBatch(subscriber, SMALL_BATCH); + } + + @Test + public void shouldFilterIncomingEventsForBatch() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + final Subscription subscription = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()) + .withSubscriptionPolicy( + buildBatchPolicy() + .withBatchSize(2) + .withBatchTime(3) + .withBatchVolume(1024) + .build()) + .withFilter(MESSAGE_NAME_FILTER) + .build()); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), BOB.asJson()); + hermes.api().publishUntilSuccess(topic.getQualifiedName(), ALICE.asJson()); + + // then + expectSingleBatch(subscriber, SINGLE_MESSAGE_FILTERED); + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> + hermes + .api() + .getConsumersMetrics() + .expectStatus() + .isOk() + .expectBody(String.class) + .value( + (body) -> + assertThatMetrics(body) .contains("hermes_consumers_subscription_filtered_out_total") .withLabels( - "group", topic.getName().getGroupName(), - "subscription", subscription.getName(), - "topic", topic.getName().getName() - ) - .withValue(1.0) - ) - ); - } - - @Test - public void shouldCommitFilteredMessagesForBatch() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - final Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()) - .withSubscriptionPolicy(buildBatchPolicy() - .withBatchSize(10) - .withBatchTime(Integer.MAX_VALUE) - .withBatchVolume(1024) - .build()) - .withFilter(MESSAGE_NAME_FILTER) - .build()); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), ALICE.asJson()); - hermes.api().publishUntilSuccess(topic.getQualifiedName(), ALICE.asJson()); - hermes.api().publishUntilSuccess(topic.getQualifiedName(), ALICE.asJson()); - - // then - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> - hermes.api().getConsumersMetrics() - .expectStatus() - .isOk() - .expectBody(String.class) - .value((body) -> assertThatMetrics(body) + "group", topic.getName().getGroupName(), + "subscription", subscription.getName(), + "topic", topic.getName().getName()) + .withValue(1.0))); + } + + @Test + public void shouldCommitFilteredMessagesForBatch() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + final Subscription subscription = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()) + .withSubscriptionPolicy( + buildBatchPolicy() + .withBatchSize(10) + .withBatchTime(Integer.MAX_VALUE) + .withBatchVolume(1024) + .build()) + .withFilter(MESSAGE_NAME_FILTER) + .build()); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), ALICE.asJson()); + hermes.api().publishUntilSuccess(topic.getQualifiedName(), ALICE.asJson()); + hermes.api().publishUntilSuccess(topic.getQualifiedName(), ALICE.asJson()); + + // then + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> + hermes + .api() + .getConsumersMetrics() + .expectStatus() + .isOk() + .expectBody(String.class) + .value( + (body) -> + assertThatMetrics(body) .contains("hermes_consumers_subscription_filtered_out_total") .withLabels( - "group", topic.getName().getGroupName(), - "subscription", subscription.getName(), - "topic", topic.getName().getName() - ) - .withValue(3.0) - ) - ); - hermes.api().waitUntilConsumerCommitsOffset(topic.getQualifiedName(), subscription.getName()); - } - - @Test - public void shouldDeliverBatchInGivenTimePeriod() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestSubscriber subscriber = subscribers.createSubscriber(); - - Subscription subscription = subscription(topic.getQualifiedName(), - "subscription", subscriber.getEndpoint()) - .withSubscriptionPolicy(buildBatchPolicy() - .withBatchSize(100) - .withBatchTime(1) - .withBatchVolume(1024) - .build() - ).build(); - - hermes.initHelper().createSubscription(subscription); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), SINGLE_MESSAGE.body()); - - // then - expectSingleBatch(subscriber, SINGLE_MESSAGE); - } - - @Test - public void shouldDeliverBatchInGivenVolume() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestSubscriber subscriber = subscribers.createSubscriber(); - - int batchVolumeThatFitsOneMessageOnly = 150; - - Subscription subscription = subscription(topic.getQualifiedName(), - "subscription", subscriber.getEndpoint()) - .withSubscriptionPolicy(buildBatchPolicy() - .withBatchSize(100) - .withBatchTime(Integer.MAX_VALUE) - .withBatchVolume(batchVolumeThatFitsOneMessageOnly) - .build() - ).build(); - - hermes.initHelper().createSubscription(subscription); - - // when publishing more than buffer capacity - hermes.api().publishUntilSuccess(topic.getQualifiedName(), SINGLE_MESSAGE.body()); - hermes.api().publishUntilSuccess(topic.getQualifiedName(), SINGLE_MESSAGE.body()); - - // then we expect to receive batch that has desired batch volume (one message only) - expectSingleBatch(subscriber, SINGLE_MESSAGE); - } - - @Test - public void shouldDeliverAvroMessagesAsJsonBatch() { - // given - AvroUser user = new AvroUser("Bob", 50, "blue"); - - Topic topic = hermes.initHelper().createTopicWithSchema( - topicWithSchema(topicWithRandomName().build(), user.getSchemaAsString()) - ); - - TestSubscriber subscriber = subscribers.createSubscriber(); - - Subscription subscription = subscription(topic.getQualifiedName(), - "subscription", subscriber.getEndpoint()) - .withSubscriptionPolicy(buildBatchPolicy() - .withBatchSize(2) - .withBatchTime(Integer.MAX_VALUE) - .withBatchVolume(1024) - .build() - ).build(); - - hermes.initHelper().createSubscription(subscription); - - TestMessage[] avroBatch = {user.asTestMessage(), user.asTestMessage()}; - - // when - publishAll(topic.getQualifiedName(), avroBatch); - - // then - expectSingleBatch(subscriber, avroBatch); - } - - - @Test - public void shouldPassSubscriptionHeaders() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestSubscriber subscriber = subscribers.createSubscriber(); - - BatchSubscriptionPolicy policy = buildBatchPolicy() - .withBatchSize(100) - .withBatchTime(1) - .withBatchVolume(1024) - .build(); - Subscription subscription = subscription(topic, "batchSubscription") - .withEndpoint(subscriber.getEndpoint()) - .withContentType(ContentType.JSON) - .withSubscriptionPolicy(policy) - .withHeader("MY-HEADER", "myHeaderValue") - .withHeader("MY-OTHER-HEADER", "myOtherHeaderValue") - .build(); - - hermes.initHelper().createSubscription(subscription); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), SINGLE_MESSAGE.body()); - - // then - subscriber.waitUntilRequestReceived(request -> { - Assertions.assertThat(request.getHeader("MY-HEADER")).isEqualTo("myHeaderValue"); - Assertions.assertThat(request.getHeader("MY-OTHER-HEADER")).isEqualTo("myOtherHeaderValue"); + "group", topic.getName().getGroupName(), + "subscription", subscription.getName(), + "topic", topic.getName().getName()) + .withValue(3.0))); + hermes.api().waitUntilConsumerCommitsOffset(topic.getQualifiedName(), subscription.getName()); + } + + @Test + public void shouldDeliverBatchInGivenTimePeriod() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestSubscriber subscriber = subscribers.createSubscriber(); + + Subscription subscription = + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) + .withSubscriptionPolicy( + buildBatchPolicy() + .withBatchSize(100) + .withBatchTime(1) + .withBatchVolume(1024) + .build()) + .build(); + + hermes.initHelper().createSubscription(subscription); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), SINGLE_MESSAGE.body()); + + // then + expectSingleBatch(subscriber, SINGLE_MESSAGE); + } + + @Test + public void shouldDeliverBatchInGivenVolume() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestSubscriber subscriber = subscribers.createSubscriber(); + + int batchVolumeThatFitsOneMessageOnly = 150; + + Subscription subscription = + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) + .withSubscriptionPolicy( + buildBatchPolicy() + .withBatchSize(100) + .withBatchTime(Integer.MAX_VALUE) + .withBatchVolume(batchVolumeThatFitsOneMessageOnly) + .build()) + .build(); + + hermes.initHelper().createSubscription(subscription); + + // when publishing more than buffer capacity + hermes.api().publishUntilSuccess(topic.getQualifiedName(), SINGLE_MESSAGE.body()); + hermes.api().publishUntilSuccess(topic.getQualifiedName(), SINGLE_MESSAGE.body()); + + // then we expect to receive batch that has desired batch volume (one message only) + expectSingleBatch(subscriber, SINGLE_MESSAGE); + } + + @Test + public void shouldDeliverAvroMessagesAsJsonBatch() { + // given + AvroUser user = new AvroUser("Bob", 50, "blue"); + + Topic topic = + hermes + .initHelper() + .createTopicWithSchema( + topicWithSchema(topicWithRandomName().build(), user.getSchemaAsString())); + + TestSubscriber subscriber = subscribers.createSubscriber(); + + Subscription subscription = + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) + .withSubscriptionPolicy( + buildBatchPolicy() + .withBatchSize(2) + .withBatchTime(Integer.MAX_VALUE) + .withBatchVolume(1024) + .build()) + .build(); + + hermes.initHelper().createSubscription(subscription); + + TestMessage[] avroBatch = {user.asTestMessage(), user.asTestMessage()}; + + // when + publishAll(topic.getQualifiedName(), avroBatch); + + // then + expectSingleBatch(subscriber, avroBatch); + } + + @Test + public void shouldPassSubscriptionHeaders() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestSubscriber subscriber = subscribers.createSubscriber(); + + BatchSubscriptionPolicy policy = + buildBatchPolicy().withBatchSize(100).withBatchTime(1).withBatchVolume(1024).build(); + Subscription subscription = + subscription(topic, "batchSubscription") + .withEndpoint(subscriber.getEndpoint()) + .withContentType(ContentType.JSON) + .withSubscriptionPolicy(policy) + .withHeader("MY-HEADER", "myHeaderValue") + .withHeader("MY-OTHER-HEADER", "myOtherHeaderValue") + .build(); + + hermes.initHelper().createSubscription(subscription); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), SINGLE_MESSAGE.body()); + + // then + subscriber.waitUntilRequestReceived( + request -> { + Assertions.assertThat(request.getHeader("MY-HEADER")).isEqualTo("myHeaderValue"); + Assertions.assertThat(request.getHeader("MY-OTHER-HEADER")) + .isEqualTo("myOtherHeaderValue"); }); - } - - @Test - public void shouldAttachSubscriptionIdentityHeadersWhenItIsEnabled() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - - TestSubscriber subscriber = subscribers.createSubscriber(); - - BatchSubscriptionPolicy policy = buildBatchPolicy() - .withBatchSize(100) - .withBatchTime(1) - .withBatchVolume(1024) - .build(); - Subscription subscription = subscription(topic, "batchSubscription") - .withEndpoint(subscriber.getEndpoint()) - .withContentType(ContentType.JSON) - .withSubscriptionPolicy(policy) - .withAttachingIdentityHeadersEnabled(true) - .build(); - - hermes.initHelper().createSubscription(subscription); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), SINGLE_MESSAGE.body()); - - // then - subscriber.waitUntilRequestReceived(request -> { - Assertions.assertThat(request.getHeader("Hermes-Topic-Name")).isEqualTo(topic.getQualifiedName()); - Assertions.assertThat(request.getHeader("Hermes-Subscription-Name")).isEqualTo("batchSubscription"); + } + + @Test + public void shouldAttachSubscriptionIdentityHeadersWhenItIsEnabled() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + + TestSubscriber subscriber = subscribers.createSubscriber(); + + BatchSubscriptionPolicy policy = + buildBatchPolicy().withBatchSize(100).withBatchTime(1).withBatchVolume(1024).build(); + Subscription subscription = + subscription(topic, "batchSubscription") + .withEndpoint(subscriber.getEndpoint()) + .withContentType(ContentType.JSON) + .withSubscriptionPolicy(policy) + .withAttachingIdentityHeadersEnabled(true) + .build(); + + hermes.initHelper().createSubscription(subscription); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), SINGLE_MESSAGE.body()); + + // then + subscriber.waitUntilRequestReceived( + request -> { + Assertions.assertThat(request.getHeader("Hermes-Topic-Name")) + .isEqualTo(topic.getQualifiedName()); + Assertions.assertThat(request.getHeader("Hermes-Subscription-Name")) + .isEqualTo("batchSubscription"); }); - } + } - @Test - public void shouldNotAttachSubscriptionIdentityHeadersWhenItIsDisabled() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + @Test + public void shouldNotAttachSubscriptionIdentityHeadersWhenItIsDisabled() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestSubscriber subscriber = subscribers.createSubscriber(); + TestSubscriber subscriber = subscribers.createSubscriber(); - BatchSubscriptionPolicy policy = buildBatchPolicy() - .withBatchSize(100) - .withBatchTime(1) - .withBatchVolume(1024) - .build(); + BatchSubscriptionPolicy policy = + buildBatchPolicy().withBatchSize(100).withBatchTime(1).withBatchVolume(1024).build(); - Subscription subscription = subscription(topic, "batchSubscription") - .withEndpoint(subscriber.getEndpoint()) - .withContentType(ContentType.JSON) - .withSubscriptionPolicy(policy) - .withAttachingIdentityHeadersEnabled(false) - .build(); + Subscription subscription = + subscription(topic, "batchSubscription") + .withEndpoint(subscriber.getEndpoint()) + .withContentType(ContentType.JSON) + .withSubscriptionPolicy(policy) + .withAttachingIdentityHeadersEnabled(false) + .build(); - hermes.initHelper().createSubscription(subscription); + hermes.initHelper().createSubscription(subscription); - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), SINGLE_MESSAGE.body()); + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), SINGLE_MESSAGE.body()); - // then - subscriber.waitUntilRequestReceived(request -> { - Assertions.assertThat(request.getHeader("Hermes-Topic-Name")).isNull(); - Assertions.assertThat(request.getHeader("Hermes-Subscription-Name")).isNull(); + // then + subscriber.waitUntilRequestReceived( + request -> { + Assertions.assertThat(request.getHeader("Hermes-Topic-Name")).isNull(); + Assertions.assertThat(request.getHeader("Hermes-Subscription-Name")).isNull(); }); - } - - @Test - public void shouldTimeoutRequestToSlowlyRespondingClient() { - //given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - - // response chunk every 500ms, total 5s - int chunks = 10; - int totalResponseDuration = 5000; - - TestSubscriber subscriber = subscribers.createSubscriber((service, endpoint) -> { - service.addStubMapping( - post(urlEqualTo(endpoint)) - .inScenario("slowAndFast") - .whenScenarioStateIs(STARTED) - .willSetStateTo("slow") - .willReturn( - aResponse() - .withStatus(200) - .withBody("I am very slow!") - .withChunkedDribbleDelay(chunks, totalResponseDuration) - ).build() - ); - - service.addStubMapping( - post(urlEqualTo(endpoint)) - .inScenario("slowAndFast") - .whenScenarioStateIs("slow") - .willReturn( - aResponse() - .withStatus(200) - .withFixedDelay(0) - ).build() - ); + } + + @Test + public void shouldTimeoutRequestToSlowlyRespondingClient() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + + // response chunk every 500ms, total 5s + int chunks = 10; + int totalResponseDuration = 5000; + + TestSubscriber subscriber = + subscribers.createSubscriber( + (service, endpoint) -> { + service.addStubMapping( + post(urlEqualTo(endpoint)) + .inScenario("slowAndFast") + .whenScenarioStateIs(STARTED) + .willSetStateTo("slow") + .willReturn( + aResponse() + .withStatus(200) + .withBody("I am very slow!") + .withChunkedDribbleDelay(chunks, totalResponseDuration)) + .build()); + + service.addStubMapping( + post(urlEqualTo(endpoint)) + .inScenario("slowAndFast") + .whenScenarioStateIs("slow") + .willReturn(aResponse().withStatus(200).withFixedDelay(0)) + .build()); + }); + + Subscription subscription = + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) + .withSubscriptionPolicy( + buildBatchPolicy() + .withBatchSize(1) + .withBatchTime(1) + .withBatchVolume(1024) + .withRequestTimeout(1000) + .build()) + .build(); + + hermes.initHelper().createSubscription(subscription); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), SINGLE_MESSAGE.body()); + + // then + // first request is retried because of timeout (with socket / idle timeout only, the request + // wouldn't be timed out because + // there are chunks of response every 500ms which is smaller than 1s timeout) + subscriber.waitUntilReceived(Duration.ofSeconds(5), 2); + Assertions.assertThat(subscriber.getLastReceivedRequest().getHeader("Hermes-Retry-Count")) + .isEqualTo("1"); + } + + private void publishAll(String topicQualifiedName, TestMessage... messages) { + stream(messages) + .forEach(message -> hermes.api().publishUntilSuccess(topicQualifiedName, message.body())); + } + + private void expectSingleBatch(TestSubscriber subscriber, TestMessage... expectedContents) { + subscriber.waitUntilRequestReceived( + message -> { + List> batch = readBatch(message.getBodyAsString()); + Assertions.assertThat(batch).hasSize(expectedContents.length); + for (int i = 0; i < expectedContents.length; i++) { + Assertions.assertThat(batch.get(i).get("message")) + .isEqualTo(expectedContents[i].getContent()); + Assertions.assertThat((String) ((Map) batch.get(i).get("metadata")).get("id")) + .isNotEmpty(); + } }); - - Subscription subscription = subscription(topic.getQualifiedName(), - "subscription", subscriber.getEndpoint()) - .withSubscriptionPolicy(buildBatchPolicy() - .withBatchSize(1) - .withBatchTime(1) - .withBatchVolume(1024) - .withRequestTimeout(1000) - .build() - ).build(); - - hermes.initHelper().createSubscription(subscription); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), SINGLE_MESSAGE.body()); - - // then - // first request is retried because of timeout (with socket / idle timeout only, the request wouldn't be timed out because - // there are chunks of response every 500ms which is smaller than 1s timeout) - subscriber.waitUntilReceived(Duration.ofSeconds(5), 2); - Assertions.assertThat(subscriber.getLastReceivedRequest().getHeader("Hermes-Retry-Count")).isEqualTo("1"); - } - - private void publishAll(String topicQualifiedName, TestMessage... messages) { - stream(messages).forEach(message -> hermes.api().publishUntilSuccess(topicQualifiedName, message.body())); - } - - private void expectSingleBatch(TestSubscriber subscriber, TestMessage... expectedContents) { - subscriber.waitUntilRequestReceived(message -> { - List> batch = readBatch(message.getBodyAsString()); - Assertions.assertThat(batch).hasSize(expectedContents.length); - for (int i = 0; i < expectedContents.length; i++) { - Assertions.assertThat(batch.get(i).get("message")).isEqualTo(expectedContents[i].getContent()); - Assertions.assertThat((String) ((Map) batch.get(i).get("metadata")).get("id")).isNotEmpty(); - } - }); - } - - private BatchSubscriptionPolicy.Builder buildBatchPolicy() { - return batchSubscriptionPolicy() - .applyDefaults() - .withMessageTtl(100) - .withRequestTimeout(100) - .withMessageBackoff(10); - } - - @SuppressWarnings("unchecked") - private List> readBatch(String message) { - try { - return mapper.readValue(message, List.class); - } catch (IOException e) { - throw new RuntimeException(e); - } + } + + private BatchSubscriptionPolicy.Builder buildBatchPolicy() { + return batchSubscriptionPolicy() + .applyDefaults() + .withMessageTtl(100) + .withRequestTimeout(100) + .withMessageBackoff(10); + } + + @SuppressWarnings("unchecked") + private List> readBatch(String message) { + try { + return mapper.readValue(message, List.class); + } catch (IOException e) { + throw new RuntimeException(e); } + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/BatchRetryPolicyTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/BatchRetryPolicyTest.java index 77c9a42074..dbd5f413b0 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/BatchRetryPolicyTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/BatchRetryPolicyTest.java @@ -1,8 +1,22 @@ package pl.allegro.tech.hermes.integrationtests; +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.containing; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static org.apache.hc.core5.http.HttpStatus.SC_BAD_REQUEST; +import static org.apache.hc.core5.http.HttpStatus.SC_CREATED; +import static org.apache.hc.core5.http.HttpStatus.SC_INTERNAL_SERVER_ERROR; +import static pl.allegro.tech.hermes.api.BatchSubscriptionPolicy.Builder.batchSubscriptionPolicy; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + import com.fasterxml.jackson.databind.ObjectMapper; import com.github.tomakehurst.wiremock.stubbing.Scenario; import com.github.tomakehurst.wiremock.verification.LoggedRequest; +import java.io.IOException; +import java.time.Duration; +import java.util.List; +import java.util.Map; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -15,219 +29,238 @@ import pl.allegro.tech.hermes.integrationtests.subscriber.TestSubscribersExtension; import pl.allegro.tech.hermes.test.helper.message.TestMessage; -import java.io.IOException; -import java.time.Duration; -import java.util.List; -import java.util.Map; - -import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; -import static com.github.tomakehurst.wiremock.client.WireMock.containing; -import static com.github.tomakehurst.wiremock.client.WireMock.post; -import static org.apache.hc.core5.http.HttpStatus.SC_BAD_REQUEST; -import static org.apache.hc.core5.http.HttpStatus.SC_CREATED; -import static org.apache.hc.core5.http.HttpStatus.SC_INTERNAL_SERVER_ERROR; -import static pl.allegro.tech.hermes.api.BatchSubscriptionPolicy.Builder.batchSubscriptionPolicy; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class BatchRetryPolicyTest { - private static final String HEALTHY = "healthy"; - private static final String failedRequestBody = "{\"body\":\"failed\"}"; - private static final String successfulRequestBody = "{\"body\":\"successful\"}"; - - private final ObjectMapper mapper = new ObjectMapper(); - - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - @RegisterExtension - public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); - - @Test - public void shouldRetryUntilRequestSuccessfulAndSendRetryCounterInHeader() { - //given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - - TestSubscriber subscriber = subscribers.createSubscriber((service, endpoint) -> { - service.addStubMapping(post(endpoint) - .inScenario(topic.getQualifiedName()) - .whenScenarioStateIs(Scenario.STARTED) - .willReturn(aResponse().withStatus(SC_INTERNAL_SERVER_ERROR)) - .willSetStateTo(HEALTHY) - .build() - ); - - service.addStubMapping(post(endpoint) - .inScenario(topic.getQualifiedName()) - .whenScenarioStateIs(HEALTHY) - .willReturn(aResponse().withStatus(SC_CREATED)).build()); - }); - - createSingleMessageBatchSubscription(topic, subscriber.getEndpoint()); - - //when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), TestMessage.simple().body()); - - //then - subscriber.waitUntilRequestsReceived(requests -> { - Assertions.assertThat(requests).hasSize(2); - Assertions.assertThat(requests.get(0).header("Hermes-Retry-Count").containsValue("0")).isTrue(); - Assertions.assertThat(requests.get(1).header("Hermes-Retry-Count").containsValue("1")).isTrue(); - }); - } - - @Test - public void shouldNotRetryIfRequestSuccessful() { - //given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestMessage message = TestMessage.simple(); - - TestSubscriber subscriber = subscribers.createSubscriber(); - - createSingleMessageBatchSubscription(topic, subscriber.getEndpoint()); - - //when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - - //then - subscriber.waitUntilReceived(Duration.ofSeconds(5), 1); - } - - @Test - public void shouldRetryUntilTtlExceeded() { - //given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - - TestSubscriber subscriber = subscribers.createSubscriber((service, endpoint) -> { - service.addStubMapping((post(endpoint)) - .withRequestBody(containing("failed")) - .willReturn(aResponse().withStatus(SC_INTERNAL_SERVER_ERROR)).build()); - - service.addStubMapping((post(endpoint)) - .withRequestBody(containing("successful")) - .willReturn(aResponse().withStatus(SC_CREATED)).build()); - }); - - createSingleMessageBatchSubscription(topic, subscriber.getEndpoint(), 1, 10); - - //when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), failedRequestBody); - hermes.api().publishUntilSuccess(topic.getQualifiedName(), successfulRequestBody); - - //then - subscriber.waitUntilRequestsReceived(requests -> - Assertions.assertThat(requests) - .extracting(LoggedRequest::getBodyAsString) - .extracting(this::readMessage) - .containsSequence(failedRequestBody, failedRequestBody, successfulRequestBody)); - } - - @Test - public void shouldRetryOnClientErrors() { - //given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - - TestSubscriber subscriber = subscribers.createSubscriber((service, endpoint) -> { - service.addStubMapping(post(endpoint) - .inScenario(topic.getQualifiedName()) - .whenScenarioStateIs(Scenario.STARTED) - .willReturn(aResponse().withStatus(SC_BAD_REQUEST)) - .willSetStateTo(HEALTHY).build()); - - service.addStubMapping(post(endpoint) - .inScenario(topic.getQualifiedName()) - .whenScenarioStateIs(HEALTHY) - .willReturn(aResponse().withStatus(SC_CREATED)).build()); - }); - - createSingleMessageBatchSubscription(topic, subscriber.getEndpoint(), true); - - //when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), TestMessage.simple().body()); - - //then - subscriber.waitUntilRequestsReceived(requests -> Assertions.assertThat(requests.size()).isEqualTo(2)); - } - - @Test - public void shouldNotRetryOnClientErrors() { - //given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - - - TestSubscriber subscriber = subscribers.createSubscriber((service, endpoint) -> { - - service.addStubMapping(post(endpoint) - .withRequestBody(containing("failed")) - .willReturn(aResponse().withStatus(SC_BAD_REQUEST)).build()); - - service.addStubMapping(post(endpoint) - .withRequestBody(containing("successful")) - .willReturn(aResponse().withStatus(SC_CREATED)).build()); + private static final String HEALTHY = "healthy"; + private static final String failedRequestBody = "{\"body\":\"failed\"}"; + private static final String successfulRequestBody = "{\"body\":\"successful\"}"; + + private final ObjectMapper mapper = new ObjectMapper(); + + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); + + @RegisterExtension + public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); + + @Test + public void shouldRetryUntilRequestSuccessfulAndSendRetryCounterInHeader() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + + TestSubscriber subscriber = + subscribers.createSubscriber( + (service, endpoint) -> { + service.addStubMapping( + post(endpoint) + .inScenario(topic.getQualifiedName()) + .whenScenarioStateIs(Scenario.STARTED) + .willReturn(aResponse().withStatus(SC_INTERNAL_SERVER_ERROR)) + .willSetStateTo(HEALTHY) + .build()); + + service.addStubMapping( + post(endpoint) + .inScenario(topic.getQualifiedName()) + .whenScenarioStateIs(HEALTHY) + .willReturn(aResponse().withStatus(SC_CREATED)) + .build()); + }); + + createSingleMessageBatchSubscription(topic, subscriber.getEndpoint()); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), TestMessage.simple().body()); + + // then + subscriber.waitUntilRequestsReceived( + requests -> { + Assertions.assertThat(requests).hasSize(2); + Assertions.assertThat(requests.get(0).header("Hermes-Retry-Count").containsValue("0")) + .isTrue(); + Assertions.assertThat(requests.get(1).header("Hermes-Retry-Count").containsValue("1")) + .isTrue(); }); - - createSingleMessageBatchSubscription(topic, subscriber.getEndpoint()); - - //when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), failedRequestBody); - hermes.api().publishUntilSuccess(topic.getQualifiedName(), successfulRequestBody); - - //then - subscriber.waitUntilRequestsReceived(requests -> - Assertions.assertThat(requests) - .extracting(LoggedRequest::getBodyAsString) - .extracting(this::readMessage) - .containsExactly(failedRequestBody, successfulRequestBody)); - } - - - private void createSingleMessageBatchSubscription(Topic topic, String endpoint, int messageTtl, int messageBackoff) { - createBatchSubscription( - topic, endpoint, messageTtl, messageBackoff, 1, 1, 200, false - ); - } - - private void createSingleMessageBatchSubscription(Topic topic, String endpoint) { - createSingleMessageBatchSubscription(topic, endpoint, false); + } + + @Test + public void shouldNotRetryIfRequestSuccessful() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestMessage message = TestMessage.simple(); + + TestSubscriber subscriber = subscribers.createSubscriber(); + + createSingleMessageBatchSubscription(topic, subscriber.getEndpoint()); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + + // then + subscriber.waitUntilReceived(Duration.ofSeconds(5), 1); + } + + @Test + public void shouldRetryUntilTtlExceeded() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + + TestSubscriber subscriber = + subscribers.createSubscriber( + (service, endpoint) -> { + service.addStubMapping( + (post(endpoint)) + .withRequestBody(containing("failed")) + .willReturn(aResponse().withStatus(SC_INTERNAL_SERVER_ERROR)) + .build()); + + service.addStubMapping( + (post(endpoint)) + .withRequestBody(containing("successful")) + .willReturn(aResponse().withStatus(SC_CREATED)) + .build()); + }); + + createSingleMessageBatchSubscription(topic, subscriber.getEndpoint(), 1, 10); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), failedRequestBody); + hermes.api().publishUntilSuccess(topic.getQualifiedName(), successfulRequestBody); + + // then + subscriber.waitUntilRequestsReceived( + requests -> + Assertions.assertThat(requests) + .extracting(LoggedRequest::getBodyAsString) + .extracting(this::readMessage) + .containsSequence(failedRequestBody, failedRequestBody, successfulRequestBody)); + } + + @Test + public void shouldRetryOnClientErrors() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + + TestSubscriber subscriber = + subscribers.createSubscriber( + (service, endpoint) -> { + service.addStubMapping( + post(endpoint) + .inScenario(topic.getQualifiedName()) + .whenScenarioStateIs(Scenario.STARTED) + .willReturn(aResponse().withStatus(SC_BAD_REQUEST)) + .willSetStateTo(HEALTHY) + .build()); + + service.addStubMapping( + post(endpoint) + .inScenario(topic.getQualifiedName()) + .whenScenarioStateIs(HEALTHY) + .willReturn(aResponse().withStatus(SC_CREATED)) + .build()); + }); + + createSingleMessageBatchSubscription(topic, subscriber.getEndpoint(), true); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), TestMessage.simple().body()); + + // then + subscriber.waitUntilRequestsReceived( + requests -> Assertions.assertThat(requests.size()).isEqualTo(2)); + } + + @Test + public void shouldNotRetryOnClientErrors() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + + TestSubscriber subscriber = + subscribers.createSubscriber( + (service, endpoint) -> { + service.addStubMapping( + post(endpoint) + .withRequestBody(containing("failed")) + .willReturn(aResponse().withStatus(SC_BAD_REQUEST)) + .build()); + + service.addStubMapping( + post(endpoint) + .withRequestBody(containing("successful")) + .willReturn(aResponse().withStatus(SC_CREATED)) + .build()); + }); + + createSingleMessageBatchSubscription(topic, subscriber.getEndpoint()); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), failedRequestBody); + hermes.api().publishUntilSuccess(topic.getQualifiedName(), successfulRequestBody); + + // then + subscriber.waitUntilRequestsReceived( + requests -> + Assertions.assertThat(requests) + .extracting(LoggedRequest::getBodyAsString) + .extracting(this::readMessage) + .containsExactly(failedRequestBody, successfulRequestBody)); + } + + private void createSingleMessageBatchSubscription( + Topic topic, String endpoint, int messageTtl, int messageBackoff) { + createBatchSubscription(topic, endpoint, messageTtl, messageBackoff, 1, 1, 200, false); + } + + private void createSingleMessageBatchSubscription(Topic topic, String endpoint) { + createSingleMessageBatchSubscription(topic, endpoint, false); + } + + private void createSingleMessageBatchSubscription( + Topic topic, String endpoint, boolean retryOnClientErrors) { + createBatchSubscription(topic, endpoint, 1, 10, 1, 1, 500, retryOnClientErrors); + } + + public void createBatchSubscription( + Topic topic, + String endpoint, + int messageTtl, + int messageBackoff, + int batchSize, + int batchTime, + int batchVolume, + boolean retryOnClientErrors) { + BatchSubscriptionPolicy policy = + batchSubscriptionPolicy() + .applyDefaults() + .withMessageTtl(messageTtl) + .withMessageBackoff(messageBackoff) + .withBatchSize(batchSize) + .withBatchTime(batchTime) + .withBatchVolume(batchVolume) + .withClientErrorRetry(retryOnClientErrors) + .withRequestTimeout(500) + .build(); + + createBatchSubscription(topic, endpoint, policy); + } + + public void createBatchSubscription( + Topic topic, String endpoint, BatchSubscriptionPolicy policy) { + Subscription subscription = + subscription(topic, "batchSubscription") + .withEndpoint(endpoint) + .withContentType(ContentType.JSON) + .withSubscriptionPolicy(policy) + .build(); + + hermes.initHelper().createSubscription(subscription); + } + + private String readMessage(String body) { + try { + return mapper.writeValueAsString( + ((Map) mapper.readValue(body, List.class).get(0)).get("message")); + } catch (IOException e) { + throw new RuntimeException(e); } - - private void createSingleMessageBatchSubscription(Topic topic, String endpoint, boolean retryOnClientErrors) { - createBatchSubscription(topic, endpoint, 1, 10, 1, 1, 500, retryOnClientErrors); - } - - public void createBatchSubscription(Topic topic, String endpoint, int messageTtl, int messageBackoff, int batchSize, int batchTime, - int batchVolume, boolean retryOnClientErrors) { - BatchSubscriptionPolicy policy = batchSubscriptionPolicy() - .applyDefaults() - .withMessageTtl(messageTtl) - .withMessageBackoff(messageBackoff) - .withBatchSize(batchSize) - .withBatchTime(batchTime) - .withBatchVolume(batchVolume) - .withClientErrorRetry(retryOnClientErrors) - .withRequestTimeout(500) - .build(); - - createBatchSubscription(topic, endpoint, policy); - } - - public void createBatchSubscription(Topic topic, String endpoint, BatchSubscriptionPolicy policy) { - Subscription subscription = subscription(topic, "batchSubscription") - .withEndpoint(endpoint) - .withContentType(ContentType.JSON) - .withSubscriptionPolicy(policy) - .build(); - - hermes.initHelper().createSubscription(subscription); - } - - private String readMessage(String body) { - try { - return mapper.writeValueAsString(((Map) mapper.readValue(body, List.class).get(0)).get("message")); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/BroadcastDeliveryTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/BroadcastDeliveryTest.java index b6c51c3903..bf14a48a73 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/BroadcastDeliveryTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/BroadcastDeliveryTest.java @@ -1,5 +1,16 @@ package pl.allegro.tech.hermes.integrationtests; +import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.waitAtMost; +import static pl.allegro.tech.hermes.api.SubscriptionPolicy.Builder.subscriptionPolicy; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + +import java.time.Duration; +import java.util.List; +import java.util.stream.Stream; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -13,115 +24,113 @@ import pl.allegro.tech.hermes.integrationtests.subscriber.TestSubscribersExtension; import pl.allegro.tech.hermes.test.helper.message.TestMessage; -import java.time.Duration; -import java.util.List; -import java.util.stream.Stream; - -import static java.util.stream.Collectors.joining; -import static java.util.stream.Collectors.toList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.waitAtMost; -import static pl.allegro.tech.hermes.api.SubscriptionPolicy.Builder.subscriptionPolicy; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class BroadcastDeliveryTest { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - @RegisterExtension - public static final TestSubscribersExtension subscribersFactory = new TestSubscribersExtension(); - - @Test - public void shouldPublishAndConsumeMessageByAllServices() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestMessage message = TestMessage.random(); - - List subscribers = succeedingSubscribers(4); - String endpointUrl = setUpSubscribersAndGetEndpoint(subscribers); - - hermes.initHelper().createSubscription( - broadcastSubscription(topic, "subscription", endpointUrl) - ); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - - // then - subscribers.forEach(s -> s.waitUntilReceived(message.body())); - } - - @Test - public void shouldPublishAndRetryOnlyForUndeliveredConsumers() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestMessage message = TestMessage.random(); - - List subscribers = succeedingSubscribers(3); - TestSubscriber retryingSubscriber = subscribersFactory.createSubscriberWithRetry(message.body(), 1); - subscribers.add(retryingSubscriber); - - String endpointUrl = setUpSubscribersAndGetEndpoint(subscribers); - - hermes.initHelper().createSubscription( - broadcastSubscription(topic, "subscription", endpointUrl) - ); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - - // then - subscribers.forEach(s -> s.waitUntilReceived(message.body())); - retryingSubscriber.waitUntilReceived(Duration.ofMinutes(1), 2); - Assertions.assertThat(retryingSubscriber.getLastReceivedRequest().getHeader("Hermes-Retry-Count")).isEqualTo("1"); - } - - @Test - public void shouldNotRetryForBadRequestsFromConsumers() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestMessage message = TestMessage.random(); - - List subscribers = succeedingSubscribers(3); - subscribers.add(subscribersFactory.createSubscriber(400)); - - String endpointUrl = setUpSubscribersAndGetEndpoint(subscribers); - - - hermes.initHelper().createSubscription( - broadcastSubscription(topic, "subscription", endpointUrl) - ); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - - // then - subscribers.forEach(s -> s.waitUntilReceived(message.body())); - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> { - long discarded = hermes.api() - .getSubscriptionMetrics(topic.getQualifiedName(), "subscription") - .expectBody(SubscriptionMetrics.class).returnResult().getResponseBody().getDiscarded(); - assertThat(discarded).isEqualTo(1); - }); - } - - private List succeedingSubscribers(int subscribersCount) { - return Stream.generate(subscribersFactory::createSubscriber).limit(subscribersCount).collect(toList()); - } - - private Subscription broadcastSubscription(Topic topic, String subscriptionName, String endpoint) { - return subscription(topic, subscriptionName) - .withEndpoint(endpoint) - .withContentType(ContentType.JSON) - .withSubscriptionPolicy(subscriptionPolicy().applyDefaults().build()) - .withMode(SubscriptionMode.BROADCAST) - .build(); - } - - private String setUpSubscribersAndGetEndpoint(List subscribers) { - return subscribers.stream().map(TestSubscriber::getEndpoint).collect(joining(";")); - } - + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); + + @RegisterExtension + public static final TestSubscribersExtension subscribersFactory = new TestSubscribersExtension(); + + @Test + public void shouldPublishAndConsumeMessageByAllServices() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestMessage message = TestMessage.random(); + + List subscribers = succeedingSubscribers(4); + String endpointUrl = setUpSubscribersAndGetEndpoint(subscribers); + + hermes + .initHelper() + .createSubscription(broadcastSubscription(topic, "subscription", endpointUrl)); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + + // then + subscribers.forEach(s -> s.waitUntilReceived(message.body())); + } + + @Test + public void shouldPublishAndRetryOnlyForUndeliveredConsumers() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestMessage message = TestMessage.random(); + + List subscribers = succeedingSubscribers(3); + TestSubscriber retryingSubscriber = + subscribersFactory.createSubscriberWithRetry(message.body(), 1); + subscribers.add(retryingSubscriber); + + String endpointUrl = setUpSubscribersAndGetEndpoint(subscribers); + + hermes + .initHelper() + .createSubscription(broadcastSubscription(topic, "subscription", endpointUrl)); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + + // then + subscribers.forEach(s -> s.waitUntilReceived(message.body())); + retryingSubscriber.waitUntilReceived(Duration.ofMinutes(1), 2); + Assertions.assertThat( + retryingSubscriber.getLastReceivedRequest().getHeader("Hermes-Retry-Count")) + .isEqualTo("1"); + } + + @Test + public void shouldNotRetryForBadRequestsFromConsumers() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestMessage message = TestMessage.random(); + + List subscribers = succeedingSubscribers(3); + subscribers.add(subscribersFactory.createSubscriber(400)); + + String endpointUrl = setUpSubscribersAndGetEndpoint(subscribers); + + hermes + .initHelper() + .createSubscription(broadcastSubscription(topic, "subscription", endpointUrl)); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + + // then + subscribers.forEach(s -> s.waitUntilReceived(message.body())); + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + long discarded = + hermes + .api() + .getSubscriptionMetrics(topic.getQualifiedName(), "subscription") + .expectBody(SubscriptionMetrics.class) + .returnResult() + .getResponseBody() + .getDiscarded(); + assertThat(discarded).isEqualTo(1); + }); + } + + private List succeedingSubscribers(int subscribersCount) { + return Stream.generate(subscribersFactory::createSubscriber) + .limit(subscribersCount) + .collect(toList()); + } + + private Subscription broadcastSubscription( + Topic topic, String subscriptionName, String endpoint) { + return subscription(topic, subscriptionName) + .withEndpoint(endpoint) + .withContentType(ContentType.JSON) + .withSubscriptionPolicy(subscriptionPolicy().applyDefaults().build()) + .withMode(SubscriptionMode.BROADCAST) + .build(); + } + + private String setUpSubscribersAndGetEndpoint(List subscribers) { + return subscribers.stream().map(TestSubscriber::getEndpoint).collect(joining(";")); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/ConsumerProfilingTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/ConsumerProfilingTest.java index 64148f67d0..5294afba5a 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/ConsumerProfilingTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/ConsumerProfilingTest.java @@ -1,8 +1,15 @@ package pl.allegro.tech.hermes.integrationtests; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.waitAtMost; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.read.ListAppender; +import java.time.Duration; +import java.util.List; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -19,229 +26,288 @@ import pl.allegro.tech.hermes.integrationtests.subscriber.TestSubscribersExtension; import pl.allegro.tech.hermes.test.helper.message.TestMessage; -import java.time.Duration; -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.waitAtMost; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class ConsumerProfilingTest { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - @RegisterExtension - public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); - - private ListAppender listAppender; - - @BeforeEach - void createLogAppender() { - Logger logWatcher = (Logger) LoggerFactory.getLogger(DefaultConsumerProfiler.class); - - listAppender = new ListAppender<>(); - listAppender.start(); - - logWatcher.addAppender(listAppender); - } - - @AfterEach - void teardown() { - ((Logger) LoggerFactory.getLogger(DefaultConsumerProfiler.class)).detachAndStopAllAppenders(); - hermes.clearManagementData(); - } - - @AfterAll - static void teardownClass() { - hermes.clearManagementData(); - } - - @Test - public void shouldNotProfileWhenProfilingIsDisabled() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()) - .withProfilingEnabled(false).build()); - TestMessage message = TestMessage.random(); - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - - // when - subscriber.waitUntilReceived(message.body()); - - // then - List logsList = listAppender.list.stream() - .filter(log -> log.getFormattedMessage().contains(subscription.getQualifiedName().toString())).toList(); - assertThat(logsList).hasSize(0); - } - - @Test - public void shouldProfileEmptyRun() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - - // when - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()) - .withProfilingEnabled(true).build()); - - // then - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> { - List logsList = listAppender.list.stream() - .filter(log -> log.getFormattedMessage().contains(subscription.getQualifiedName().toString())).toList(); - assertThat(logsList).hasSizeGreaterThan(0); - assertThat(logsList.get(0).getFormattedMessage()).contains( - String.format("Consumer profiler measurements for subscription %s and %s run:", subscription.getQualifiedName(), ConsumerRun.EMPTY), - Measurement.SIGNALS_AND_SEMAPHORE_ACQUIRE, - Measurement.MESSAGE_RECEIVER_NEXT, - Measurement.MESSAGE_CONVERSION, - "partialMeasurements", - Measurement.SIGNALS_INTERRUPT_RUN - ); - }); - } - - @Test - public void shouldNotProfileRunsBelowThreshold() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()) - .withProfilingEnabled(true) - .withProfilingThresholdMs(100_000) - .build()); - TestMessage message = TestMessage.random(); - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - - // when - subscriber.waitUntilReceived(message.body()); - - // then - List logsList = listAppender.list.stream() - .filter(log -> log.getFormattedMessage().contains(subscription.getQualifiedName().toString())).toList(); - assertThat(logsList).hasSize(0); - } - - @Test - public void shouldProfileSuccessfulMessageProcessing() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()) - .withProfilingEnabled(true).build()); - TestMessage message = TestMessage.random(); - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - - // when - subscriber.waitUntilReceived(message.body()); - - // then - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> { - List logsList = listAppender.list.stream() - .filter(log -> log.getFormattedMessage().contains(ConsumerRun.DELIVERED.name())).toList(); - assertThat(logsList).hasSizeGreaterThan(0); - assertThat(logsList.get(0).getFormattedMessage()).contains( - String.format("Consumer profiler measurements for subscription %s and %s run:", subscription.getQualifiedName(), ConsumerRun.DELIVERED), - Measurement.SIGNALS_AND_SEMAPHORE_ACQUIRE, - Measurement.MESSAGE_RECEIVER_NEXT, - Measurement.MESSAGE_CONVERSION, - Measurement.OFFER_INFLIGHT_OFFSET, - Measurement.TRACKERS_LOG_INFLIGHT, - Measurement.SCHEDULE_MESSAGE_SENDING, - Measurement.ACQUIRE_RATE_LIMITER, - Measurement.MESSAGE_SENDER_SEND, - Measurement.HANDLERS, - "partialMeasurements", - Measurement.SIGNALS_INTERRUPT_RUN - ); - }); - } - - @Test - public void shouldProfileDiscardedMessageProcessing() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestSubscriber subscriber = subscribers.createSubscriber(400); - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()) - .withProfilingEnabled(true) - .build()); - TestMessage message = TestMessage.random(); - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - - // when - subscriber.waitUntilReceived(message.body()); - - // then - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> { - List logsList = listAppender.list.stream() - .filter(log -> log.getFormattedMessage().contains(ConsumerRun.DISCARDED.name())).toList(); - assertThat(logsList).hasSizeGreaterThan(0); - assertThat(logsList.get(0).getFormattedMessage()).contains( - String.format("Consumer profiler measurements for subscription %s and %s run:", subscription.getQualifiedName(), ConsumerRun.DISCARDED), - Measurement.SIGNALS_AND_SEMAPHORE_ACQUIRE, - Measurement.MESSAGE_RECEIVER_NEXT, - Measurement.MESSAGE_CONVERSION, - Measurement.OFFER_INFLIGHT_OFFSET, - Measurement.TRACKERS_LOG_INFLIGHT, - Measurement.SCHEDULE_MESSAGE_SENDING, - Measurement.ACQUIRE_RATE_LIMITER, - Measurement.MESSAGE_SENDER_SEND, - Measurement.HANDLERS, - "partialMeasurements", - Measurement.SIGNALS_INTERRUPT_RUN - ); - }); - } - - @Test - public void shouldProfileRetriedMessageProcessing() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestMessage message = TestMessage.random(); - TestSubscriber subscriber = subscribers.createSubscriberWithRetry(message.body(), 1); - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()) - .withProfilingEnabled(true) - .build()); - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - - // when - subscriber.waitUntilReceived(Duration.ofSeconds(5), 2); - - // then - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> { - List retriedLogsList = listAppender.list.stream() - .filter(log -> log.getFormattedMessage().contains(ConsumerRun.RETRIED.name())).toList(); - assertThat(retriedLogsList).hasSizeGreaterThan(0); - assertThat(retriedLogsList.get(0).getFormattedMessage()).contains( - String.format("Consumer profiler measurements for subscription %s and %s run:", subscription.getQualifiedName(), ConsumerRun.RETRIED), - Measurement.SIGNALS_AND_SEMAPHORE_ACQUIRE, - Measurement.MESSAGE_RECEIVER_NEXT, - Measurement.MESSAGE_CONVERSION, - Measurement.OFFER_INFLIGHT_OFFSET, - Measurement.TRACKERS_LOG_INFLIGHT, - Measurement.SCHEDULE_MESSAGE_SENDING, - Measurement.ACQUIRE_RATE_LIMITER, - Measurement.MESSAGE_SENDER_SEND, - Measurement.HANDLERS, - "partialMeasurements", - Measurement.SIGNALS_INTERRUPT_RUN - ); - }); - - - // and - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> { - List processedLogsList = listAppender.list.stream() - .filter(log -> log.getFormattedMessage().contains(ConsumerRun.DELIVERED.name())).toList(); - assertThat(processedLogsList).hasSizeGreaterThan(0); - assertThat(processedLogsList.get(0).getFormattedMessage()).contains( - String.format("Consumer profiler measurements for subscription %s and %s run:", subscription.getQualifiedName(), ConsumerRun.DELIVERED), - Measurement.SCHEDULE_RESEND, - Measurement.MESSAGE_SENDER_SEND, - Measurement.HANDLERS, - "retryDelayMillis 1000" - ); - }); - } + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); + + @RegisterExtension + public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); + + private ListAppender listAppender; + + @BeforeEach + void createLogAppender() { + Logger logWatcher = (Logger) LoggerFactory.getLogger(DefaultConsumerProfiler.class); + + listAppender = new ListAppender<>(); + listAppender.start(); + + logWatcher.addAppender(listAppender); + } + + @AfterEach + void teardown() { + ((Logger) LoggerFactory.getLogger(DefaultConsumerProfiler.class)).detachAndStopAllAppenders(); + hermes.clearManagementData(); + } + + @AfterAll + static void teardownClass() { + hermes.clearManagementData(); + } + + @Test + public void shouldNotProfileWhenProfilingIsDisabled() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()) + .withProfilingEnabled(false) + .build()); + TestMessage message = TestMessage.random(); + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + + // when + subscriber.waitUntilReceived(message.body()); + + // then + List logsList = + listAppender.list.stream() + .filter( + log -> + log.getFormattedMessage().contains(subscription.getQualifiedName().toString())) + .toList(); + assertThat(logsList).hasSize(0); + } + + @Test + public void shouldProfileEmptyRun() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + + // when + Subscription subscription = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName()).withProfilingEnabled(true).build()); + + // then + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + List logsList = + listAppender.list.stream() + .filter( + log -> + log.getFormattedMessage() + .contains(subscription.getQualifiedName().toString())) + .toList(); + assertThat(logsList).hasSizeGreaterThan(0); + assertThat(logsList.get(0).getFormattedMessage()) + .contains( + String.format( + "Consumer profiler measurements for subscription %s and %s run:", + subscription.getQualifiedName(), ConsumerRun.EMPTY), + Measurement.SIGNALS_AND_SEMAPHORE_ACQUIRE, + Measurement.MESSAGE_RECEIVER_NEXT, + Measurement.MESSAGE_CONVERSION, + "partialMeasurements", + Measurement.SIGNALS_INTERRUPT_RUN); + }); + } + + @Test + public void shouldNotProfileRunsBelowThreshold() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()) + .withProfilingEnabled(true) + .withProfilingThresholdMs(100_000) + .build()); + TestMessage message = TestMessage.random(); + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + + // when + subscriber.waitUntilReceived(message.body()); + + // then + List logsList = + listAppender.list.stream() + .filter( + log -> + log.getFormattedMessage().contains(subscription.getQualifiedName().toString())) + .toList(); + assertThat(logsList).hasSize(0); + } + + @Test + public void shouldProfileSuccessfulMessageProcessing() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()) + .withProfilingEnabled(true) + .build()); + TestMessage message = TestMessage.random(); + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + + // when + subscriber.waitUntilReceived(message.body()); + + // then + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + List logsList = + listAppender.list.stream() + .filter( + log -> log.getFormattedMessage().contains(ConsumerRun.DELIVERED.name())) + .toList(); + assertThat(logsList).hasSizeGreaterThan(0); + assertThat(logsList.get(0).getFormattedMessage()) + .contains( + String.format( + "Consumer profiler measurements for subscription %s and %s run:", + subscription.getQualifiedName(), ConsumerRun.DELIVERED), + Measurement.SIGNALS_AND_SEMAPHORE_ACQUIRE, + Measurement.MESSAGE_RECEIVER_NEXT, + Measurement.MESSAGE_CONVERSION, + Measurement.OFFER_INFLIGHT_OFFSET, + Measurement.TRACKERS_LOG_INFLIGHT, + Measurement.SCHEDULE_MESSAGE_SENDING, + Measurement.ACQUIRE_RATE_LIMITER, + Measurement.MESSAGE_SENDER_SEND, + Measurement.HANDLERS, + "partialMeasurements", + Measurement.SIGNALS_INTERRUPT_RUN); + }); + } + + @Test + public void shouldProfileDiscardedMessageProcessing() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestSubscriber subscriber = subscribers.createSubscriber(400); + Subscription subscription = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()) + .withProfilingEnabled(true) + .build()); + TestMessage message = TestMessage.random(); + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + + // when + subscriber.waitUntilReceived(message.body()); + + // then + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + List logsList = + listAppender.list.stream() + .filter( + log -> log.getFormattedMessage().contains(ConsumerRun.DISCARDED.name())) + .toList(); + assertThat(logsList).hasSizeGreaterThan(0); + assertThat(logsList.get(0).getFormattedMessage()) + .contains( + String.format( + "Consumer profiler measurements for subscription %s and %s run:", + subscription.getQualifiedName(), ConsumerRun.DISCARDED), + Measurement.SIGNALS_AND_SEMAPHORE_ACQUIRE, + Measurement.MESSAGE_RECEIVER_NEXT, + Measurement.MESSAGE_CONVERSION, + Measurement.OFFER_INFLIGHT_OFFSET, + Measurement.TRACKERS_LOG_INFLIGHT, + Measurement.SCHEDULE_MESSAGE_SENDING, + Measurement.ACQUIRE_RATE_LIMITER, + Measurement.MESSAGE_SENDER_SEND, + Measurement.HANDLERS, + "partialMeasurements", + Measurement.SIGNALS_INTERRUPT_RUN); + }); + } + + @Test + public void shouldProfileRetriedMessageProcessing() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestMessage message = TestMessage.random(); + TestSubscriber subscriber = subscribers.createSubscriberWithRetry(message.body(), 1); + Subscription subscription = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()) + .withProfilingEnabled(true) + .build()); + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + + // when + subscriber.waitUntilReceived(Duration.ofSeconds(5), 2); + + // then + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + List retriedLogsList = + listAppender.list.stream() + .filter(log -> log.getFormattedMessage().contains(ConsumerRun.RETRIED.name())) + .toList(); + assertThat(retriedLogsList).hasSizeGreaterThan(0); + assertThat(retriedLogsList.get(0).getFormattedMessage()) + .contains( + String.format( + "Consumer profiler measurements for subscription %s and %s run:", + subscription.getQualifiedName(), ConsumerRun.RETRIED), + Measurement.SIGNALS_AND_SEMAPHORE_ACQUIRE, + Measurement.MESSAGE_RECEIVER_NEXT, + Measurement.MESSAGE_CONVERSION, + Measurement.OFFER_INFLIGHT_OFFSET, + Measurement.TRACKERS_LOG_INFLIGHT, + Measurement.SCHEDULE_MESSAGE_SENDING, + Measurement.ACQUIRE_RATE_LIMITER, + Measurement.MESSAGE_SENDER_SEND, + Measurement.HANDLERS, + "partialMeasurements", + Measurement.SIGNALS_INTERRUPT_RUN); + }); + + // and + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + List processedLogsList = + listAppender.list.stream() + .filter( + log -> log.getFormattedMessage().contains(ConsumerRun.DELIVERED.name())) + .toList(); + assertThat(processedLogsList).hasSizeGreaterThan(0); + assertThat(processedLogsList.get(0).getFormattedMessage()) + .contains( + String.format( + "Consumer profiler measurements for subscription %s and %s run:", + subscription.getQualifiedName(), ConsumerRun.DELIVERED), + Measurement.SCHEDULE_RESEND, + Measurement.MESSAGE_SENDER_SEND, + Measurement.HANDLERS, + "retryDelayMillis 1000"); + }); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/ConsumingHttp2Test.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/ConsumingHttp2Test.java index 68d23f2b5a..45a7b87537 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/ConsumingHttp2Test.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/ConsumingHttp2Test.java @@ -1,5 +1,9 @@ package pl.allegro.tech.hermes.integrationtests; +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + import com.github.tomakehurst.wiremock.verification.LoggedRequest; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -9,37 +13,33 @@ import pl.allegro.tech.hermes.integrationtests.subscriber.TestSubscribersExtension; import pl.allegro.tech.hermes.test.helper.message.TestMessage; -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class ConsumingHttp2Test { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - @RegisterExtension - public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); - - @Test - public void shouldDeliverMessageUsingHttp2() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - hermes.initHelper().createSubscription( - subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) - .withHttp2Enabled(true) - .build() - ); - TestMessage message = TestMessage.of("hello", "world"); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - - // then - subscriber.waitUntilReceived(message.body()); - LoggedRequest lastReceivedRequest = subscriber.getLastReceivedRequest(); - assertThat(lastReceivedRequest.getHeader("Keep-Alive")).isNull(); - assertThat(lastReceivedRequest.getProtocol()).isEqualTo("HTTP/2.0"); - } + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); + + @RegisterExtension + public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); + + @Test + public void shouldDeliverMessageUsingHttp2() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) + .withHttp2Enabled(true) + .build()); + TestMessage message = TestMessage.of("hello", "world"); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + + // then + subscriber.waitUntilReceived(message.body()); + LoggedRequest lastReceivedRequest = subscriber.getLastReceivedRequest(); + assertThat(lastReceivedRequest.getHeader("Keep-Alive")).isNull(); + assertThat(lastReceivedRequest.getProtocol()).isEqualTo("HTTP/2.0"); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/FilteringAvroTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/FilteringAvroTest.java index 5bfa5b2ede..36c1e171fa 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/FilteringAvroTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/FilteringAvroTest.java @@ -1,5 +1,12 @@ package pl.allegro.tech.hermes.integrationtests; +import static com.google.common.collect.ImmutableMap.of; +import static pl.allegro.tech.hermes.api.ContentType.AVRO; +import static pl.allegro.tech.hermes.api.TopicWithSchema.topicWithSchema; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + +import java.util.Set; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import pl.allegro.tech.hermes.api.ContentType; @@ -12,82 +19,74 @@ import pl.allegro.tech.hermes.integrationtests.subscriber.TestSubscribersExtension; import pl.allegro.tech.hermes.test.helper.avro.AvroUser; -import java.util.Set; - -import static com.google.common.collect.ImmutableMap.of; -import static pl.allegro.tech.hermes.api.ContentType.AVRO; -import static pl.allegro.tech.hermes.api.TopicWithSchema.topicWithSchema; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class FilteringAvroTest { - private static final MessageFilterSpecification MESSAGE_NAME_FILTER = - new MessageFilterSpecification(of("type", "avropath", "path", ".name", "matcher", "Bob")); - - private static final MessageFilterSpecification MESSAGE_COLOR_FILTER = - new MessageFilterSpecification(of("type", "avropath", "path", ".favoriteColor", "matcher", "grey")); - + private static final MessageFilterSpecification MESSAGE_NAME_FILTER = + new MessageFilterSpecification(of("type", "avropath", "path", ".name", "matcher", "Bob")); - private static final AvroUser BOB = new AvroUser("Bob", 50, "blue"); - private static final AvroUser ALICE = new AvroUser("Alice", 20, "magenta"); - private static final AvroUser BOB_GREY = new AvroUser("Bob", 50, "grey"); + private static final MessageFilterSpecification MESSAGE_COLOR_FILTER = + new MessageFilterSpecification( + of("type", "avropath", "path", ".favoriteColor", "matcher", "grey")); - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); + private static final AvroUser BOB = new AvroUser("Bob", 50, "blue"); + private static final AvroUser ALICE = new AvroUser("Alice", 20, "magenta"); + private static final AvroUser BOB_GREY = new AvroUser("Bob", 50, "grey"); - @RegisterExtension - public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); - @Test - public void shouldFilterIncomingEvents() { - // given - TopicWithSchema topicWithSchema = topicWithSchema(topicWithRandomName() - .withContentType(AVRO) - .build(), BOB.getSchemaAsString()); + @RegisterExtension + public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); - Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); + @Test + public void shouldFilterIncomingEvents() { + // given + TopicWithSchema topicWithSchema = + topicWithSchema( + topicWithRandomName().withContentType(AVRO).build(), BOB.getSchemaAsString()); - TestSubscriber subscriber = subscribers.createSubscriber(); - final Subscription subscription = subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) - .withContentType(ContentType.JSON) - .withFilter(MESSAGE_NAME_FILTER) - .build(); + Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); - hermes.initHelper().createSubscription(subscription); + TestSubscriber subscriber = subscribers.createSubscriber(); + final Subscription subscription = + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) + .withContentType(ContentType.JSON) + .withFilter(MESSAGE_NAME_FILTER) + .build(); - // when - hermes.api().publishJSONUntilSuccess(topic.getQualifiedName(), ALICE.asJson()); - hermes.api().publishJSONUntilSuccess(topic.getQualifiedName(), BOB.asJson()); + hermes.initHelper().createSubscription(subscription); - // then - subscriber.waitUntilAllReceivedStrict(Set.of(BOB.asJson())); - } + // when + hermes.api().publishJSONUntilSuccess(topic.getQualifiedName(), ALICE.asJson()); + hermes.api().publishJSONUntilSuccess(topic.getQualifiedName(), BOB.asJson()); - @Test - public void shouldChainMultipleFilters() { - // given - TopicWithSchema topicWithSchema = topicWithSchema(topicWithRandomName() - .withContentType(AVRO) - .build(), BOB.getSchemaAsString()); + // then + subscriber.waitUntilAllReceivedStrict(Set.of(BOB.asJson())); + } - Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); + @Test + public void shouldChainMultipleFilters() { + // given + TopicWithSchema topicWithSchema = + topicWithSchema( + topicWithRandomName().withContentType(AVRO).build(), BOB.getSchemaAsString()); - TestSubscriber subscriber = subscribers.createSubscriber(); - final Subscription subscription = subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) - .withContentType(ContentType.JSON) - .withFilter(MESSAGE_NAME_FILTER) - .withFilter(MESSAGE_COLOR_FILTER) - .build(); + Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); - hermes.initHelper().createSubscription(subscription); + TestSubscriber subscriber = subscribers.createSubscriber(); + final Subscription subscription = + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) + .withContentType(ContentType.JSON) + .withFilter(MESSAGE_NAME_FILTER) + .withFilter(MESSAGE_COLOR_FILTER) + .build(); - // when - hermes.api().publishJSONUntilSuccess(topic.getQualifiedName(), BOB.asJson()); - hermes.api().publishJSONUntilSuccess(topic.getQualifiedName(), BOB_GREY.asJson()); + hermes.initHelper().createSubscription(subscription); - // then - subscriber.waitUntilAllReceivedStrict(Set.of(BOB_GREY.asJson())); - } + // when + hermes.api().publishJSONUntilSuccess(topic.getQualifiedName(), BOB.asJson()); + hermes.api().publishJSONUntilSuccess(topic.getQualifiedName(), BOB_GREY.asJson()); + // then + subscriber.waitUntilAllReceivedStrict(Set.of(BOB_GREY.asJson())); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/FilteringHeadersTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/FilteringHeadersTest.java index 19f83925eb..e4d9b3da99 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/FilteringHeadersTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/FilteringHeadersTest.java @@ -1,5 +1,13 @@ package pl.allegro.tech.hermes.integrationtests; +import static com.google.common.collect.ImmutableMap.of; +import static jakarta.ws.rs.core.MediaType.TEXT_PLAIN; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; +import static pl.allegro.tech.hermes.utils.Headers.createHeaders; + +import java.util.Map; +import java.util.Set; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import pl.allegro.tech.hermes.api.MessageFilterSpecification; @@ -9,51 +17,77 @@ import pl.allegro.tech.hermes.integrationtests.subscriber.TestSubscribersExtension; import pl.allegro.tech.hermes.test.helper.avro.AvroUser; -import java.util.Map; -import java.util.Set; - -import static com.google.common.collect.ImmutableMap.of; -import static jakarta.ws.rs.core.MediaType.TEXT_PLAIN; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; -import static pl.allegro.tech.hermes.utils.Headers.createHeaders; - public class FilteringHeadersTest { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); - @RegisterExtension - public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); + @RegisterExtension + public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); - private static final MessageFilterSpecification TRACE_ID_HEADER_FILTER = - new MessageFilterSpecification(of("type", "header", "header", "Trace-Id", "matcher", "^vte.*")); + private static final MessageFilterSpecification TRACE_ID_HEADER_FILTER = + new MessageFilterSpecification( + of("type", "header", "header", "Trace-Id", "matcher", "^vte.*")); - private static final MessageFilterSpecification SPAN_ID_HEADER_FILTER = - new MessageFilterSpecification(of("type", "header", "header", "Span-Id", "matcher", ".*span$")); + private static final MessageFilterSpecification SPAN_ID_HEADER_FILTER = + new MessageFilterSpecification( + of("type", "header", "header", "Span-Id", "matcher", ".*span$")); - private static final AvroUser ALICE = new AvroUser("Alice", 20, "blue"); - private static final AvroUser BOB = new AvroUser("Bob", 30, "red"); + private static final AvroUser ALICE = new AvroUser("Alice", 20, "blue"); + private static final AvroUser BOB = new AvroUser("Bob", 30, "red"); - @Test - public void shouldFilterIncomingEventsByHeaders() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + @Test + public void shouldFilterIncomingEventsByHeaders() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()) + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()) .withFilter(TRACE_ID_HEADER_FILTER) .withFilter(SPAN_ID_HEADER_FILTER) .build()); - // when - hermes.api().publish(topic.getQualifiedName(), ALICE.asJson(), createHeaders(Map.of("Trace-Id", "vte12", "Span-Id", "my-span", "Content-Type", TEXT_PLAIN))).expectStatus().is2xxSuccessful(); - hermes.api().publish(topic.getQualifiedName(), BOB.asJson(), createHeaders(Map.of("Trace-Id", "vte12"))).expectStatus().is2xxSuccessful(); - hermes.api().publish(topic.getQualifiedName(), BOB.asJson(), createHeaders(Map.of("Span-Id", "my-span"))).expectStatus().is2xxSuccessful(); - hermes.api().publish(topic.getQualifiedName(), BOB.asJson(), createHeaders(Map.of("Trace-Id", "vte12", "Span-Id", "span-1"))).expectStatus().is2xxSuccessful(); - hermes.api().publish(topic.getQualifiedName(), BOB.asJson(), createHeaders(Map.of("Trace-Id", "invalid", "Span-Id", "my-span"))).expectStatus().is2xxSuccessful(); + // when + hermes + .api() + .publish( + topic.getQualifiedName(), + ALICE.asJson(), + createHeaders( + Map.of("Trace-Id", "vte12", "Span-Id", "my-span", "Content-Type", TEXT_PLAIN))) + .expectStatus() + .is2xxSuccessful(); + hermes + .api() + .publish(topic.getQualifiedName(), BOB.asJson(), createHeaders(Map.of("Trace-Id", "vte12"))) + .expectStatus() + .is2xxSuccessful(); + hermes + .api() + .publish( + topic.getQualifiedName(), BOB.asJson(), createHeaders(Map.of("Span-Id", "my-span"))) + .expectStatus() + .is2xxSuccessful(); + hermes + .api() + .publish( + topic.getQualifiedName(), + BOB.asJson(), + createHeaders(Map.of("Trace-Id", "vte12", "Span-Id", "span-1"))) + .expectStatus() + .is2xxSuccessful(); + hermes + .api() + .publish( + topic.getQualifiedName(), + BOB.asJson(), + createHeaders(Map.of("Trace-Id", "invalid", "Span-Id", "my-span"))) + .expectStatus() + .is2xxSuccessful(); - // then - subscriber.waitUntilAllReceivedStrict(Set.of(ALICE.asJson())); - } + // then + subscriber.waitUntilAllReceivedStrict(Set.of(ALICE.asJson())); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/FilteringJsonTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/FilteringJsonTest.java index b0ffcaf4c0..84c3a78469 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/FilteringJsonTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/FilteringJsonTest.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.integrationtests; +import static com.google.common.collect.ImmutableMap.of; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + +import java.util.Set; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import pl.allegro.tech.hermes.api.MessageFilterSpecification; @@ -10,89 +15,93 @@ import pl.allegro.tech.hermes.integrationtests.subscriber.TestSubscribersExtension; import pl.allegro.tech.hermes.test.helper.avro.AvroUser; -import java.util.Set; - -import static com.google.common.collect.ImmutableMap.of; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class FilteringJsonTest { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); - @RegisterExtension - public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); + @RegisterExtension + public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); - private static final MessageFilterSpecification MESSAGE_NAME_FILTER = - new MessageFilterSpecification(of("type", "jsonpath", "path", ".name", "matcher", "^Bob.*")); + private static final MessageFilterSpecification MESSAGE_NAME_FILTER = + new MessageFilterSpecification(of("type", "jsonpath", "path", ".name", "matcher", "^Bob.*")); - private static final MessageFilterSpecification MESSAGE_COLOR_FILTER = - new MessageFilterSpecification(of("type", "jsonpath", "path", ".favoriteColor", "matcher", "grey")); + private static final MessageFilterSpecification MESSAGE_COLOR_FILTER = + new MessageFilterSpecification( + of("type", "jsonpath", "path", ".favoriteColor", "matcher", "grey")); - static final AvroUser BOB = new AvroUser("Bob", 50, "blue"); - static final AvroUser ALICE = new AvroUser("Alice", 20, "magenta"); - static final AvroUser ALICE_GREY = new AvroUser("Alice", 20, "grey"); - static final AvroUser BOB_GREY = new AvroUser("Bob", 50, "grey"); + static final AvroUser BOB = new AvroUser("Bob", 50, "blue"); + static final AvroUser ALICE = new AvroUser("Alice", 20, "magenta"); + static final AvroUser ALICE_GREY = new AvroUser("Alice", 20, "grey"); + static final AvroUser BOB_GREY = new AvroUser("Bob", 50, "grey"); - private static final SubscriptionPolicy SUBSCRIPTION_POLICY = new SubscriptionPolicy(100, 2000, 1000, 1000, true, 100, null, 0, 1, 600); + private static final SubscriptionPolicy SUBSCRIPTION_POLICY = + new SubscriptionPolicy(100, 2000, 1000, 1000, true, 100, null, 0, 1, 600); - @Test - public void shouldFilterIncomingEvents() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()) + @Test + public void shouldFilterIncomingEvents() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()) .withSubscriptionPolicy(SUBSCRIPTION_POLICY) .withFilter(MESSAGE_NAME_FILTER) .build()); - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), ALICE.asJson()); - hermes.api().publishUntilSuccess(topic.getQualifiedName(), BOB.asJson()); - - // then - subscriber.waitUntilAllReceivedStrict(Set.of(BOB.asJson())); - } - - @Test - public void shouldChainFilters() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()) + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), ALICE.asJson()); + hermes.api().publishUntilSuccess(topic.getQualifiedName(), BOB.asJson()); + + // then + subscriber.waitUntilAllReceivedStrict(Set.of(BOB.asJson())); + } + + @Test + public void shouldChainFilters() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()) .withSubscriptionPolicy(SUBSCRIPTION_POLICY) .withFilter(MESSAGE_NAME_FILTER) .withFilter(MESSAGE_COLOR_FILTER) .build()); - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), ALICE.asJson()); - hermes.api().publishUntilSuccess(topic.getQualifiedName(), ALICE_GREY.asJson()); - hermes.api().publishUntilSuccess(topic.getQualifiedName(), BOB.asJson()); - hermes.api().publishUntilSuccess(topic.getQualifiedName(), BOB_GREY.asJson()); - - // then - subscriber.waitUntilAllReceivedStrict(Set.of(BOB_GREY.asJson())); - } - - @Test - public void shouldPassSubscriptionHeadersWhenFilteringIsEnabledForIncomingEvents() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()) + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), ALICE.asJson()); + hermes.api().publishUntilSuccess(topic.getQualifiedName(), ALICE_GREY.asJson()); + hermes.api().publishUntilSuccess(topic.getQualifiedName(), BOB.asJson()); + hermes.api().publishUntilSuccess(topic.getQualifiedName(), BOB_GREY.asJson()); + + // then + subscriber.waitUntilAllReceivedStrict(Set.of(BOB_GREY.asJson())); + } + + @Test + public void shouldPassSubscriptionHeadersWhenFilteringIsEnabledForIncomingEvents() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()) .withSubscriptionPolicy(SUBSCRIPTION_POLICY) .withFilter(MESSAGE_NAME_FILTER) .withHeader("MY-HEADER", "myHeaderValue") .build()); - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), ALICE.asJson()); - hermes.api().publishUntilSuccess(topic.getQualifiedName(), BOB.asJson()); + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), ALICE.asJson()); + hermes.api().publishUntilSuccess(topic.getQualifiedName(), BOB.asJson()); - // then - subscriber.waitUntilAllReceivedStrict(Set.of(BOB.asJson())); - subscriber.waitUntilMessageWithHeaderReceived("MY-HEADER", "myHeaderValue"); - } + // then + subscriber.waitUntilAllReceivedStrict(Set.of(BOB.asJson())); + subscriber.waitUntilMessageWithHeaderReceived("MY-HEADER", "myHeaderValue"); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/GooglePubSubConsumingTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/GooglePubSubConsumingTest.java index e418da908e..0240b4d43d 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/GooglePubSubConsumingTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/GooglePubSubConsumingTest.java @@ -1,6 +1,12 @@ package pl.allegro.tech.hermes.integrationtests; +import static pl.allegro.tech.hermes.integrationtests.assertions.HermesAssertions.assertThat; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + import com.google.pubsub.v1.ReceivedMessage; +import java.io.IOException; +import java.util.List; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -10,45 +16,39 @@ import pl.allegro.tech.hermes.integrationtests.subscriber.TestGooglePubSubSubscriber; import pl.allegro.tech.hermes.test.helper.message.TestMessage; -import java.io.IOException; -import java.util.List; - -import static pl.allegro.tech.hermes.integrationtests.assertions.HermesAssertions.assertThat; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class GooglePubSubConsumingTest { - @Order(0) - @RegisterExtension - public static final GooglePubSubExtension googlePubSub = new GooglePubSubExtension(); - - @Order(1) - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension() - .withGooglePubSub(googlePubSub); - - @Test - public void shouldDeliverMessageToGooglePubSub() throws IOException { - // given - TestGooglePubSubSubscriber subscriber = googlePubSub.subscriber(); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - hermes.initHelper().createSubscription( - subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()).build() - ); - TestMessage message = TestMessage.of("hello", "world"); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - - // then - subscriber.waitUntilAnyMessageReceived(); - List allReceivedMessages = subscriber.getAllReceivedMessages(); - assertThat(allReceivedMessages).hasSize(1); - assertThat(allReceivedMessages.get(0).getMessage()) - .hasAttribute("tn") - .hasAttribute("id") - .hasAttribute("ts") - .hasBody(message.body()); - } + @Order(0) + @RegisterExtension + public static final GooglePubSubExtension googlePubSub = new GooglePubSubExtension(); + + @Order(1) + @RegisterExtension + public static final HermesExtension hermes = new HermesExtension().withGooglePubSub(googlePubSub); + + @Test + public void shouldDeliverMessageToGooglePubSub() throws IOException { + // given + TestGooglePubSubSubscriber subscriber = googlePubSub.subscriber(); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) + .build()); + TestMessage message = TestMessage.of("hello", "world"); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + + // then + subscriber.waitUntilAnyMessageReceived(); + List allReceivedMessages = subscriber.getAllReceivedMessages(); + assertThat(allReceivedMessages).hasSize(1); + assertThat(allReceivedMessages.get(0).getMessage()) + .hasAttribute("tn") + .hasAttribute("id") + .hasAttribute("ts") + .hasBody(message.body()); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/HermesClientPublishingTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/HermesClientPublishingTest.java index 813e0d9496..27bca9c938 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/HermesClientPublishingTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/HermesClientPublishingTest.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.integrationtests; +import static jakarta.ws.rs.client.ClientBuilder.newClient; +import static org.assertj.core.api.Assertions.assertThat; +import static pl.allegro.tech.hermes.client.HermesClientBuilder.hermesClient; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + import jakarta.ws.rs.core.UriBuilder; import okhttp3.OkHttpClient; import org.junit.jupiter.api.Test; @@ -13,66 +18,62 @@ import pl.allegro.tech.hermes.integrationtests.setup.HermesExtension; import pl.allegro.tech.hermes.test.helper.message.TestMessage; -import static jakarta.ws.rs.client.ClientBuilder.newClient; -import static org.assertj.core.api.Assertions.assertThat; -import static pl.allegro.tech.hermes.client.HermesClientBuilder.hermesClient; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class HermesClientPublishingTest { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - private static final String FRONTEND_URI = "http://localhost:{frontendPort}"; - private final TestMessage message = TestMessage.of("hello", "world"); + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); + private static final String FRONTEND_URI = "http://localhost:{frontendPort}"; + private final TestMessage message = TestMessage.of("hello", "world"); - @Test - public void shouldPublishUsingJerseyClient() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - HermesClient client = hermesClient( - new JerseyHermesSender(newClient())).withURI(UriBuilder.fromUri(FRONTEND_URI).build(hermes.getFrontendPort()) - ).build(); + @Test + public void shouldPublishUsingJerseyClient() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + HermesClient client = + hermesClient(new JerseyHermesSender(newClient())) + .withURI(UriBuilder.fromUri(FRONTEND_URI).build(hermes.getFrontendPort())) + .build(); - // when & then - runTestSuiteForHermesClient(client, topic); - } + // when & then + runTestSuiteForHermesClient(client, topic); + } - @Test - public void shouldPublishUsingOkHttpClient() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - HermesClient client = hermesClient( - new OkHttpHermesSender(new OkHttpClient())).withURI(UriBuilder.fromUri(FRONTEND_URI).build(hermes.getFrontendPort()) - ).build(); + @Test + public void shouldPublishUsingOkHttpClient() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + HermesClient client = + hermesClient(new OkHttpHermesSender(new OkHttpClient())) + .withURI(UriBuilder.fromUri(FRONTEND_URI).build(hermes.getFrontendPort())) + .build(); - // when & then - runTestSuiteForHermesClient(client, topic); - } + // when & then + runTestSuiteForHermesClient(client, topic); + } - private void runTestSuiteForHermesClient(HermesClient client, Topic topic) { - shouldPublishUsingHermesClient(client, topic); - shouldNotPublishUsingHermesClient(client); - } + private void runTestSuiteForHermesClient(HermesClient client, Topic topic) { + shouldPublishUsingHermesClient(client, topic); + shouldNotPublishUsingHermesClient(client); + } - private void shouldPublishUsingHermesClient(HermesClient client, Topic topic) { - // when - HermesResponse response = client.publish(topic.getQualifiedName(), message.body()).join(); + private void shouldPublishUsingHermesClient(HermesClient client, Topic topic) { + // when + HermesResponse response = client.publish(topic.getQualifiedName(), message.body()).join(); - // then - assertThat(response.isSuccess()).isTrue(); - assertThat(response.getMessageId()).isNotEmpty(); - assertThat(response.getDebugLog()).contains("Sending message").contains("succeeded"); - } + // then + assertThat(response.isSuccess()).isTrue(); + assertThat(response.getMessageId()).isNotEmpty(); + assertThat(response.getDebugLog()).contains("Sending message").contains("succeeded"); + } - private void shouldNotPublishUsingHermesClient(HermesClient client) { - // given - TopicName topic = new TopicName("not", "existing"); + private void shouldNotPublishUsingHermesClient(HermesClient client) { + // given + TopicName topic = new TopicName("not", "existing"); - // when - HermesResponse response = client.publish(topic.qualifiedName(), message.body()).join(); + // when + HermesResponse response = client.publish(topic.qualifiedName(), message.body()).join(); - // then - assertThat(response.isFailure()).isTrue(); - assertThat(response.getDebugLog()).contains("Sending message").contains("failed"); - } + // then + assertThat(response.isFailure()).isTrue(); + assertThat(response.getDebugLog()).contains("Sending message").contains("failed"); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/JmsConsumingTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/JmsConsumingTest.java index 8df3d87b51..e002f80639 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/JmsConsumingTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/JmsConsumingTest.java @@ -1,5 +1,9 @@ package pl.allegro.tech.hermes.integrationtests; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + +import java.util.UUID; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -12,91 +16,94 @@ import pl.allegro.tech.hermes.integrationtests.subscriber.TestJmsSubscriber; import pl.allegro.tech.hermes.test.helper.message.TestMessage; -import java.util.UUID; - -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class JmsConsumingTest { - private static final String JMS_TOPIC_NAME = "hermes"; - - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - private static final JmsStarter starter = new JmsStarter(); - - @BeforeAll - static void startJms() throws Exception { - starter.start(); - } - - @AfterAll - static void stopJms() throws Exception { - starter.stop(); - } - - @Test - public void shouldConsumeMessageOnJMSEndpoint() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestJmsSubscriber subscriber = new TestJmsSubscriber(JMS_TOPIC_NAME); - hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName(), jmsEndpointAddress()).build()); - - // when - hermes.api().publish(topic.getQualifiedName(), TestMessage.simple().body()); - - // then - subscriber.waitUntilReceived(TestMessage.simple().body()); - } - - @Test - public void shouldPublishAndConsumeJmsMessageWithTraceId() { - // given - String traceId = UUID.randomUUID().toString(); - HttpHeaders headers = new HttpHeaders(); - headers.add("Trace-Id", traceId); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestJmsSubscriber subscriber = new TestJmsSubscriber(JMS_TOPIC_NAME); - hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName(), jmsEndpointAddress()).build()); - - // when - hermes.api().publish(topic.getQualifiedName(), TestMessage.simple().body(), headers); - - // then - subscriber.waitUntilMessageWithHeaderReceived("TraceId", traceId); - } - - @Test - public void shouldPublishAndConsumeJmsMessageWithTraceHeaders() { - // given - TraceContext trace = TraceContext.random(); - HttpHeaders headers = new HttpHeaders(); - addTraceHeaders(trace, headers); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestJmsSubscriber subscriber = new TestJmsSubscriber(JMS_TOPIC_NAME); - hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName(), jmsEndpointAddress()).build()); - - // when - hermes.api().publish(topic.getQualifiedName(), TestMessage.simple().body(), headers); - - // then - subscriber.waitUntilMessageWithHeaderReceived("TraceId", trace.traceId()); - subscriber.waitUntilMessageWithHeaderReceived("SpanId", trace.spanId()); - subscriber.waitUntilMessageWithHeaderReceived("ParentSpanId", trace.parentSpanId()); - subscriber.waitUntilMessageWithHeaderReceived("TraceSampled", trace.traceSampled()); - subscriber.waitUntilMessageWithHeaderReceived("TraceReported", trace.traceReported()); - } - - private void addTraceHeaders(TraceContext trace, HttpHeaders headers) { - headers.add("Trace-Id", trace.traceId()); - headers.add("Span-Id", trace.spanId()); - headers.add("Parent-Span-Id", trace.parentSpanId()); - headers.add("Trace-Sampled", trace.traceSampled()); - headers.add("Trace-Reported", trace.traceReported()); - } - - private String jmsEndpointAddress() { - return "jms://guest:guest@localhost:5445/" + JmsConsumingTest.JMS_TOPIC_NAME; - } + private static final String JMS_TOPIC_NAME = "hermes"; + + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); + + private static final JmsStarter starter = new JmsStarter(); + + @BeforeAll + static void startJms() throws Exception { + starter.start(); + } + + @AfterAll + static void stopJms() throws Exception { + starter.stop(); + } + + @Test + public void shouldConsumeMessageOnJMSEndpoint() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestJmsSubscriber subscriber = new TestJmsSubscriber(JMS_TOPIC_NAME); + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), jmsEndpointAddress()).build()); + + // when + hermes.api().publish(topic.getQualifiedName(), TestMessage.simple().body()); + + // then + subscriber.waitUntilReceived(TestMessage.simple().body()); + } + + @Test + public void shouldPublishAndConsumeJmsMessageWithTraceId() { + // given + String traceId = UUID.randomUUID().toString(); + HttpHeaders headers = new HttpHeaders(); + headers.add("Trace-Id", traceId); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestJmsSubscriber subscriber = new TestJmsSubscriber(JMS_TOPIC_NAME); + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), jmsEndpointAddress()).build()); + + // when + hermes.api().publish(topic.getQualifiedName(), TestMessage.simple().body(), headers); + + // then + subscriber.waitUntilMessageWithHeaderReceived("TraceId", traceId); + } + + @Test + public void shouldPublishAndConsumeJmsMessageWithTraceHeaders() { + // given + TraceContext trace = TraceContext.random(); + HttpHeaders headers = new HttpHeaders(); + addTraceHeaders(trace, headers); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestJmsSubscriber subscriber = new TestJmsSubscriber(JMS_TOPIC_NAME); + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), jmsEndpointAddress()).build()); + + // when + hermes.api().publish(topic.getQualifiedName(), TestMessage.simple().body(), headers); + + // then + subscriber.waitUntilMessageWithHeaderReceived("TraceId", trace.traceId()); + subscriber.waitUntilMessageWithHeaderReceived("SpanId", trace.spanId()); + subscriber.waitUntilMessageWithHeaderReceived("ParentSpanId", trace.parentSpanId()); + subscriber.waitUntilMessageWithHeaderReceived("TraceSampled", trace.traceSampled()); + subscriber.waitUntilMessageWithHeaderReceived("TraceReported", trace.traceReported()); + } + + private void addTraceHeaders(TraceContext trace, HttpHeaders headers) { + headers.add("Trace-Id", trace.traceId()); + headers.add("Span-Id", trace.spanId()); + headers.add("Parent-Span-Id", trace.parentSpanId()); + headers.add("Trace-Sampled", trace.traceSampled()); + headers.add("Trace-Reported", trace.traceReported()); + } + + private String jmsEndpointAddress() { + return "jms://guest:guest@localhost:5445/" + JmsConsumingTest.JMS_TOPIC_NAME; + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/KafkaProducerMetricsTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/KafkaProducerMetricsTest.java index 71555d636d..73b1444b8e 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/KafkaProducerMetricsTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/KafkaProducerMetricsTest.java @@ -1,47 +1,50 @@ package pl.allegro.tech.hermes.integrationtests; +import static org.awaitility.Awaitility.await; +import static pl.allegro.tech.hermes.integrationtests.assertions.HermesAssertions.assertThatMetrics; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.integrationtests.assertions.PrometheusMetricsAssertion; import pl.allegro.tech.hermes.integrationtests.setup.HermesExtension; -import java.util.concurrent.TimeUnit; - -import static org.awaitility.Awaitility.await; -import static pl.allegro.tech.hermes.integrationtests.assertions.HermesAssertions.assertThatMetrics; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class KafkaProducerMetricsTest { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - @Test - public void shouldRegisterSendMetrics() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - - double initialMetricValue = assertMetricsContainTotalSendMetric().withInitialValue(); - - // when - hermes.api().publish(topic.getQualifiedName(), "hello world"); - hermes.api().publish(topic.getQualifiedName(), "hello world"); - - // then - await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> - assertMetricsContainTotalSendMetric().withValue(initialMetricValue + 2.0)); - } - - PrometheusMetricsAssertion.PrometheusMetricAssertion assertMetricsContainTotalSendMetric() { - return assertThatMetrics(hermes.api() - .getFrontendMetrics().expectStatus().isOk() - .expectBody(String.class).returnResult().getResponseBody()) - .contains("hermes_frontend_kafka_producer_ack_leader_record_send_total") - .withLabels( - "storageDc", "dc", - "sender", "default" - ); - } - - + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); + + @Test + public void shouldRegisterSendMetrics() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + + double initialMetricValue = assertMetricsContainTotalSendMetric().withInitialValue(); + + // when + hermes.api().publish(topic.getQualifiedName(), "hello world"); + hermes.api().publish(topic.getQualifiedName(), "hello world"); + + // then + await() + .atMost(10, TimeUnit.SECONDS) + .untilAsserted( + () -> assertMetricsContainTotalSendMetric().withValue(initialMetricValue + 2.0)); + } + + PrometheusMetricsAssertion.PrometheusMetricAssertion assertMetricsContainTotalSendMetric() { + return assertThatMetrics( + hermes + .api() + .getFrontendMetrics() + .expectStatus() + .isOk() + .expectBody(String.class) + .returnResult() + .getResponseBody()) + .contains("hermes_frontend_kafka_producer_ack_leader_record_send_total") + .withLabels( + "storageDc", "dc", + "sender", "default"); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/KafkaRetransmissionServiceTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/KafkaRetransmissionServiceTest.java index 7bc280419d..94c7eb8f43 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/KafkaRetransmissionServiceTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/KafkaRetransmissionServiceTest.java @@ -1,7 +1,17 @@ package pl.allegro.tech.hermes.integrationtests; +import static java.util.stream.IntStream.range; +import static org.assertj.core.api.Assertions.assertThat; +import static pl.allegro.tech.hermes.api.PatchData.patchData; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.Multimaps; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.springframework.test.web.reactive.server.WebTestClient; @@ -18,140 +28,171 @@ import pl.allegro.tech.hermes.test.helper.avro.AvroUser; import pl.allegro.tech.hermes.test.helper.message.TestMessage; -import java.time.OffsetDateTime; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import static java.util.stream.IntStream.range; -import static org.assertj.core.api.Assertions.assertThat; -import static pl.allegro.tech.hermes.api.PatchData.patchData; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class KafkaRetransmissionServiceTest { - private static final String PRIMARY_KAFKA_CLUSTER_NAME = "primary-dc"; - - private final AvroUser avroUser = new AvroUser(); - - private final List messages = new ArrayList<>() {{ - range(0, 4).forEach(i -> add(TestMessage.random().body())); - }}; - - private final List messages2 = new ArrayList<>() {{ - range(5, 6).forEach(i -> add(TestMessage.random().body())); - }}; - - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - @RegisterExtension - public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); - - @Test - public void shouldMoveOffsetNearGivenTimestamp() throws InterruptedException { - // given - final TestSubscriber subscriber = subscribers.createSubscriber(); - final Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - final Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()).build()); - publishAndConsumeMessages(messages, topic, subscriber); - Thread.sleep(1000); //wait 1s because our date time format has seconds precision - final OffsetRetransmissionDate retransmissionDate = new OffsetRetransmissionDate(OffsetDateTime.now()); - Thread.sleep(1000); - publishAndConsumeMessages(messages2, topic, subscriber); - hermes.api().waitUntilConsumerCommitsOffset(topic.getQualifiedName(), subscription.getName()); - - // when - WebTestClient.ResponseSpec response = hermes.api().retransmit(topic.getQualifiedName(), subscription.getName(), retransmissionDate, false); + private static final String PRIMARY_KAFKA_CLUSTER_NAME = "primary-dc"; - // then - response.expectStatus().isOk(); - messages2.forEach(subscriber::waitUntilReceived); - } - - @Test - public void shouldMoveOffsetInDryRunMode() throws InterruptedException { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - final Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()).build()); - // 4 messages - publishAndConsumeMessages(messages, topic, subscriber); - Thread.sleep(2000); - final OffsetRetransmissionDate retransmissionDate = new OffsetRetransmissionDate(OffsetDateTime.now()); - publishAndConsumeMessages(messages2, topic, subscriber); - hermes.api().waitUntilConsumerCommitsOffset(topic.getQualifiedName(), subscription.getName()); - subscriber.reset(); - - // when - WebTestClient.ResponseSpec response = hermes.api().retransmit(topic.getQualifiedName(), subscription.getName(), retransmissionDate, true); - - // then - response.expectStatus().isOk(); - MultiDCOffsetChangeSummary summary = response.expectBody(MultiDCOffsetChangeSummary.class).returnResult().getResponseBody(); - - assert summary != null; - Long offsetSum = summary.getPartitionOffsetListPerBrokerName().get(PRIMARY_KAFKA_CLUSTER_NAME).stream() - .collect(Collectors.summarizingLong(PartitionOffset::getOffset)).getSum(); - assertThat(offsetSum).isEqualTo(4); - subscriber.noMessagesReceived(); - } - - @Test - public void shouldMoveOffsetInDryRunModeForTopicsMigratedToAvro() throws InterruptedException { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()).build()); - - hermes.api().publish(topic.getQualifiedName(), TestMessage.simple().body()); - hermes.api().waitUntilConsumerCommitsOffset(topic.getQualifiedName(), subscription.getName()); - - Thread.sleep(1000); //wait 1s because our date time format has seconds precision - final OffsetRetransmissionDate retransmissionDate = new OffsetRetransmissionDate(OffsetDateTime.now()); - - hermes.api().publish(topic.getQualifiedName(), TestMessage.simple().body()); - hermes.api().waitUntilConsumerCommitsOffset(topic.getQualifiedName(), subscription.getName()); - - PatchData patch = patchData() - .set("contentType", ContentType.AVRO) - .set("migratedFromJsonType", true) - .set("schema", avroUser.getSchemaAsString()) - .build(); - - hermes.api().updateTopic(topic.getQualifiedName(), patch).expectStatus().isOk(); - - hermes.api().publishAvroUntilSuccess(topic.getQualifiedName(), avroUser.asBytes()); - hermes.api().waitUntilConsumerCommitsOffset(topic.getQualifiedName(), subscription.getName()); - - // when - WebTestClient.ResponseSpec response = hermes.api().retransmit(topic.getQualifiedName(), subscription.getName(), retransmissionDate, true); - - // then - response.expectStatus().isOk(); - MultiDCOffsetChangeSummary summary = response.expectBody(MultiDCOffsetChangeSummary.class).returnResult().getResponseBody(); - assert summary != null; - PartitionOffsetsPerKafkaTopic offsets = PartitionOffsetsPerKafkaTopic.from( - summary.getPartitionOffsetListPerBrokerName().get(PRIMARY_KAFKA_CLUSTER_NAME) - ); - assertThat((Long) offsets.jsonPartitionOffsets.stream().mapToLong(PartitionOffset::getOffset).sum()).isEqualTo(1); - assertThat((Long) offsets.avroPartitionOffsets.stream().mapToLong(PartitionOffset::getOffset).sum()).isEqualTo(0); - } + private final AvroUser avroUser = new AvroUser(); - private void publishAndConsumeMessages(List messages, Topic topic, TestSubscriber subscriber) { - messages.forEach(message -> hermes.api().publish(topic.getQualifiedName(), message)); - messages.forEach(subscriber::waitUntilReceived); - } - - private record PartitionOffsetsPerKafkaTopic(List avroPartitionOffsets, - List jsonPartitionOffsets) { + private final List messages = + new ArrayList<>() { + { + range(0, 4).forEach(i -> add(TestMessage.random().body())); + } + }; - private static PartitionOffsetsPerKafkaTopic from(List all) { - ImmutableListMultimap partitionOffsets = Multimaps.index( - all, p -> p.getTopic().asString().endsWith("_avro") - ); - return new PartitionOffsetsPerKafkaTopic(partitionOffsets.get(true), partitionOffsets.get(false)); - } + private final List messages2 = + new ArrayList<>() { + { + range(5, 6).forEach(i -> add(TestMessage.random().body())); } + }; + + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); + + @RegisterExtension + public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); + + @Test + public void shouldMoveOffsetNearGivenTimestamp() throws InterruptedException { + // given + final TestSubscriber subscriber = subscribers.createSubscriber(); + final Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + final Subscription subscription = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()).build()); + publishAndConsumeMessages(messages, topic, subscriber); + Thread.sleep(1000); // wait 1s because our date time format has seconds precision + final OffsetRetransmissionDate retransmissionDate = + new OffsetRetransmissionDate(OffsetDateTime.now()); + Thread.sleep(1000); + publishAndConsumeMessages(messages2, topic, subscriber); + hermes.api().waitUntilConsumerCommitsOffset(topic.getQualifiedName(), subscription.getName()); + + // when + WebTestClient.ResponseSpec response = + hermes + .api() + .retransmit( + topic.getQualifiedName(), subscription.getName(), retransmissionDate, false); + + // then + response.expectStatus().isOk(); + messages2.forEach(subscriber::waitUntilReceived); + } + + @Test + public void shouldMoveOffsetInDryRunMode() throws InterruptedException { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + final Subscription subscription = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()).build()); + // 4 messages + publishAndConsumeMessages(messages, topic, subscriber); + Thread.sleep(2000); + final OffsetRetransmissionDate retransmissionDate = + new OffsetRetransmissionDate(OffsetDateTime.now()); + publishAndConsumeMessages(messages2, topic, subscriber); + hermes.api().waitUntilConsumerCommitsOffset(topic.getQualifiedName(), subscription.getName()); + subscriber.reset(); + + // when + WebTestClient.ResponseSpec response = + hermes + .api() + .retransmit(topic.getQualifiedName(), subscription.getName(), retransmissionDate, true); + + // then + response.expectStatus().isOk(); + MultiDCOffsetChangeSummary summary = + response.expectBody(MultiDCOffsetChangeSummary.class).returnResult().getResponseBody(); + + assert summary != null; + Long offsetSum = + summary.getPartitionOffsetListPerBrokerName().get(PRIMARY_KAFKA_CLUSTER_NAME).stream() + .collect(Collectors.summarizingLong(PartitionOffset::getOffset)) + .getSum(); + assertThat(offsetSum).isEqualTo(4); + subscriber.noMessagesReceived(); + } + + @Test + public void shouldMoveOffsetInDryRunModeForTopicsMigratedToAvro() throws InterruptedException { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()).build()); + + hermes.api().publish(topic.getQualifiedName(), TestMessage.simple().body()); + hermes.api().waitUntilConsumerCommitsOffset(topic.getQualifiedName(), subscription.getName()); + + Thread.sleep(1000); // wait 1s because our date time format has seconds precision + final OffsetRetransmissionDate retransmissionDate = + new OffsetRetransmissionDate(OffsetDateTime.now()); + + hermes.api().publish(topic.getQualifiedName(), TestMessage.simple().body()); + hermes.api().waitUntilConsumerCommitsOffset(topic.getQualifiedName(), subscription.getName()); + + PatchData patch = + patchData() + .set("contentType", ContentType.AVRO) + .set("migratedFromJsonType", true) + .set("schema", avroUser.getSchemaAsString()) + .build(); + + hermes.api().updateTopic(topic.getQualifiedName(), patch).expectStatus().isOk(); + + hermes.api().publishAvroUntilSuccess(topic.getQualifiedName(), avroUser.asBytes()); + hermes.api().waitUntilConsumerCommitsOffset(topic.getQualifiedName(), subscription.getName()); + + // when + WebTestClient.ResponseSpec response = + hermes + .api() + .retransmit(topic.getQualifiedName(), subscription.getName(), retransmissionDate, true); + + // then + response.expectStatus().isOk(); + MultiDCOffsetChangeSummary summary = + response.expectBody(MultiDCOffsetChangeSummary.class).returnResult().getResponseBody(); + assert summary != null; + PartitionOffsetsPerKafkaTopic offsets = + PartitionOffsetsPerKafkaTopic.from( + summary.getPartitionOffsetListPerBrokerName().get(PRIMARY_KAFKA_CLUSTER_NAME)); + assertThat( + (Long) + offsets.jsonPartitionOffsets.stream().mapToLong(PartitionOffset::getOffset).sum()) + .isEqualTo(1); + assertThat( + (Long) + offsets.avroPartitionOffsets.stream().mapToLong(PartitionOffset::getOffset).sum()) + .isEqualTo(0); + } + + private void publishAndConsumeMessages( + List messages, Topic topic, TestSubscriber subscriber) { + messages.forEach(message -> hermes.api().publish(topic.getQualifiedName(), message)); + messages.forEach(subscriber::waitUntilReceived); + } + + private record PartitionOffsetsPerKafkaTopic( + List avroPartitionOffsets, List jsonPartitionOffsets) { + + private static PartitionOffsetsPerKafkaTopic from(List all) { + ImmutableListMultimap partitionOffsets = + Multimaps.index(all, p -> p.getTopic().asString().endsWith("_avro")); + return new PartitionOffsetsPerKafkaTopic( + partitionOffsets.get(true), partitionOffsets.get(false)); + } + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/KafkaSingleMessageReaderTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/KafkaSingleMessageReaderTest.java index 007c18f472..5bf337789c 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/KafkaSingleMessageReaderTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/KafkaSingleMessageReaderTest.java @@ -1,5 +1,14 @@ package pl.allegro.tech.hermes.integrationtests; +import static java.util.stream.IntStream.range; +import static org.assertj.core.api.Assertions.assertThat; +import static pl.allegro.tech.hermes.api.ContentType.AVRO; +import static pl.allegro.tech.hermes.api.TopicWithSchema.topicWithSchema; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + +import java.util.ArrayList; +import java.util.List; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.springframework.test.web.reactive.server.WebTestClient; @@ -11,138 +20,162 @@ import pl.allegro.tech.hermes.test.helper.avro.AvroUser; import pl.allegro.tech.hermes.test.helper.message.TestMessage; -import java.util.ArrayList; -import java.util.List; - -import static java.util.stream.IntStream.range; -import static org.assertj.core.api.Assertions.assertThat; -import static pl.allegro.tech.hermes.api.ContentType.AVRO; -import static pl.allegro.tech.hermes.api.TopicWithSchema.topicWithSchema; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class KafkaSingleMessageReaderTest { - private static final int NUMBER_OF_PARTITIONS = 2; - - private static final String PRIMARY_KAFKA_CLUSTER_NAME = "primary-dc"; - - private final AvroUser avroUser = new AvroUser("Bob", 50, "blue"); - - private final List messages = new ArrayList<>() {{ - range(0, 3).forEach(i -> add(TestMessage.random().body())); - }}; - - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); + private static final int NUMBER_OF_PARTITIONS = 2; - @RegisterExtension - public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); + private static final String PRIMARY_KAFKA_CLUSTER_NAME = "primary-dc"; - @Test - public void shouldFetchSingleMessageByTopicPartitionAndOffset() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()).build()); - messages.forEach(message -> hermes.api().publish(topic.getQualifiedName(), message)); - messages.forEach(subscriber::waitUntilReceived); + private final AvroUser avroUser = new AvroUser("Bob", 50, "blue"); - // when - List previews = fetchPreviewsFromAllPartitions(topic.getQualifiedName(), 10, true); - - // then - assertThat(previews).containsAll(messages); - } - - @Test - public void shouldFetchSingleAvroMessage() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - TopicWithSchema topic = topicWithSchema(topicWithRandomName().withContentType(AVRO).build(), avroUser.getSchemaAsString()); - hermes.initHelper().createTopicWithSchema(topic); - hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()).build()); - - hermes.api().publishAvro(topic.getQualifiedName(), avroUser.asBytes()); - subscriber.waitUntilAnyMessageReceived(); - - // when - List previews = fetchPreviewsFromAllPartitions(topic.getQualifiedName(), 10, false); - - // then - boolean isMessagePresent = previews.stream().anyMatch(preview -> preview.contains(avroUser.getName())); - assertThat(isMessagePresent).isTrue(); - } - - @Test - public void shouldFetchSingleAvroMessageWithSchemaAwareSerialization() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - TopicWithSchema topic = topicWithSchema(topicWithRandomName().withContentType(AVRO).withSchemaIdAwareSerialization().build(), avroUser.getSchemaAsString()); - hermes.initHelper().createTopicWithSchema(topic); - hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()).build()); - - hermes.api().publishAvro(topic.getQualifiedName(), avroUser.asBytes()); - subscriber.waitUntilAnyMessageReceived(); - - // when - List previews = fetchPreviewsFromAllPartitions(topic.getQualifiedName(), 10, false); - - // then - boolean isMessagePresent = previews.stream().anyMatch(preview -> preview.contains(avroUser.getName())); - assertThat(isMessagePresent).isTrue(); - } - - @Test - public void shouldReturnNotFoundErrorForNonExistingOffset() { - // given - final long nonExistingOffset = 10L; - TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()).build()); - messages.forEach(message -> hermes.api().publish(topic.getQualifiedName(), message)); - messages.forEach(subscriber::waitUntilReceived); - - // when - WebTestClient.ResponseSpec response = hermes.api().getPreview(topic.getQualifiedName(), PRIMARY_KAFKA_CLUSTER_NAME, 0, nonExistingOffset); - - // then - response.expectStatus().isNotFound(); - } - - @Test - public void shouldReturnNotFoundErrorForNonExistingPartition() { - // given - int nonExistingPartition = 20; - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - - // when - WebTestClient.ResponseSpec response = hermes.api().getPreview(topic.getQualifiedName(), PRIMARY_KAFKA_CLUSTER_NAME, nonExistingPartition, 0L); - - // then - response.expectStatus().isNotFound(); - } - - - private List fetchPreviewsFromAllPartitions(String qualifiedTopicName, int upToOffset, boolean unwrap) { - List result = new ArrayList<>(); - for (int partition = 0; partition < NUMBER_OF_PARTITIONS; partition++) { - long offset = 0; - while (offset <= upToOffset) { - try { - String wrappedMessage = hermes.api().getPreview(qualifiedTopicName, PRIMARY_KAFKA_CLUSTER_NAME, partition, offset).expectBody(String.class).returnResult().getResponseBody(); - result.add(unwrap ? unwrap(wrappedMessage) : wrappedMessage); - offset++; - } catch (Exception e) { - break; - } - } + private final List messages = + new ArrayList<>() { + { + range(0, 3).forEach(i -> add(TestMessage.random().body())); } - return result; + }; + + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); + + @RegisterExtension + public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); + + @Test + public void shouldFetchSingleMessageByTopicPartitionAndOffset() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()).build()); + messages.forEach(message -> hermes.api().publish(topic.getQualifiedName(), message)); + messages.forEach(subscriber::waitUntilReceived); + + // when + List previews = fetchPreviewsFromAllPartitions(topic.getQualifiedName(), 10, true); + + // then + assertThat(previews).containsAll(messages); + } + + @Test + public void shouldFetchSingleAvroMessage() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + TopicWithSchema topic = + topicWithSchema( + topicWithRandomName().withContentType(AVRO).build(), avroUser.getSchemaAsString()); + hermes.initHelper().createTopicWithSchema(topic); + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()).build()); + + hermes.api().publishAvro(topic.getQualifiedName(), avroUser.asBytes()); + subscriber.waitUntilAnyMessageReceived(); + + // when + List previews = fetchPreviewsFromAllPartitions(topic.getQualifiedName(), 10, false); + + // then + boolean isMessagePresent = + previews.stream().anyMatch(preview -> preview.contains(avroUser.getName())); + assertThat(isMessagePresent).isTrue(); + } + + @Test + public void shouldFetchSingleAvroMessageWithSchemaAwareSerialization() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + TopicWithSchema topic = + topicWithSchema( + topicWithRandomName().withContentType(AVRO).withSchemaIdAwareSerialization().build(), + avroUser.getSchemaAsString()); + hermes.initHelper().createTopicWithSchema(topic); + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()).build()); + + hermes.api().publishAvro(topic.getQualifiedName(), avroUser.asBytes()); + subscriber.waitUntilAnyMessageReceived(); + + // when + List previews = fetchPreviewsFromAllPartitions(topic.getQualifiedName(), 10, false); + + // then + boolean isMessagePresent = + previews.stream().anyMatch(preview -> preview.contains(avroUser.getName())); + assertThat(isMessagePresent).isTrue(); + } + + @Test + public void shouldReturnNotFoundErrorForNonExistingOffset() { + // given + final long nonExistingOffset = 10L; + TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()).build()); + messages.forEach(message -> hermes.api().publish(topic.getQualifiedName(), message)); + messages.forEach(subscriber::waitUntilReceived); + + // when + WebTestClient.ResponseSpec response = + hermes + .api() + .getPreview(topic.getQualifiedName(), PRIMARY_KAFKA_CLUSTER_NAME, 0, nonExistingOffset); + + // then + response.expectStatus().isNotFound(); + } + + @Test + public void shouldReturnNotFoundErrorForNonExistingPartition() { + // given + int nonExistingPartition = 20; + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + + // when + WebTestClient.ResponseSpec response = + hermes + .api() + .getPreview( + topic.getQualifiedName(), PRIMARY_KAFKA_CLUSTER_NAME, nonExistingPartition, 0L); + + // then + response.expectStatus().isNotFound(); + } + + private List fetchPreviewsFromAllPartitions( + String qualifiedTopicName, int upToOffset, boolean unwrap) { + List result = new ArrayList<>(); + for (int partition = 0; partition < NUMBER_OF_PARTITIONS; partition++) { + long offset = 0; + while (offset <= upToOffset) { + try { + String wrappedMessage = + hermes + .api() + .getPreview(qualifiedTopicName, PRIMARY_KAFKA_CLUSTER_NAME, partition, offset) + .expectBody(String.class) + .returnResult() + .getResponseBody(); + result.add(unwrap ? unwrap(wrappedMessage) : wrappedMessage); + offset++; + } catch (Exception e) { + break; + } + } } + return result; + } - private String unwrap(String wrappedMessage) { - String msg = wrappedMessage.split("\"message\":", 2)[1]; - return msg.substring(0, msg.length() - 1); - } + private String unwrap(String wrappedMessage) { + String msg = wrappedMessage.split("\"message\":", 2)[1]; + return msg.substring(0, msg.length() - 1); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/MetricsTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/MetricsTest.java index 5559711bde..0030fae3ac 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/MetricsTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/MetricsTest.java @@ -1,5 +1,21 @@ package pl.allegro.tech.hermes.integrationtests; +import static java.lang.Integer.MAX_VALUE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.waitAtMost; +import static pl.allegro.tech.hermes.api.BatchSubscriptionPolicy.Builder.batchSubscriptionPolicy; +import static pl.allegro.tech.hermes.api.SubscriptionPolicy.Builder.subscriptionPolicy; +import static pl.allegro.tech.hermes.integrationtests.assertions.HermesAssertions.assertThatMetrics; +import static pl.allegro.tech.hermes.integrationtests.prometheus.SubscriptionMetrics.subscriptionMetrics; +import static pl.allegro.tech.hermes.integrationtests.prometheus.TopicMetrics.topicMetrics; +import static pl.allegro.tech.hermes.test.helper.builder.GroupBuilder.groupWithRandomName; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + +import java.time.Duration; +import java.util.Map; +import java.util.UUID; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -19,461 +35,496 @@ import pl.allegro.tech.hermes.integrationtests.subscriber.TestSubscribersExtension; import pl.allegro.tech.hermes.test.helper.message.TestMessage; -import java.time.Duration; -import java.util.Map; -import java.util.UUID; - -import static java.lang.Integer.MAX_VALUE; -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.waitAtMost; -import static pl.allegro.tech.hermes.api.BatchSubscriptionPolicy.Builder.batchSubscriptionPolicy; -import static pl.allegro.tech.hermes.api.SubscriptionPolicy.Builder.subscriptionPolicy; -import static pl.allegro.tech.hermes.integrationtests.assertions.HermesAssertions.assertThatMetrics; -import static pl.allegro.tech.hermes.integrationtests.prometheus.SubscriptionMetrics.subscriptionMetrics; -import static pl.allegro.tech.hermes.integrationtests.prometheus.TopicMetrics.topicMetrics; -import static pl.allegro.tech.hermes.test.helper.builder.GroupBuilder.groupWithRandomName; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class MetricsTest { - @Order(0) - @RegisterExtension - public static final PrometheusExtension prometheus = new PrometheusExtension(); - - @Order(1) - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension() - .withPrometheus(prometheus); - - @RegisterExtension - public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); - - @Test - public void shouldIncreaseTopicMetricsAfterMessageHasBeenPublished() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - hermes.initHelper().createSubscription( - subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()).build() - ); - TestMessage message = TestMessage.simple(); - int attempts = hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> { - // when - WebTestClient.ResponseSpec response = hermes.api().getTopicMetrics(topic.getQualifiedName()); - - // then - response.expectStatus().is2xxSuccessful(); - TopicMetrics metrics = response.expectBody(TopicMetrics.class).returnResult().getResponseBody(); - assertThat(metrics).isNotNull(); - assertThat(metrics.getPublished()).isBetween(1L, (long) attempts); - assertThat(metrics.getVolume()).isGreaterThan(1); - assertThat(metrics.getSubscriptions()).isEqualTo(1); - }); - } - - @Test - public void shouldIncreaseSubscriptionDeliveredMetricsAfterMessageDelivered() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription( - subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()).build() - ); - TestMessage message = TestMessage.simple(); - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - subscriber.waitUntilReceived(message.body()); - - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> { - // when - WebTestClient.ResponseSpec response = hermes.api().getSubscriptionMetrics(topic.getQualifiedName(), subscription.getName()); - - // then - response.expectStatus().is2xxSuccessful(); - SubscriptionMetrics metrics = response.expectBody(SubscriptionMetrics.class).returnResult().getResponseBody(); - assertThat(metrics).isNotNull(); - // potentially there were retries, therefore we cannot assume that only one message was delivered - assertThat(metrics.getDelivered()).isGreaterThan(0); - assertThat(metrics.getDiscarded()).isEqualTo(0); - assertThat(metrics.getVolume()).isGreaterThan(1); - }); - } - - @Test - public void shouldNotCreateNewSubscriptionWhenAskedForNonExistingMetrics() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - String randomSubscriptionName = UUID.randomUUID().toString(); - - // when - hermes.api().getSubscriptionMetrics(topic.getQualifiedName(), randomSubscriptionName); - - // then - hermes.api().listSubscriptions(topic.getQualifiedName()) - .expectStatus() - .isOk() - .expectBodyList(String.class) - .doesNotContain(randomSubscriptionName); - } - - @Test - public void shouldNotCreateNewTopicWhenAskingForNonExistingMetrics() { - // given - Group group = hermes.initHelper().createGroup(groupWithRandomName().build()); - TopicName nonexistentTopicName = new TopicName(group.getGroupName(), "nonexistentTopic"); - - // when - hermes.api().getTopicMetrics(nonexistentTopicName.qualifiedName()); - - // then - hermes.api().listTopics(group.getGroupName()) - .expectStatus() - .isOk() - .expectBodyList(String.class) - .doesNotContain(nonexistentTopicName.qualifiedName()); - } - - @Test - public void shouldReportHttpErrorCodeMetrics() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(404); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription( + @Order(0) + @RegisterExtension + public static final PrometheusExtension prometheus = new PrometheusExtension(); + + @Order(1) + @RegisterExtension + public static final HermesExtension hermes = new HermesExtension().withPrometheus(prometheus); + + @RegisterExtension + public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); + + @Test + public void shouldIncreaseTopicMetricsAfterMessageHasBeenPublished() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) + .build()); + TestMessage message = TestMessage.simple(); + int attempts = hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + // when + WebTestClient.ResponseSpec response = + hermes.api().getTopicMetrics(topic.getQualifiedName()); + + // then + response.expectStatus().is2xxSuccessful(); + TopicMetrics metrics = + response.expectBody(TopicMetrics.class).returnResult().getResponseBody(); + assertThat(metrics).isNotNull(); + assertThat(metrics.getPublished()).isBetween(1L, (long) attempts); + assertThat(metrics.getVolume()).isGreaterThan(1); + assertThat(metrics.getSubscriptions()).isEqualTo(1); + }); + } + + @Test + public void shouldIncreaseSubscriptionDeliveredMetricsAfterMessageDelivered() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) + .build()); + TestMessage message = TestMessage.simple(); + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + subscriber.waitUntilReceived(message.body()); + + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + // when + WebTestClient.ResponseSpec response = + hermes + .api() + .getSubscriptionMetrics(topic.getQualifiedName(), subscription.getName()); + + // then + response.expectStatus().is2xxSuccessful(); + SubscriptionMetrics metrics = + response.expectBody(SubscriptionMetrics.class).returnResult().getResponseBody(); + assertThat(metrics).isNotNull(); + // potentially there were retries, therefore we cannot assume that only one message + // was delivered + assertThat(metrics.getDelivered()).isGreaterThan(0); + assertThat(metrics.getDiscarded()).isEqualTo(0); + assertThat(metrics.getVolume()).isGreaterThan(1); + }); + } + + @Test + public void shouldNotCreateNewSubscriptionWhenAskedForNonExistingMetrics() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + String randomSubscriptionName = UUID.randomUUID().toString(); + + // when + hermes.api().getSubscriptionMetrics(topic.getQualifiedName(), randomSubscriptionName); + + // then + hermes + .api() + .listSubscriptions(topic.getQualifiedName()) + .expectStatus() + .isOk() + .expectBodyList(String.class) + .doesNotContain(randomSubscriptionName); + } + + @Test + public void shouldNotCreateNewTopicWhenAskingForNonExistingMetrics() { + // given + Group group = hermes.initHelper().createGroup(groupWithRandomName().build()); + TopicName nonexistentTopicName = new TopicName(group.getGroupName(), "nonexistentTopic"); + + // when + hermes.api().getTopicMetrics(nonexistentTopicName.qualifiedName()); + + // then + hermes + .api() + .listTopics(group.getGroupName()) + .expectStatus() + .isOk() + .expectBodyList(String.class) + .doesNotContain(nonexistentTopicName.qualifiedName()); + } + + @Test + public void shouldReportHttpErrorCodeMetrics() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(404); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + hermes + .initHelper() + .createSubscription( subscription(topic, "subscription") - .withEndpoint(subscriber.getEndpoint()) - .withSubscriptionPolicy( - subscriptionPolicy() - .applyDefaults() - .withMessageTtl(0) - .build() - ) - .build() - ); - TestMessage message = TestMessage.simple(); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - - // then - subscriber.waitUntilReceived(message.body()); - hermes.api().getConsumersMetrics() - .expectStatus() - .isOk() - .expectBody(String.class) - .value((body) -> assertThatMetrics(body) - .contains("hermes_consumers_subscription_http_status_codes_total") - .withLabels( - "group", topic.getName().getGroupName(), - "status_code", "404", - "subscription", subscription.getName(), - "topic", topic.getName().getName() - ) - .withValue(1.0) - ); - } - - @Test - public void shouldReportHttpSuccessCodeMetrics() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription( + .withEndpoint(subscriber.getEndpoint()) + .withSubscriptionPolicy( + subscriptionPolicy().applyDefaults().withMessageTtl(0).build()) + .build()); + TestMessage message = TestMessage.simple(); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + + // then + subscriber.waitUntilReceived(message.body()); + hermes + .api() + .getConsumersMetrics() + .expectStatus() + .isOk() + .expectBody(String.class) + .value( + (body) -> + assertThatMetrics(body) + .contains("hermes_consumers_subscription_http_status_codes_total") + .withLabels( + "group", topic.getName().getGroupName(), + "status_code", "404", + "subscription", subscription.getName(), + "topic", topic.getName().getName()) + .withValue(1.0)); + } + + @Test + public void shouldReportHttpSuccessCodeMetrics() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + hermes + .initHelper() + .createSubscription( subscription(topic, "subscription") - .withEndpoint(subscriber.getEndpoint()) - .withSubscriptionPolicy( - subscriptionPolicy() - .applyDefaults() - .withMessageTtl(0) - .build() - ) - .build() - ); - TestMessage message = TestMessage.simple(); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - - // then - subscriber.waitUntilReceived(message.body()); - hermes.api().getConsumersMetrics() - .expectStatus() - .isOk() - .expectBody(String.class) - .value((body) -> assertThatMetrics(body) - .contains("hermes_consumers_subscription_http_status_codes_total") - .withLabels( - "group", topic.getName().getGroupName(), - "status_code", "200", - "subscription", subscription.getName(), - "topic", topic.getName().getName() - ) - .withValue(1.0) - ); - } - - @Test - public void shouldReportMetricForFilteredSubscription() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - final Subscription subscription = hermes.initHelper().createSubscription( + .withEndpoint(subscriber.getEndpoint()) + .withSubscriptionPolicy( + subscriptionPolicy().applyDefaults().withMessageTtl(0).build()) + .build()); + TestMessage message = TestMessage.simple(); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + + // then + subscriber.waitUntilReceived(message.body()); + hermes + .api() + .getConsumersMetrics() + .expectStatus() + .isOk() + .expectBody(String.class) + .value( + (body) -> + assertThatMetrics(body) + .contains("hermes_consumers_subscription_http_status_codes_total") + .withLabels( + "group", topic.getName().getGroupName(), + "status_code", "200", + "subscription", subscription.getName(), + "topic", topic.getName().getName()) + .withValue(1.0)); + } + + @Test + public void shouldReportMetricForFilteredSubscription() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + final Subscription subscription = + hermes + .initHelper() + .createSubscription( subscription(topic, "subscription") - .withEndpoint(subscriber.getEndpoint()) - .withSubscriptionPolicy( - subscriptionPolicy() - .applyDefaults() - .withMessageTtl(0) - .build() - ) - .withFilter(filterMatchingHeaderByPattern("Trace-Id", "^vte.*")) - .build() - ); - TestMessage unfiltered = TestMessage.of("msg", "unfiltered"); - TestMessage filtered = TestMessage.of("msg", "filtered"); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), unfiltered.body(), header("Trace-Id", "vte12")); - hermes.api().publishUntilSuccess(topic.getQualifiedName(), filtered.body(), header("Trace-Id", "otherTraceId")); - - // then - subscriber.waitUntilReceived(unfiltered.body()); - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> - hermes.api().getConsumersMetrics() - .expectStatus() - .isOk() - .expectBody(String.class) - .value((body) -> assertThatMetrics(body) + .withEndpoint(subscriber.getEndpoint()) + .withSubscriptionPolicy( + subscriptionPolicy().applyDefaults().withMessageTtl(0).build()) + .withFilter(filterMatchingHeaderByPattern("Trace-Id", "^vte.*")) + .build()); + TestMessage unfiltered = TestMessage.of("msg", "unfiltered"); + TestMessage filtered = TestMessage.of("msg", "filtered"); + + // when + hermes + .api() + .publishUntilSuccess( + topic.getQualifiedName(), unfiltered.body(), header("Trace-Id", "vte12")); + hermes + .api() + .publishUntilSuccess( + topic.getQualifiedName(), filtered.body(), header("Trace-Id", "otherTraceId")); + + // then + subscriber.waitUntilReceived(unfiltered.body()); + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> + hermes + .api() + .getConsumersMetrics() + .expectStatus() + .isOk() + .expectBody(String.class) + .value( + (body) -> + assertThatMetrics(body) .contains("hermes_consumers_subscription_filtered_out_total") .withLabels( - "group", topic.getName().getGroupName(), - "subscription", subscription.getName(), - "topic", topic.getName().getName() - ) - .withValue(1.0) - ) - ); - } - - @Test - public void shouldNotIncreaseInflightForFilteredSubscription() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(503); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - final Subscription subscription = hermes.initHelper().createSubscription( + "group", topic.getName().getGroupName(), + "subscription", subscription.getName(), + "topic", topic.getName().getName()) + .withValue(1.0))); + } + + @Test + public void shouldNotIncreaseInflightForFilteredSubscription() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(503); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + final Subscription subscription = + hermes + .initHelper() + .createSubscription( subscription(topic, "subscription") - .withEndpoint(subscriber.getEndpoint()) - .withSubscriptionPolicy( - subscriptionPolicy() - .withMessageTtl(3600) - .withInflightSize(1) - .build() - ) - .withFilter(filterMatchingHeaderByPattern("Trace-Id", "^vte.*")) - .build() - ); - TestMessage unfiltered = TestMessage.of("msg", "unfiltered"); - TestMessage filtered = TestMessage.of("msg", "filtered"); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), filtered.body(), header("Trace-Id", "otherTraceId")); - hermes.api().publishUntilSuccess(topic.getQualifiedName(), filtered.body(), header("Trace-Id", "otherTraceId")); - - // then - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> - hermes.api().getConsumersMetrics() - .expectStatus() - .isOk() - .expectBody(String.class) - .value((body) -> assertThatMetrics(body) + .withEndpoint(subscriber.getEndpoint()) + .withSubscriptionPolicy( + subscriptionPolicy().withMessageTtl(3600).withInflightSize(1).build()) + .withFilter(filterMatchingHeaderByPattern("Trace-Id", "^vte.*")) + .build()); + TestMessage unfiltered = TestMessage.of("msg", "unfiltered"); + TestMessage filtered = TestMessage.of("msg", "filtered"); + + // when + hermes + .api() + .publishUntilSuccess( + topic.getQualifiedName(), filtered.body(), header("Trace-Id", "otherTraceId")); + hermes + .api() + .publishUntilSuccess( + topic.getQualifiedName(), filtered.body(), header("Trace-Id", "otherTraceId")); + + // then + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> + hermes + .api() + .getConsumersMetrics() + .expectStatus() + .isOk() + .expectBody(String.class) + .value( + (body) -> + assertThatMetrics(body) .contains("hermes_consumers_subscription_inflight") .withLabels( - "group", topic.getName().getGroupName(), - "subscription", subscription.getName(), - "topic", topic.getName().getName() - ) - .withValue(0.0) - ) - ); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), unfiltered.body(), header("Trace-Id", "vte12")); - hermes.api().publishUntilSuccess(topic.getQualifiedName(), unfiltered.body(), header("Trace-Id", "vte12")); - - // then - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> - hermes.api().getConsumersMetrics() - .expectStatus() - .isOk() - .expectBody(String.class) - .value((body) -> assertThatMetrics(body) + "group", topic.getName().getGroupName(), + "subscription", subscription.getName(), + "topic", topic.getName().getName()) + .withValue(0.0))); + + // when + hermes + .api() + .publishUntilSuccess( + topic.getQualifiedName(), unfiltered.body(), header("Trace-Id", "vte12")); + hermes + .api() + .publishUntilSuccess( + topic.getQualifiedName(), unfiltered.body(), header("Trace-Id", "vte12")); + + // then + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> + hermes + .api() + .getConsumersMetrics() + .expectStatus() + .isOk() + .expectBody(String.class) + .value( + (body) -> + assertThatMetrics(body) .contains("hermes_consumers_subscription_inflight") .withLabels( - "group", topic.getName().getGroupName(), - "subscription", subscription.getName(), - "topic", topic.getName().getName() - ) - .withValue(1.0) - ) - ); - } - - @Test - public void shouldReportMetricsForSuccessfulBatchDelivery() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - final Subscription subscription = hermes.initHelper().createSubscription( + "group", topic.getName().getGroupName(), + "subscription", subscription.getName(), + "topic", topic.getName().getName()) + .withValue(1.0))); + } + + @Test + public void shouldReportMetricsForSuccessfulBatchDelivery() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + final Subscription subscription = + hermes + .initHelper() + .createSubscription( subscription(topic, "subscription") - .withEndpoint(subscriber.getEndpoint()) - .withSubscriptionPolicy( - batchSubscriptionPolicy() - .withBatchSize(2) - .withMessageTtl(MAX_VALUE) - .withRequestTimeout(MAX_VALUE) - .withBatchTime(MAX_VALUE) - .withBatchVolume(1024) - .build() - ) - .build() - ); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), TestMessage.of("key1", "message").body()); - hermes.api().publishUntilSuccess(topic.getQualifiedName(), TestMessage.of("key2", "message").body()); - - // then - subscriber.waitUntilAnyMessageReceived(); - hermes.api().getConsumersMetrics() - .expectStatus() - .isOk() - .expectBody(String.class) - .value((body) -> { - assertThatMetrics(body) - .contains("hermes_consumers_subscription_delivered_total") - .withLabels( - "group", topic.getName().getGroupName(), - "subscription", subscription.getName(), - "topic", topic.getName().getName() - ) - .withValue(2.0); - assertThatMetrics(body) - .contains("hermes_consumers_subscription_batches_total") - .withLabels( - "group", topic.getName().getGroupName(), - "subscription", subscription.getName(), - "topic", topic.getName().getName() - ) - .withValue(1.0); - assertThatMetrics(body) - .contains("hermes_consumers_subscription_http_status_codes_total") - .withLabels( - "group", topic.getName().getGroupName(), - "status_code", "200", - "subscription", subscription.getName(), - "topic", topic.getName().getName() - ) - .withValue(1.0); - } - ); - } - - @Test - public void shouldReportMetricsForFailedBatchDelivery() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(404); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - final Subscription subscription = hermes.initHelper().createSubscription( + .withEndpoint(subscriber.getEndpoint()) + .withSubscriptionPolicy( + batchSubscriptionPolicy() + .withBatchSize(2) + .withMessageTtl(MAX_VALUE) + .withRequestTimeout(MAX_VALUE) + .withBatchTime(MAX_VALUE) + .withBatchVolume(1024) + .build()) + .build()); + + // when + hermes + .api() + .publishUntilSuccess(topic.getQualifiedName(), TestMessage.of("key1", "message").body()); + hermes + .api() + .publishUntilSuccess(topic.getQualifiedName(), TestMessage.of("key2", "message").body()); + + // then + subscriber.waitUntilAnyMessageReceived(); + hermes + .api() + .getConsumersMetrics() + .expectStatus() + .isOk() + .expectBody(String.class) + .value( + (body) -> { + assertThatMetrics(body) + .contains("hermes_consumers_subscription_delivered_total") + .withLabels( + "group", topic.getName().getGroupName(), + "subscription", subscription.getName(), + "topic", topic.getName().getName()) + .withValue(2.0); + assertThatMetrics(body) + .contains("hermes_consumers_subscription_batches_total") + .withLabels( + "group", topic.getName().getGroupName(), + "subscription", subscription.getName(), + "topic", topic.getName().getName()) + .withValue(1.0); + assertThatMetrics(body) + .contains("hermes_consumers_subscription_http_status_codes_total") + .withLabels( + "group", topic.getName().getGroupName(), + "status_code", "200", + "subscription", subscription.getName(), + "topic", topic.getName().getName()) + .withValue(1.0); + }); + } + + @Test + public void shouldReportMetricsForFailedBatchDelivery() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(404); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + final Subscription subscription = + hermes + .initHelper() + .createSubscription( subscription(topic, "subscription") - .withEndpoint(subscriber.getEndpoint()) - .withSubscriptionPolicy( - batchSubscriptionPolicy() - .withBatchSize(2) - .withMessageTtl(0) - .withRequestTimeout(MAX_VALUE) - .withBatchTime(MAX_VALUE) - .withBatchVolume(1024) - .build() - ) - .build() - ); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), TestMessage.of("key1", "message").body()); - hermes.api().publishUntilSuccess(topic.getQualifiedName(), TestMessage.of("key2", "message").body()); - - // then - subscriber.waitUntilAnyMessageReceived(); - hermes.api().getConsumersMetrics() - .expectStatus() - .isOk() - .expectBody(String.class) - .value((body) -> assertThatMetrics(body) - .contains("hermes_consumers_subscription_http_status_codes_total") - .withLabels( - "group", topic.getName().getGroupName(), - "status_code", "404", - "subscription", subscription.getName(), - "topic", topic.getName().getName() - ) - .withValue(1.0) - ); - } - - @Test - public void shouldReadTopicMetricsFromPrometheus() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - prometheus.stubTopicMetrics( - topicMetrics(topic.getName()) - .withRate(10) - .withDeliveryRate(15) - .build() - ); - - // when - WebTestClient.ResponseSpec response = hermes.api().getTopicMetrics(topic.getQualifiedName()); - - // then - response.expectStatus().is2xxSuccessful(); - TopicMetrics metrics = response.expectBody(TopicMetrics.class).returnResult().getResponseBody(); - assertThat(metrics).isNotNull(); - assertThat(metrics.getRate()).isEqualTo(MetricDecimalValue.of("10.0")); - assertThat(metrics.getDeliveryRate()).isEqualTo(MetricDecimalValue.of("15.0")); - } - - @Test - public void shouldReadSubscriptionMetricsFromPrometheus() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription( - subscriptionWithRandomName(topic.getName(), "http://endpoint2").build() - ); - prometheus.stubSubscriptionMetrics( - subscriptionMetrics(subscription.getQualifiedName()) - .withRate(15) - .build() - ); - - // when - WebTestClient.ResponseSpec response = hermes.api().getSubscriptionMetrics(topic.getQualifiedName(), subscription.getName()); - - // then - response.expectStatus().is2xxSuccessful(); - SubscriptionMetrics metrics = response.expectBody(SubscriptionMetrics.class).returnResult().getResponseBody(); - assertThat(metrics).isNotNull(); - assertThat(metrics.getRate()).isEqualTo(MetricDecimalValue.of("15.0")); - } - - private static HttpHeaders header(String key, String value) { - HttpHeaders headers = new HttpHeaders(); - headers.add(key, value); - return headers; - } - - private static MessageFilterSpecification filterMatchingHeaderByPattern(String headerName, String pattern) { - return new MessageFilterSpecification(Map.of("type", "header", "header", headerName, "matcher", pattern)); - } + .withEndpoint(subscriber.getEndpoint()) + .withSubscriptionPolicy( + batchSubscriptionPolicy() + .withBatchSize(2) + .withMessageTtl(0) + .withRequestTimeout(MAX_VALUE) + .withBatchTime(MAX_VALUE) + .withBatchVolume(1024) + .build()) + .build()); + + // when + hermes + .api() + .publishUntilSuccess(topic.getQualifiedName(), TestMessage.of("key1", "message").body()); + hermes + .api() + .publishUntilSuccess(topic.getQualifiedName(), TestMessage.of("key2", "message").body()); + + // then + subscriber.waitUntilAnyMessageReceived(); + hermes + .api() + .getConsumersMetrics() + .expectStatus() + .isOk() + .expectBody(String.class) + .value( + (body) -> + assertThatMetrics(body) + .contains("hermes_consumers_subscription_http_status_codes_total") + .withLabels( + "group", topic.getName().getGroupName(), + "status_code", "404", + "subscription", subscription.getName(), + "topic", topic.getName().getName()) + .withValue(1.0)); + } + + @Test + public void shouldReadTopicMetricsFromPrometheus() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + prometheus.stubTopicMetrics( + topicMetrics(topic.getName()).withRate(10).withDeliveryRate(15).build()); + + // when + WebTestClient.ResponseSpec response = hermes.api().getTopicMetrics(topic.getQualifiedName()); + + // then + response.expectStatus().is2xxSuccessful(); + TopicMetrics metrics = response.expectBody(TopicMetrics.class).returnResult().getResponseBody(); + assertThat(metrics).isNotNull(); + assertThat(metrics.getRate()).isEqualTo(MetricDecimalValue.of("10.0")); + assertThat(metrics.getDeliveryRate()).isEqualTo(MetricDecimalValue.of("15.0")); + } + + @Test + public void shouldReadSubscriptionMetricsFromPrometheus() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), "http://endpoint2").build()); + prometheus.stubSubscriptionMetrics( + subscriptionMetrics(subscription.getQualifiedName()).withRate(15).build()); + + // when + WebTestClient.ResponseSpec response = + hermes.api().getSubscriptionMetrics(topic.getQualifiedName(), subscription.getName()); + + // then + response.expectStatus().is2xxSuccessful(); + SubscriptionMetrics metrics = + response.expectBody(SubscriptionMetrics.class).returnResult().getResponseBody(); + assertThat(metrics).isNotNull(); + assertThat(metrics.getRate()).isEqualTo(MetricDecimalValue.of("15.0")); + } + + private static HttpHeaders header(String key, String value) { + HttpHeaders headers = new HttpHeaders(); + headers.add(key, value); + return headers; + } + + private static MessageFilterSpecification filterMatchingHeaderByPattern( + String headerName, String pattern) { + return new MessageFilterSpecification( + Map.of("type", "header", "header", headerName, "matcher", pattern)); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/OAuthIntegrationTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/OAuthIntegrationTest.java index 886cd85e45..1c7d8678ec 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/OAuthIntegrationTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/OAuthIntegrationTest.java @@ -1,5 +1,17 @@ package pl.allegro.tech.hermes.integrationtests; +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.not; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static org.apache.hc.core5.http.HttpStatus.SC_OK; +import static org.apache.hc.core5.http.HttpStatus.SC_UNAUTHORIZED; +import static pl.allegro.tech.hermes.api.SubscriptionOAuthPolicy.clientCredentialsGrantOAuthPolicy; +import static pl.allegro.tech.hermes.api.SubscriptionOAuthPolicy.passwordGrantOAuthPolicy; +import static pl.allegro.tech.hermes.test.helper.builder.OAuthProviderBuilder.oAuthProvider; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -14,170 +26,178 @@ import pl.allegro.tech.hermes.test.helper.oauth.server.OAuthResourceOwner; import pl.allegro.tech.hermes.test.helper.oauth.server.OAuthTestServer; -import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; -import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; -import static com.github.tomakehurst.wiremock.client.WireMock.not; -import static com.github.tomakehurst.wiremock.client.WireMock.post; -import static org.apache.hc.core5.http.HttpStatus.SC_OK; -import static org.apache.hc.core5.http.HttpStatus.SC_UNAUTHORIZED; -import static pl.allegro.tech.hermes.api.SubscriptionOAuthPolicy.clientCredentialsGrantOAuthPolicy; -import static pl.allegro.tech.hermes.api.SubscriptionOAuthPolicy.passwordGrantOAuthPolicy; -import static pl.allegro.tech.hermes.test.helper.builder.OAuthProviderBuilder.oAuthProvider; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class OAuthIntegrationTest { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - @RegisterExtension - public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); - - private static final OAuthTestServer oAuthTestServer = new OAuthTestServer(); - - private static final OAuthClient oAuthClient = new OAuthClient("client1", "secret1"); - - private static final OAuthResourceOwner resourceOwner = new OAuthResourceOwner("testUser1", "password1"); - - private final SubscriptionOAuthPolicy usernamePasswordOAuthPolicy = passwordGrantOAuthPolicy("provider1") - .withUsername(resourceOwner.username()) - .withPassword(resourceOwner.password()) - .build(); - - private final SubscriptionOAuthPolicy clientCredentialsOAuthPolicy = clientCredentialsGrantOAuthPolicy("provider1") - .build(); - - @BeforeAll - public static void initialize() { - oAuthTestServer.start(); - hermes.initHelper().createOAuthProvider( - oAuthProvider("provider1") - .withTokenEndpoint(oAuthTestServer.getTokenEndpoint()) - .withClientId(oAuthClient.clientId()) - .withClientSecret(oAuthClient.secret()) - .withRequestTimeout(10 * 1000) - .withTokenRequestInitialDelay(100) - .withTokenRequestMaxDelay(10000) - .build() - ); - } - - @AfterAll - public static void tearDown() { - oAuthTestServer.stop(); - } - - @BeforeEach - public void initializeAlways() { - oAuthTestServer.reset(); - } - - @Test - public void shouldSendMessageToUsernamePasswordGrantOAuthSecuredEndpoint() { - // given - String token = oAuthTestServer.stubAccessTokenForPasswordGrant(oAuthClient, resourceOwner); - TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - hermes.initHelper().createSubscription( - subscription(topic, "subscription1") - .withEndpoint(subscriber.getEndpoint()) - .withOAuthPolicy(usernamePasswordOAuthPolicy) - .build() - ); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), "hello world"); - - // then - subscriber.waitUntilMessageWithHeaderReceived("Authorization", "Bearer " + token); - } - - @Test - public void shouldRequestNewTokenIfPreviousIsInvalidForUsernamePasswordGrantOAuthSecuredEndpoint() { - // given - String invalidToken = oAuthTestServer.stubAccessTokenForPasswordGrant(oAuthClient, resourceOwner); - TestSubscriber subscriber = subscribers.createSubscriber((service, endpoint) -> { - service.addStubMapping((post(endpoint)) - .withHeader("Authorization", equalTo("Bearer " + invalidToken)) - .willReturn(aResponse().withStatus(SC_UNAUTHORIZED)).build()); - - service.addStubMapping((post(endpoint)) - .withHeader("Authorization", not(equalTo("Bearer " + invalidToken))) - .willReturn(aResponse().withStatus(SC_OK)).build()); - }); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - hermes.initHelper().createSubscription( - subscription(topic, "subscription2") - .withEndpoint(subscriber.getEndpoint()) - .withOAuthPolicy(usernamePasswordOAuthPolicy) - .build() - ); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), "hello world"); - - // then - subscriber.waitUntilMessageWithHeaderReceived("Authorization", "Bearer " + invalidToken); - - // and when - String validToken = oAuthTestServer.stubAccessTokenForPasswordGrant(oAuthClient, resourceOwner); - - // then - subscriber.waitUntilMessageWithHeaderReceived("Authorization", "Bearer " + validToken); - } - - @Test - public void shouldSendMessageToClientCredentialsGrantOAuthSecuredEndpoint() { - // given - String token = oAuthTestServer.stubAccessTokenForClientCredentials(oAuthClient); - TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - hermes.initHelper().createSubscription( - subscription(topic, "subscription3") - .withEndpoint(subscriber.getEndpoint()) - .withOAuthPolicy(clientCredentialsOAuthPolicy) - .build() - ); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), "hello world"); - - // then - subscriber.waitUntilMessageWithHeaderReceived("Authorization", "Bearer " + token); - } - - @Test - public void shouldRequestNewTokenIfPreviousIsInvalidForClientCredentialsGrantSecuredEndpoint() { - // given - String invalidToken = oAuthTestServer.stubAccessTokenForClientCredentials(oAuthClient); - TestSubscriber subscriber = subscribers.createSubscriber((service, endpoint) -> { - service.addStubMapping((post(endpoint)) - .withHeader("Authorization", equalTo("Bearer " + invalidToken)) - .willReturn(aResponse().withStatus(SC_UNAUTHORIZED)).build()); - - service.addStubMapping((post(endpoint)) - .withHeader("Authorization", not(equalTo("Bearer " + invalidToken))) - .willReturn(aResponse().withStatus(SC_OK)).build()); - }); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - hermes.initHelper().createSubscription( - subscription(topic, "subscription4") - .withEndpoint(subscriber.getEndpoint()) - .withOAuthPolicy(clientCredentialsOAuthPolicy) - .build() - ); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), "hello world"); - - // then - subscriber.waitUntilMessageWithHeaderReceived("Authorization", "Bearer " + invalidToken); - - // and when - String validToken = oAuthTestServer.stubAccessTokenForClientCredentials(oAuthClient); - - // then - subscriber.waitUntilMessageWithHeaderReceived("Authorization", "Bearer " + validToken); - } + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); + + @RegisterExtension + public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); + + private static final OAuthTestServer oAuthTestServer = new OAuthTestServer(); + + private static final OAuthClient oAuthClient = new OAuthClient("client1", "secret1"); + + private static final OAuthResourceOwner resourceOwner = + new OAuthResourceOwner("testUser1", "password1"); + + private final SubscriptionOAuthPolicy usernamePasswordOAuthPolicy = + passwordGrantOAuthPolicy("provider1") + .withUsername(resourceOwner.username()) + .withPassword(resourceOwner.password()) + .build(); + + private final SubscriptionOAuthPolicy clientCredentialsOAuthPolicy = + clientCredentialsGrantOAuthPolicy("provider1").build(); + + @BeforeAll + public static void initialize() { + oAuthTestServer.start(); + hermes + .initHelper() + .createOAuthProvider( + oAuthProvider("provider1") + .withTokenEndpoint(oAuthTestServer.getTokenEndpoint()) + .withClientId(oAuthClient.clientId()) + .withClientSecret(oAuthClient.secret()) + .withRequestTimeout(10 * 1000) + .withTokenRequestInitialDelay(100) + .withTokenRequestMaxDelay(10000) + .build()); + } + + @AfterAll + public static void tearDown() { + oAuthTestServer.stop(); + } + + @BeforeEach + public void initializeAlways() { + oAuthTestServer.reset(); + } + + @Test + public void shouldSendMessageToUsernamePasswordGrantOAuthSecuredEndpoint() { + // given + String token = oAuthTestServer.stubAccessTokenForPasswordGrant(oAuthClient, resourceOwner); + TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + hermes + .initHelper() + .createSubscription( + subscription(topic, "subscription1") + .withEndpoint(subscriber.getEndpoint()) + .withOAuthPolicy(usernamePasswordOAuthPolicy) + .build()); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), "hello world"); + + // then + subscriber.waitUntilMessageWithHeaderReceived("Authorization", "Bearer " + token); + } + + @Test + public void + shouldRequestNewTokenIfPreviousIsInvalidForUsernamePasswordGrantOAuthSecuredEndpoint() { + // given + String invalidToken = + oAuthTestServer.stubAccessTokenForPasswordGrant(oAuthClient, resourceOwner); + TestSubscriber subscriber = + subscribers.createSubscriber( + (service, endpoint) -> { + service.addStubMapping( + (post(endpoint)) + .withHeader("Authorization", equalTo("Bearer " + invalidToken)) + .willReturn(aResponse().withStatus(SC_UNAUTHORIZED)) + .build()); + + service.addStubMapping( + (post(endpoint)) + .withHeader("Authorization", not(equalTo("Bearer " + invalidToken))) + .willReturn(aResponse().withStatus(SC_OK)) + .build()); + }); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + hermes + .initHelper() + .createSubscription( + subscription(topic, "subscription2") + .withEndpoint(subscriber.getEndpoint()) + .withOAuthPolicy(usernamePasswordOAuthPolicy) + .build()); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), "hello world"); + + // then + subscriber.waitUntilMessageWithHeaderReceived("Authorization", "Bearer " + invalidToken); + + // and when + String validToken = oAuthTestServer.stubAccessTokenForPasswordGrant(oAuthClient, resourceOwner); + + // then + subscriber.waitUntilMessageWithHeaderReceived("Authorization", "Bearer " + validToken); + } + + @Test + public void shouldSendMessageToClientCredentialsGrantOAuthSecuredEndpoint() { + // given + String token = oAuthTestServer.stubAccessTokenForClientCredentials(oAuthClient); + TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + hermes + .initHelper() + .createSubscription( + subscription(topic, "subscription3") + .withEndpoint(subscriber.getEndpoint()) + .withOAuthPolicy(clientCredentialsOAuthPolicy) + .build()); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), "hello world"); + + // then + subscriber.waitUntilMessageWithHeaderReceived("Authorization", "Bearer " + token); + } + + @Test + public void shouldRequestNewTokenIfPreviousIsInvalidForClientCredentialsGrantSecuredEndpoint() { + // given + String invalidToken = oAuthTestServer.stubAccessTokenForClientCredentials(oAuthClient); + TestSubscriber subscriber = + subscribers.createSubscriber( + (service, endpoint) -> { + service.addStubMapping( + (post(endpoint)) + .withHeader("Authorization", equalTo("Bearer " + invalidToken)) + .willReturn(aResponse().withStatus(SC_UNAUTHORIZED)) + .build()); + + service.addStubMapping( + (post(endpoint)) + .withHeader("Authorization", not(equalTo("Bearer " + invalidToken))) + .willReturn(aResponse().withStatus(SC_OK)) + .build()); + }); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + hermes + .initHelper() + .createSubscription( + subscription(topic, "subscription4") + .withEndpoint(subscriber.getEndpoint()) + .withOAuthPolicy(clientCredentialsOAuthPolicy) + .build()); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), "hello world"); + + // then + subscriber.waitUntilMessageWithHeaderReceived("Authorization", "Bearer " + invalidToken); + + // and when + String validToken = oAuthTestServer.stubAccessTokenForClientCredentials(oAuthClient); + + // then + subscriber.waitUntilMessageWithHeaderReceived("Authorization", "Bearer " + validToken); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingAndConsumingTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingAndConsumingTest.java index f154ff77da..835f4dfeb9 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingAndConsumingTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingAndConsumingTest.java @@ -1,6 +1,19 @@ package pl.allegro.tech.hermes.integrationtests; +import static jakarta.ws.rs.core.Response.Status.CREATED; +import static org.awaitility.Awaitility.waitAtMost; +import static pl.allegro.tech.hermes.api.SubscriptionPolicy.Builder.subscriptionPolicy; +import static pl.allegro.tech.hermes.integrationtests.assertions.HermesAssertions.assertThat; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; +import static pl.allegro.tech.hermes.utils.Headers.createHeaders; + import jakarta.ws.rs.core.Response; +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.time.Duration; +import java.util.Map; +import java.util.UUID; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -20,398 +33,469 @@ import pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder; import pl.allegro.tech.hermes.test.helper.message.TestMessage; -import java.io.IOException; -import java.net.SocketTimeoutException; -import java.time.Duration; -import java.util.Map; -import java.util.UUID; - -import static jakarta.ws.rs.core.Response.Status.CREATED; -import static org.awaitility.Awaitility.waitAtMost; -import static pl.allegro.tech.hermes.api.SubscriptionPolicy.Builder.subscriptionPolicy; -import static pl.allegro.tech.hermes.integrationtests.assertions.HermesAssertions.assertThat; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; -import static pl.allegro.tech.hermes.utils.Headers.createHeaders; - public class PublishingAndConsumingTest { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - @RegisterExtension - public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); - - @Test - public void shouldPublishAndConsumeMessage() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - hermes.initHelper().createSubscription(subscription(topic.getQualifiedName(), "subscription1", subscriber.getEndpoint()).build()); - TestMessage message = TestMessage.of("hello", "world"); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - - // then - subscriber.waitUntilReceived(message.body()); - } + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); - @Test - public void shouldConsumeMessagesOnMultipleSubscriptions() { - // given - TestMessage message = TestMessage.of("hello", "world"); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestSubscriber subscriber1 = subscribers.createSubscriber(); - TestSubscriber subscriber2 = subscribers.createSubscriber(); - hermes.initHelper().createSubscription(subscription(topic.getQualifiedName(), "subscription1", subscriber1.getEndpoint()).build()); - hermes.initHelper().createSubscription(subscription(topic.getQualifiedName(), "subscription2", subscriber2.getEndpoint()).build()); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - - // then - subscriber1.waitUntilReceived(message.body()); - subscriber2.waitUntilReceived(message.body()); - } - - @Test - public void shouldPublishMessageToEndpointWithURIInterpolatedFromMessageBody() { - // given - TestMessage message = TestMessage.of("template", "hello"); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - - TestSubscriber subscriber = subscribers.createSubscriber("/hello/"); - String interpolatedEndpoint = subscriber.getEndpoint().replace("/hello/", "/{template}/"); - hermes.initHelper().createSubscription(subscription(topic.getQualifiedName(), "subscription", interpolatedEndpoint).build()); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - - // then - subscriber.waitUntilReceived(message.body()); - } + @RegisterExtension + public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); - @Test - public void shouldTreatMessageWithInvalidInterpolationAsUndelivered() { - // given - TestMessage message = TestMessage.of("hello", "world"); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestSubscriber subscriber = subscribers.createSubscriber("/hello/"); - - String interpolatedEndpoint = subscriber.getEndpoint().replace("/hello/", "/{template}/"); - hermes.initHelper().createSubscription( - subscription(topic.getQualifiedName(), "subscription", interpolatedEndpoint) - .withSubscriptionPolicy(SubscriptionPolicy.Builder.subscriptionPolicy().applyDefaults().withMessageTtl(1).build()) - .build()); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - - // then - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> { - long discarded = hermes.api() - .getSubscriptionMetrics(topic.getQualifiedName(), "subscription") - .expectBody(SubscriptionMetrics.class).returnResult().getResponseBody().getDiscarded(); - assertThat(discarded).isEqualTo(1); - }); - subscriber.noMessagesReceived(); - } - - @Test - public void shouldPassSubscriptionFixedHeaders() { - // given - TestMessage message = TestMessage.of("hello", "world"); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestSubscriber subscriber = subscribers.createSubscriber(); - Subscription subscription = SubscriptionBuilder.subscriptionWithRandomName(topic.getName()) - .withEndpoint(subscriber.getEndpoint()) - .withHeader("MY-HEADER", "myHeader123") - .build(); - hermes.initHelper().createSubscription(subscription); + @Test + public void shouldPublishAndConsumeMessage() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription1", subscriber.getEndpoint()) + .build()); + TestMessage message = TestMessage.of("hello", "world"); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + + // then + subscriber.waitUntilReceived(message.body()); + } + + @Test + public void shouldConsumeMessagesOnMultipleSubscriptions() { + // given + TestMessage message = TestMessage.of("hello", "world"); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestSubscriber subscriber1 = subscribers.createSubscriber(); + TestSubscriber subscriber2 = subscribers.createSubscriber(); + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription1", subscriber1.getEndpoint()) + .build()); + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription2", subscriber2.getEndpoint()) + .build()); - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + + // then + subscriber1.waitUntilReceived(message.body()); + subscriber2.waitUntilReceived(message.body()); + } + + @Test + public void shouldPublishMessageToEndpointWithURIInterpolatedFromMessageBody() { + // given + TestMessage message = TestMessage.of("template", "hello"); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + + TestSubscriber subscriber = subscribers.createSubscriber("/hello/"); + String interpolatedEndpoint = subscriber.getEndpoint().replace("/hello/", "/{template}/"); + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription", interpolatedEndpoint).build()); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + + // then + subscriber.waitUntilReceived(message.body()); + } + + @Test + public void shouldTreatMessageWithInvalidInterpolationAsUndelivered() { + // given + TestMessage message = TestMessage.of("hello", "world"); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestSubscriber subscriber = subscribers.createSubscriber("/hello/"); + + String interpolatedEndpoint = subscriber.getEndpoint().replace("/hello/", "/{template}/"); + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription", interpolatedEndpoint) + .withSubscriptionPolicy( + SubscriptionPolicy.Builder.subscriptionPolicy() + .applyDefaults() + .withMessageTtl(1) + .build()) + .build()); - // then - subscriber.waitUntilRequestReceived(request -> { - assertThat(request).hasHeaderValue("MY-HEADER", "myHeader123"); - assertThat(request.getHeader("Hermes-Message-Id")).isNotEmpty(); + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + + // then + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + long discarded = + hermes + .api() + .getSubscriptionMetrics(topic.getQualifiedName(), "subscription") + .expectBody(SubscriptionMetrics.class) + .returnResult() + .getResponseBody() + .getDiscarded(); + assertThat(discarded).isEqualTo(1); + }); + subscriber.noMessagesReceived(); + } + + @Test + public void shouldPassSubscriptionFixedHeaders() { + // given + TestMessage message = TestMessage.of("hello", "world"); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestSubscriber subscriber = subscribers.createSubscriber(); + Subscription subscription = + SubscriptionBuilder.subscriptionWithRandomName(topic.getName()) + .withEndpoint(subscriber.getEndpoint()) + .withHeader("MY-HEADER", "myHeader123") + .build(); + hermes.initHelper().createSubscription(subscription); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + + // then + subscriber.waitUntilRequestReceived( + request -> { + assertThat(request).hasHeaderValue("MY-HEADER", "myHeader123"); + assertThat(request.getHeader("Hermes-Message-Id")).isNotEmpty(); }); - } - - @Test - public void shouldRetryWithDelayOnRetryAfterEndpointResponse() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestMessage message = TestMessage.of("hello", "world"); - int retryAfterSeconds = 1; - TestSubscriber subscriber = subscribers.createSubscriberWithRetry(message.body(), retryAfterSeconds); - hermes.initHelper().createSubscription(subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()).build()); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - - // then - subscriber.waitUntilMessageWithHeaderReceived("Hermes-Retry-Count", "0"); - subscriber.waitUntilMessageWithHeaderReceived("Hermes-Retry-Count", "1"); - assertThat(subscriber.durationBetweenFirstAndLastRequest()).isGreaterThanOrEqualTo(Duration.ofSeconds(retryAfterSeconds)); - } - - @Test - public void shouldNotPublishMessageIfContentLengthDoNotMatch() throws IOException, InterruptedException { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestSubscriber subscriber = subscribers.createSubscriber(); - - // when - publishEventWithInvalidContentLength(topic.getQualifiedName()); - - // then - subscriber.noMessagesReceived(); - } + } + + @Test + public void shouldRetryWithDelayOnRetryAfterEndpointResponse() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestMessage message = TestMessage.of("hello", "world"); + int retryAfterSeconds = 1; + TestSubscriber subscriber = + subscribers.createSubscriberWithRetry(message.body(), retryAfterSeconds); + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) + .build()); - @Test - public void shouldConsumeMessageWithKeepAliveHeader() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestSubscriber subscriber = subscribers.createSubscriber(); - hermes.initHelper().createSubscription(subscription(topic.getQualifiedName(), "subscription1", subscriber.getEndpoint()).build()); + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + + // then + subscriber.waitUntilMessageWithHeaderReceived("Hermes-Retry-Count", "0"); + subscriber.waitUntilMessageWithHeaderReceived("Hermes-Retry-Count", "1"); + assertThat(subscriber.durationBetweenFirstAndLastRequest()) + .isGreaterThanOrEqualTo(Duration.ofSeconds(retryAfterSeconds)); + } + + @Test + public void shouldNotPublishMessageIfContentLengthDoNotMatch() + throws IOException, InterruptedException { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestSubscriber subscriber = subscribers.createSubscriber(); + + // when + publishEventWithInvalidContentLength(topic.getQualifiedName()); + + // then + subscriber.noMessagesReceived(); + } + + @Test + public void shouldConsumeMessageWithKeepAliveHeader() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestSubscriber subscriber = subscribers.createSubscriber(); + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription1", subscriber.getEndpoint()) + .build()); - TestMessage message = TestMessage.of("hello", "world"); + TestMessage message = TestMessage.of("hello", "world"); - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - // then - subscriber.waitUntilRequestReceived(request -> { - assertThat(request.getBodyAsString()).isEqualTo(message.body()); - assertThat(request).hasHeaderValue("Keep-Alive", "true"); + // then + subscriber.waitUntilRequestReceived( + request -> { + assertThat(request.getBodyAsString()).isEqualTo(message.body()); + assertThat(request).hasHeaderValue("Keep-Alive", "true"); }); - } - - @Test - public void shouldMarkSubscriptionAsActiveAfterReceivingFirstMessage() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestSubscriber subscriber = subscribers.createSubscriber(); - hermes.initHelper().createSubscription(subscription(topic.getQualifiedName(), "subscription1", subscriber.getEndpoint()).build()); - - TestMessage message = TestMessage.of("hello", "world"); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + } + + @Test + public void shouldMarkSubscriptionAsActiveAfterReceivingFirstMessage() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestSubscriber subscriber = subscribers.createSubscriber(); + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription1", subscriber.getEndpoint()) + .build()); - // then - subscriber.waitUntilReceived(message.body()); + TestMessage message = TestMessage.of("hello", "world"); - assertThat(hermes.api().getSubscription(topic.getQualifiedName(), "subscription1").getState()).isEqualTo(Subscription.State.ACTIVE); - } + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - @Test - public void shouldNotConsumeMessagesWhenSubscriptionIsSuspended() { - // given - String subscriptionName = "subscription1"; - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestSubscriber subscriber = subscribers.createSubscriber(); + // then + subscriber.waitUntilReceived(message.body()); - hermes.initHelper().createSubscription(subscription(topic.getQualifiedName(), subscriptionName, subscriber.getEndpoint()).build()); + assertThat(hermes.api().getSubscription(topic.getQualifiedName(), "subscription1").getState()) + .isEqualTo(Subscription.State.ACTIVE); + } - // when - hermes.api().waitUntilSubscriptionActivated(topic.getQualifiedName(), subscriptionName); - hermes.api().suspendSubscription(topic, subscriptionName); - hermes.api().waitUntilSubscriptionSuspended(topic.getQualifiedName(), subscriptionName); - hermes.api().publishUntilSuccess(topic.getQualifiedName(), TestMessage.of("hello", "world").body()); + @Test + public void shouldNotConsumeMessagesWhenSubscriptionIsSuspended() { + // given + String subscriptionName = "subscription1"; + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestSubscriber subscriber = subscribers.createSubscriber(); - // then - subscriber.noMessagesReceived(); - } - - @Test - public void shouldNotCreateTopicWhenPublishingToNonExistingTopic() { - // given - TopicName nonExisting = TopicName.fromQualifiedName("nonExistingGroup.nonExistingTopic8326"); + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), subscriptionName, subscriber.getEndpoint()) + .build()); - // when - WebTestClient.ResponseSpec responseForNonExisting = hermes.api().publish(nonExisting.qualifiedName(), TestMessage.simple().body()); + // when + hermes.api().waitUntilSubscriptionActivated(topic.getQualifiedName(), subscriptionName); + hermes.api().suspendSubscription(topic, subscriptionName); + hermes.api().waitUntilSubscriptionSuspended(topic.getQualifiedName(), subscriptionName); + hermes + .api() + .publishUntilSuccess(topic.getQualifiedName(), TestMessage.of("hello", "world").body()); + + // then + subscriber.noMessagesReceived(); + } + + @Test + public void shouldNotCreateTopicWhenPublishingToNonExistingTopic() { + // given + TopicName nonExisting = TopicName.fromQualifiedName("nonExistingGroup.nonExistingTopic8326"); + + // when + WebTestClient.ResponseSpec responseForNonExisting = + hermes.api().publish(nonExisting.qualifiedName(), TestMessage.simple().body()); + + // then + responseForNonExisting.expectStatus().isNotFound(); + hermes.api().getTopicResponse(nonExisting.qualifiedName()).expectStatus().isNotFound(); + } + + @Test + public void shouldNotOverrideHeadersAddedByMetadataAppendersWithSubscriptionHeaders() { + // given + TestMessage message = TestMessage.of("hello", "world"); + String subscriptionName = "subscription"; + + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestSubscriber subscriber = subscribers.createSubscriber(); + + hermes + .initHelper() + .createSubscription( + SubscriptionBuilder.subscription(topic, subscriptionName) + .withEndpoint(subscriber.getEndpoint()) + .withHeader("Trace-Id", "defaultValue") + .build()); - // then - responseForNonExisting.expectStatus().isNotFound(); - hermes.api().getTopicResponse(nonExisting.qualifiedName()).expectStatus().isNotFound(); - } + // when + hermes + .api() + .publishUntilSuccess( + topic.getQualifiedName(), + message.body(), + createHeaders(Map.of("Trace-Id", "valueFromRequest"))); + + // then + subscriber.waitUntilRequestReceived( + request -> { + assertThat(request.getBodyAsString()).isEqualTo(message.body()); + assertThat(request.getHeader("Trace-Id")).isEqualTo("valueFromRequest"); + }); + } - @Test - public void shouldNotOverrideHeadersAddedByMetadataAppendersWithSubscriptionHeaders() { - // given - TestMessage message = TestMessage.of("hello", "world"); - String subscriptionName = "subscription"; + @Test + public void shouldAttachSubscriptionIdentityHeadersWhenItIsEnabled() { + // given + TestMessage message = TestMessage.of("hello", "world"); + String subscriptionName = "subscription"; - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestSubscriber subscriber = subscribers.createSubscriber(); - hermes.initHelper().createSubscription( - SubscriptionBuilder.subscription(topic, subscriptionName) - .withEndpoint(subscriber.getEndpoint()) - .withHeader("Trace-Id", "defaultValue") - .build()); + hermes + .initHelper() + .createSubscription( + SubscriptionBuilder.subscription(topic, subscriptionName) + .withEndpoint(subscriber.getEndpoint()) + .withAttachingIdentityHeadersEnabled(true) + .build()); - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body(), createHeaders(Map.of("Trace-Id", "valueFromRequest"))); + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - // then - subscriber.waitUntilRequestReceived(request -> { - assertThat(request.getBodyAsString()).isEqualTo(message.body()); - assertThat(request.getHeader("Trace-Id")).isEqualTo("valueFromRequest"); + // then + subscriber.waitUntilRequestReceived( + request -> { + assertThat(request.getBodyAsString()).isEqualTo(message.body()); + assertThat(request.getHeader("Hermes-Topic-Name")).isEqualTo(topic.getQualifiedName()); + assertThat(request.getHeader("Hermes-Subscription-Name")).isEqualTo(subscriptionName); }); - } + } + + @Test + public void shouldNotAttachSubscriptionIdentityHeadersWhenItIsDisabled() { + // given + TestMessage message = TestMessage.of("hello", "world"); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestSubscriber subscriber = subscribers.createSubscriber(); + + hermes + .initHelper() + .createSubscription( + SubscriptionBuilder.subscription(topic, "subscription") + .withEndpoint(subscriber.getEndpoint()) + .withAttachingIdentityHeadersEnabled(false) + .build()); - @Test - public void shouldAttachSubscriptionIdentityHeadersWhenItIsEnabled() { - // given - TestMessage message = TestMessage.of("hello", "world"); - String subscriptionName = "subscription"; - - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestSubscriber subscriber = subscribers.createSubscriber(); - - hermes.initHelper().createSubscription( - SubscriptionBuilder.subscription(topic, subscriptionName) - .withEndpoint(subscriber.getEndpoint()) - .withAttachingIdentityHeadersEnabled(true) - .build()); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - - // then - subscriber.waitUntilRequestReceived(request -> { - assertThat(request.getBodyAsString()).isEqualTo(message.body()); - assertThat(request.getHeader("Hermes-Topic-Name")).isEqualTo(topic.getQualifiedName()); - assertThat(request.getHeader("Hermes-Subscription-Name")).isEqualTo(subscriptionName); - }); - } + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - @Test - public void shouldNotAttachSubscriptionIdentityHeadersWhenItIsDisabled() { - // given - TestMessage message = TestMessage.of("hello", "world"); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestSubscriber subscriber = subscribers.createSubscriber(); - - hermes.initHelper().createSubscription( - SubscriptionBuilder.subscription(topic, "subscription") - .withEndpoint(subscriber.getEndpoint()) - .withAttachingIdentityHeadersEnabled(false) - .build()); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - - // then - subscriber.waitUntilRequestReceived(request -> { - assertThat(request.getBodyAsString()).isEqualTo(message.body()); - Assertions.assertThat(request.getHeader("Hermes-Topic-Name")).isNull(); - Assertions.assertThat(request.getHeader("Hermes-Subscription-Name")).isNull(); + // then + subscriber.waitUntilRequestReceived( + request -> { + assertThat(request.getBodyAsString()).isEqualTo(message.body()); + Assertions.assertThat(request.getHeader("Hermes-Topic-Name")).isNull(); + Assertions.assertThat(request.getHeader("Hermes-Subscription-Name")).isNull(); }); - } - - @Test - public void shouldPublishAndConsumeMessageWithTraceId() { - // given - TestMessage message = TestMessage.of("hello", "world"); - String traceId = UUID.randomUUID().toString(); - - // and - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestSubscriber subscriber = subscribers.createSubscriber(); - hermes.initHelper().createSubscription(subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()).build()); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body(), createHeaders(Map.of("Trace-Id", traceId))); + } + + @Test + public void shouldPublishAndConsumeMessageWithTraceId() { + // given + TestMessage message = TestMessage.of("hello", "world"); + String traceId = UUID.randomUUID().toString(); + + // and + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestSubscriber subscriber = subscribers.createSubscriber(); + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) + .build()); - // then - subscriber.waitUntilRequestReceived(request -> { - assertThat(request.getBodyAsString()).isEqualTo(message.body()); - assertThat(request).hasHeaderValue("Trace-Id", traceId); + // when + hermes + .api() + .publishUntilSuccess( + topic.getQualifiedName(), message.body(), createHeaders(Map.of("Trace-Id", traceId))); + + // then + subscriber.waitUntilRequestReceived( + request -> { + assertThat(request.getBodyAsString()).isEqualTo(message.body()); + assertThat(request).hasHeaderValue("Trace-Id", traceId); }); - } - - @Test - public void shouldPublishAndConsumeMessageWithTraceAndSpanHeaders() { - // given - TestMessage message = TestMessage.of("hello", "world"); - TraceContext trace = TraceContext.random(); - - // and - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestSubscriber subscriber = subscribers.createSubscriber(); - hermes.initHelper().createSubscription(subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()).build()); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body(), createHeaders(TraceHeaders.fromTraceContext(trace))); + } + + @Test + public void shouldPublishAndConsumeMessageWithTraceAndSpanHeaders() { + // given + TestMessage message = TestMessage.of("hello", "world"); + TraceContext trace = TraceContext.random(); + + // and + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestSubscriber subscriber = subscribers.createSubscriber(); + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) + .build()); - // then - subscriber.waitUntilRequestReceived(request -> { - assertThat(request.getBodyAsString()).isEqualTo(message.body()); - assertThat(request).containsAllHeaders(trace.asMap()); + // when + hermes + .api() + .publishUntilSuccess( + topic.getQualifiedName(), + message.body(), + createHeaders(TraceHeaders.fromTraceContext(trace))); + + // then + subscriber.waitUntilRequestReceived( + request -> { + assertThat(request.getBodyAsString()).isEqualTo(message.body()); + assertThat(request).containsAllHeaders(trace.asMap()); }); - } - - @Test - public void shouldPublishWithDelayAndConsumeMessage() { - // given - int delay = 2000; - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestSubscriber subscriber = subscribers.createSubscriber(); - - hermes.initHelper().createSubscription( - SubscriptionBuilder.subscription(topic, "subscription") + } + + @Test + public void shouldPublishWithDelayAndConsumeMessage() { + // given + int delay = 2000; + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestSubscriber subscriber = subscribers.createSubscriber(); + + hermes + .initHelper() + .createSubscription( + SubscriptionBuilder.subscription(topic, "subscription") .withEndpoint(subscriber.getEndpoint()) .withContentType(ContentType.JSON) - .withSubscriptionPolicy(subscriptionPolicy().applyDefaults() - .withSendingDelay(delay) - .build()) + .withSubscriptionPolicy( + subscriptionPolicy().applyDefaults().withSendingDelay(delay).build()) .withMode(SubscriptionMode.ANYCAST) .build()); - - TestMessage message = TestMessage.of("hello", "world"); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - long publishedTime = System.currentTimeMillis(); - - // then - subscriber.waitUntilReceived(message.body()); - long receivedTime = System.currentTimeMillis(); - assertThat(receivedTime - publishedTime).isGreaterThanOrEqualTo(delay); - } - - @Test - public void shouldPublishMessageUsingChunkedEncoding() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - - // when - Response response = hermes.api().publishChunked(topic.getQualifiedName(), "{}"); - - // then - assertThat(response).hasStatus(CREATED); + TestMessage message = TestMessage.of("hello", "world"); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + long publishedTime = System.currentTimeMillis(); + + // then + subscriber.waitUntilReceived(message.body()); + long receivedTime = System.currentTimeMillis(); + assertThat(receivedTime - publishedTime).isGreaterThanOrEqualTo(delay); + } + + @Test + public void shouldPublishMessageUsingChunkedEncoding() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + + // when + Response response = hermes.api().publishChunked(topic.getQualifiedName(), "{}"); + + // then + assertThat(response).hasStatus(CREATED); + } + + /* + The only way we managed to bypass in-built server side Content-Length validation + is to close the connection before entire message is delivered. Unfortunately we + cannot verify response status code because connection is already closed. We tried + manually sending different Content-Length than the actual message but servlet + container is smart enough to not invoke ReadListener.onAllDataRead() in that case. + */ + private void publishEventWithInvalidContentLength(String topic) + throws IOException, InterruptedException { + try { + hermes.api().publishSlowly(500, 100, 0, topic); + } catch (SocketTimeoutException e) { + // this is expected } - - /* - The only way we managed to bypass in-built server side Content-Length validation - is to close the connection before entire message is delivered. Unfortunately we - cannot verify response status code because connection is already closed. We tried - manually sending different Content-Length than the actual message but servlet - container is smart enough to not invoke ReadListener.onAllDataRead() in that case. - */ - private void publishEventWithInvalidContentLength(String topic) throws IOException, InterruptedException { - try { - hermes.api().publishSlowly(500, 100, 0, topic); - } catch (SocketTimeoutException e) { - // this is expected - } - } - + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingAuthenticationTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingAuthenticationTest.java index fb6ec99094..1ec391f295 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingAuthenticationTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingAuthenticationTest.java @@ -1,5 +1,17 @@ package pl.allegro.tech.hermes.integrationtests; +import static org.awaitility.Awaitility.waitAtMost; +import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.AUTH_PASSWORD; +import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.AUTH_USERNAME; +import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_AUTHENTICATION_ENABLED; +import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_AUTHENTICATION_MODE; +import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_SSL_ENABLED; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Base64; +import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.springframework.http.HttpHeaders; @@ -9,89 +21,90 @@ import pl.allegro.tech.hermes.test.helper.message.TestMessage; import pl.allegro.tech.hermes.utils.Headers; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.Base64; -import java.util.Map; - -import static org.awaitility.Awaitility.waitAtMost; -import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.AUTH_PASSWORD; -import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.AUTH_USERNAME; -import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_AUTHENTICATION_ENABLED; -import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_AUTHENTICATION_MODE; -import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_SSL_ENABLED; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class PublishingAuthenticationTest { - private static final String USERNAME = "someUser"; - private static final String PASSWORD = "somePassword123"; - private static final String MESSAGE = TestMessage.of("hello", "world").body(); - - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension() - .withFrontendProfile("authRequired") - .withFrontendProperty(AUTH_USERNAME, USERNAME) - .withFrontendProperty(AUTH_PASSWORD, PASSWORD) - .withFrontendProperty(FRONTEND_SSL_ENABLED, false) - .withFrontendProperty(FRONTEND_AUTHENTICATION_MODE, "pro_active") - .withFrontendProperty(FRONTEND_AUTHENTICATION_ENABLED, true); - - @Test - public void shouldAuthenticateUsingBasicAuth() { - //given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> { - // when - WebTestClient.ResponseSpec response = hermes.api().publish( - topic.getQualifiedName(), - MESSAGE, - createAuthorizationHeader(USERNAME, PASSWORD) - ); - - // then - response.expectStatus().isCreated(); - }); - } - - @Test - public void shouldNotAuthenticateUserWithInvalidCredentials() { - //given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> { - // when - WebTestClient.ResponseSpec response = hermes.api().publish( - topic.getQualifiedName(), - MESSAGE, - createAuthorizationHeader(USERNAME, "someInvalidPassword") - ); - - // then - response.expectStatus().isUnauthorized(); - }); - } - - @Test - public void shouldNotAuthenticateUserWithoutCredentials() { - //given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> { - // when - WebTestClient.ResponseSpec response = hermes.api().publish(topic.getQualifiedName(), MESSAGE); - - // then - response.expectStatus().isUnauthorized(); - }); - } - - private static HttpHeaders createAuthorizationHeader(String username, String password) { - String credentials = username + ":" + password; - Map headers = Map.of( - "Authorization", "Basic " + Base64.getEncoder().encodeToString(credentials.getBytes(StandardCharsets.UTF_8)) - ); - return Headers.createHeaders(headers); - } + private static final String USERNAME = "someUser"; + private static final String PASSWORD = "somePassword123"; + private static final String MESSAGE = TestMessage.of("hello", "world").body(); + + @RegisterExtension + public static final HermesExtension hermes = + new HermesExtension() + .withFrontendProfile("authRequired") + .withFrontendProperty(AUTH_USERNAME, USERNAME) + .withFrontendProperty(AUTH_PASSWORD, PASSWORD) + .withFrontendProperty(FRONTEND_SSL_ENABLED, false) + .withFrontendProperty(FRONTEND_AUTHENTICATION_MODE, "pro_active") + .withFrontendProperty(FRONTEND_AUTHENTICATION_ENABLED, true); + + @Test + public void shouldAuthenticateUsingBasicAuth() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + // when + WebTestClient.ResponseSpec response = + hermes + .api() + .publish( + topic.getQualifiedName(), + MESSAGE, + createAuthorizationHeader(USERNAME, PASSWORD)); + + // then + response.expectStatus().isCreated(); + }); + } + + @Test + public void shouldNotAuthenticateUserWithInvalidCredentials() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + // when + WebTestClient.ResponseSpec response = + hermes + .api() + .publish( + topic.getQualifiedName(), + MESSAGE, + createAuthorizationHeader(USERNAME, "someInvalidPassword")); + + // then + response.expectStatus().isUnauthorized(); + }); + } + + @Test + public void shouldNotAuthenticateUserWithoutCredentials() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + // when + WebTestClient.ResponseSpec response = + hermes.api().publish(topic.getQualifiedName(), MESSAGE); + + // then + response.expectStatus().isUnauthorized(); + }); + } + + private static HttpHeaders createAuthorizationHeader(String username, String password) { + String credentials = username + ":" + password; + Map headers = + Map.of( + "Authorization", + "Basic " + + Base64.getEncoder().encodeToString(credentials.getBytes(StandardCharsets.UTF_8))); + return Headers.createHeaders(headers); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingAvroTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingAvroTest.java index 706572a6f4..9dd6e576f3 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingAvroTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingAvroTest.java @@ -1,5 +1,27 @@ package pl.allegro.tech.hermes.integrationtests; +import static jakarta.ws.rs.core.MediaType.TEXT_PLAIN; +import static java.util.Collections.singletonMap; +import static net.javacrumbs.jsonunit.fluent.JsonFluentAssert.assertThatJson; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.waitAtMost; +import static pl.allegro.tech.hermes.api.AvroMediaType.AVRO_JSON; +import static pl.allegro.tech.hermes.api.ContentType.AVRO; +import static pl.allegro.tech.hermes.api.ContentType.JSON; +import static pl.allegro.tech.hermes.api.PatchData.patchData; +import static pl.allegro.tech.hermes.api.TopicWithSchema.topicWithSchema; +import static pl.allegro.tech.hermes.client.HermesMessage.hermesMessage; +import static pl.allegro.tech.hermes.consumers.supervisor.process.Signal.SignalType.UPDATE_SUBSCRIPTION; +import static pl.allegro.tech.hermes.test.helper.avro.AvroUserSchemaLoader.load; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; +import static pl.allegro.tech.hermes.test.helper.endpoint.TimeoutAdjuster.adjust; +import static pl.allegro.tech.hermes.utils.Headers.createHeaders; + +import java.time.Clock; +import java.time.Duration; +import java.util.Map; +import java.util.UUID; import net.javacrumbs.jsonunit.core.Option; import org.apache.avro.Schema; import org.junit.jupiter.api.Test; @@ -20,535 +42,641 @@ import pl.allegro.tech.hermes.test.helper.avro.AvroUserSchemaLoader; import pl.allegro.tech.hermes.test.helper.message.TestMessage; -import java.time.Clock; -import java.time.Duration; -import java.util.Map; -import java.util.UUID; - -import static jakarta.ws.rs.core.MediaType.TEXT_PLAIN; -import static java.util.Collections.singletonMap; -import static net.javacrumbs.jsonunit.fluent.JsonFluentAssert.assertThatJson; -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.waitAtMost; -import static pl.allegro.tech.hermes.api.AvroMediaType.AVRO_JSON; -import static pl.allegro.tech.hermes.api.ContentType.AVRO; -import static pl.allegro.tech.hermes.api.ContentType.JSON; -import static pl.allegro.tech.hermes.api.PatchData.patchData; -import static pl.allegro.tech.hermes.api.TopicWithSchema.topicWithSchema; -import static pl.allegro.tech.hermes.client.HermesMessage.hermesMessage; -import static pl.allegro.tech.hermes.consumers.supervisor.process.Signal.SignalType.UPDATE_SUBSCRIPTION; -import static pl.allegro.tech.hermes.test.helper.avro.AvroUserSchemaLoader.load; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; -import static pl.allegro.tech.hermes.test.helper.endpoint.TimeoutAdjuster.adjust; -import static pl.allegro.tech.hermes.utils.Headers.createHeaders; - public class PublishingAvroTest { - private static final Logger logger = LoggerFactory.getLogger(PublishingAvroTest.class); - - private final Clock clock = Clock.systemDefaultZone(); - - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - @RegisterExtension - public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); - - private static final AvroUser user = new AvroUser("Bob", 50, "blue"); - - - @Test - public void shouldPublishAvroAndConsumeJsonMessage() { - // given - TopicWithSchema topicWithSchema = topicWithSchema(topicWithRandomName() - .withContentType(AVRO) - .build(), user.getSchemaAsString()); - Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); - - TestSubscriber subscriber = subscribers.createSubscriber(); - - hermes.initHelper().createSubscription( - subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()).build() - ); - - // when - hermes.api().publishAvroUntilSuccess(topic.getQualifiedName(), user.asBytes()); - - // then - subscriber.waitUntilReceived(user.asJson()); - } - - @Test - public void shouldNotPublishAvroWhenMessageIsNotJsonOrAvro() { - // given - TopicWithSchema topicWithSchema = topicWithSchema(topicWithRandomName() - .withContentType(AVRO) - .build(), user.getSchemaAsString()); - Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); - - TestSubscriber subscriber = subscribers.createSubscriber(); + private static final Logger logger = LoggerFactory.getLogger(PublishingAvroTest.class); - hermes.initHelper().createSubscription( - subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()).build() - ); + private final Clock clock = Clock.systemDefaultZone(); - // when - WebTestClient.ResponseSpec response = hermes.api().publish(topic.getQualifiedName(), user.asJson(), createHeaders(Map.of("Content-Type", TEXT_PLAIN))); + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); - // then - response.expectStatus().isBadRequest(); - } + @RegisterExtension + public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); - @Test - public void shouldPublishAvroAndConsumeAvroMessage() { - // given - TopicWithSchema topicWithSchema = topicWithSchema(topicWithRandomName() - .withContentType(AVRO) - .build(), user.getSchemaAsString()); - Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); - - TestSubscriber subscriber = subscribers.createSubscriber(); + private static final AvroUser user = new AvroUser("Bob", 50, "blue"); - hermes.initHelper().createSubscription( - subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) - .withContentType(AVRO) - .build() - ); + @Test + public void shouldPublishAvroAndConsumeJsonMessage() { + // given + TopicWithSchema topicWithSchema = + topicWithSchema( + topicWithRandomName().withContentType(AVRO).build(), user.getSchemaAsString()); + Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); - // when - hermes.api().publishAvroUntilSuccess(topic.getQualifiedName(), user.asBytes()); + TestSubscriber subscriber = subscribers.createSubscriber(); - // then - subscriber.waitUntilRequestReceived(request -> assertBodyDeserializesIntoUser(request.getBodyAsString(), user)); - } - - @Test - public void shouldSendAvroAfterSubscriptionContentTypeChanged() { - // given avro topic - TopicWithSchema topicWithSchema = topicWithSchema(topicWithRandomName() - .withContentType(AVRO) - .build(), user.getSchemaAsString()); - Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); - - TestSubscriber subscriber = subscribers.createSubscriber(); - - // and subscription with json content type - hermes.initHelper().createSubscription( - subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) - .withContentType(JSON) - .build() - ); - - // when first message is published - hermes.api().publishAvroUntilSuccess(topic.getQualifiedName(), user.asBytes()); - - // then it is consumed as json - subscriber.waitUntilRequestReceived(request -> - assertThatJson(request.getBodyAsString()).when(Option.IGNORING_EXTRA_FIELDS).isEqualTo(user.asJson())); - subscriber.reset(); - - //when subscription content type is changed to avro - hermes.api().updateSubscription(topic, "subscription", patchData().set("contentType", ContentType.AVRO).build()); - long currentTime = clock.millis(); - waitUntilSubscriptionContentTypeChanged(topic, "subscription", ContentType.AVRO); - waitUntilConsumersUpdateSubscription(currentTime, topic, "subscription"); - - // and second message is published - hermes.api().publishAvroUntilSuccess(topic.getQualifiedName(), user.asBytes()); - - // then it is consumed as avro - subscriber.waitUntilRequestReceived(request -> assertBodyDeserializesIntoUser(request.getBodyAsString(), user)); - } - - @Test - public void shouldGetBadRequestForPublishingInvalidMessageWithSchema() { - // given - TopicWithSchema topicWithSchema = topicWithSchema(topicWithRandomName() - .withContentType(AVRO) - .build(), user.getSchemaAsString()); - Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) + .build()); - // when - WebTestClient.ResponseSpec response = hermes.api().publishAvro(topic.getQualifiedName(), "invalidMessage".getBytes()); + // when + hermes.api().publishAvroUntilSuccess(topic.getQualifiedName(), user.asBytes()); - // then - response.expectStatus().isBadRequest(); - } + // then + subscriber.waitUntilReceived(user.asJson()); + } - @Test - public void shouldIgnoreValidationDryRunSettingForAvroTopic() { - // given - TopicWithSchema topicWithSchema = topicWithSchema(topicWithRandomName() - .withContentType(AVRO) - .withJsonToAvroDryRun(true) - .build(), user.getSchemaAsString()); - Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); + @Test + public void shouldNotPublishAvroWhenMessageIsNotJsonOrAvro() { + // given + TopicWithSchema topicWithSchema = + topicWithSchema( + topicWithRandomName().withContentType(AVRO).build(), user.getSchemaAsString()); + Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); - // when - WebTestClient.ResponseSpec response = hermes.api().publishAvro(topic.getQualifiedName(), "invalidMessage".getBytes()); + TestSubscriber subscriber = subscribers.createSubscriber(); - // then - response.expectStatus().isBadRequest(); - } + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) + .build()); - @Test - public void shouldPublishJsonMessageConvertedToAvroForAvroTopics() { - // given - TopicWithSchema topicWithSchema = topicWithSchema(topicWithRandomName() + // when + WebTestClient.ResponseSpec response = + hermes + .api() + .publish( + topic.getQualifiedName(), + user.asJson(), + createHeaders(Map.of("Content-Type", TEXT_PLAIN))); + + // then + response.expectStatus().isBadRequest(); + } + + @Test + public void shouldPublishAvroAndConsumeAvroMessage() { + // given + TopicWithSchema topicWithSchema = + topicWithSchema( + topicWithRandomName().withContentType(AVRO).build(), user.getSchemaAsString()); + Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); + + TestSubscriber subscriber = subscribers.createSubscriber(); + + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) .withContentType(AVRO) - .build(), user.getSchemaAsString()); - Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); - - // when & then - hermes.api().publishJSONUntilSuccess(topic.getQualifiedName(), user.asJson()); - } + .build()); - @Test - public void shouldPublishAvroEncodedJsonMessageConvertedToAvroForAvroTopics() { - // given - TopicWithSchema topicWithSchema = topicWithSchema(topicWithRandomName() - .withContentType(AVRO) - .build(), user.getSchemaAsString()); - Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); + // when + hermes.api().publishAvroUntilSuccess(topic.getQualifiedName(), user.asBytes()); + + // then + subscriber.waitUntilRequestReceived( + request -> assertBodyDeserializesIntoUser(request.getBodyAsString(), user)); + } + + @Test + public void shouldSendAvroAfterSubscriptionContentTypeChanged() { + // given avro topic + TopicWithSchema topicWithSchema = + topicWithSchema( + topicWithRandomName().withContentType(AVRO).build(), user.getSchemaAsString()); + Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); + + TestSubscriber subscriber = subscribers.createSubscriber(); + + // and subscription with json content type + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) + .withContentType(JSON) + .build()); - // when & then - hermes.api().publishUntilSuccess(topic.getQualifiedName(), user.asAvroEncodedJson(), createHeaders(singletonMap("Content-Type", AVRO_JSON))); - } + // when first message is published + hermes.api().publishAvroUntilSuccess(topic.getQualifiedName(), user.asBytes()); + + // then it is consumed as json + subscriber.waitUntilRequestReceived( + request -> + assertThatJson(request.getBodyAsString()) + .when(Option.IGNORING_EXTRA_FIELDS) + .isEqualTo(user.asJson())); + subscriber.reset(); + + // when subscription content type is changed to avro + hermes + .api() + .updateSubscription( + topic, "subscription", patchData().set("contentType", ContentType.AVRO).build()); + long currentTime = clock.millis(); + waitUntilSubscriptionContentTypeChanged(topic, "subscription", ContentType.AVRO); + waitUntilConsumersUpdateSubscription(currentTime, topic, "subscription"); + + // and second message is published + hermes.api().publishAvroUntilSuccess(topic.getQualifiedName(), user.asBytes()); + + // then it is consumed as avro + subscriber.waitUntilRequestReceived( + request -> assertBodyDeserializesIntoUser(request.getBodyAsString(), user)); + } + + @Test + public void shouldGetBadRequestForPublishingInvalidMessageWithSchema() { + // given + TopicWithSchema topicWithSchema = + topicWithSchema( + topicWithRandomName().withContentType(AVRO).build(), user.getSchemaAsString()); + Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); + + // when + WebTestClient.ResponseSpec response = + hermes.api().publishAvro(topic.getQualifiedName(), "invalidMessage".getBytes()); + + // then + response.expectStatus().isBadRequest(); + } + + @Test + public void shouldIgnoreValidationDryRunSettingForAvroTopic() { + // given + TopicWithSchema topicWithSchema = + topicWithSchema( + topicWithRandomName().withContentType(AVRO).withJsonToAvroDryRun(true).build(), + user.getSchemaAsString()); + Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); + + // when + WebTestClient.ResponseSpec response = + hermes.api().publishAvro(topic.getQualifiedName(), "invalidMessage".getBytes()); + + // then + response.expectStatus().isBadRequest(); + } + + @Test + public void shouldPublishJsonMessageConvertedToAvroForAvroTopics() { + // given + TopicWithSchema topicWithSchema = + topicWithSchema( + topicWithRandomName().withContentType(AVRO).build(), user.getSchemaAsString()); + Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); + + // when & then + hermes.api().publishJSONUntilSuccess(topic.getQualifiedName(), user.asJson()); + } + + @Test + public void shouldPublishAvroEncodedJsonMessageConvertedToAvroForAvroTopics() { + // given + TopicWithSchema topicWithSchema = + topicWithSchema( + topicWithRandomName().withContentType(AVRO).build(), user.getSchemaAsString()); + Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); + + // when & then + hermes + .api() + .publishUntilSuccess( + topic.getQualifiedName(), + user.asAvroEncodedJson(), + createHeaders(singletonMap("Content-Type", AVRO_JSON))); + } + + @Test + public void shouldGetBadRequestForJsonNotMatchingWithAvroSchemaAndAvroContentType() { + // given + TopicWithSchema topicWithSchema = + topicWithSchema( + topicWithRandomName().withContentType(AVRO).build(), user.getSchemaAsString()); + Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); + String message = "{\"__metadata\":null,\"name\":\"john\",\"age\":\"string instead of int\"}"; + + // when / then + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + WebTestClient.ResponseSpec response = + hermes + .api() + .publish( + topic.getQualifiedName(), + message, + createHeaders(Map.of("Content-Type", AVRO_JSON))); + response.expectStatus().isBadRequest(); + response + .expectBody(String.class) + .isEqualTo( + "{" + + "\"message\":\"Invalid message: Failed to convert to AVRO: Expected int. Got VALUE_STRING.\"," + + "\"code\":\"VALIDATION_ERROR\"" + + "}"); + }); + } + + @Test + public void shouldGetBadRequestForJsonNotMachingWithAvroSchemaAndJsonContentType() { + // given + TopicWithSchema topicWithSchema = + topicWithSchema( + topicWithRandomName().withContentType(AVRO).build(), user.getSchemaAsString()); + Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); + + // when + String message = "{\"name\":\"john\",\"age\":\"string instead of int\"}"; + WebTestClient.ResponseSpec response = + hermes.api().publishJSON(topic.getQualifiedName(), message); + + // then + response.expectStatus().isBadRequest(); + response + .expectBody(String.class) + .isEqualTo( + "{" + + "\"message\":\"Invalid message: Failed to convert JSON to Avro: Field age is expected to be type: java.lang.Number\"," + + "\"code\":\"VALIDATION_ERROR\"" + + "}"); + } + + @Test + public void shouldGetBadRequestForInvalidJsonWithAvroSchema() { + // given + TopicWithSchema topicWithSchema = + topicWithSchema( + topicWithRandomName().withContentType(AVRO).build(), user.getSchemaAsString()); + Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); + + // when + WebTestClient.ResponseSpec response = + hermes.api().publishJSON(topic.getQualifiedName(), "{\"name\":\"Bob\""); + + // then + response.expectStatus().isBadRequest(); + } + + @Test + public void shouldReturnBadRequestOnMissingSchemaAtSpecifiedVersion() { + TopicWithSchema topicWithSchema = + topicWithSchema( + topicWithRandomName().withContentType(AVRO).build(), user.getSchemaAsString()); + Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); + + // when + HermesMessage message = hermesMessage(topic.getQualifiedName(), user.asBytes()).avro(2).build(); + + WebTestClient.ResponseSpec response = + hermes + .api() + .publishAvro( + topic.getQualifiedName(), message.getBody(), createHeaders(message.getHeaders())); + + // then + response.expectStatus().isBadRequest(); + response + .expectBody(String.class) + .isEqualTo( + "{\"message\":\"Given schema version '2' does not exist\",\"code\":\"SCHEMA_VERSION_DOES_NOT_EXIST\"}"); + } + + @Test + public void shouldPublishJsonIncompatibleWithSchemaWhileJsonToAvroDryRunModeIsEnabled() { + // given + Topic topic = + hermes + .initHelper() + .createTopic( + topicWithRandomName().withJsonToAvroDryRun(true).withContentType(JSON).build()); + hermes.api().ensureSchemaSaved(topic.getQualifiedName(), false, user.getSchemaAsString()); + + TestSubscriber subscriber = subscribers.createSubscriber(); + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) + .build()); - @Test - public void shouldGetBadRequestForJsonNotMatchingWithAvroSchemaAndAvroContentType() { - // given - TopicWithSchema topicWithSchema = topicWithSchema(topicWithRandomName() - .withContentType(AVRO) - .build(), user.getSchemaAsString()); - Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); - String message = "{\"__metadata\":null,\"name\":\"john\",\"age\":\"string instead of int\"}"; - - // when / then - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> { - WebTestClient.ResponseSpec response = hermes.api().publish(topic.getQualifiedName(), message, createHeaders(Map.of("Content-Type", AVRO_JSON))); - response.expectStatus().isBadRequest(); - response.expectBody(String.class).isEqualTo( - "{" + "\"message\":\"Invalid message: Failed to convert to AVRO: Expected int. Got VALUE_STRING.\"," - + "\"code\":\"VALIDATION_ERROR\"" - + "}" - ); - }); - } + TestMessage message = TestMessage.random(); + + // when + hermes.api().publishJSONUntilSuccess(topic.getQualifiedName(), message.body()); + + // then + subscriber.waitUntilReceived(message.body()); + } + + @Test + public void + shouldPublishJsonCompatibleWithSchemaWithoutMetadataWhileJsonToAvroDryRunModeIsEnabled() { + // given + Topic topic = + hermes + .initHelper() + .createTopic( + topicWithRandomName().withJsonToAvroDryRun(true).withContentType(JSON).build()); + + Schema schema = AvroUserSchemaLoader.load("/schema/user_no_metadata.avsc"); + hermes.api().ensureSchemaSaved(topic.getQualifiedName(), false, schema.toString()); + + TestSubscriber subscriber = subscribers.createSubscriber(); + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) + .build()); - @Test - public void shouldGetBadRequestForJsonNotMachingWithAvroSchemaAndJsonContentType() { - // given - TopicWithSchema topicWithSchema = topicWithSchema(topicWithRandomName() - .withContentType(AVRO) - .build(), user.getSchemaAsString()); - Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); - - // when - String message = "{\"name\":\"john\",\"age\":\"string instead of int\"}"; - WebTestClient.ResponseSpec response = hermes.api().publishJSON(topic.getQualifiedName(), message); - - // then - response.expectStatus().isBadRequest(); - response.expectBody(String.class).isEqualTo( - "{" + "\"message\":\"Invalid message: Failed to convert JSON to Avro: Field age is expected to be type: java.lang.Number\"," - + "\"code\":\"VALIDATION_ERROR\"" - + "}" - ); - } - - @Test - public void shouldGetBadRequestForInvalidJsonWithAvroSchema() { - // given - TopicWithSchema topicWithSchema = topicWithSchema(topicWithRandomName() - .withContentType(AVRO) - .build(), user.getSchemaAsString()); - Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); + // when + hermes.api().publishJSONUntilSuccess(topic.getQualifiedName(), user.asJson()); - // when - WebTestClient.ResponseSpec response = hermes.api().publishJSON(topic.getQualifiedName(), "{\"name\":\"Bob\""); + // then + subscriber.waitUntilReceived(user.asJson()); + } - // then - response.expectStatus().isBadRequest(); - } + @Test + public void shouldPublishAvroAndConsumeJsonMessageWithTraceId() { + // given + final String traceId = UUID.randomUUID().toString(); - @Test - public void shouldReturnBadRequestOnMissingSchemaAtSpecifiedVersion() { - TopicWithSchema topicWithSchema = topicWithSchema(topicWithRandomName() - .withContentType(AVRO) - .build(), user.getSchemaAsString()); - Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); - - // when - HermesMessage message = hermesMessage(topic.getQualifiedName(), user.asBytes()) - .avro(2) - .build(); - - WebTestClient.ResponseSpec response = hermes.api().publishAvro(topic.getQualifiedName(), message.getBody(), createHeaders(message.getHeaders())); - - // then - response.expectStatus().isBadRequest(); - response.expectBody(String.class) - .isEqualTo("{\"message\":\"Given schema version '2' does not exist\",\"code\":\"SCHEMA_VERSION_DOES_NOT_EXIST\"}"); - } - - @Test - public void shouldPublishJsonIncompatibleWithSchemaWhileJsonToAvroDryRunModeIsEnabled() { - // given - Topic topic = hermes.initHelper().createTopic( - topicWithRandomName() - .withJsonToAvroDryRun(true) - .withContentType(JSON) - .build()); - hermes.api().ensureSchemaSaved(topic.getQualifiedName(), false, user.getSchemaAsString()); - - TestSubscriber subscriber = subscribers.createSubscriber(); - hermes.initHelper().createSubscription(subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()).build()); - - TestMessage message = TestMessage.random(); - - // when - hermes.api().publishJSONUntilSuccess(topic.getQualifiedName(), message.body()); - - // then - subscriber.waitUntilReceived(message.body()); - } - - @Test - public void shouldPublishJsonCompatibleWithSchemaWithoutMetadataWhileJsonToAvroDryRunModeIsEnabled() { - // given - Topic topic = hermes.initHelper().createTopic( - topicWithRandomName() - .withJsonToAvroDryRun(true) - .withContentType(JSON) - .build()); - - Schema schema = AvroUserSchemaLoader.load("/schema/user_no_metadata.avsc"); - hermes.api().ensureSchemaSaved(topic.getQualifiedName(), false, schema.toString()); - - TestSubscriber subscriber = subscribers.createSubscriber(); - hermes.initHelper().createSubscription(subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()).build()); - - // when - hermes.api().publishJSONUntilSuccess(topic.getQualifiedName(), user.asJson()); - - // then - subscriber.waitUntilReceived(user.asJson()); - } - - @Test - public void shouldPublishAvroAndConsumeJsonMessageWithTraceId() { - // given - final String traceId = UUID.randomUUID().toString(); - - // and - TopicWithSchema topicWithSchema = topicWithSchema(topicWithRandomName() - .withContentType(AVRO) - .build(), user.getSchemaAsString()); + // and + TopicWithSchema topicWithSchema = + topicWithSchema( + topicWithRandomName().withContentType(AVRO).build(), user.getSchemaAsString()); - Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); + Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); - TestSubscriber subscriber = subscribers.createSubscriber(); - hermes.initHelper().createSubscription(subscription(topic.getQualifiedName(), - "subscription", subscriber.getEndpoint()) + TestSubscriber subscriber = subscribers.createSubscriber(); + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) .build()); - // when - hermes.api().publishAvroUntilSuccess(topic.getQualifiedName(), user.asBytes(), createHeaders(singletonMap("Trace-Id", traceId))); - - // then - subscriber.waitUntilRequestReceived(request -> { - assertThat(request.getBodyAsString()).isEqualTo(user.asJson()); - assertThat(request.getHeader("Trace-Id")).isEqualTo(traceId); + // when + hermes + .api() + .publishAvroUntilSuccess( + topic.getQualifiedName(), + user.asBytes(), + createHeaders(singletonMap("Trace-Id", traceId))); + + // then + subscriber.waitUntilRequestReceived( + request -> { + assertThat(request.getBodyAsString()).isEqualTo(user.asJson()); + assertThat(request.getHeader("Trace-Id")).isEqualTo(traceId); }); - } - - @Test - public void shouldUseExplicitSchemaVersionWhenPublishingAndConsuming() { - // given - TopicWithSchema topicWithSchema = topicWithSchema(topicWithRandomName() - .withContentType(AVRO) - .withSchemaIdAwareSerialization() - .build(), user.getSchemaAsString()); - - Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); - - TestSubscriber subscriber = subscribers.createSubscriber(); - hermes.initHelper().createSubscription(subscription(topic.getQualifiedName(), - "subscription", subscriber.getEndpoint()) + } + + @Test + public void shouldUseExplicitSchemaVersionWhenPublishingAndConsuming() { + // given + TopicWithSchema topicWithSchema = + topicWithSchema( + topicWithRandomName().withContentType(AVRO).withSchemaIdAwareSerialization().build(), + user.getSchemaAsString()); + + Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); + + TestSubscriber subscriber = subscribers.createSubscriber(); + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) .withContentType(AVRO) .build()); - hermes.api().ensureSchemaSaved(topic.getQualifiedName(), false, load("/schema/user_v2.avsc").toString()); - - // when - HermesMessage message = hermesMessage(topic.getQualifiedName(), user.asBytes()) - .avro(1) - .build(); - - hermes.api().publishAvroUntilSuccess(topic.getQualifiedName(), message.getBody(), createHeaders(message.getHeaders())); - - // then - subscriber.waitUntilRequestReceived(request -> { - assertThat(request.getHeaders().getHeader(HermesMessage.SCHEMA_VERSION_HEADER).firstValue()).isEqualTo("1"); - assertBodyDeserializesIntoUser(request.getBodyAsString(), user); + hermes + .api() + .ensureSchemaSaved( + topic.getQualifiedName(), false, load("/schema/user_v2.avsc").toString()); + + // when + HermesMessage message = hermesMessage(topic.getQualifiedName(), user.asBytes()).avro(1).build(); + + hermes + .api() + .publishAvroUntilSuccess( + topic.getQualifiedName(), message.getBody(), createHeaders(message.getHeaders())); + + // then + subscriber.waitUntilRequestReceived( + request -> { + assertThat( + request.getHeaders().getHeader(HermesMessage.SCHEMA_VERSION_HEADER).firstValue()) + .isEqualTo("1"); + assertBodyDeserializesIntoUser(request.getBodyAsString(), user); }); - } - - @Test - public void shouldUseExplicitSchemaVersionWhenPublishingAndConsumingWithLowercaseHeader() { - // given - TopicWithSchema topicWithSchema = topicWithSchema(topicWithRandomName() - .withContentType(AVRO) - .withSchemaIdAwareSerialization() - .build(), user.getSchemaAsString()); - - Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); - - TestSubscriber subscriber = subscribers.createSubscriber(); - hermes.initHelper().createSubscription(subscription(topic.getQualifiedName(), - "subscription", subscriber.getEndpoint()) + } + + @Test + public void shouldUseExplicitSchemaVersionWhenPublishingAndConsumingWithLowercaseHeader() { + // given + TopicWithSchema topicWithSchema = + topicWithSchema( + topicWithRandomName().withContentType(AVRO).withSchemaIdAwareSerialization().build(), + user.getSchemaAsString()); + + Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); + + TestSubscriber subscriber = subscribers.createSubscriber(); + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) .withContentType(AVRO) .build()); - hermes.api().ensureSchemaSaved(topic.getQualifiedName(), false, load("/schema/user_v2.avsc").toString()); - - // when - HermesMessage message = hermesMessage(topic.getQualifiedName(), user.asBytes()) - .withHeader("schema-version", "1") - .build(); - - hermes.api().publishAvroUntilSuccess(topic.getQualifiedName(), message.getBody(), createHeaders(message.getHeaders())); - - // then - subscriber.waitUntilRequestReceived(request -> { - assertThat(request.getHeaders().getHeader(HermesMessage.SCHEMA_VERSION_HEADER).firstValue()).isEqualTo("1"); - assertBodyDeserializesIntoUser(request.getBodyAsString(), user); + hermes + .api() + .ensureSchemaSaved( + topic.getQualifiedName(), false, load("/schema/user_v2.avsc").toString()); + + // when + HermesMessage message = + hermesMessage(topic.getQualifiedName(), user.asBytes()) + .withHeader("schema-version", "1") + .build(); + + hermes + .api() + .publishAvroUntilSuccess( + topic.getQualifiedName(), message.getBody(), createHeaders(message.getHeaders())); + + // then + subscriber.waitUntilRequestReceived( + request -> { + assertThat( + request.getHeaders().getHeader(HermesMessage.SCHEMA_VERSION_HEADER).firstValue()) + .isEqualTo("1"); + assertBodyDeserializesIntoUser(request.getBodyAsString(), user); }); - } - - @Test - public void shouldUpdateSchemaAndUseItImmediately() { - // given - TopicWithSchema topicWithSchema = topicWithSchema(topicWithRandomName() - .withContentType(AVRO) - .build(), user.getSchemaAsString()); - - Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); - - TestSubscriber subscriber = subscribers.createSubscriber(); - hermes.initHelper().createSubscription(subscription(topic.getQualifiedName(), - "subscription", subscriber.getEndpoint()) + } + + @Test + public void shouldUpdateSchemaAndUseItImmediately() { + // given + TopicWithSchema topicWithSchema = + topicWithSchema( + topicWithRandomName().withContentType(AVRO).build(), user.getSchemaAsString()); + + Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); + + TestSubscriber subscriber = subscribers.createSubscriber(); + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) .withContentType(AVRO) .build()); - HermesMessage message = hermesMessage(topic.getQualifiedName(), user.asBytes()).build(); - - // when message is published with schema version 1 - hermes.api().publishAvroUntilSuccess(topic.getQualifiedName(), message.getBody(), createHeaders(message.getHeaders())); - - // then it is consumed with schema version 1 - subscriber.waitUntilRequestReceived(request -> { - assertThat(request.getHeaders().getHeader(HermesMessage.SCHEMA_VERSION_HEADER).firstValue()).isEqualTo("1"); - assertBodyDeserializesIntoUser(request.getBodyAsString(), user); + HermesMessage message = hermesMessage(topic.getQualifiedName(), user.asBytes()).build(); + + // when message is published with schema version 1 + hermes + .api() + .publishAvroUntilSuccess( + topic.getQualifiedName(), message.getBody(), createHeaders(message.getHeaders())); + + // then it is consumed with schema version 1 + subscriber.waitUntilRequestReceived( + request -> { + assertThat( + request.getHeaders().getHeader(HermesMessage.SCHEMA_VERSION_HEADER).firstValue()) + .isEqualTo("1"); + assertBodyDeserializesIntoUser(request.getBodyAsString(), user); }); - subscriber.reset(); - - Schema schemaV2 = load("/schema/user_v2.avsc"); - AvroUser userV2 = new AvroUser(CompiledSchema.of(schemaV2, 2, 2), "Bob", 50, "blue"); - HermesMessage messageV2 = hermesMessage(topic.getQualifiedName(), userV2.asBytes()).build(); - - // when schema is updated to version 2 - hermes.api().ensureSchemaSaved(topic.getQualifiedName(), false, schemaV2.toString()); - - // and messages is published with schema version 2 - hermes.api().publishAvroUntilSuccess(topic.getQualifiedName(), messageV2.getBody(), createHeaders(messageV2.getHeaders())); - - // then it is consumed with schema version 2 - subscriber.waitUntilRequestReceived(request -> { - assertThat(request.getHeaders().getHeader(HermesMessage.SCHEMA_VERSION_HEADER).firstValue()).isEqualTo("2"); - assertBodyDeserializesIntoUser(request.getBodyAsString(), userV2); + subscriber.reset(); + + Schema schemaV2 = load("/schema/user_v2.avsc"); + AvroUser userV2 = new AvroUser(CompiledSchema.of(schemaV2, 2, 2), "Bob", 50, "blue"); + HermesMessage messageV2 = hermesMessage(topic.getQualifiedName(), userV2.asBytes()).build(); + + // when schema is updated to version 2 + hermes.api().ensureSchemaSaved(topic.getQualifiedName(), false, schemaV2.toString()); + + // and messages is published with schema version 2 + hermes + .api() + .publishAvroUntilSuccess( + topic.getQualifiedName(), messageV2.getBody(), createHeaders(messageV2.getHeaders())); + + // then it is consumed with schema version 2 + subscriber.waitUntilRequestReceived( + request -> { + assertThat( + request.getHeaders().getHeader(HermesMessage.SCHEMA_VERSION_HEADER).firstValue()) + .isEqualTo("2"); + assertBodyDeserializesIntoUser(request.getBodyAsString(), userV2); }); - } - - @Test - public void shouldPublishAndConsumeJsonMessageAfterMigrationFromJsonToAvro() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().withContentType(JSON).build()); - - TestSubscriber subscriber = subscribers.createSubscriber(); - hermes.initHelper().createSubscription(subscription(topic.getQualifiedName(), - "subscription", subscriber.getEndpoint()).build()); - - final TestMessage beforeMigrationMessage = new AvroUser("Bob", 50, "blue").asTestMessage(); - final AvroUser afterMigrationMessage = new AvroUser("Barney", 35, "yellow"); - - hermes.api().publishUntilSuccess(topic.getQualifiedName(), beforeMigrationMessage.body()); - subscriber.waitUntilReceived(beforeMigrationMessage.body()); - subscriber.reset(); - - hermes.api().waitUntilConsumerCommitsOffset(topic.getQualifiedName(), "subscription"); - - PatchData patch = patchData() - .set("contentType", ContentType.AVRO) - .set("migratedFromJsonType", true) - .set("schema", user.getSchemaAsString()) - .build(); - hermes.api().updateTopic(topic.getQualifiedName(), patch); - - // when - hermes.api().publishJSONUntilSuccess(topic.getQualifiedName(), afterMigrationMessage.asTestMessage().withEmptyAvroMetadata().body()); - - // then - subscriber.waitUntilReceived(afterMigrationMessage.asTestMessage().body()); - } + } + + @Test + public void shouldPublishAndConsumeJsonMessageAfterMigrationFromJsonToAvro() { + // given + Topic topic = + hermes.initHelper().createTopic(topicWithRandomName().withContentType(JSON).build()); + + TestSubscriber subscriber = subscribers.createSubscriber(); + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) + .build()); - @Test - public void shouldSendMessageIdHeaderToSubscriber() { - // given - TopicWithSchema topicWithSchema = topicWithSchema(topicWithRandomName() - .withContentType(AVRO) - .build(), user.getSchemaAsString()); - Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); - TestSubscriber subscriber = subscribers.createSubscriber(); - hermes.initHelper().createSubscription(subscription(topic.getQualifiedName(), - "subscription", subscriber.getEndpoint()) + final TestMessage beforeMigrationMessage = new AvroUser("Bob", 50, "blue").asTestMessage(); + final AvroUser afterMigrationMessage = new AvroUser("Barney", 35, "yellow"); + + hermes.api().publishUntilSuccess(topic.getQualifiedName(), beforeMigrationMessage.body()); + subscriber.waitUntilReceived(beforeMigrationMessage.body()); + subscriber.reset(); + + hermes.api().waitUntilConsumerCommitsOffset(topic.getQualifiedName(), "subscription"); + + PatchData patch = + patchData() + .set("contentType", ContentType.AVRO) + .set("migratedFromJsonType", true) + .set("schema", user.getSchemaAsString()) + .build(); + hermes.api().updateTopic(topic.getQualifiedName(), patch); + + // when + hermes + .api() + .publishJSONUntilSuccess( + topic.getQualifiedName(), + afterMigrationMessage.asTestMessage().withEmptyAvroMetadata().body()); + + // then + subscriber.waitUntilReceived(afterMigrationMessage.asTestMessage().body()); + } + + @Test + public void shouldSendMessageIdHeaderToSubscriber() { + // given + TopicWithSchema topicWithSchema = + topicWithSchema( + topicWithRandomName().withContentType(AVRO).build(), user.getSchemaAsString()); + Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); + TestSubscriber subscriber = subscribers.createSubscriber(); + hermes + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) .withContentType(AVRO) .build()); - String traceId = UUID.randomUUID().toString(); - - // when - hermes.api().publishAvroUntilSuccess(topic.getQualifiedName(), user.asBytes(), createHeaders(singletonMap("Trace-Id", traceId))); - - // then - subscriber.waitUntilRequestReceived(request -> { - String messageId = request.getHeader("Hermes-Message-Id"); - assertThat(messageId).isNotBlank(); - assertThat(request.getHeader("messageId")).isEqualTo(messageId); - assertThat(request.getHeader("Trace-Id")).isEqualTo(traceId); - }); - } - - private void assertBodyDeserializesIntoUser(String body, AvroUser user) { - AvroUser avroUser = AvroUser.create(user.getCompiledSchema(), body.getBytes()); - assertThat(avroUser.getName()).isEqualTo(user.getName()); - assertThat(avroUser.getAge()).isEqualTo(user.getAge()); - assertThat(avroUser.getFavoriteColor()).isEqualTo(user.getFavoriteColor()); - } - - private void waitUntilSubscriptionContentTypeChanged(Topic topic, String subscription, ContentType expected) { - waitAtMost(adjust(Duration.ofSeconds(10))).until(() -> { - ContentType actual = hermes.api().getSubscription(topic.getQualifiedName(), subscription).getContentType(); - logger.info("Expecting {} subscription endpoint address. Actual {}", expected, actual); - return expected.equals(actual); + String traceId = UUID.randomUUID().toString(); + + // when + hermes + .api() + .publishAvroUntilSuccess( + topic.getQualifiedName(), + user.asBytes(), + createHeaders(singletonMap("Trace-Id", traceId))); + + // then + subscriber.waitUntilRequestReceived( + request -> { + String messageId = request.getHeader("Hermes-Message-Id"); + assertThat(messageId).isNotBlank(); + assertThat(request.getHeader("messageId")).isEqualTo(messageId); + assertThat(request.getHeader("Trace-Id")).isEqualTo(traceId); }); - } - - private void waitUntilConsumersUpdateSubscription(final long currentTime, Topic topic, String subscription) { - waitAtMost(adjust(Duration.ofSeconds(10))).until(() -> + } + + private void assertBodyDeserializesIntoUser(String body, AvroUser user) { + AvroUser avroUser = AvroUser.create(user.getCompiledSchema(), body.getBytes()); + assertThat(avroUser.getName()).isEqualTo(user.getName()); + assertThat(avroUser.getAge()).isEqualTo(user.getAge()); + assertThat(avroUser.getFavoriteColor()).isEqualTo(user.getFavoriteColor()); + } + + private void waitUntilSubscriptionContentTypeChanged( + Topic topic, String subscription, ContentType expected) { + waitAtMost(adjust(Duration.ofSeconds(10))) + .until( + () -> { + ContentType actual = + hermes + .api() + .getSubscription(topic.getQualifiedName(), subscription) + .getContentType(); + logger.info( + "Expecting {} subscription endpoint address. Actual {}", expected, actual); + return expected.equals(actual); + }); + } + + private void waitUntilConsumersUpdateSubscription( + final long currentTime, Topic topic, String subscription) { + waitAtMost(adjust(Duration.ofSeconds(10))) + .until( + () -> hermes.api().getRunningSubscriptionsStatus().stream() - .filter(sub -> sub.getQualifiedName().equals(topic.getQualifiedName() + "$" + subscription)) - .anyMatch(sub -> sub.getSignalTimesheet().getOrDefault(UPDATE_SUBSCRIPTION, 0L) > currentTime)); - } + .filter( + sub -> + sub.getQualifiedName() + .equals(topic.getQualifiedName() + "$" + subscription)) + .anyMatch( + sub -> + sub.getSignalTimesheet().getOrDefault(UPDATE_SUBSCRIPTION, 0L) + > currentTime)); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingTest.java index 54ab67272b..7f8b7e61f8 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingTest.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.integrationtests; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + import org.apache.commons.lang3.StringUtils; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -9,32 +11,34 @@ import pl.allegro.tech.hermes.integrationtests.setup.HermesExtension; import pl.allegro.tech.hermes.test.helper.message.TestMessage; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class PublishingTest { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - @Test - public void shouldReturn429ForQuotaViolation() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - - // Frontend is configured in integration test suite to block publisher after 50_000 kb/sec - TestMessage message = TestMessage.of("content", StringUtils.repeat("X", 60_000)); - - hermes.api().publishUntilStatus(topic.getQualifiedName(), message.body(), HttpStatus.TOO_MANY_REQUESTS.value()); - } - - @Test - public void shouldReturn4xxForTooLargeContent() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().withMaxMessageSize(2048).build()); - - // when - WebTestClient.ResponseSpec response = hermes.api().publish(topic.getQualifiedName(), StringUtils.repeat("X", 2555)); - - // then - response.expectStatus().isBadRequest(); - } + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); + + @Test + public void shouldReturn429ForQuotaViolation() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + + // Frontend is configured in integration test suite to block publisher after 50_000 kb/sec + TestMessage message = TestMessage.of("content", StringUtils.repeat("X", 60_000)); + + hermes + .api() + .publishUntilStatus( + topic.getQualifiedName(), message.body(), HttpStatus.TOO_MANY_REQUESTS.value()); + } + + @Test + public void shouldReturn4xxForTooLargeContent() { + // given + Topic topic = + hermes.initHelper().createTopic(topicWithRandomName().withMaxMessageSize(2048).build()); + + // when + WebTestClient.ResponseSpec response = + hermes.api().publish(topic.getQualifiedName(), StringUtils.repeat("X", 2555)); + + // then + response.expectStatus().isBadRequest(); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingTimeoutTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingTimeoutTest.java index 8a50d79d44..4b3975b451 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingTimeoutTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingTimeoutTest.java @@ -1,76 +1,96 @@ package pl.allegro.tech.hermes.integrationtests; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + +import java.io.IOException; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.slf4j.LoggerFactory; import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.integrationtests.setup.HermesExtension; -import java.io.IOException; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class PublishingTimeoutTest { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); - @Test - public void shouldHandleRequestTimeout() throws IOException, InterruptedException { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - int clientTimeout = 5000; - int pauseTimeBetweenChunks = 300; - int delayBeforeSendingFirstData = 0; + @Test + public void shouldHandleRequestTimeout() throws IOException, InterruptedException { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + int clientTimeout = 5000; + int pauseTimeBetweenChunks = 300; + int delayBeforeSendingFirstData = 0; - // when - long start = System.currentTimeMillis(); - String response = hermes.api().publishSlowly(clientTimeout, pauseTimeBetweenChunks, delayBeforeSendingFirstData, topic.getQualifiedName()); - long elapsed = System.currentTimeMillis() - start; + // when + long start = System.currentTimeMillis(); + String response = + hermes + .api() + .publishSlowly( + clientTimeout, + pauseTimeBetweenChunks, + delayBeforeSendingFirstData, + topic.getQualifiedName()); + long elapsed = System.currentTimeMillis() - start; - //then - assertThat(response).contains("408 Request Time-out"); - assertThat(elapsed).isLessThan(2500); - } + // then + assertThat(response).contains("408 Request Time-out"); + assertThat(elapsed).isLessThan(2500); + } - @Test - public void shouldCloseConnectionAfterSendingDelayData() { - //given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - int clientTimeout = 5000; - int pauseTimeBetweenChunks = 0; - int delayBeforeSendingFirstData = 3000; + @Test + public void shouldCloseConnectionAfterSendingDelayData() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + int clientTimeout = 5000; + int pauseTimeBetweenChunks = 0; + int delayBeforeSendingFirstData = 3000; - //when - Exception thrown = assertThrows(Exception.class, () -> - hermes.api().publishSlowly( - clientTimeout, pauseTimeBetweenChunks, delayBeforeSendingFirstData, topic.getQualifiedName() - )); + // when + Exception thrown = + assertThrows( + Exception.class, + () -> + hermes + .api() + .publishSlowly( + clientTimeout, + pauseTimeBetweenChunks, + delayBeforeSendingFirstData, + topic.getQualifiedName())); - // then - LoggerFactory.getLogger(PublishingTimeoutTest.class).error("Caught exception", thrown); - assertThat(thrown.getMessage()).containsAnyOf("Broken pipe", "Connection reset by peer"); - } + // then + LoggerFactory.getLogger(PublishingTimeoutTest.class).error("Caught exception", thrown); + assertThat(thrown.getMessage()).containsAnyOf("Broken pipe", "Connection reset by peer"); + } - @Test - public void shouldHandleTimeoutForSlowRequestWithChunkedEncoding() throws IOException, InterruptedException { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - int clientTimeout = 5000; - int pauseTimeBetweenChunks = 300; - int delayBeforeSendingFirstData = 0; - boolean chunkedEncoding = true; + @Test + public void shouldHandleTimeoutForSlowRequestWithChunkedEncoding() + throws IOException, InterruptedException { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + int clientTimeout = 5000; + int pauseTimeBetweenChunks = 300; + int delayBeforeSendingFirstData = 0; + boolean chunkedEncoding = true; - // when - long start = System.currentTimeMillis(); - String response = hermes.api().publishSlowly( - clientTimeout, pauseTimeBetweenChunks, delayBeforeSendingFirstData, topic.getQualifiedName(), chunkedEncoding); - long elapsed = System.currentTimeMillis() - start; + // when + long start = System.currentTimeMillis(); + String response = + hermes + .api() + .publishSlowly( + clientTimeout, + pauseTimeBetweenChunks, + delayBeforeSendingFirstData, + topic.getQualifiedName(), + chunkedEncoding); + long elapsed = System.currentTimeMillis() - start; - // then - assertThat(response).contains("408 Request Time-out"); - assertThat(elapsed).isLessThan(2500); - } + // then + assertThat(response).contains("408 Request Time-out"); + assertThat(elapsed).isLessThan(2500); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingWithFailoverTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingWithFailoverTest.java index 319808285e..0f4dcb0c66 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingWithFailoverTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingWithFailoverTest.java @@ -1,5 +1,8 @@ package pl.allegro.tech.hermes.integrationtests; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.springframework.test.web.reactive.server.WebTestClient; @@ -9,36 +12,36 @@ import pl.allegro.tech.hermes.integrationtests.subscriber.TestSubscribersExtension; import pl.allegro.tech.hermes.test.helper.message.TestMessage; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class PublishingWithFailoverTest { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - @RegisterExtension - public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); - - @Test - public void shouldReturn202IfKafkaFailedToRespondButMessageCanBeBufferedInMemory() { - //given - TestSubscriber subscriber = subscribers.createSubscriber(); - TestMessage message = TestMessage.of("hello", "world"); - - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()).build()); - //we must send first message to a working kafka because producer need to fetch metadata - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); - subscriber.waitUntilReceived(message.body()); - - //when - hermes.cutOffConnectionsBetweenBrokersAndClients(); - WebTestClient.ResponseSpec response = hermes.api().publish(topic.getQualifiedName(), message.body()); - hermes.restoreConnectionsBetweenBrokersAndClients(); - - //then - response.expectStatus().isAccepted(); - subscriber.waitUntilReceived(message.body()); - } -} \ No newline at end of file + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); + + @RegisterExtension + public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); + + @Test + public void shouldReturn202IfKafkaFailedToRespondButMessageCanBeBufferedInMemory() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + TestMessage message = TestMessage.of("hello", "world"); + + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()).build()); + // we must send first message to a working kafka because producer need to fetch metadata + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message.body()); + subscriber.waitUntilReceived(message.body()); + + // when + hermes.cutOffConnectionsBetweenBrokersAndClients(); + WebTestClient.ResponseSpec response = + hermes.api().publish(topic.getQualifiedName(), message.body()); + hermes.restoreConnectionsBetweenBrokersAndClients(); + + // then + response.expectStatus().isAccepted(); + subscriber.waitUntilReceived(message.body()); + } +} diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/ReadinessCheckTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/ReadinessCheckTest.java index e4f65f4b33..ff0979f8bd 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/ReadinessCheckTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/ReadinessCheckTest.java @@ -1,41 +1,47 @@ package pl.allegro.tech.hermes.integrationtests; +import static org.awaitility.Awaitility.waitAtMost; +import static pl.allegro.tech.hermes.infrastructure.dc.DefaultDatacenterNameProvider.DEFAULT_DC_NAME; + +import java.time.Duration; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import pl.allegro.tech.hermes.integrationtests.setup.HermesExtension; -import java.time.Duration; - -import static org.awaitility.Awaitility.waitAtMost; -import static pl.allegro.tech.hermes.infrastructure.dc.DefaultDatacenterNameProvider.DEFAULT_DC_NAME; - public class ReadinessCheckTest { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - @Test - public void shouldRespectReadinessStatusSetByAdmin() { - // when - hermes.api().setReadiness(DEFAULT_DC_NAME, false).expectStatus().isAccepted(); - - // then - waitAtMost(Duration.ofSeconds(5)).untilAsserted(() -> - hermes.api() - .getFrontendReadiness() - .expectStatus().is5xxServerError() - .expectBody(String.class).isEqualTo("NOT_READY") - ); - - // when - hermes.api().setReadiness(DEFAULT_DC_NAME, true).expectStatus().isAccepted(); - - // then - waitAtMost(Duration.ofSeconds(5)).untilAsserted(() -> - hermes.api() - .getFrontendReadiness() - .expectStatus().isOk() - .expectBody(String.class).isEqualTo("READY") - ); - } + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); + + @Test + public void shouldRespectReadinessStatusSetByAdmin() { + // when + hermes.api().setReadiness(DEFAULT_DC_NAME, false).expectStatus().isAccepted(); + + // then + waitAtMost(Duration.ofSeconds(5)) + .untilAsserted( + () -> + hermes + .api() + .getFrontendReadiness() + .expectStatus() + .is5xxServerError() + .expectBody(String.class) + .isEqualTo("NOT_READY")); + + // when + hermes.api().setReadiness(DEFAULT_DC_NAME, true).expectStatus().isAccepted(); + + // then + waitAtMost(Duration.ofSeconds(5)) + .untilAsserted( + () -> + hermes + .api() + .getFrontendReadiness() + .expectStatus() + .isOk() + .expectBody(String.class) + .isEqualTo("READY")); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/TopicAuthorizationTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/TopicAuthorizationTest.java index 9f0fc17700..bdca3bd712 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/TopicAuthorizationTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/TopicAuthorizationTest.java @@ -1,5 +1,17 @@ package pl.allegro.tech.hermes.integrationtests; +import static org.awaitility.Awaitility.waitAtMost; +import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.AUTH_PASSWORD; +import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.AUTH_USERNAME; +import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_AUTHENTICATION_ENABLED; +import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_AUTHENTICATION_MODE; +import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_SSL_ENABLED; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Base64; +import java.util.Map; +import java.util.stream.Stream; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; @@ -11,184 +23,169 @@ import pl.allegro.tech.hermes.test.helper.message.TestMessage; import pl.allegro.tech.hermes.utils.Headers; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.Base64; -import java.util.Map; -import java.util.stream.Stream; - -import static org.awaitility.Awaitility.waitAtMost; -import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.AUTH_PASSWORD; -import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.AUTH_USERNAME; -import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_AUTHENTICATION_ENABLED; -import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_AUTHENTICATION_MODE; -import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_SSL_ENABLED; - public class TopicAuthorizationTest { - private static final String USERNAME = "someUser"; - private static final String PASSWORD = "somePassword123"; - private static final String MESSAGE = TestMessage.of("hello", "world").body(); - private static final String USERNAME2 = "foobar"; - - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension() - .withFrontendProfile("authNonRequired") - .withFrontendProperty(AUTH_USERNAME, USERNAME) - .withFrontendProperty(AUTH_PASSWORD, PASSWORD) - .withFrontendProperty(FRONTEND_SSL_ENABLED, false) - .withFrontendProperty(FRONTEND_AUTHENTICATION_MODE, "pro_active") - .withFrontendProperty(FRONTEND_AUTHENTICATION_ENABLED, true); - - @ParameterizedTest - @MethodSource("publishWhenAuthenticatedTopics") - public void shouldPublishWhenAuthenticated(Topic topic) { - // given - hermes.initHelper().createTopic(topic); - - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> { - // when - WebTestClient.ResponseSpec response = hermes.api().publish( - topic.getQualifiedName(), - MESSAGE, - createAuthorizationHeader(USERNAME, PASSWORD) - ); - - // then - response.expectStatus().isCreated(); - }); - } - - static Stream publishWhenAuthenticatedTopics() { - return Stream.of( - TopicBuilder.topicWithRandomName() - .build(), - TopicBuilder.topicWithRandomName() - .withPublisher(USERNAME) - .withAuthEnabled() - .build(), - TopicBuilder.topicWithRandomName() - .withPublisher(USERNAME) - .withPublisher(USERNAME2) - .withAuthEnabled() - .build(), - TopicBuilder.topicWithRandomName() - .withPublisher(USERNAME) - .withAuthEnabled() - .withUnauthenticatedAccessDisabled() - .build(), - TopicBuilder.topicWithRandomName() - .withPublisher(USERNAME) - .withPublisher(USERNAME2) - .withAuthEnabled() - .withUnauthenticatedAccessDisabled() - .build() - ); - } - - @ParameterizedTest - @MethodSource("publishAsGuestWhenAuthIsNotRequiredTopics") - public void shouldPublishAsGuestWhenAuthIsNotRequired(Topic topic) { - // given - hermes.initHelper().createTopic(topic); - - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> { - // when - WebTestClient.ResponseSpec response = hermes.api().publish(topic.getQualifiedName(), MESSAGE); - - // then - response.expectStatus().isCreated(); - }); - } - - static Stream publishAsGuestWhenAuthIsNotRequiredTopics() { - return Stream.of( - TopicBuilder.topicWithRandomName() - .build(), - TopicBuilder.topicWithRandomName() - .withAuthEnabled() - .build(), - TopicBuilder.topicWithRandomName() - .withPublisher(USERNAME2) - .withAuthEnabled() - .build() - ); - } - - @ParameterizedTest - @MethodSource("notPublishAsGuestWhenAuthIsRequiredTopics") - public void shouldNotPublishAsGuestWhenAuthIsRequired(Topic topic) { - // given - hermes.initHelper().createTopic(topic); - - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> { - // when - WebTestClient.ResponseSpec response = hermes.api().publish(topic.getQualifiedName(), MESSAGE); - - // then - response.expectStatus().isForbidden(); - }); - } - - static Stream notPublishAsGuestWhenAuthIsRequiredTopics() { - return Stream.of( - TopicBuilder.topicWithRandomName() - .withAuthEnabled() - .withUnauthenticatedAccessDisabled() - .build(), - TopicBuilder.topicWithRandomName() - .withPublisher(USERNAME2) - .withAuthEnabled() - .withUnauthenticatedAccessDisabled() - .build() - ); - } - - @ParameterizedTest - @MethodSource("notPublishWithoutPermissionWhenAuthenticatedTopics") - public void shouldNotPublishWithoutPermissionWhenAuthenticated(Topic topic) { - // given - hermes.initHelper().createTopic(topic); - - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> { - // when - WebTestClient.ResponseSpec response = hermes.api().publish( - topic.getQualifiedName(), - MESSAGE, - createAuthorizationHeader(USERNAME, PASSWORD) - ); - - // then - response.expectStatus().isForbidden(); - }); - } - - static Stream notPublishWithoutPermissionWhenAuthenticatedTopics() { - return Stream.of( - TopicBuilder.topicWithRandomName() - .withAuthEnabled() - .build(), - TopicBuilder.topicWithRandomName() - .withPublisher(USERNAME2) - .withAuthEnabled() - .build(), - TopicBuilder.topicWithRandomName() - .withAuthEnabled() - .withUnauthenticatedAccessDisabled() - .build(), - TopicBuilder.topicWithRandomName() - .withPublisher(USERNAME2) - .withAuthEnabled() - .withUnauthenticatedAccessDisabled() - .build() - ); - } - - private static HttpHeaders createAuthorizationHeader(String username, String password) { - String credentials = username + ":" + password; - Map headers = Map.of( - "Authorization", "Basic " + Base64.getEncoder().encodeToString(credentials.getBytes(StandardCharsets.UTF_8)) - ); - return Headers.createHeaders(headers); - } + private static final String USERNAME = "someUser"; + private static final String PASSWORD = "somePassword123"; + private static final String MESSAGE = TestMessage.of("hello", "world").body(); + private static final String USERNAME2 = "foobar"; + + @RegisterExtension + public static final HermesExtension hermes = + new HermesExtension() + .withFrontendProfile("authNonRequired") + .withFrontendProperty(AUTH_USERNAME, USERNAME) + .withFrontendProperty(AUTH_PASSWORD, PASSWORD) + .withFrontendProperty(FRONTEND_SSL_ENABLED, false) + .withFrontendProperty(FRONTEND_AUTHENTICATION_MODE, "pro_active") + .withFrontendProperty(FRONTEND_AUTHENTICATION_ENABLED, true); + + @ParameterizedTest + @MethodSource("publishWhenAuthenticatedTopics") + public void shouldPublishWhenAuthenticated(Topic topic) { + // given + hermes.initHelper().createTopic(topic); + + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + // when + WebTestClient.ResponseSpec response = + hermes + .api() + .publish( + topic.getQualifiedName(), + MESSAGE, + createAuthorizationHeader(USERNAME, PASSWORD)); + + // then + response.expectStatus().isCreated(); + }); + } + + static Stream publishWhenAuthenticatedTopics() { + return Stream.of( + TopicBuilder.topicWithRandomName().build(), + TopicBuilder.topicWithRandomName().withPublisher(USERNAME).withAuthEnabled().build(), + TopicBuilder.topicWithRandomName() + .withPublisher(USERNAME) + .withPublisher(USERNAME2) + .withAuthEnabled() + .build(), + TopicBuilder.topicWithRandomName() + .withPublisher(USERNAME) + .withAuthEnabled() + .withUnauthenticatedAccessDisabled() + .build(), + TopicBuilder.topicWithRandomName() + .withPublisher(USERNAME) + .withPublisher(USERNAME2) + .withAuthEnabled() + .withUnauthenticatedAccessDisabled() + .build()); + } + + @ParameterizedTest + @MethodSource("publishAsGuestWhenAuthIsNotRequiredTopics") + public void shouldPublishAsGuestWhenAuthIsNotRequired(Topic topic) { + // given + hermes.initHelper().createTopic(topic); + + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + // when + WebTestClient.ResponseSpec response = + hermes.api().publish(topic.getQualifiedName(), MESSAGE); + + // then + response.expectStatus().isCreated(); + }); + } + + static Stream publishAsGuestWhenAuthIsNotRequiredTopics() { + return Stream.of( + TopicBuilder.topicWithRandomName().build(), + TopicBuilder.topicWithRandomName().withAuthEnabled().build(), + TopicBuilder.topicWithRandomName().withPublisher(USERNAME2).withAuthEnabled().build()); + } + + @ParameterizedTest + @MethodSource("notPublishAsGuestWhenAuthIsRequiredTopics") + public void shouldNotPublishAsGuestWhenAuthIsRequired(Topic topic) { + // given + hermes.initHelper().createTopic(topic); + + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + // when + WebTestClient.ResponseSpec response = + hermes.api().publish(topic.getQualifiedName(), MESSAGE); + + // then + response.expectStatus().isForbidden(); + }); + } + + static Stream notPublishAsGuestWhenAuthIsRequiredTopics() { + return Stream.of( + TopicBuilder.topicWithRandomName() + .withAuthEnabled() + .withUnauthenticatedAccessDisabled() + .build(), + TopicBuilder.topicWithRandomName() + .withPublisher(USERNAME2) + .withAuthEnabled() + .withUnauthenticatedAccessDisabled() + .build()); + } + + @ParameterizedTest + @MethodSource("notPublishWithoutPermissionWhenAuthenticatedTopics") + public void shouldNotPublishWithoutPermissionWhenAuthenticated(Topic topic) { + // given + hermes.initHelper().createTopic(topic); + + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + // when + WebTestClient.ResponseSpec response = + hermes + .api() + .publish( + topic.getQualifiedName(), + MESSAGE, + createAuthorizationHeader(USERNAME, PASSWORD)); + + // then + response.expectStatus().isForbidden(); + }); + } + + static Stream notPublishWithoutPermissionWhenAuthenticatedTopics() { + return Stream.of( + TopicBuilder.topicWithRandomName().withAuthEnabled().build(), + TopicBuilder.topicWithRandomName().withPublisher(USERNAME2).withAuthEnabled().build(), + TopicBuilder.topicWithRandomName() + .withAuthEnabled() + .withUnauthenticatedAccessDisabled() + .build(), + TopicBuilder.topicWithRandomName() + .withPublisher(USERNAME2) + .withAuthEnabled() + .withUnauthenticatedAccessDisabled() + .build()); + } + + private static HttpHeaders createAuthorizationHeader(String username, String password) { + String credentials = username + ":" + password; + Map headers = + Map.of( + "Authorization", + "Basic " + + Base64.getEncoder().encodeToString(credentials.getBytes(StandardCharsets.UTF_8))); + return Headers.createHeaders(headers); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/TopicBlacklistTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/TopicBlacklistTest.java index 3b98dd7d54..d060d4ae59 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/TopicBlacklistTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/TopicBlacklistTest.java @@ -1,5 +1,12 @@ package pl.allegro.tech.hermes.integrationtests; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.waitAtMost; +import static pl.allegro.tech.hermes.api.BlacklistStatus.BLACKLISTED; +import static pl.allegro.tech.hermes.api.BlacklistStatus.NOT_BLACKLISTED; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + +import java.time.Duration; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.springframework.test.web.reactive.server.WebTestClient; @@ -7,100 +14,99 @@ import pl.allegro.tech.hermes.integrationtests.setup.HermesExtension; import pl.allegro.tech.hermes.test.helper.message.TestMessage; -import java.time.Duration; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.waitAtMost; -import static pl.allegro.tech.hermes.api.BlacklistStatus.BLACKLISTED; -import static pl.allegro.tech.hermes.api.BlacklistStatus.NOT_BLACKLISTED; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class TopicBlacklistTest { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - @Test - public void shouldRefuseMessageOnBlacklistedTopic() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestMessage message = TestMessage.of("hello", "world"); - - // when - hermes.api().blacklistTopic(topic.getQualifiedName()); - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> { - WebTestClient.ResponseSpec response = hermes.api().publish(topic.getQualifiedName(), message.body()); - - // then - response.expectStatus().isForbidden(); - }); - } - - @Test - public void shouldAcceptMessageOnUnblacklistedTopic() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestMessage message = TestMessage.of("hello", "world"); - hermes.api().blacklistTopic(topic.getQualifiedName()); - - // when - hermes.api().unblacklistTopic(topic.getQualifiedName()); - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> { - WebTestClient.ResponseSpec response = hermes.api().publish(topic.getQualifiedName(), message.body()); - - // then - response.expectStatus().is2xxSuccessful(); - }); - } - - @Test - public void shouldBlacklistNonExistingTopic() { - // expect - WebTestClient.ResponseSpec response = hermes.api().blacklistTopicResponse("nonExisting.topic"); - - // then - response.expectStatus().isOk(); - } - - - @Test - public void shouldUnBlacklistTopic() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - hermes.api().blacklistTopic(topic.getQualifiedName()); - - // when - WebTestClient.ResponseSpec response = hermes.api().unblacklistTopicResponse(topic.getQualifiedName()); - - // then - response.expectStatus().isOk(); - } - - @Test - public void shouldReportValidStatusOfTopic() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - - // when - hermes.api().blacklistTopic(topic.getQualifiedName()); - - // then - assertThat(hermes.api().isTopicBlacklisted(topic.getQualifiedName())).isEqualTo(BLACKLISTED); - - // when - hermes.api().unblacklistTopic(topic.getQualifiedName()); - - // then - assertThat(hermes.api().isTopicBlacklisted(topic.getQualifiedName())).isEqualTo(NOT_BLACKLISTED); - } - - @Test - public void shouldReturnErrorOnNonBlacklistedUnblacklist() { - // when - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - WebTestClient.ResponseSpec response = hermes.api().unblacklistTopicResponse(topic.getQualifiedName()); - - // then - response.expectStatus().isBadRequest(); - } -} \ No newline at end of file + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); + + @Test + public void shouldRefuseMessageOnBlacklistedTopic() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestMessage message = TestMessage.of("hello", "world"); + + // when + hermes.api().blacklistTopic(topic.getQualifiedName()); + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + WebTestClient.ResponseSpec response = + hermes.api().publish(topic.getQualifiedName(), message.body()); + + // then + response.expectStatus().isForbidden(); + }); + } + + @Test + public void shouldAcceptMessageOnUnblacklistedTopic() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestMessage message = TestMessage.of("hello", "world"); + hermes.api().blacklistTopic(topic.getQualifiedName()); + + // when + hermes.api().unblacklistTopic(topic.getQualifiedName()); + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + WebTestClient.ResponseSpec response = + hermes.api().publish(topic.getQualifiedName(), message.body()); + + // then + response.expectStatus().is2xxSuccessful(); + }); + } + + @Test + public void shouldBlacklistNonExistingTopic() { + // expect + WebTestClient.ResponseSpec response = hermes.api().blacklistTopicResponse("nonExisting.topic"); + + // then + response.expectStatus().isOk(); + } + + @Test + public void shouldUnBlacklistTopic() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + hermes.api().blacklistTopic(topic.getQualifiedName()); + + // when + WebTestClient.ResponseSpec response = + hermes.api().unblacklistTopicResponse(topic.getQualifiedName()); + + // then + response.expectStatus().isOk(); + } + + @Test + public void shouldReportValidStatusOfTopic() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + + // when + hermes.api().blacklistTopic(topic.getQualifiedName()); + + // then + assertThat(hermes.api().isTopicBlacklisted(topic.getQualifiedName())).isEqualTo(BLACKLISTED); + + // when + hermes.api().unblacklistTopic(topic.getQualifiedName()); + + // then + assertThat(hermes.api().isTopicBlacklisted(topic.getQualifiedName())) + .isEqualTo(NOT_BLACKLISTED); + } + + @Test + public void shouldReturnErrorOnNonBlacklistedUnblacklist() { + // when + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + WebTestClient.ResponseSpec response = + hermes.api().unblacklistTopicResponse(topic.getQualifiedName()); + + // then + response.expectStatus().isBadRequest(); + } +} diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/UndeliveredLogTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/UndeliveredLogTest.java index cd7b0e7341..54f6995595 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/UndeliveredLogTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/UndeliveredLogTest.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.integrationtests; +import static org.awaitility.Awaitility.waitAtMost; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + +import java.time.Duration; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import pl.allegro.tech.hermes.api.Subscription; @@ -7,31 +12,33 @@ import pl.allegro.tech.hermes.integrationtests.setup.HermesExtension; import pl.allegro.tech.hermes.test.helper.message.TestMessage; -import java.time.Duration; - -import static org.awaitility.Awaitility.waitAtMost; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class UndeliveredLogTest { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - private static final String INVALID_ENDPOINT_URL = "http://localhost:60000"; - - @Test - public void shouldLogUndeliveredMessage() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName(), INVALID_ENDPOINT_URL).build()); - - // when - hermes.api().publishUntilSuccess(topic.getQualifiedName(), TestMessage.simple().body()); - - // then - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> - hermes.api().getLatestUndeliveredMessage(topic.getQualifiedName(), subscription.getName()).expectStatus().is2xxSuccessful() - ); - } -} \ No newline at end of file + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); + + private static final String INVALID_ENDPOINT_URL = "http://localhost:60000"; + + @Test + public void shouldLogUndeliveredMessage() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), INVALID_ENDPOINT_URL).build()); + + // when + hermes.api().publishUntilSuccess(topic.getQualifiedName(), TestMessage.simple().body()); + + // then + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> + hermes + .api() + .getLatestUndeliveredMessage(topic.getQualifiedName(), subscription.getName()) + .expectStatus() + .is2xxSuccessful()); + } +} diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/FiltersVerificationTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/FiltersVerificationTest.java index 18ac9f767e..511ba0a470 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/FiltersVerificationTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/FiltersVerificationTest.java @@ -1,5 +1,15 @@ package pl.allegro.tech.hermes.integrationtests.management; +import static com.google.common.collect.ImmutableMap.of; +import static java.util.Collections.singletonList; +import static org.assertj.core.api.Assertions.assertThat; +import static pl.allegro.tech.hermes.api.ContentType.AVRO; +import static pl.allegro.tech.hermes.api.MessageFiltersVerificationResult.VerificationStatus.ERROR; +import static pl.allegro.tech.hermes.api.MessageFiltersVerificationResult.VerificationStatus.MATCHED; +import static pl.allegro.tech.hermes.api.MessageFiltersVerificationResult.VerificationStatus.NOT_MATCHED; +import static pl.allegro.tech.hermes.api.TopicWithSchema.topicWithSchema; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import pl.allegro.tech.hermes.api.MessageFilterSpecification; @@ -11,84 +21,80 @@ import pl.allegro.tech.hermes.test.helper.avro.AvroUser; import pl.allegro.tech.hermes.test.helper.avro.AvroUserSchemaLoader; -import static com.google.common.collect.ImmutableMap.of; -import static java.util.Collections.singletonList; -import static org.assertj.core.api.Assertions.assertThat; -import static pl.allegro.tech.hermes.api.ContentType.AVRO; -import static pl.allegro.tech.hermes.api.MessageFiltersVerificationResult.VerificationStatus.ERROR; -import static pl.allegro.tech.hermes.api.MessageFiltersVerificationResult.VerificationStatus.MATCHED; -import static pl.allegro.tech.hermes.api.MessageFiltersVerificationResult.VerificationStatus.NOT_MATCHED; -import static pl.allegro.tech.hermes.api.TopicWithSchema.topicWithSchema; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class FiltersVerificationTest { - private static final MessageFilterSpecification FILTER_MATCHING_USERS_WITH_NAME_BOB = new MessageFilterSpecification(of( - "type", "avropath", - "path", ".name", - "matcher", "Bob" - )); - - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - @Test - void shouldReturnMatchedWhenGivenMessageMatchesFilter() { - //given - Topic topic = createTopicWithAvroUserSchema(); - AvroUser bob = new AvroUser("Bob", 50, "blue"); - - //when - MessageFiltersVerificationResult result = verifyFilters(topic, FILTER_MATCHING_USERS_WITH_NAME_BOB, bob.asJson().getBytes()); - - //then - assertThat(result.getStatus()).isEqualTo(MATCHED); - assertThat(result.getErrorMessage()).isNull(); - } - - @Test - void shouldReturnNotMatchedWhenGivenMessageDoesNotMatchFilter() { - //given - Topic topic = createTopicWithAvroUserSchema(); - AvroUser alice = new AvroUser("Alice", 50, "blue"); - - //when - MessageFiltersVerificationResult result = verifyFilters(topic, FILTER_MATCHING_USERS_WITH_NAME_BOB, alice.asJson().getBytes()); - - //then - assertThat(result.getStatus()).isEqualTo(NOT_MATCHED); - assertThat(result.getErrorMessage()).isNull(); - } - - @Test - void shouldReturnErrorWhenGivenMessageIsInvalid() { - //given - Topic topic = createTopicWithAvroUserSchema(); - - //when - MessageFiltersVerificationResult result = verifyFilters(topic, FILTER_MATCHING_USERS_WITH_NAME_BOB, "xyz".getBytes()); - - //then - assertThat(result.getStatus()).isEqualTo(ERROR); - assertThat(result.getErrorMessage()).contains("Failed to parse json to map format."); - } - - private Topic createTopicWithAvroUserSchema() { - String schema = AvroUserSchemaLoader.load().toString(); - - TopicWithSchema topicWithSchema = topicWithSchema(topicWithRandomName() - .withContentType(AVRO) - .build(), schema); - - return hermes.initHelper().createTopicWithSchema(topicWithSchema); - } - - private MessageFiltersVerificationResult verifyFilters(Topic topic, MessageFilterSpecification specification, byte[] message) { - return hermes.api().verifyFilters( - topic.getQualifiedName(), - new MessageFiltersVerificationInput(singletonList(specification), message) - ) - .expectStatus().isOk() - .expectBody(MessageFiltersVerificationResult.class) - .returnResult().getResponseBody(); - } + private static final MessageFilterSpecification FILTER_MATCHING_USERS_WITH_NAME_BOB = + new MessageFilterSpecification( + of( + "type", "avropath", + "path", ".name", + "matcher", "Bob")); + + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); + + @Test + void shouldReturnMatchedWhenGivenMessageMatchesFilter() { + // given + Topic topic = createTopicWithAvroUserSchema(); + AvroUser bob = new AvroUser("Bob", 50, "blue"); + + // when + MessageFiltersVerificationResult result = + verifyFilters(topic, FILTER_MATCHING_USERS_WITH_NAME_BOB, bob.asJson().getBytes()); + + // then + assertThat(result.getStatus()).isEqualTo(MATCHED); + assertThat(result.getErrorMessage()).isNull(); + } + + @Test + void shouldReturnNotMatchedWhenGivenMessageDoesNotMatchFilter() { + // given + Topic topic = createTopicWithAvroUserSchema(); + AvroUser alice = new AvroUser("Alice", 50, "blue"); + + // when + MessageFiltersVerificationResult result = + verifyFilters(topic, FILTER_MATCHING_USERS_WITH_NAME_BOB, alice.asJson().getBytes()); + + // then + assertThat(result.getStatus()).isEqualTo(NOT_MATCHED); + assertThat(result.getErrorMessage()).isNull(); + } + + @Test + void shouldReturnErrorWhenGivenMessageIsInvalid() { + // given + Topic topic = createTopicWithAvroUserSchema(); + + // when + MessageFiltersVerificationResult result = + verifyFilters(topic, FILTER_MATCHING_USERS_WITH_NAME_BOB, "xyz".getBytes()); + + // then + assertThat(result.getStatus()).isEqualTo(ERROR); + assertThat(result.getErrorMessage()).contains("Failed to parse json to map format."); + } + + private Topic createTopicWithAvroUserSchema() { + String schema = AvroUserSchemaLoader.load().toString(); + + TopicWithSchema topicWithSchema = + topicWithSchema(topicWithRandomName().withContentType(AVRO).build(), schema); + + return hermes.initHelper().createTopicWithSchema(topicWithSchema); + } + + private MessageFiltersVerificationResult verifyFilters( + Topic topic, MessageFilterSpecification specification, byte[] message) { + return hermes + .api() + .verifyFilters( + topic.getQualifiedName(), + new MessageFiltersVerificationInput(singletonList(specification), message)) + .expectStatus() + .isOk() + .expectBody(MessageFiltersVerificationResult.class) + .returnResult() + .getResponseBody(); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/GroupManagementTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/GroupManagementTest.java index b61e16cbef..e690dc492d 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/GroupManagementTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/GroupManagementTest.java @@ -1,17 +1,5 @@ package pl.allegro.tech.hermes.integrationtests.management; -import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; -import org.springframework.test.web.reactive.server.WebTestClient; -import pl.allegro.tech.hermes.api.Group; -import pl.allegro.tech.hermes.api.Topic; -import pl.allegro.tech.hermes.integrationtests.setup.HermesExtension; -import pl.allegro.tech.hermes.management.TestSecurityProvider; - -import java.time.Duration; -import java.util.stream.Stream; - import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.waitAtMost; import static pl.allegro.tech.hermes.api.ErrorCode.GROUP_NAME_IS_INVALID; @@ -22,139 +10,155 @@ import static pl.allegro.tech.hermes.test.helper.builder.GroupBuilder.groupWithRandomName; import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; +import java.time.Duration; +import java.util.stream.Stream; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.springframework.test.web.reactive.server.WebTestClient; +import pl.allegro.tech.hermes.api.Group; +import pl.allegro.tech.hermes.api.Topic; +import pl.allegro.tech.hermes.integrationtests.setup.HermesExtension; +import pl.allegro.tech.hermes.management.TestSecurityProvider; + public class GroupManagementTest { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - @Test - public void shouldEmitAuditEventWhenGroupCreated() { - //when - Group group = hermes.initHelper().createGroup(groupWithRandomName().build()); - - //then - assertThat( - auditEvents.getLastReceivedRequest().getBodyAsString() - ).contains("CREATED", group.getGroupName()); - } - - @Test - public void shouldEmitAuditEventWhenGroupRemoved() { - //given - Group group = hermes.initHelper().createGroup(groupWithRandomName().build()); - - //when - hermes.api().deleteGroup(group.getGroupName()); - - //then - assertThat( - auditEvents.getLastReceivedRequest().getBodyAsString() - ).contains("REMOVED", group.getGroupName()); - } - - @Test - public void shouldEmitAuditEventWhenGroupUpdated() { - //given - Group group = hermes.initHelper().createGroup(groupWithRandomName().build()); - - //when - hermes.api().updateGroup(group.getGroupName(), group); - - //then - assertThat( - auditEvents.getLastReceivedRequest().getBodyAsString() - ).contains("UPDATED", group.getGroupName()); - } - - @Test - public void shouldCreateGroup() { - // given when - Group group = groupWithRandomName().build(); - WebTestClient.ResponseSpec response = hermes.api().createGroup(group); - - // then - response.expectStatus().isCreated(); - - assertThat(hermes.api().getGroups()).contains(group.getGroupName()); - } - - @Test - public void shouldListGroups() { - // given - Group group1 = hermes.initHelper().createGroup(groupWithRandomName().build()); - Group group2 = hermes.initHelper().createGroup(groupWithRandomName().build()); - - // when then - Assertions.assertThat(hermes.api().getGroups()).containsOnlyOnce(group1.getGroupName(), group2.getGroupName()); - } - - @Test - public void shouldReturnBadRequestStatusWhenAttemptToCreateGroupWithInvalidCharactersWasMade() { - // given - Group groupWithNameWithSpaces = group("group;` name with spaces").build(); - - // when - WebTestClient.ResponseSpec response = hermes.api().createGroup(groupWithNameWithSpaces); - - // then - response.expectStatus().isBadRequest(); - assertThat(getErrorCode(response)).isEqualTo(GROUP_NAME_IS_INVALID); - } - - @Test - public void shouldRemoveGroup() { - // given - Group group = hermes.initHelper().createGroup(groupWithRandomName().build()); - - // when - WebTestClient.ResponseSpec response = hermes.api().deleteGroup(group.getGroupName()); - - // then - response.expectStatus().isOk(); - waitAtMost(Duration.ofSeconds(10)) - .untilAsserted(() -> Assertions.assertThat(hermes.api().getGroups()).doesNotContain(group.getGroupName())); - } - - @Test - public void shouldAllowNonAdminUserToRemoveGroup() { - // given - TestSecurityProvider.setUserIsAdmin(false); - Group group = hermes.initHelper().createGroup(groupWithRandomName().build()); - - // when - WebTestClient.ResponseSpec response = hermes.api().deleteGroup(group.getGroupName()); - - // then - response.expectStatus().isOk(); - - waitAtMost(Duration.ofSeconds(10)) - .untilAsserted(() -> Assertions.assertThat(hermes.api().getGroups()).doesNotContain(group.getGroupName())); - - // cleanup - TestSecurityProvider.reset(); - } - - @Test - public void shouldNotAllowOnRemovingNonEmptyGroup() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - - // when - WebTestClient.ResponseSpec response = hermes.api().deleteGroup(topic.getName().getGroupName()); - - // then - response.expectStatus().isForbidden(); - assertThat(getErrorCode(response)).isEqualTo(GROUP_NOT_EMPTY); - } - - @Test - public void shouldNotAllowDollarSigns() { - Stream.of("$name", "na$me", "name$").forEach(name -> { - // when - WebTestClient.ResponseSpec response = hermes.api().createGroup(group(name).build()); - - // then - response.expectStatus().isBadRequest(); - }); - } + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); + + @Test + public void shouldEmitAuditEventWhenGroupCreated() { + // when + Group group = hermes.initHelper().createGroup(groupWithRandomName().build()); + + // then + assertThat(auditEvents.getLastReceivedRequest().getBodyAsString()) + .contains("CREATED", group.getGroupName()); + } + + @Test + public void shouldEmitAuditEventWhenGroupRemoved() { + // given + Group group = hermes.initHelper().createGroup(groupWithRandomName().build()); + + // when + hermes.api().deleteGroup(group.getGroupName()); + + // then + assertThat(auditEvents.getLastReceivedRequest().getBodyAsString()) + .contains("REMOVED", group.getGroupName()); + } + + @Test + public void shouldEmitAuditEventWhenGroupUpdated() { + // given + Group group = hermes.initHelper().createGroup(groupWithRandomName().build()); + + // when + hermes.api().updateGroup(group.getGroupName(), group); + + // then + assertThat(auditEvents.getLastReceivedRequest().getBodyAsString()) + .contains("UPDATED", group.getGroupName()); + } + + @Test + public void shouldCreateGroup() { + // given when + Group group = groupWithRandomName().build(); + WebTestClient.ResponseSpec response = hermes.api().createGroup(group); + + // then + response.expectStatus().isCreated(); + + assertThat(hermes.api().getGroups()).contains(group.getGroupName()); + } + + @Test + public void shouldListGroups() { + // given + Group group1 = hermes.initHelper().createGroup(groupWithRandomName().build()); + Group group2 = hermes.initHelper().createGroup(groupWithRandomName().build()); + + // when then + Assertions.assertThat(hermes.api().getGroups()) + .containsOnlyOnce(group1.getGroupName(), group2.getGroupName()); + } + + @Test + public void shouldReturnBadRequestStatusWhenAttemptToCreateGroupWithInvalidCharactersWasMade() { + // given + Group groupWithNameWithSpaces = group("group;` name with spaces").build(); + + // when + WebTestClient.ResponseSpec response = hermes.api().createGroup(groupWithNameWithSpaces); + + // then + response.expectStatus().isBadRequest(); + assertThat(getErrorCode(response)).isEqualTo(GROUP_NAME_IS_INVALID); + } + + @Test + public void shouldRemoveGroup() { + // given + Group group = hermes.initHelper().createGroup(groupWithRandomName().build()); + + // when + WebTestClient.ResponseSpec response = hermes.api().deleteGroup(group.getGroupName()); + + // then + response.expectStatus().isOk(); + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> + Assertions.assertThat(hermes.api().getGroups()) + .doesNotContain(group.getGroupName())); + } + + @Test + public void shouldAllowNonAdminUserToRemoveGroup() { + // given + TestSecurityProvider.setUserIsAdmin(false); + Group group = hermes.initHelper().createGroup(groupWithRandomName().build()); + + // when + WebTestClient.ResponseSpec response = hermes.api().deleteGroup(group.getGroupName()); + + // then + response.expectStatus().isOk(); + + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> + Assertions.assertThat(hermes.api().getGroups()) + .doesNotContain(group.getGroupName())); + + // cleanup + TestSecurityProvider.reset(); + } + + @Test + public void shouldNotAllowOnRemovingNonEmptyGroup() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + + // when + WebTestClient.ResponseSpec response = hermes.api().deleteGroup(topic.getName().getGroupName()); + + // then + response.expectStatus().isForbidden(); + assertThat(getErrorCode(response)).isEqualTo(GROUP_NOT_EMPTY); + } + + @Test + public void shouldNotAllowDollarSigns() { + Stream.of("$name", "na$me", "name$") + .forEach( + name -> { + // when + WebTestClient.ResponseSpec response = hermes.api().createGroup(group(name).build()); + + // then + response.expectStatus().isBadRequest(); + }); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/HealthCheckTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/HealthCheckTest.java index ca1447cbfa..0eefed51c6 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/HealthCheckTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/HealthCheckTest.java @@ -1,26 +1,23 @@ package pl.allegro.tech.hermes.integrationtests.management; +import static org.awaitility.Awaitility.await; + +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.springframework.test.web.reactive.server.WebTestClient; import pl.allegro.tech.hermes.integrationtests.setup.HermesExtension; -import java.util.concurrent.TimeUnit; - -import static org.awaitility.Awaitility.await; - public class HealthCheckTest { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); - @Test - public void shouldManagementBeHealthy() { - // given - WebTestClient.ResponseSpec response = hermes.api().getManagementHealth(); + @Test + public void shouldManagementBeHealthy() { + // given + WebTestClient.ResponseSpec response = hermes.api().getManagementHealth(); - // when & then - await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> - response.expectStatus().isOk()); - } + // when & then + await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> response.expectStatus().isOk()); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/ListClientsForTopicTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/ListClientsForTopicTest.java index 69c7b72185..dcc8469689 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/ListClientsForTopicTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/ListClientsForTopicTest.java @@ -1,58 +1,90 @@ package pl.allegro.tech.hermes.integrationtests.management; +import static pl.allegro.tech.hermes.integrationtests.assertions.HermesAssertions.assertThat; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + +import java.util.Arrays; +import java.util.List; +import java.util.Objects; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import pl.allegro.tech.hermes.api.OwnerId; import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.integrationtests.setup.HermesExtension; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; +public class ListClientsForTopicTest { -import static pl.allegro.tech.hermes.integrationtests.assertions.HermesAssertions.assertThat; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); -public class ListClientsForTopicTest { + @Test + public void shouldListClientsForTopic() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + List expectedResponse = Arrays.asList("Smurfs", "Admins"); + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName()) + .withOwner(new OwnerId("Plaintext", "Smurfs")) + .build()); + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName()) + .withOwner(new OwnerId("Plaintext", "Admins")) + .build()); + + // then + assertThat(listClientsForTopic(topic.getQualifiedName())) + .containsExactlyInAnyOrderElementsOf(expectedResponse); + } + + @Test + public void shouldListClientsForTopicWithoutRepeating() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName()) + .withOwner(new OwnerId("Plaintext", "Team A")) + .build()); + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName()) + .withOwner(new OwnerId("Plaintext", "Team A")) + .build()); + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName()) + .withOwner(new OwnerId("Plaintext", "Team B")) + .build()); + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName()) + .withOwner(new OwnerId("Plaintext", "Team B")) + .build()); + List expectedResponse = Arrays.asList("Team A", "Team B"); + + // then + assertThat(listClientsForTopic(topic.getQualifiedName())) + .containsExactlyInAnyOrderElementsOf(expectedResponse); + } - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - @Test - public void shouldListClientsForTopic() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - List expectedResponse = Arrays.asList("Smurfs", "Admins"); - hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).withOwner(new OwnerId("Plaintext", "Smurfs")).build()); - hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).withOwner(new OwnerId("Plaintext", "Admins")).build()); - - // then - assertThat(listClientsForTopic(topic.getQualifiedName())).containsExactlyInAnyOrderElementsOf(expectedResponse); - } - - @Test - public void shouldListClientsForTopicWithoutRepeating() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).withOwner(new OwnerId("Plaintext", "Team A")).build()); - hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).withOwner(new OwnerId("Plaintext", "Team A")).build()); - hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).withOwner(new OwnerId("Plaintext", "Team B")).build()); - hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).withOwner(new OwnerId("Plaintext", "Team B")).build()); - List expectedResponse = Arrays.asList("Team A", "Team B"); - - // then - assertThat(listClientsForTopic(topic.getQualifiedName())).containsExactlyInAnyOrderElementsOf(expectedResponse); - } - - private List listClientsForTopic(String topicQualifiedName) { - return Arrays.asList(Objects.requireNonNull( - hermes.api().getAllTopicClients(topicQualifiedName) - .expectStatus() - .is2xxSuccessful() - .expectBody(String[].class) - .returnResult() - .getResponseBody()) - ); - } + private List listClientsForTopic(String topicQualifiedName) { + return Arrays.asList( + Objects.requireNonNull( + hermes + .api() + .getAllTopicClients(topicQualifiedName) + .expectStatus() + .is2xxSuccessful() + .expectBody(String[].class) + .returnResult() + .getResponseBody())); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/ListSubscriptionForOwnerTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/ListSubscriptionForOwnerTest.java index 74672f9ffa..a37b2aee86 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/ListSubscriptionForOwnerTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/ListSubscriptionForOwnerTest.java @@ -1,5 +1,12 @@ package pl.allegro.tech.hermes.integrationtests.management; +import static org.assertj.core.api.Assertions.assertThat; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -9,95 +16,149 @@ import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.integrationtests.setup.HermesExtension; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - -import static org.assertj.core.api.Assertions.assertThat; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class ListSubscriptionForOwnerTest { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - private static Topic topic; - - @BeforeAll - public static void createTopic() { - topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - } - - @Test - public void shouldListSubscriptionsForOwnerId() { - // given - Subscription subscription1 = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).withOwner(new OwnerId("Plaintext", "ListSubForOwner - Team A")).build()); - Subscription subscription2 = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).withOwner(new OwnerId("Plaintext", "ListSubForOwner - Team A")).build()); - Subscription subscription3 = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).withOwner(new OwnerId("Plaintext", "ListSubForOwner - Team B")).build()); - - // then - assertThat(listSubscriptionsForOwner("ListSubForOwner - Team A")).containsOnly(subscription1.getName(), subscription2.getName()); - assertThat(listSubscriptionsForOwner("ListSubForOwner - Team B")).containsExactly(subscription3.getName()); - } - - @Test - public void shouldListSubscriptionAfterNewSubscriptionIsAdded() { - // given - assertThat(listSubscriptionsForOwner("ListSubForOwner - Team C")).isEmpty(); - - // when - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).withOwner(new OwnerId("Plaintext", "ListSubForOwner - Team C")).build()); - - // then - assertThat(listSubscriptionsForOwner("ListSubForOwner - Team C")).containsExactly(subscription.getName()); - } - - @Test - public void shouldListSubscriptionAfterOwnerIsChanged() { - // given - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).withOwner(new OwnerId("Plaintext", "ListSubForOwner - Team D")).build()); - - // then - assertThat(listSubscriptionsForOwner("ListSubForOwner - Team D")).containsExactly(subscription.getName()); - assertThat(listSubscriptionsForOwner("ListSubForOwner - Team E")).isEmpty(); - - // when - hermes.api().updateSubscription(topic, subscription.getName(), - PatchData.patchData().set("owner", new OwnerId("Plaintext", "ListSubForOwner - Team E")).build()); - - // then - assertThat(listSubscriptionsForOwner("ListSubForOwner - Team D")).isEmpty(); - assertThat(listSubscriptionsForOwner("ListSubForOwner - Team E")).containsExactly(subscription.getName()); - } - - @Test - public void shouldNotListTopicAfterIsDeleted() { - // given - Subscription subscription1 = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).withOwner(new OwnerId("Plaintext", "ListSubForOwner - Team F")).build()); - Subscription subscription2 = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).withOwner(new OwnerId("Plaintext", "ListSubForOwner - Team G")).build()); - - // then - assertThat(listSubscriptionsForOwner("ListSubForOwner - Team F")).containsExactly(subscription1.getName()); - assertThat(listSubscriptionsForOwner("ListSubForOwner - Team G")).containsExactly(subscription2.getName()); - - // when - hermes.api().deleteSubscription(topic.getQualifiedName(), subscription1.getName()).expectStatus().is2xxSuccessful(); - - // then - assertThat(listSubscriptionsForOwner("ListSubForOwner - Team F")).isEmpty(); - assertThat(listSubscriptionsForOwner("ListSubForOwner - Team G")).containsExactly(subscription2.getName()); - } - - private List listSubscriptionsForOwner(String ownerId) { - return Objects.requireNonNull(hermes.api().getSubscriptionsForOwner("Plaintext", ownerId) - .expectStatus() - .is2xxSuccessful() - .expectBodyList(Subscription.class) - .returnResult() - .getResponseBody()) - .stream() - .map(Subscription::getName) - .collect(Collectors.toList()); - } + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); + + private static Topic topic; + + @BeforeAll + public static void createTopic() { + topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + } + + @Test + public void shouldListSubscriptionsForOwnerId() { + // given + Subscription subscription1 = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName()) + .withOwner(new OwnerId("Plaintext", "ListSubForOwner - Team A")) + .build()); + Subscription subscription2 = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName()) + .withOwner(new OwnerId("Plaintext", "ListSubForOwner - Team A")) + .build()); + Subscription subscription3 = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName()) + .withOwner(new OwnerId("Plaintext", "ListSubForOwner - Team B")) + .build()); + + // then + assertThat(listSubscriptionsForOwner("ListSubForOwner - Team A")) + .containsOnly(subscription1.getName(), subscription2.getName()); + assertThat(listSubscriptionsForOwner("ListSubForOwner - Team B")) + .containsExactly(subscription3.getName()); + } + + @Test + public void shouldListSubscriptionAfterNewSubscriptionIsAdded() { + // given + assertThat(listSubscriptionsForOwner("ListSubForOwner - Team C")).isEmpty(); + + // when + Subscription subscription = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName()) + .withOwner(new OwnerId("Plaintext", "ListSubForOwner - Team C")) + .build()); + + // then + assertThat(listSubscriptionsForOwner("ListSubForOwner - Team C")) + .containsExactly(subscription.getName()); + } + + @Test + public void shouldListSubscriptionAfterOwnerIsChanged() { + // given + Subscription subscription = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName()) + .withOwner(new OwnerId("Plaintext", "ListSubForOwner - Team D")) + .build()); + + // then + assertThat(listSubscriptionsForOwner("ListSubForOwner - Team D")) + .containsExactly(subscription.getName()); + assertThat(listSubscriptionsForOwner("ListSubForOwner - Team E")).isEmpty(); + + // when + hermes + .api() + .updateSubscription( + topic, + subscription.getName(), + PatchData.patchData() + .set("owner", new OwnerId("Plaintext", "ListSubForOwner - Team E")) + .build()); + + // then + assertThat(listSubscriptionsForOwner("ListSubForOwner - Team D")).isEmpty(); + assertThat(listSubscriptionsForOwner("ListSubForOwner - Team E")) + .containsExactly(subscription.getName()); + } + + @Test + public void shouldNotListTopicAfterIsDeleted() { + // given + Subscription subscription1 = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName()) + .withOwner(new OwnerId("Plaintext", "ListSubForOwner - Team F")) + .build()); + Subscription subscription2 = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName()) + .withOwner(new OwnerId("Plaintext", "ListSubForOwner - Team G")) + .build()); + + // then + assertThat(listSubscriptionsForOwner("ListSubForOwner - Team F")) + .containsExactly(subscription1.getName()); + assertThat(listSubscriptionsForOwner("ListSubForOwner - Team G")) + .containsExactly(subscription2.getName()); + + // when + hermes + .api() + .deleteSubscription(topic.getQualifiedName(), subscription1.getName()) + .expectStatus() + .is2xxSuccessful(); + + // then + assertThat(listSubscriptionsForOwner("ListSubForOwner - Team F")).isEmpty(); + assertThat(listSubscriptionsForOwner("ListSubForOwner - Team G")) + .containsExactly(subscription2.getName()); + } + + private List listSubscriptionsForOwner(String ownerId) { + return Objects.requireNonNull( + hermes + .api() + .getSubscriptionsForOwner("Plaintext", ownerId) + .expectStatus() + .is2xxSuccessful() + .expectBodyList(Subscription.class) + .returnResult() + .getResponseBody()) + .stream() + .map(Subscription::getName) + .collect(Collectors.toList()); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/ListTopicForOwnerTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/ListTopicForOwnerTest.java index d765a59bc1..ab74274be6 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/ListTopicForOwnerTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/ListTopicForOwnerTest.java @@ -1,5 +1,11 @@ package pl.allegro.tech.hermes.integrationtests.management; +import static org.assertj.core.api.Assertions.assertThat; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import pl.allegro.tech.hermes.api.OwnerId; @@ -7,83 +13,124 @@ import pl.allegro.tech.hermes.api.Topic; import pl.allegro.tech.hermes.integrationtests.setup.HermesExtension; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - -import static org.assertj.core.api.Assertions.assertThat; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class ListTopicForOwnerTest { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - @Test - public void shouldListTopicsForOwnerId() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().withOwner(new OwnerId("Plaintext", "ListTopicForOwner - Team A")).build()); - Topic topic2 = hermes.initHelper().createTopic(topicWithRandomName().withOwner(new OwnerId("Plaintext", "ListTopicForOwner - Team A")).build()); - Topic topic3 = hermes.initHelper().createTopic(topicWithRandomName().withOwner(new OwnerId("Plaintext", "ListTopicForOwner - Team B")).build()); - - // expect - assertThat(listTopicsForOwner("ListTopicForOwner - Team A")).containsExactly(topic.getQualifiedName(), topic2.getQualifiedName()); - assertThat(listTopicsForOwner("ListTopicForOwner - Team B")).containsExactly(topic3.getQualifiedName()); - } - - @Test - public void shouldListTopicAfterNewTopicIsAdded() { - // expect empty list on start - assertThat(listTopicsForOwner("ListTopicForOwner - Team C")).isEmpty(); - - // when - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().withOwner(new OwnerId("Plaintext", "ListTopicForOwner - Team C")).build()); - - // then - assertThat(listTopicsForOwner("ListTopicForOwner - Team C")).containsExactly(topic.getQualifiedName()); - } - - @Test - public void shouldListTopicAfterOwnerIsChanged() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().withOwner(new OwnerId("Plaintext", "ListTopicForOwner - Team D")).build()); - - // then - assertThat(listTopicsForOwner("ListTopicForOwner - Team D")).containsExactly(topic.getQualifiedName()); - assertThat(listTopicsForOwner("ListTopicForOwner - Team E")).isEmpty(); - - // when - hermes.api().updateTopic(topic.getQualifiedName(), - PatchData.patchData().set("owner", new OwnerId("Plaintext", "ListTopicForOwner - Team E")).build()); - - // then - assertThat(listTopicsForOwner("ListTopicForOwner - Team D")).isEmpty(); - assertThat(listTopicsForOwner("ListTopicForOwner - Team E")).containsExactly(topic.getQualifiedName()); - } - - @Test - public void shouldNotListTopicAfterTopicIsDeleted() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().withOwner(new OwnerId("Plaintext", "ListTopicForOwner - Team F")).build()); - assertThat(listTopicsForOwner("ListTopicForOwner - Team F")).containsExactly(topic.getQualifiedName()); - - // when - hermes.api().deleteTopic(topic.getQualifiedName()).expectStatus().is2xxSuccessful(); - - // then - assertThat(listTopicsForOwner("ListTopicForOwner - Team F")).isEmpty(); - } - - private List listTopicsForOwner(String ownerId) { - return Objects.requireNonNull(hermes.api().getTopicsForOwner("Plaintext", ownerId) - .expectStatus() - .is2xxSuccessful() - .expectBodyList(Topic.class) - .returnResult() - .getResponseBody()) - .stream() - .map(Topic::getQualifiedName) - .collect(Collectors.toList()); - } + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); + + @Test + public void shouldListTopicsForOwnerId() { + // given + Topic topic = + hermes + .initHelper() + .createTopic( + topicWithRandomName() + .withOwner(new OwnerId("Plaintext", "ListTopicForOwner - Team A")) + .build()); + Topic topic2 = + hermes + .initHelper() + .createTopic( + topicWithRandomName() + .withOwner(new OwnerId("Plaintext", "ListTopicForOwner - Team A")) + .build()); + Topic topic3 = + hermes + .initHelper() + .createTopic( + topicWithRandomName() + .withOwner(new OwnerId("Plaintext", "ListTopicForOwner - Team B")) + .build()); + + // expect + assertThat(listTopicsForOwner("ListTopicForOwner - Team A")) + .containsExactly(topic.getQualifiedName(), topic2.getQualifiedName()); + assertThat(listTopicsForOwner("ListTopicForOwner - Team B")) + .containsExactly(topic3.getQualifiedName()); + } + + @Test + public void shouldListTopicAfterNewTopicIsAdded() { + // expect empty list on start + assertThat(listTopicsForOwner("ListTopicForOwner - Team C")).isEmpty(); + + // when + Topic topic = + hermes + .initHelper() + .createTopic( + topicWithRandomName() + .withOwner(new OwnerId("Plaintext", "ListTopicForOwner - Team C")) + .build()); + + // then + assertThat(listTopicsForOwner("ListTopicForOwner - Team C")) + .containsExactly(topic.getQualifiedName()); + } + + @Test + public void shouldListTopicAfterOwnerIsChanged() { + // given + Topic topic = + hermes + .initHelper() + .createTopic( + topicWithRandomName() + .withOwner(new OwnerId("Plaintext", "ListTopicForOwner - Team D")) + .build()); + + // then + assertThat(listTopicsForOwner("ListTopicForOwner - Team D")) + .containsExactly(topic.getQualifiedName()); + assertThat(listTopicsForOwner("ListTopicForOwner - Team E")).isEmpty(); + + // when + hermes + .api() + .updateTopic( + topic.getQualifiedName(), + PatchData.patchData() + .set("owner", new OwnerId("Plaintext", "ListTopicForOwner - Team E")) + .build()); + + // then + assertThat(listTopicsForOwner("ListTopicForOwner - Team D")).isEmpty(); + assertThat(listTopicsForOwner("ListTopicForOwner - Team E")) + .containsExactly(topic.getQualifiedName()); + } + + @Test + public void shouldNotListTopicAfterTopicIsDeleted() { + // given + Topic topic = + hermes + .initHelper() + .createTopic( + topicWithRandomName() + .withOwner(new OwnerId("Plaintext", "ListTopicForOwner - Team F")) + .build()); + assertThat(listTopicsForOwner("ListTopicForOwner - Team F")) + .containsExactly(topic.getQualifiedName()); + + // when + hermes.api().deleteTopic(topic.getQualifiedName()).expectStatus().is2xxSuccessful(); + + // then + assertThat(listTopicsForOwner("ListTopicForOwner - Team F")).isEmpty(); + } + + private List listTopicsForOwner(String ownerId) { + return Objects.requireNonNull( + hermes + .api() + .getTopicsForOwner("Plaintext", ownerId) + .expectStatus() + .is2xxSuccessful() + .expectBodyList(Topic.class) + .returnResult() + .getResponseBody()) + .stream() + .map(Topic::getQualifiedName) + .collect(Collectors.toList()); + } } - diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/ListUnhealthySubscriptionsForOwnerTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/ListUnhealthySubscriptionsForOwnerTest.java index c5a24d86cd..924ca0f593 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/ListUnhealthySubscriptionsForOwnerTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/ListUnhealthySubscriptionsForOwnerTest.java @@ -1,5 +1,15 @@ package pl.allegro.tech.hermes.integrationtests.management; +import static org.assertj.core.api.Assertions.assertThat; +import static pl.allegro.tech.hermes.api.MonitoringDetails.Severity; +import static pl.allegro.tech.hermes.api.SubscriptionHealthProblem.malfunctioning; +import static pl.allegro.tech.hermes.integrationtests.prometheus.SubscriptionMetrics.subscriptionMetrics; +import static pl.allegro.tech.hermes.integrationtests.prometheus.TopicMetrics.topicMetrics; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + +import java.time.Duration; +import java.util.Set; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Order; @@ -14,374 +24,332 @@ import pl.allegro.tech.hermes.integrationtests.prometheus.PrometheusExtension; import pl.allegro.tech.hermes.integrationtests.setup.HermesExtension; -import java.time.Duration; -import java.util.Set; - -import static org.assertj.core.api.Assertions.assertThat; -import static pl.allegro.tech.hermes.api.MonitoringDetails.Severity; -import static pl.allegro.tech.hermes.api.SubscriptionHealthProblem.malfunctioning; -import static pl.allegro.tech.hermes.integrationtests.prometheus.SubscriptionMetrics.subscriptionMetrics; -import static pl.allegro.tech.hermes.integrationtests.prometheus.TopicMetrics.topicMetrics; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class ListUnhealthySubscriptionsForOwnerTest { - @Order(0) - @RegisterExtension - public static final PrometheusExtension prometheus = new PrometheusExtension(); + @Order(0) + @RegisterExtension + public static final PrometheusExtension prometheus = new PrometheusExtension(); - @Order(1) - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension() - .withPrometheus(prometheus); + @Order(1) + @RegisterExtension + public static final HermesExtension hermes = new HermesExtension().withPrometheus(prometheus); - @BeforeAll - static void setup() { - hermes.clearManagementData(); - } + @BeforeAll + static void setup() { + hermes.clearManagementData(); + } - @AfterEach - void cleanup() { - hermes.clearManagementData(); - } + @AfterEach + void cleanup() { + hermes.clearManagementData(); + } - @Test - public void shouldNotListHealthySubscriptions() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription healthySubscription = createSubscriptionOwnedBy("Team A", topic); - prometheus.stubTopicMetrics( - topicMetrics(topic.getName()) - .withRate(100) - .withDeliveryRate(100) - .build() - ); - prometheus.stubSubscriptionMetrics( - subscriptionMetrics(healthySubscription.getQualifiedName()) - .withRate(100) - .build() - ); + @Test + public void shouldNotListHealthySubscriptions() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription healthySubscription = createSubscriptionOwnedBy("Team A", topic); + prometheus.stubTopicMetrics( + topicMetrics(topic.getName()).withRate(100).withDeliveryRate(100).build()); + prometheus.stubSubscriptionMetrics( + subscriptionMetrics(healthySubscription.getQualifiedName()).withRate(100).build()); - // when / then - hermes.api().listUnhealthyForOwner("Team A") - .expectStatus() - .isOk() - .expectBodyList(UnhealthySubscription.class) - .hasSize(0); - hermes.api().listUnhealthyForOwnerAsPlainText("Team A") - .expectStatus() - .isOk() - .expectBody() - .isEmpty(); - } + // when / then + hermes + .api() + .listUnhealthyForOwner("Team A") + .expectStatus() + .isOk() + .expectBodyList(UnhealthySubscription.class) + .hasSize(0); + hermes + .api() + .listUnhealthyForOwnerAsPlainText("Team A") + .expectStatus() + .isOk() + .expectBody() + .isEmpty(); + } - @Test - public void shouldReturnOnlyUnhealthySubscriptionOfSingleOwner() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription healthySubscription = createSubscriptionOwnedBy("Team A", topic); - Subscription unhealthySubscription = createSubscriptionOwnedBy("Team A", topic); - prometheus.stubTopicMetrics( - topicMetrics(topic.getName()) - .withRate(100) - .withDeliveryRate(50) - .build() - ); - prometheus.stubSubscriptionMetrics( - subscriptionMetrics(healthySubscription.getQualifiedName()) - .withRate(100) - .build() - ); - prometheus.stubSubscriptionMetrics( - subscriptionMetrics(unhealthySubscription.getQualifiedName()) - .withRate(50) - .with500Rate(11) - .build() - ); + @Test + public void shouldReturnOnlyUnhealthySubscriptionOfSingleOwner() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription healthySubscription = createSubscriptionOwnedBy("Team A", topic); + Subscription unhealthySubscription = createSubscriptionOwnedBy("Team A", topic); + prometheus.stubTopicMetrics( + topicMetrics(topic.getName()).withRate(100).withDeliveryRate(50).build()); + prometheus.stubSubscriptionMetrics( + subscriptionMetrics(healthySubscription.getQualifiedName()).withRate(100).build()); + prometheus.stubSubscriptionMetrics( + subscriptionMetrics(unhealthySubscription.getQualifiedName()) + .withRate(50) + .with500Rate(11) + .build()); - // when / then - hermes.api().listUnhealthyForOwner("Team A") - .expectStatus() - .isOk() - .expectBodyList(UnhealthySubscription.class) - .contains(malfunctioningSubscription(unhealthySubscription, 11)); - hermes.api().listUnhealthyForOwnerAsPlainText("Team A") - .expectStatus() - .isOk() - .expectBody(String.class) - .isEqualTo("%s - Consuming service returns a lot of 5xx codes for subscription %s, currently 11 5xx/s" - .formatted(unhealthySubscription.getName(), unhealthySubscription.getQualifiedName().toString()) - ); - } + // when / then + hermes + .api() + .listUnhealthyForOwner("Team A") + .expectStatus() + .isOk() + .expectBodyList(UnhealthySubscription.class) + .contains(malfunctioningSubscription(unhealthySubscription, 11)); + hermes + .api() + .listUnhealthyForOwnerAsPlainText("Team A") + .expectStatus() + .isOk() + .expectBody(String.class) + .isEqualTo( + "%s - Consuming service returns a lot of 5xx codes for subscription %s, currently 11 5xx/s" + .formatted( + unhealthySubscription.getName(), + unhealthySubscription.getQualifiedName().toString())); + } - @Test - public void shouldReportAllUnhealthySubscriptions() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription healthySubscription = createSubscriptionOwnedBy("Team A", topic); - Subscription unhealthySubscription = createSubscriptionOwnedBy("Team A", topic); - prometheus.stubTopicMetrics( - topicMetrics(topic.getName()) - .withRate(100) - .withDeliveryRate(50) - .build() - ); - prometheus.stubSubscriptionMetrics( - subscriptionMetrics(healthySubscription.getQualifiedName()) - .withRate(100) - .build() - ); - prometheus.stubSubscriptionMetrics( - subscriptionMetrics(unhealthySubscription.getQualifiedName()) - .withRate(50) - .with500Rate(11) - .build() - ); + @Test + public void shouldReportAllUnhealthySubscriptions() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription healthySubscription = createSubscriptionOwnedBy("Team A", topic); + Subscription unhealthySubscription = createSubscriptionOwnedBy("Team A", topic); + prometheus.stubTopicMetrics( + topicMetrics(topic.getName()).withRate(100).withDeliveryRate(50).build()); + prometheus.stubSubscriptionMetrics( + subscriptionMetrics(healthySubscription.getQualifiedName()).withRate(100).build()); + prometheus.stubSubscriptionMetrics( + subscriptionMetrics(unhealthySubscription.getQualifiedName()) + .withRate(50) + .with500Rate(11) + .build()); - // when / then - hermes.api().listUnhealthy() - .expectStatus() - .isOk() - .expectBodyList(UnhealthySubscription.class) - .contains(malfunctioningSubscription(unhealthySubscription, 11)); - hermes.api().listUnhealthyAsPlainText() - .expectStatus() - .isOk() - .expectBody(String.class) - .isEqualTo("%s - Consuming service returns a lot of 5xx codes for subscription %s, currently 11 5xx/s" - .formatted(unhealthySubscription.getName(), unhealthySubscription.getQualifiedName().toString()) - ); - } + // when / then + hermes + .api() + .listUnhealthy() + .expectStatus() + .isOk() + .expectBodyList(UnhealthySubscription.class) + .contains(malfunctioningSubscription(unhealthySubscription, 11)); + hermes + .api() + .listUnhealthyAsPlainText() + .expectStatus() + .isOk() + .expectBody(String.class) + .isEqualTo( + "%s - Consuming service returns a lot of 5xx codes for subscription %s, currently 11 5xx/s" + .formatted( + unhealthySubscription.getName(), + unhealthySubscription.getQualifiedName().toString())); + } - @Test - public void shouldReturnUnhealthySubscriptionsFilteredByTopic() { - // given - Topic topic1 = hermes.initHelper().createTopic(topicWithRandomName().build()); - Topic topic2 = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription unhealthySubscriptionTopic1 = createSubscriptionOwnedBy("Team A", topic1); - Subscription unhealthySubscriptionTopic2 = createSubscriptionOwnedBy("Team A", topic2); - prometheus.stubTopicMetrics( - topicMetrics(topic1.getName()) - .withRate(100) - .withDeliveryRate(50) - .build() - ); - prometheus.stubTopicMetrics( - topicMetrics(topic2.getName()) - .withRate(100) - .withDeliveryRate(50) - .build() - ); - prometheus.stubSubscriptionMetrics( - subscriptionMetrics(unhealthySubscriptionTopic1.getQualifiedName()) - .withRate(50) - .with500Rate(11) - .build() - ); - prometheus.stubSubscriptionMetrics( - subscriptionMetrics(unhealthySubscriptionTopic2.getQualifiedName()) - .withRate(50) - .with500Rate(11) - .build() - ); + @Test + public void shouldReturnUnhealthySubscriptionsFilteredByTopic() { + // given + Topic topic1 = hermes.initHelper().createTopic(topicWithRandomName().build()); + Topic topic2 = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription unhealthySubscriptionTopic1 = createSubscriptionOwnedBy("Team A", topic1); + Subscription unhealthySubscriptionTopic2 = createSubscriptionOwnedBy("Team A", topic2); + prometheus.stubTopicMetrics( + topicMetrics(topic1.getName()).withRate(100).withDeliveryRate(50).build()); + prometheus.stubTopicMetrics( + topicMetrics(topic2.getName()).withRate(100).withDeliveryRate(50).build()); + prometheus.stubSubscriptionMetrics( + subscriptionMetrics(unhealthySubscriptionTopic1.getQualifiedName()) + .withRate(50) + .with500Rate(11) + .build()); + prometheus.stubSubscriptionMetrics( + subscriptionMetrics(unhealthySubscriptionTopic2.getQualifiedName()) + .withRate(50) + .with500Rate(11) + .build()); - // when / then - hermes.api().listUnhealthyForTopic(topic2.getQualifiedName()) - .expectStatus() - .isOk() - .expectBodyList(UnhealthySubscription.class) - .contains(malfunctioningSubscription(unhealthySubscriptionTopic2, 11)); - hermes.api().listUnhealthyForTopicAsPlainText(topic2.getQualifiedName()) - .expectStatus() - .isOk() - .expectBody(String.class) - .isEqualTo("%s - Consuming service returns a lot of 5xx codes for subscription %s, currently 11 5xx/s" - .formatted(unhealthySubscriptionTopic2.getName(), unhealthySubscriptionTopic2.getQualifiedName().toString()) - ); - } + // when / then + hermes + .api() + .listUnhealthyForTopic(topic2.getQualifiedName()) + .expectStatus() + .isOk() + .expectBodyList(UnhealthySubscription.class) + .contains(malfunctioningSubscription(unhealthySubscriptionTopic2, 11)); + hermes + .api() + .listUnhealthyForTopicAsPlainText(topic2.getQualifiedName()) + .expectStatus() + .isOk() + .expectBody(String.class) + .isEqualTo( + "%s - Consuming service returns a lot of 5xx codes for subscription %s, currently 11 5xx/s" + .formatted( + unhealthySubscriptionTopic2.getName(), + unhealthySubscriptionTopic2.getQualifiedName().toString())); + } - @Test - public void shouldReturnSpecificUnhealthySubscription() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription healthySubscription = createSubscriptionOwnedBy("Team A", topic); - Subscription unhealthySubscription = createSubscriptionOwnedBy("Team A", topic); - prometheus.stubTopicMetrics( - topicMetrics(topic.getName()) - .withRate(100) - .withDeliveryRate(50) - .build() - ); - prometheus.stubSubscriptionMetrics( - subscriptionMetrics(healthySubscription.getQualifiedName()) - .withRate(100) - .build() - ); - prometheus.stubSubscriptionMetrics( - subscriptionMetrics(unhealthySubscription.getQualifiedName()) - .withRate(50) - .with500Rate(11) - .build() - ); + @Test + public void shouldReturnSpecificUnhealthySubscription() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription healthySubscription = createSubscriptionOwnedBy("Team A", topic); + Subscription unhealthySubscription = createSubscriptionOwnedBy("Team A", topic); + prometheus.stubTopicMetrics( + topicMetrics(topic.getName()).withRate(100).withDeliveryRate(50).build()); + prometheus.stubSubscriptionMetrics( + subscriptionMetrics(healthySubscription.getQualifiedName()).withRate(100).build()); + prometheus.stubSubscriptionMetrics( + subscriptionMetrics(unhealthySubscription.getQualifiedName()) + .withRate(50) + .with500Rate(11) + .build()); - // when / then - hermes.api().listUnhealthyForSubscription(topic.getQualifiedName(), healthySubscription.getName()) - .expectStatus() - .isOk() - .expectBodyList(UnhealthySubscription.class) - .hasSize(0); - hermes.api().listUnhealthyForSubscriptionAsPlainText(topic.getQualifiedName(), healthySubscription.getName()) - .expectStatus() - .isOk() - .expectBody() - .isEmpty(); - hermes.api().listUnhealthyForSubscription(topic.getQualifiedName(), unhealthySubscription.getName()) - .expectStatus() - .isOk() - .expectBodyList(UnhealthySubscription.class) - .contains(malfunctioningSubscription(unhealthySubscription, 11)); - hermes.api().listUnhealthyForSubscriptionAsPlainText(topic.getQualifiedName(), unhealthySubscription.getName()) - .expectStatus() - .isOk() - .expectBody(String.class) - .isEqualTo("%s - Consuming service returns a lot of 5xx codes for subscription %s, currently 11 5xx/s" - .formatted(unhealthySubscription.getName(), unhealthySubscription.getQualifiedName().toString()) - ); - } + // when / then + hermes + .api() + .listUnhealthyForSubscription(topic.getQualifiedName(), healthySubscription.getName()) + .expectStatus() + .isOk() + .expectBodyList(UnhealthySubscription.class) + .hasSize(0); + hermes + .api() + .listUnhealthyForSubscriptionAsPlainText( + topic.getQualifiedName(), healthySubscription.getName()) + .expectStatus() + .isOk() + .expectBody() + .isEmpty(); + hermes + .api() + .listUnhealthyForSubscription(topic.getQualifiedName(), unhealthySubscription.getName()) + .expectStatus() + .isOk() + .expectBodyList(UnhealthySubscription.class) + .contains(malfunctioningSubscription(unhealthySubscription, 11)); + hermes + .api() + .listUnhealthyForSubscriptionAsPlainText( + topic.getQualifiedName(), unhealthySubscription.getName()) + .expectStatus() + .isOk() + .expectBody(String.class) + .isEqualTo( + "%s - Consuming service returns a lot of 5xx codes for subscription %s, currently 11 5xx/s" + .formatted( + unhealthySubscription.getName(), + unhealthySubscription.getQualifiedName().toString())); + } - @Test - public void shouldReportUnhealthySubscriptionsDisrespectingSeverity() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription1 = createSubscriptionOwnedBy("Team A", topic, Severity.CRITICAL); - Subscription subscription2 = createSubscriptionOwnedBy("Team A", topic, Severity.IMPORTANT); - Subscription subscription3 = createSubscriptionOwnedBy("Team A", topic, Severity.NON_IMPORTANT); - prometheus.stubTopicMetrics( - topicMetrics(topic.getName()) - .withRate(100) - .withDeliveryRate(50) - .build() - ); - prometheus.stubSubscriptionMetrics( - subscriptionMetrics(subscription1.getQualifiedName()) - .withRate(50) - .with500Rate(11) - .build() - ); - prometheus.stubSubscriptionMetrics( - subscriptionMetrics(subscription2.getQualifiedName()) - .withRate(50) - .with500Rate(11) - .build() - ); - prometheus.stubSubscriptionMetrics( - subscriptionMetrics(subscription3.getQualifiedName()) - .withRate(50) - .with500Rate(11) - .build() - ); + @Test + public void shouldReportUnhealthySubscriptionsDisrespectingSeverity() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription1 = createSubscriptionOwnedBy("Team A", topic, Severity.CRITICAL); + Subscription subscription2 = createSubscriptionOwnedBy("Team A", topic, Severity.IMPORTANT); + Subscription subscription3 = createSubscriptionOwnedBy("Team A", topic, Severity.NON_IMPORTANT); + prometheus.stubTopicMetrics( + topicMetrics(topic.getName()).withRate(100).withDeliveryRate(50).build()); + prometheus.stubSubscriptionMetrics( + subscriptionMetrics(subscription1.getQualifiedName()).withRate(50).with500Rate(11).build()); + prometheus.stubSubscriptionMetrics( + subscriptionMetrics(subscription2.getQualifiedName()).withRate(50).with500Rate(11).build()); + prometheus.stubSubscriptionMetrics( + subscriptionMetrics(subscription3.getQualifiedName()).withRate(50).with500Rate(11).build()); - // when / then - hermes.api().listUnhealthyForOwner("Team A") - .expectStatus() - .isOk() - .expectBodyList(UnhealthySubscription.class) - .contains( - malfunctioningSubscription(subscription1, 11), - malfunctioningSubscription(subscription2, 11), - malfunctioningSubscription(subscription3, 11) - ); - hermes.api().listUnhealthyForOwnerAsPlainText("Team A") - .expectStatus() - .isOk() - .expectBody(String.class) - .isEqualTo(""" + // when / then + hermes + .api() + .listUnhealthyForOwner("Team A") + .expectStatus() + .isOk() + .expectBodyList(UnhealthySubscription.class) + .contains( + malfunctioningSubscription(subscription1, 11), + malfunctioningSubscription(subscription2, 11), + malfunctioningSubscription(subscription3, 11)); + hermes + .api() + .listUnhealthyForOwnerAsPlainText("Team A") + .expectStatus() + .isOk() + .expectBody(String.class) + .isEqualTo( + """ %s - Consuming service returns a lot of 5xx codes for subscription %s, currently 11 5xx/s\r %s - Consuming service returns a lot of 5xx codes for subscription %s, currently 11 5xx/s\r %s - Consuming service returns a lot of 5xx codes for subscription %s, currently 11 5xx/s""" - .formatted( - subscription1.getName(), subscription1.getQualifiedName().toString(), - subscription2.getName(), subscription2.getQualifiedName().toString(), - subscription3.getName(), subscription3.getQualifiedName().toString() - ) - ); - } + .formatted( + subscription1.getName(), subscription1.getQualifiedName().toString(), + subscription2.getName(), subscription2.getQualifiedName().toString(), + subscription3.getName(), subscription3.getQualifiedName().toString())); + } - @Test - public void shouldTimeoutUnhealthySubscriptionsRequest() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - createSubscriptionOwnedBy("Team A", topic); - prometheus.stubDelay(Duration.ofMillis(3000)); + @Test + public void shouldTimeoutUnhealthySubscriptionsRequest() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + createSubscriptionOwnedBy("Team A", topic); + prometheus.stubDelay(Duration.ofMillis(3000)); - // when - long start = System.currentTimeMillis(); - WebTestClient.ResponseSpec response = hermes.api().listUnhealthyAsPlainText(); - long end = System.currentTimeMillis(); + // when + long start = System.currentTimeMillis(); + WebTestClient.ResponseSpec response = hermes.api().listUnhealthyAsPlainText(); + long end = System.currentTimeMillis(); - // then - response.expectStatus() - .isOk() - .expectBody() - .isEmpty(); - assertThat(end - start).isLessThan(3000); - } + // then + response.expectStatus().isOk().expectBody().isEmpty(); + assertThat(end - start).isLessThan(3000); + } - @Test - public void shouldReportSuspendedSubscriptionAsHealthy() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription unhealthySubscription = createSubscriptionOwnedBy("Team A", topic); - prometheus.stubTopicMetrics( - topicMetrics(topic.getName()) - .withRate(100) - .withDeliveryRate(50) - .build() - ); - prometheus.stubSubscriptionMetrics( - subscriptionMetrics(unhealthySubscription.getQualifiedName()) - .withRate(50) - .with500Rate(11) - .build() - ); + @Test + public void shouldReportSuspendedSubscriptionAsHealthy() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription unhealthySubscription = createSubscriptionOwnedBy("Team A", topic); + prometheus.stubTopicMetrics( + topicMetrics(topic.getName()).withRate(100).withDeliveryRate(50).build()); + prometheus.stubSubscriptionMetrics( + subscriptionMetrics(unhealthySubscription.getQualifiedName()) + .withRate(50) + .with500Rate(11) + .build()); - // when - hermes.api().suspendSubscription(topic, unhealthySubscription.getName()); + // when + hermes.api().suspendSubscription(topic, unhealthySubscription.getName()); - // then - hermes.api().listUnhealthy() - .expectStatus() - .isOk() - .expectBodyList(UnhealthySubscription.class) - .hasSize(0); - hermes.api().listUnhealthyAsPlainText() - .expectStatus() - .isOk() - .expectBody() - .isEmpty(); - } + // then + hermes + .api() + .listUnhealthy() + .expectStatus() + .isOk() + .expectBodyList(UnhealthySubscription.class) + .hasSize(0); + hermes.api().listUnhealthyAsPlainText().expectStatus().isOk().expectBody().isEmpty(); + } - private Subscription createSubscriptionOwnedBy(String ownerId, Topic topic) { - return createSubscriptionOwnedBy(ownerId, topic, Severity.IMPORTANT); - } + private Subscription createSubscriptionOwnedBy(String ownerId, Topic topic) { + return createSubscriptionOwnedBy(ownerId, topic, Severity.IMPORTANT); + } - private Subscription createSubscriptionOwnedBy(String ownerId, Topic topic, Severity severity) { - return hermes.initHelper().createSubscription( - subscriptionWithRandomName(topic.getName()) - .withOwner(new OwnerId("Plaintext", ownerId)) - .withMonitoringDetails(new MonitoringDetails(severity, "")) - .withEndpoint("http://localhost:8080") - .build() - ); - } + private Subscription createSubscriptionOwnedBy(String ownerId, Topic topic, Severity severity) { + return hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName()) + .withOwner(new OwnerId("Plaintext", ownerId)) + .withMonitoringDetails(new MonitoringDetails(severity, "")) + .withEndpoint("http://localhost:8080") + .build()); + } - private static UnhealthySubscription malfunctioningSubscription(Subscription subscription, double code5xxErrorsRate) { - return new UnhealthySubscription( - subscription.getName(), - subscription.getTopicName().qualifiedName(), - subscription.getMonitoringDetails().getSeverity(), - Set.of(malfunctioning(code5xxErrorsRate, subscription.getQualifiedName().toString())) - ); - } + private static UnhealthySubscription malfunctioningSubscription( + Subscription subscription, double code5xxErrorsRate) { + return new UnhealthySubscription( + subscription.getName(), + subscription.getTopicName().qualifiedName(), + subscription.getMonitoringDetails().getSeverity(), + Set.of(malfunctioning(code5xxErrorsRate, subscription.getQualifiedName().toString()))); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/MessagePreviewIntegrationTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/MessagePreviewIntegrationTest.java index 026de4a9fc..b3fe546b85 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/MessagePreviewIntegrationTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/MessagePreviewIntegrationTest.java @@ -1,5 +1,14 @@ package pl.allegro.tech.hermes.integrationtests.management; +import static net.javacrumbs.jsonunit.fluent.JsonFluentAssert.assertThatJson; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static pl.allegro.tech.hermes.api.ContentType.AVRO; +import static pl.allegro.tech.hermes.api.TopicWithSchema.topicWithSchema; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + +import java.time.Duration; +import java.util.List; import net.javacrumbs.jsonunit.core.Option; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -9,46 +18,43 @@ import pl.allegro.tech.hermes.integrationtests.setup.HermesExtension; import pl.allegro.tech.hermes.test.helper.avro.AvroUser; -import java.time.Duration; -import java.util.List; - -import static net.javacrumbs.jsonunit.fluent.JsonFluentAssert.assertThatJson; -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; -import static pl.allegro.tech.hermes.api.ContentType.AVRO; -import static pl.allegro.tech.hermes.api.TopicWithSchema.topicWithSchema; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class MessagePreviewIntegrationTest { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - @Test - public void shouldReturnAvroMessageWithSchemaAwareSerialization() { - // given - AvroUser avroUser = new AvroUser("Bob", 50, "blue"); - TopicWithSchema topicWithSchema = topicWithSchema(topicWithRandomName() - .withContentType(AVRO) - .withSchemaIdAwareSerialization() - .build(), avroUser.getSchemaAsString()); - - Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); - - hermes.api().publishAvroUntilSuccess(topic.getQualifiedName(), avroUser.asBytes()); - - await().atMost(Duration.ofSeconds(10)).untilAsserted(() -> { - // when - List previews = hermes.api().getPreview(topic.getQualifiedName()) - .expectStatus().isOk() - .expectBodyList(MessageTextPreview.class).returnResult().getResponseBody(); - - // then - assertThat(previews).hasSize(1); - assertThatJson(previews.get(0).getContent()) - .when(Option.IGNORING_EXTRA_FIELDS) - .isEqualTo(avroUser.asJson()); - } - ); - } + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); + + @Test + public void shouldReturnAvroMessageWithSchemaAwareSerialization() { + // given + AvroUser avroUser = new AvroUser("Bob", 50, "blue"); + TopicWithSchema topicWithSchema = + topicWithSchema( + topicWithRandomName().withContentType(AVRO).withSchemaIdAwareSerialization().build(), + avroUser.getSchemaAsString()); + + Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); + + hermes.api().publishAvroUntilSuccess(topic.getQualifiedName(), avroUser.asBytes()); + + await() + .atMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + // when + List previews = + hermes + .api() + .getPreview(topic.getQualifiedName()) + .expectStatus() + .isOk() + .expectBodyList(MessageTextPreview.class) + .returnResult() + .getResponseBody(); + + // then + assertThat(previews).hasSize(1); + assertThatJson(previews.get(0).getContent()) + .when(Option.IGNORING_EXTRA_FIELDS) + .isEqualTo(avroUser.asJson()); + }); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/OAuthProviderManagementTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/OAuthProviderManagementTest.java index dec298de98..4bc203518f 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/OAuthProviderManagementTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/OAuthProviderManagementTest.java @@ -1,5 +1,9 @@ package pl.allegro.tech.hermes.integrationtests.management; +import static org.assertj.core.api.Assertions.assertThat; +import static pl.allegro.tech.hermes.api.PatchData.patchData; +import static pl.allegro.tech.hermes.test.helper.builder.OAuthProviderBuilder.oAuthProvider; + import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.springframework.test.web.reactive.server.WebTestClient.ResponseSpec; @@ -9,84 +13,101 @@ import pl.allegro.tech.hermes.api.PatchData; import pl.allegro.tech.hermes.integrationtests.setup.HermesExtension; -import static org.assertj.core.api.Assertions.assertThat; -import static pl.allegro.tech.hermes.api.PatchData.patchData; -import static pl.allegro.tech.hermes.test.helper.builder.OAuthProviderBuilder.oAuthProvider; - public class OAuthProviderManagementTest { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - @Test - public void shouldCreateOAuthProvider() { - // given - OAuthProvider oAuthProvider = oAuthProvider("myProvider").withSocketTimeout(200).build(); - - // when - ResponseSpec response = hermes.api().createOAuthProvider(oAuthProvider); - - // then - response.expectStatus().isCreated(); - OAuthProvider createdProvider = hermes.api().getOAuthProvider(oAuthProvider.getName()) - .expectStatus().isOk() - .expectBody(OAuthProvider.class).returnResult().getResponseBody(); - assertThat(createdProvider).isNotNull(); - assertThat(createdProvider.getSocketTimeout()).isEqualTo(200); - } - - @Test - public void shouldNotAllowCreatingTheSameOAuthProviderTwice() { - // given - OAuthProvider oAuthProvider = oAuthProvider("originalProvider").build(); - hermes.initHelper().createOAuthProvider(oAuthProvider); - - // when - ResponseSpec secondTryResponse = hermes.api().createOAuthProvider(oAuthProvider); - - // then - ErrorDescription error = secondTryResponse - .expectStatus().isBadRequest() - .expectBody(ErrorDescription.class).returnResult().getResponseBody(); - assertThat(error).isNotNull(); - assertThat(error.getCode()).isEqualTo(ErrorCode.OAUTH_PROVIDER_ALREADY_EXISTS); - } - - @Test - public void shouldUpdateOAuthProvider() { - // given - OAuthProvider oAuthProvider = oAuthProvider("myOtherProvider").build(); - hermes.initHelper().createOAuthProvider(oAuthProvider); - - // when - PatchData patch = patchData() - .set("tokenEndpoint", "http://other.example.com/other") - .set("socketTimeout", 100).build(); - ResponseSpec updateResponse = hermes.api().updateOAuthProvider(oAuthProvider.getName(), patch); - - // then - updateResponse.expectStatus().isOk(); - OAuthProvider updatedProvider = hermes.api().getOAuthProvider(oAuthProvider.getName()) - .expectStatus().isOk() - .expectBody(OAuthProvider.class).returnResult().getResponseBody(); - assertThat(updatedProvider).isNotNull(); - assertThat(updatedProvider.getTokenEndpoint()).isEqualTo("http://other.example.com/other"); - assertThat(updatedProvider.getSocketTimeout()).isEqualTo(100); - } - - @Test - public void shouldRemoveOAuthProvider() { - // given - OAuthProvider oAuthProvider = oAuthProvider("myProviderForRemoval").build(); - hermes.initHelper().createOAuthProvider(oAuthProvider); - - // when - ResponseSpec removalResponse = hermes.api().removeOAuthProvider(oAuthProvider.getName()); - - // then - removalResponse.expectStatus().isOk(); - hermes.api().listOAuthProvider() - .expectStatus().isOk() - .expectBodyList(String.class).doesNotContain(oAuthProvider.getName()); - } + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); + + @Test + public void shouldCreateOAuthProvider() { + // given + OAuthProvider oAuthProvider = oAuthProvider("myProvider").withSocketTimeout(200).build(); + + // when + ResponseSpec response = hermes.api().createOAuthProvider(oAuthProvider); + + // then + response.expectStatus().isCreated(); + OAuthProvider createdProvider = + hermes + .api() + .getOAuthProvider(oAuthProvider.getName()) + .expectStatus() + .isOk() + .expectBody(OAuthProvider.class) + .returnResult() + .getResponseBody(); + assertThat(createdProvider).isNotNull(); + assertThat(createdProvider.getSocketTimeout()).isEqualTo(200); + } + + @Test + public void shouldNotAllowCreatingTheSameOAuthProviderTwice() { + // given + OAuthProvider oAuthProvider = oAuthProvider("originalProvider").build(); + hermes.initHelper().createOAuthProvider(oAuthProvider); + + // when + ResponseSpec secondTryResponse = hermes.api().createOAuthProvider(oAuthProvider); + + // then + ErrorDescription error = + secondTryResponse + .expectStatus() + .isBadRequest() + .expectBody(ErrorDescription.class) + .returnResult() + .getResponseBody(); + assertThat(error).isNotNull(); + assertThat(error.getCode()).isEqualTo(ErrorCode.OAUTH_PROVIDER_ALREADY_EXISTS); + } + + @Test + public void shouldUpdateOAuthProvider() { + // given + OAuthProvider oAuthProvider = oAuthProvider("myOtherProvider").build(); + hermes.initHelper().createOAuthProvider(oAuthProvider); + + // when + PatchData patch = + patchData() + .set("tokenEndpoint", "http://other.example.com/other") + .set("socketTimeout", 100) + .build(); + ResponseSpec updateResponse = hermes.api().updateOAuthProvider(oAuthProvider.getName(), patch); + + // then + updateResponse.expectStatus().isOk(); + OAuthProvider updatedProvider = + hermes + .api() + .getOAuthProvider(oAuthProvider.getName()) + .expectStatus() + .isOk() + .expectBody(OAuthProvider.class) + .returnResult() + .getResponseBody(); + assertThat(updatedProvider).isNotNull(); + assertThat(updatedProvider.getTokenEndpoint()).isEqualTo("http://other.example.com/other"); + assertThat(updatedProvider.getSocketTimeout()).isEqualTo(100); + } + + @Test + public void shouldRemoveOAuthProvider() { + // given + OAuthProvider oAuthProvider = oAuthProvider("myProviderForRemoval").build(); + hermes.initHelper().createOAuthProvider(oAuthProvider); + + // when + ResponseSpec removalResponse = hermes.api().removeOAuthProvider(oAuthProvider.getName()); + + // then + removalResponse.expectStatus().isOk(); + hermes + .api() + .listOAuthProvider() + .expectStatus() + .isOk() + .expectBodyList(String.class) + .doesNotContain(oAuthProvider.getName()); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/OfflineRetransmissionManagementTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/OfflineRetransmissionManagementTest.java index 27739c7f72..f53fda8397 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/OfflineRetransmissionManagementTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/OfflineRetransmissionManagementTest.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.integrationtests.management; +import static org.assertj.core.api.Assertions.assertThat; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + +import java.time.Instant; +import java.util.List; import org.jetbrains.annotations.Nullable; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -13,259 +18,256 @@ import pl.allegro.tech.hermes.integrationtests.setup.HermesExtension; import pl.allegro.tech.hermes.management.TestSecurityProvider; -import java.time.Instant; -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class OfflineRetransmissionManagementTest { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - private static final String GROUP = "pl.allegro.retransmission"; - - @BeforeAll - public static void setupGroup() { - hermes.initHelper().createGroup(Group.from(GROUP)); - } - - @AfterEach - public void cleanup() { - deleteTasks(); - } - - @Test - public void shouldCreateRetransmissionTask() { - // given - Topic sourceTopic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Topic targetTopic = hermes.initHelper().createTopic(topicWithRandomName().build()); - - // when - OfflineRetransmissionRequest request = createRequest( - sourceTopic.getQualifiedName(), - targetTopic.getQualifiedName(), null); - WebTestClient.ResponseSpec response = hermes.api().createOfflineRetransmissionTask(request); - Instant now = Instant.now(); - - // then - response.expectStatus().isCreated(); - - // and - List allTasks = getOfflineRetransmissionTasks(); - assertThat(allTasks.size()).isEqualTo(1); - assertThat(allTasks.get(0).getStartTimestamp()).isEqualTo(request.getStartTimestamp()); - assertThat(allTasks.get(0).getEndTimestamp()).isEqualTo(request.getEndTimestamp()); - assertThat(allTasks.get(0).getSourceTopic()).isEqualTo(request.getSourceTopic()); - assertThat(allTasks.get(0).getTargetTopic()).isEqualTo(request.getTargetTopic()); - assertThat(allTasks.get(0).getCreatedAt()).isBefore(now); - } - - @Test - public void shouldCreateRetransmissionTaskWithViewInsteadTopic() { - // given - var targetTopic = hermes.initHelper().createTopic(topicWithRandomName().build()); - - // when - var request = createRequest(null, targetTopic.getQualifiedName(), "testViewPath"); - var response = hermes.api().createOfflineRetransmissionTask(request); - var now = Instant.now(); - - //then - response.expectStatus().isCreated(); - - // and - var allTasks = getOfflineRetransmissionTasks(); - assertThat(allTasks.size()).isEqualTo(1); - assertThat(allTasks.get(0).getStartTimestamp()).isEqualTo(request.getStartTimestamp()); - assertThat(allTasks.get(0).getEndTimestamp()).isEqualTo(request.getEndTimestamp()); - assertThat(allTasks.get(0).getSourceTopic()).isEmpty(); - assertThat(allTasks.get(0).getSourceViewPath()).hasValue("testViewPath"); - assertThat(allTasks.get(0).getTargetTopic()).isEqualTo(request.getTargetTopic()); - assertThat(allTasks.get(0).getCreatedAt()).isBefore(now); - } - - @Test - public void shouldReturnEmptyListIfThereAreNoTasks() { - // expect - assertThat(getOfflineRetransmissionTasks().size()).isEqualTo(0); - } - - @Test - public void shouldReturnClientErrorWhenRequestingRetransmissionWithEmptyData() { - // given - OfflineRetransmissionRequest request = new OfflineRetransmissionRequest( - null, - "", - "", - null, - null - ); - - //when - WebTestClient.ResponseSpec response = hermes.api().createOfflineRetransmissionTask(request); - - // then - response.expectStatus().isBadRequest(); - assertThat(response.expectBody(String.class).returnResult().getResponseBody()).contains( - List.of( - "must contain one defined source of retransmission data - source topic or source view", - "startTimestamp must not be null", - "endTimestamp must not be null") - ); - } - - @Test - public void shouldReturnClientErrorWhenRequestingRetransmissionWithNotExistingSourceTopic() { - // given - Topic targetTopic = hermes.initHelper().createTopic(topicWithRandomName().build()); - OfflineRetransmissionRequest request = createRequest("not.existing.sourceTopic", - targetTopic.getQualifiedName(), null); - - // when - WebTestClient.ResponseSpec response = hermes.api().createOfflineRetransmissionTask(request); - - // then - response.expectStatus().isBadRequest(); - assertThat(response.expectBody(String.class).returnResult().getResponseBody()) - .contains("Source topic does not exist"); - } - - @Test - public void shouldReturnClientErrorWhenRequestingRetransmissionWithNotExistingTargetTopic() { - // given - Topic sourceTopic = hermes.initHelper().createTopic(topicWithRandomName().build()); - OfflineRetransmissionRequest request = createRequest( - sourceTopic.getQualifiedName(), "not.existing.targetTopic", null); - - // when - WebTestClient.ResponseSpec response = hermes.api().createOfflineRetransmissionTask(request); - - // then - response.expectStatus().isBadRequest(); - assertThat(response.expectBody(String.class).returnResult().getResponseBody()) - .contains("Target topic does not exist"); - } - - @Test - public void shouldReturnClientErrorWhenRequestingRetransmissionWithNegativeTimeRange() { - // given - Topic sourceTopic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Topic targetTopic = hermes.initHelper().createTopic(topicWithRandomName().build()); - OfflineRetransmissionRequest request = new OfflineRetransmissionRequest( - null, - sourceTopic.getQualifiedName(), - targetTopic.getQualifiedName(), - Instant.now().toString(), - Instant.now().minusSeconds(1).toString()); - - // when - WebTestClient.ResponseSpec response = hermes.api().createOfflineRetransmissionTask(request); - - // then - response.expectStatus().isBadRequest(); - assertThat(response.expectBody(String.class).returnResult().getResponseBody()) - .contains("End timestamp must be greater than start timestamp"); - } - - @Test - public void shouldReturnClientErrorWhenRequestingRetransmissionWithTargetTopicStoredOffline() { - // given - Topic sourceTopic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Topic targetTopic = hermes.initHelper().createTopic(topicWithRandomName().withOfflineStorage(1).build()); - OfflineRetransmissionRequest request = createRequest( - sourceTopic.getQualifiedName(), targetTopic.getQualifiedName(), null); - - // when - WebTestClient.ResponseSpec response = hermes.api().createOfflineRetransmissionTask(request); - - // then - response.expectStatus().isBadRequest(); - assertThat(response.expectBody(String.class).returnResult().getResponseBody()) - .contains("Target topic must not be stored offline"); - } - - - @Test - public void shouldDeleteRetransmissionTask() { - // given - Topic sourceTopic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Topic targetTopic = hermes.initHelper().createTopic(topicWithRandomName().build()); - - OfflineRetransmissionRequest request = createRequest( - sourceTopic.getQualifiedName(), targetTopic.getQualifiedName(), null); - hermes.api().createOfflineRetransmissionTask(request); - - List allTasks = getOfflineRetransmissionTasks(); - assertThat(allTasks.size()).isEqualTo(1); - - // when - WebTestClient.ResponseSpec response = hermes.api().deleteOfflineRetransmissionTask(allTasks.get(0).getTaskId()); - - // then - response.expectStatus().isOk(); - assertThat(getOfflineRetransmissionTasks().size()).isEqualTo(0); - } - - @Test - public void shouldReturnClientErrorWhenTryingToDeleteNotExistingRetransmissionTask() { - // when - String notExistingTaskId = "notExistingId"; - WebTestClient.ResponseSpec response = hermes.api().deleteOfflineRetransmissionTask(notExistingTaskId); - - // then - response.expectStatus().isBadRequest(); - assertThat(response.expectBody(String.class).returnResult().getResponseBody()) - .contains("Retransmission task with id " + notExistingTaskId + " does not exist."); - } - - @Test - public void shouldThrowAccessDeniedWhenTryingToCreateTaskWithoutPermissionsToSourceAndTargetTopics() { - // given - Topic sourceTopic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Topic targetTopic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestSecurityProvider.setUserIsAdmin(false); - OfflineRetransmissionRequest request = createRequest( - sourceTopic.getQualifiedName(), targetTopic.getQualifiedName(), null); - - // when - WebTestClient.ResponseSpec response = hermes.api().createOfflineRetransmissionTask(request); - - // then - response.expectStatus().isForbidden(); - assertThat(response.expectBody(String.class).returnResult().getResponseBody()) - .contains("User needs permissions to source and target topics"); - assertThat(getOfflineRetransmissionTasks().size()).isEqualTo(0); - - // cleanup - TestSecurityProvider.reset(); - } - - private OfflineRetransmissionRequest createRequest(String sourceTopic, String targetTopic, String sourceViewPath) { - return new OfflineRetransmissionRequest( - sourceViewPath, - sourceTopic, - targetTopic, - Instant.now().minusSeconds(1).toString(), - Instant.now().toString() - ); - } - - private void deleteTasks() { - getOfflineRetransmissionTasks() - .forEach(task -> - hermes.api().deleteOfflineRetransmissionTask(task.getTaskId()) - .expectStatus() - .isOk() - ); - } - - @Nullable - private static List getOfflineRetransmissionTasks() { - return hermes.api().getOfflineRetransmissionTasks().expectStatus().isOk() - .expectBodyList(OfflineRetransmissionTask.class) - .returnResult() - .getResponseBody(); - } + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); + + private static final String GROUP = "pl.allegro.retransmission"; + + @BeforeAll + public static void setupGroup() { + hermes.initHelper().createGroup(Group.from(GROUP)); + } + + @AfterEach + public void cleanup() { + deleteTasks(); + } + + @Test + public void shouldCreateRetransmissionTask() { + // given + Topic sourceTopic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Topic targetTopic = hermes.initHelper().createTopic(topicWithRandomName().build()); + + // when + OfflineRetransmissionRequest request = + createRequest(sourceTopic.getQualifiedName(), targetTopic.getQualifiedName(), null); + WebTestClient.ResponseSpec response = hermes.api().createOfflineRetransmissionTask(request); + Instant now = Instant.now(); + + // then + response.expectStatus().isCreated(); + + // and + List allTasks = getOfflineRetransmissionTasks(); + assertThat(allTasks.size()).isEqualTo(1); + assertThat(allTasks.get(0).getStartTimestamp()).isEqualTo(request.getStartTimestamp()); + assertThat(allTasks.get(0).getEndTimestamp()).isEqualTo(request.getEndTimestamp()); + assertThat(allTasks.get(0).getSourceTopic()).isEqualTo(request.getSourceTopic()); + assertThat(allTasks.get(0).getTargetTopic()).isEqualTo(request.getTargetTopic()); + assertThat(allTasks.get(0).getCreatedAt()).isBefore(now); + } + + @Test + public void shouldCreateRetransmissionTaskWithViewInsteadTopic() { + // given + var targetTopic = hermes.initHelper().createTopic(topicWithRandomName().build()); + + // when + var request = createRequest(null, targetTopic.getQualifiedName(), "testViewPath"); + var response = hermes.api().createOfflineRetransmissionTask(request); + var now = Instant.now(); + + // then + response.expectStatus().isCreated(); + + // and + var allTasks = getOfflineRetransmissionTasks(); + assertThat(allTasks.size()).isEqualTo(1); + assertThat(allTasks.get(0).getStartTimestamp()).isEqualTo(request.getStartTimestamp()); + assertThat(allTasks.get(0).getEndTimestamp()).isEqualTo(request.getEndTimestamp()); + assertThat(allTasks.get(0).getSourceTopic()).isEmpty(); + assertThat(allTasks.get(0).getSourceViewPath()).hasValue("testViewPath"); + assertThat(allTasks.get(0).getTargetTopic()).isEqualTo(request.getTargetTopic()); + assertThat(allTasks.get(0).getCreatedAt()).isBefore(now); + } + + @Test + public void shouldReturnEmptyListIfThereAreNoTasks() { + // expect + assertThat(getOfflineRetransmissionTasks().size()).isEqualTo(0); + } + + @Test + public void shouldReturnClientErrorWhenRequestingRetransmissionWithEmptyData() { + // given + OfflineRetransmissionRequest request = + new OfflineRetransmissionRequest(null, "", "", null, null); + + // when + WebTestClient.ResponseSpec response = hermes.api().createOfflineRetransmissionTask(request); + + // then + response.expectStatus().isBadRequest(); + assertThat(response.expectBody(String.class).returnResult().getResponseBody()) + .contains( + List.of( + "must contain one defined source of retransmission data - source topic or source view", + "startTimestamp must not be null", + "endTimestamp must not be null")); + } + + @Test + public void shouldReturnClientErrorWhenRequestingRetransmissionWithNotExistingSourceTopic() { + // given + Topic targetTopic = hermes.initHelper().createTopic(topicWithRandomName().build()); + OfflineRetransmissionRequest request = + createRequest("not.existing.sourceTopic", targetTopic.getQualifiedName(), null); + + // when + WebTestClient.ResponseSpec response = hermes.api().createOfflineRetransmissionTask(request); + + // then + response.expectStatus().isBadRequest(); + assertThat(response.expectBody(String.class).returnResult().getResponseBody()) + .contains("Source topic does not exist"); + } + + @Test + public void shouldReturnClientErrorWhenRequestingRetransmissionWithNotExistingTargetTopic() { + // given + Topic sourceTopic = hermes.initHelper().createTopic(topicWithRandomName().build()); + OfflineRetransmissionRequest request = + createRequest(sourceTopic.getQualifiedName(), "not.existing.targetTopic", null); + + // when + WebTestClient.ResponseSpec response = hermes.api().createOfflineRetransmissionTask(request); + + // then + response.expectStatus().isBadRequest(); + assertThat(response.expectBody(String.class).returnResult().getResponseBody()) + .contains("Target topic does not exist"); + } + + @Test + public void shouldReturnClientErrorWhenRequestingRetransmissionWithNegativeTimeRange() { + // given + Topic sourceTopic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Topic targetTopic = hermes.initHelper().createTopic(topicWithRandomName().build()); + OfflineRetransmissionRequest request = + new OfflineRetransmissionRequest( + null, + sourceTopic.getQualifiedName(), + targetTopic.getQualifiedName(), + Instant.now().toString(), + Instant.now().minusSeconds(1).toString()); + + // when + WebTestClient.ResponseSpec response = hermes.api().createOfflineRetransmissionTask(request); + + // then + response.expectStatus().isBadRequest(); + assertThat(response.expectBody(String.class).returnResult().getResponseBody()) + .contains("End timestamp must be greater than start timestamp"); + } + + @Test + public void shouldReturnClientErrorWhenRequestingRetransmissionWithTargetTopicStoredOffline() { + // given + Topic sourceTopic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Topic targetTopic = + hermes.initHelper().createTopic(topicWithRandomName().withOfflineStorage(1).build()); + OfflineRetransmissionRequest request = + createRequest(sourceTopic.getQualifiedName(), targetTopic.getQualifiedName(), null); + + // when + WebTestClient.ResponseSpec response = hermes.api().createOfflineRetransmissionTask(request); + + // then + response.expectStatus().isBadRequest(); + assertThat(response.expectBody(String.class).returnResult().getResponseBody()) + .contains("Target topic must not be stored offline"); + } + + @Test + public void shouldDeleteRetransmissionTask() { + // given + Topic sourceTopic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Topic targetTopic = hermes.initHelper().createTopic(topicWithRandomName().build()); + + OfflineRetransmissionRequest request = + createRequest(sourceTopic.getQualifiedName(), targetTopic.getQualifiedName(), null); + hermes.api().createOfflineRetransmissionTask(request); + + List allTasks = getOfflineRetransmissionTasks(); + assertThat(allTasks.size()).isEqualTo(1); + + // when + WebTestClient.ResponseSpec response = + hermes.api().deleteOfflineRetransmissionTask(allTasks.get(0).getTaskId()); + + // then + response.expectStatus().isOk(); + assertThat(getOfflineRetransmissionTasks().size()).isEqualTo(0); + } + + @Test + public void shouldReturnClientErrorWhenTryingToDeleteNotExistingRetransmissionTask() { + // when + String notExistingTaskId = "notExistingId"; + WebTestClient.ResponseSpec response = + hermes.api().deleteOfflineRetransmissionTask(notExistingTaskId); + + // then + response.expectStatus().isBadRequest(); + assertThat(response.expectBody(String.class).returnResult().getResponseBody()) + .contains("Retransmission task with id " + notExistingTaskId + " does not exist."); + } + + @Test + public void + shouldThrowAccessDeniedWhenTryingToCreateTaskWithoutPermissionsToSourceAndTargetTopics() { + // given + Topic sourceTopic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Topic targetTopic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestSecurityProvider.setUserIsAdmin(false); + OfflineRetransmissionRequest request = + createRequest(sourceTopic.getQualifiedName(), targetTopic.getQualifiedName(), null); + + // when + WebTestClient.ResponseSpec response = hermes.api().createOfflineRetransmissionTask(request); + + // then + response.expectStatus().isForbidden(); + assertThat(response.expectBody(String.class).returnResult().getResponseBody()) + .contains("User needs permissions to source and target topics"); + assertThat(getOfflineRetransmissionTasks().size()).isEqualTo(0); + + // cleanup + TestSecurityProvider.reset(); + } + + private OfflineRetransmissionRequest createRequest( + String sourceTopic, String targetTopic, String sourceViewPath) { + return new OfflineRetransmissionRequest( + sourceViewPath, + sourceTopic, + targetTopic, + Instant.now().minusSeconds(1).toString(), + Instant.now().toString()); + } + + private void deleteTasks() { + getOfflineRetransmissionTasks() + .forEach( + task -> + hermes + .api() + .deleteOfflineRetransmissionTask(task.getTaskId()) + .expectStatus() + .isOk()); + } + + @Nullable + private static List getOfflineRetransmissionTasks() { + return hermes + .api() + .getOfflineRetransmissionTasks() + .expectStatus() + .isOk() + .expectBodyList(OfflineRetransmissionTask.class) + .returnResult() + .getResponseBody(); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/QueryEndpointTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/QueryEndpointTest.java index 023c092dcb..42b3c51782 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/QueryEndpointTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/QueryEndpointTest.java @@ -1,34 +1,5 @@ package pl.allegro.tech.hermes.integrationtests.management; -import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import pl.allegro.tech.hermes.api.EndpointAddress; -import pl.allegro.tech.hermes.api.Group; -import pl.allegro.tech.hermes.api.OwnerId; -import pl.allegro.tech.hermes.api.Subscription; -import pl.allegro.tech.hermes.api.SubscriptionNameWithMetrics; -import pl.allegro.tech.hermes.api.Topic; -import pl.allegro.tech.hermes.api.TopicNameWithMetrics; -import pl.allegro.tech.hermes.api.TopicWithSchema; -import pl.allegro.tech.hermes.api.TrackingMode; -import pl.allegro.tech.hermes.integrationtests.prometheus.PrometheusExtension; -import pl.allegro.tech.hermes.integrationtests.setup.HermesExtension; -import pl.allegro.tech.hermes.test.helper.avro.AvroUserSchemaLoader; -import pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static java.time.Duration.ofMinutes; import static java.util.Arrays.asList; import static java.util.stream.Collectors.toList; import static org.assertj.core.api.Assertions.assertThat; @@ -50,547 +21,759 @@ import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomNameEndedWith; import static pl.allegro.tech.hermes.test.helper.endpoint.TimeoutAdjuster.adjust; -public class QueryEndpointTest { - - private static final String SCHEMA = AvroUserSchemaLoader.load().toString(); - - @Order(0) - @RegisterExtension - public static final PrometheusExtension prometheus = new PrometheusExtension(); - - @Order(1) - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension() - .withPrometheus(prometheus); - - @Test - public void shouldReturnAllGroupsWhenQueryIsEmpty() { - // given - createGroupWithRandomName(); - createGroupWithRandomName(); - - // when - List found = hermes.api().queryGroups("{\"query\": {}}") - .expectStatus().isOk() - .expectBodyList(Group.class).returnResult().getResponseBody(); - - // then - assertThat(found) - .extracting(Group::getGroupName) - .containsExactlyInAnyOrderElementsOf(hermes.api().getGroups()); - } - - @Test - public void shouldReturnGroupsWithExactName() { - // given - Group group = createGroupWithRandomName(); - createGroupWithRandomName(); - - // when - List found = hermes.api().queryGroups("{\"query\": {\"groupName\": \"" + group.getGroupName() + "\"}}") - .expectStatus().isOk() - .expectBodyList(Group.class).returnResult().getResponseBody(); - - // then - assertThat(found) - .containsExactly(group); - } - - @Test - public void shouldReturnGroupsWithNameSuffix() { - // given - String suffix = "GroupSuffix"; - hermes.initHelper().createGroup(groupWithRandomNameEndedWith(suffix).build()); - createGroupWithRandomName(); - - // when - List found = hermes.api().queryGroups("{\"query\": {\"groupName\": {\"like\": \".*" + suffix + "\"}}}") - .expectStatus().isOk() - .expectBodyList(Group.class).returnResult().getResponseBody(); - - // then - List groupsWithSuffix = hermes.api().getGroups().stream() - .filter(name -> name.endsWith(suffix)) - .toList(); - assertThat(found) - .extracting(Group::getGroupName) - .containsExactlyInAnyOrderElementsOf(groupsWithSuffix); - } - - @Test - public void shouldReturnGroupsWithNameContainingString() { - // given - String string = "SomeString"; - hermes.initHelper().createGroup(groupWithRandomNameContaining(string).build()); - createGroupWithRandomName(); - - // when - List found = hermes.api().queryGroups("{\"query\": {\"groupName\": {\"like\": \".*" + string + ".*\"}}}") - .expectStatus().isOk() - .expectBodyList(Group.class).returnResult().getResponseBody(); - - // then - List groupsContainingString = hermes.api().getGroups().stream() - .filter(name -> name.contains(string)) - .toList(); - assertThat(found) - .extracting(Group::getGroupName) - .containsExactlyInAnyOrderElementsOf(groupsContainingString); - } - - @Test - public void shouldReturnAllTopicsWhenQueryIsEmpty() { - // given - createTopicWithRandomName(); - createTopicWithRandomName(); - - // when - List found = hermes.api().queryTopics("{\"query\": {}}") - .expectStatus().isOk() - .expectBodyList(Topic.class).returnResult().getResponseBody(); - - // then - assertThat(found) - .containsExactlyInAnyOrderElementsOf(listAllTopics()); - } - - @Test - public void shouldReturnTopicsWithExactName() { - // given - Topic topic = createTopicWithRandomName(); - createTopicWithRandomName(); - - // when - List found = hermes.api().queryTopics("{\"query\": {\"name\": \"" + topic.getQualifiedName() + "\"}}") - .expectStatus().isOk() - .expectBodyList(Topic.class).returnResult().getResponseBody(); - - // then - assertThat(found) - .containsExactly(topic); - } +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import pl.allegro.tech.hermes.api.EndpointAddress; +import pl.allegro.tech.hermes.api.Group; +import pl.allegro.tech.hermes.api.OwnerId; +import pl.allegro.tech.hermes.api.Subscription; +import pl.allegro.tech.hermes.api.SubscriptionNameWithMetrics; +import pl.allegro.tech.hermes.api.Topic; +import pl.allegro.tech.hermes.api.TopicNameWithMetrics; +import pl.allegro.tech.hermes.api.TopicWithSchema; +import pl.allegro.tech.hermes.api.TrackingMode; +import pl.allegro.tech.hermes.integrationtests.prometheus.PrometheusExtension; +import pl.allegro.tech.hermes.integrationtests.setup.HermesExtension; +import pl.allegro.tech.hermes.test.helper.avro.AvroUserSchemaLoader; +import pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder; - @Test - public void shouldReturnTopicsWithNameSuffix() { - // given - String suffix = "TopicSuffix"; - hermes.initHelper().createTopic(topicWithRandomNameEndedWith(suffix).build()); - createTopicWithRandomName(); - - // when - List found = hermes.api().queryTopics("{\"query\": {\"name\": {\"like\": \".*" + suffix + "\"}}}") - .expectStatus().isOk() - .expectBodyList(Topic.class).returnResult().getResponseBody(); - - // then - List topicsWithSuffix = listAllTopics().stream() - .filter(topic -> topic.getQualifiedName().endsWith(suffix)) - .toList(); - assertThat(found) - .containsExactlyInAnyOrderElementsOf(topicsWithSuffix); - } +public class QueryEndpointTest { - @Test - public void shouldReturnTopicsWithNameContainingString() { - // given - String string = "SomeString"; - hermes.initHelper().createTopic(topicWithRandomNameContaining(string).build()); - createTopicWithRandomName(); - - // when - List found = hermes.api().queryTopics("{\"query\": {\"name\": {\"like\": \".*" + string + ".*\"}}}") - .expectStatus().isOk() - .expectBodyList(Topic.class).returnResult().getResponseBody(); - - // then - List topicsContainingString = listAllTopics().stream() - .filter(topic -> topic.getQualifiedName().contains(string)) - .toList(); - assertThat(found) - .containsExactlyInAnyOrderElementsOf(topicsContainingString); + private static final String SCHEMA = AvroUserSchemaLoader.load().toString(); + + @Order(0) + @RegisterExtension + public static final PrometheusExtension prometheus = new PrometheusExtension(); + + @Order(1) + @RegisterExtension + public static final HermesExtension hermes = new HermesExtension().withPrometheus(prometheus); + + @Test + public void shouldReturnAllGroupsWhenQueryIsEmpty() { + // given + createGroupWithRandomName(); + createGroupWithRandomName(); + + // when + List found = + hermes + .api() + .queryGroups("{\"query\": {}}") + .expectStatus() + .isOk() + .expectBodyList(Group.class) + .returnResult() + .getResponseBody(); + + // then + assertThat(found) + .extracting(Group::getGroupName) + .containsExactlyInAnyOrderElementsOf(hermes.api().getGroups()); + } + + @Test + public void shouldReturnGroupsWithExactName() { + // given + Group group = createGroupWithRandomName(); + createGroupWithRandomName(); + + // when + List found = + hermes + .api() + .queryGroups("{\"query\": {\"groupName\": \"" + group.getGroupName() + "\"}}") + .expectStatus() + .isOk() + .expectBodyList(Group.class) + .returnResult() + .getResponseBody(); + + // then + assertThat(found).containsExactly(group); + } + + @Test + public void shouldReturnGroupsWithNameSuffix() { + // given + String suffix = "GroupSuffix"; + hermes.initHelper().createGroup(groupWithRandomNameEndedWith(suffix).build()); + createGroupWithRandomName(); + + // when + List found = + hermes + .api() + .queryGroups("{\"query\": {\"groupName\": {\"like\": \".*" + suffix + "\"}}}") + .expectStatus() + .isOk() + .expectBodyList(Group.class) + .returnResult() + .getResponseBody(); + + // then + List groupsWithSuffix = + hermes.api().getGroups().stream().filter(name -> name.endsWith(suffix)).toList(); + assertThat(found) + .extracting(Group::getGroupName) + .containsExactlyInAnyOrderElementsOf(groupsWithSuffix); + } + + @Test + public void shouldReturnGroupsWithNameContainingString() { + // given + String string = "SomeString"; + hermes.initHelper().createGroup(groupWithRandomNameContaining(string).build()); + createGroupWithRandomName(); + + // when + List found = + hermes + .api() + .queryGroups("{\"query\": {\"groupName\": {\"like\": \".*" + string + ".*\"}}}") + .expectStatus() + .isOk() + .expectBodyList(Group.class) + .returnResult() + .getResponseBody(); + + // then + List groupsContainingString = + hermes.api().getGroups().stream().filter(name -> name.contains(string)).toList(); + assertThat(found) + .extracting(Group::getGroupName) + .containsExactlyInAnyOrderElementsOf(groupsContainingString); + } + + @Test + public void shouldReturnAllTopicsWhenQueryIsEmpty() { + // given + createTopicWithRandomName(); + createTopicWithRandomName(); + + // when + List found = + hermes + .api() + .queryTopics("{\"query\": {}}") + .expectStatus() + .isOk() + .expectBodyList(Topic.class) + .returnResult() + .getResponseBody(); + + // then + assertThat(found).containsExactlyInAnyOrderElementsOf(listAllTopics()); + } + + @Test + public void shouldReturnTopicsWithExactName() { + // given + Topic topic = createTopicWithRandomName(); + createTopicWithRandomName(); + + // when + List found = + hermes + .api() + .queryTopics("{\"query\": {\"name\": \"" + topic.getQualifiedName() + "\"}}") + .expectStatus() + .isOk() + .expectBodyList(Topic.class) + .returnResult() + .getResponseBody(); + + // then + assertThat(found).containsExactly(topic); + } + + @Test + public void shouldReturnTopicsWithNameSuffix() { + // given + String suffix = "TopicSuffix"; + hermes.initHelper().createTopic(topicWithRandomNameEndedWith(suffix).build()); + createTopicWithRandomName(); + + // when + List found = + hermes + .api() + .queryTopics("{\"query\": {\"name\": {\"like\": \".*" + suffix + "\"}}}") + .expectStatus() + .isOk() + .expectBodyList(Topic.class) + .returnResult() + .getResponseBody(); + + // then + List topicsWithSuffix = + listAllTopics().stream() + .filter(topic -> topic.getQualifiedName().endsWith(suffix)) + .toList(); + assertThat(found).containsExactlyInAnyOrderElementsOf(topicsWithSuffix); + } + + @Test + public void shouldReturnTopicsWithNameContainingString() { + // given + String string = "SomeString"; + hermes.initHelper().createTopic(topicWithRandomNameContaining(string).build()); + createTopicWithRandomName(); + + // when + List found = + hermes + .api() + .queryTopics("{\"query\": {\"name\": {\"like\": \".*" + string + ".*\"}}}") + .expectStatus() + .isOk() + .expectBodyList(Topic.class) + .returnResult() + .getResponseBody(); + + // then + List topicsContainingString = + listAllTopics().stream() + .filter(topic -> topic.getQualifiedName().contains(string)) + .toList(); + assertThat(found).containsExactlyInAnyOrderElementsOf(topicsContainingString); + } + + @Test + public void shouldReturnTopicsWithAllMatchingProperties() { + // given + Topic topic1 = topicWithRandomName().withContentType(AVRO).withTrackingEnabled(false).build(); + Topic topic2 = topicWithRandomName().withContentType(JSON).withTrackingEnabled(false).build(); + Topic topic3 = topicWithRandomName().withContentType(AVRO).withTrackingEnabled(true).build(); + hermes.initHelper().createTopicWithSchema(topicWithSchema(topic1, SCHEMA)); + hermes.initHelper().createTopicWithSchema(topicWithSchema(topic3, SCHEMA)); + hermes.initHelper().createTopic(topic2); + + // when + List found = + hermes + .api() + .queryTopics( + "{\"query\": {\"and\": [{\"trackingEnabled\": \"true\"}, {\"contentType\": \"AVRO\"}]}}") + .expectStatus() + .isOk() + .expectBodyList(Topic.class) + .returnResult() + .getResponseBody(); + + // then + List topicsWithAvroAndTracking = + listAllTopics().stream() + .filter(topic -> topic.getContentType() == AVRO && topic.isTrackingEnabled()) + .toList(); + assertThat(found).containsExactlyInAnyOrderElementsOf(topicsWithAvroAndTracking); + } + + @Test + public void shouldReturnTopicsWithAtLeastOneMatchingProperty() { + // given + Topic topic1 = topicWithRandomName().withContentType(AVRO).withTrackingEnabled(false).build(); + Topic topic2 = topicWithRandomName().withContentType(JSON).withTrackingEnabled(false).build(); + Topic topic3 = topicWithRandomName().withContentType(AVRO).withTrackingEnabled(true).build(); + hermes.initHelper().createTopicWithSchema(topicWithSchema(topic1, SCHEMA)); + hermes.initHelper().createTopicWithSchema(topicWithSchema(topic3, SCHEMA)); + hermes.initHelper().createTopic(topic2); + + // when + List found = + hermes + .api() + .queryTopics( + "{\"query\": {\"or\": [{\"trackingEnabled\": \"true\"}, {\"contentType\": \"AVRO\"}]}}") + .expectStatus() + .isOk() + .expectBodyList(Topic.class) + .returnResult() + .getResponseBody(); + + // then + List topicsWithAvroOrTracking = + listAllTopics().stream() + .filter(topic -> topic.getContentType() == AVRO || topic.isTrackingEnabled()) + .toList(); + assertThat(found).containsExactlyInAnyOrderElementsOf(topicsWithAvroOrTracking); + } + + @Test + public void shouldReturnTopicsWithExactOwnerId() { + // given + Topic topic = + topicWithRandomName() + .withContentType(JSON) + .withTrackingEnabled(true) + .withOwner(new OwnerId("Plaintext", "Team Alpha")) + .build(); + hermes.initHelper().createTopic(topic); + createTopicWithRandomName(); + + // when + List found = + hermes + .api() + .queryTopics("{\"query\": {\"owner.id\": \"Team Alpha\"}}") + .expectStatus() + .isOk() + .expectBodyList(Topic.class) + .returnResult() + .getResponseBody(); + + // then + List topicsOwnerId = + listAllTopics().stream().filter(t -> t.getOwner().getId().equals("Team Alpha")).toList(); + assertThat(found).containsExactlyInAnyOrderElementsOf(topicsOwnerId); + } + + @Test + public void shouldReturnTopicsWithOwnerIdContainingString() { + // given + Topic topic = + topicWithRandomName() + .withContentType(JSON) + .withTrackingEnabled(true) + .withOwner(new OwnerId("Plaintext", "Team Alpha")) + .build(); + hermes.initHelper().createTopic(topic); + createTopicWithRandomName(); + + // when + List found = + hermes + .api() + .queryTopics("{\"query\": {\"owner.id\": {\"like\": \".*Alph.*\"}}}") + .expectStatus() + .isOk() + .expectBodyList(Topic.class) + .returnResult() + .getResponseBody(); + + // then + List topicsOwnerId = + listAllTopics().stream().filter(t -> t.getOwner().getId().contains("Alph")).toList(); + assertThat(found).containsExactlyInAnyOrderElementsOf(topicsOwnerId); + } + + private List listAllTopics() { + List groups = hermes.api().getGroups(); + List topics = new ArrayList<>(); + for (String groupName : groups) { + String[] topicNames = + hermes + .api() + .listTopics(groupName) + .expectStatus() + .isOk() + .expectBody(String[].class) + .returnResult() + .getResponseBody(); + for (String topicName : topicNames) { + TopicWithSchema topic = + hermes + .api() + .getTopicResponse(topicName) + .expectBody(TopicWithSchema.class) + .returnResult() + .getResponseBody(); + topics.add(topic.getTopic()); + } } - - @Test - public void shouldReturnTopicsWithAllMatchingProperties() { - // given - Topic topic1 = topicWithRandomName() - .withContentType(AVRO) - .withTrackingEnabled(false) - .build(); - Topic topic2 = topicWithRandomName() + return topics; + } + + public static Stream subscriptionData() { + return Stream.of( + arguments("{\"query\": {}}", asList(1, 2, 3, 4)), + arguments("{\"query\": {\"name\": \"subscription1\"}}", asList(1)), + arguments("{\"query\": {\"name\": {\"like\": \".*cription1\"}}}", asList(1)), + arguments("{\"query\": {\"name\": {\"like\": \"subscript.*\"}}}", asList(1, 2, 4)), + arguments( + "{\"query\": {\"name\": \"subscription1\", \"endpoint\": \"http://endpoint1\"}}", + asList(1)), + arguments( + "{\"query\": {\"and\": [{\"name\": \"subscription1\"}, {\"endpoint\": \"http://endpoint1\"}]}}", + asList(1)), + arguments( + "{\"query\": {\"or\": [{\"name\": \"subscription1\"}, {\"endpoint\": \"http://endpoint1\"}]}}", + asList(1, 3)), + arguments("{\"query\": {\"owner.id\": \"Team Alpha\"}}", asList(4)), + arguments("{\"query\": {\"owner.id\": {\"like\": \".*Alph.*\"}}}", asList(4)), + arguments("{\"query\": {\"endpoint\": \".*password.*\"}}", asList())); + } + + @ParameterizedTest + @MethodSource("subscriptionData") + public void shouldQuerySubscription(String query, List positions) { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + + Subscription subscription1 = + hermes + .initHelper() + .createSubscription( + enrichSubscription( + subscription(topic.getName(), "subscription1"), "http://endpoint1")); + + Subscription subscription2 = + hermes + .initHelper() + .createSubscription( + enrichSubscription( + subscription(topic.getName(), "subscription2"), "http://endpoint2")); + + Subscription subscription3 = + hermes + .initHelper() + .createSubscription( + enrichSubscription( + subscription(topic.getName(), "subTestScription3"), + "http://login:password@endpoint1")); + + Subscription subscription4 = + hermes + .initHelper() + .createSubscription( + enrichSubscription( + subscription(topic.getName(), "subscription4") + .withOwner(new OwnerId("Plaintext", "Team Alpha")), + "http://endpoint2")); + + List subscriptions = + asList(subscription1, subscription2, subscription3.anonymize(), subscription4); + + // when + List found = + hermes + .api() + .querySubscriptions(query) + .expectStatus() + .isOk() + .expectBodyList(Subscription.class) + .returnResult() + .getResponseBody(); + + // then + assertListMatches(subscriptions, found, positions); + } + + @Test + public void shouldSkipEntitiesNotContainingQueriedField() { + // given + hermes.initHelper().createGroup(groupWithRandomName().build()); + + // when + List found = + hermes + .api() + .queryGroups("{\"query\": {\"missingField\": \"xxx\"}}") + .expectStatus() + .isOk() + .expectBodyList(Group.class) + .returnResult() + .getResponseBody(); + + // then + Assertions.assertThat(found).isEmpty(); + } + + @Test + public void shouldSkipEntitiesNotContainingQueriedNestedField() { + // given + hermes.initHelper().createGroup(groupWithRandomName().build()); + + // when + List found = + hermes + .api() + .queryGroups("{\"query\": {\"missing.nested.field\": \"xxx\"}}") + .expectStatus() + .isOk() + .expectBodyList(Group.class) + .returnResult() + .getResponseBody(); + + // then + Assertions.assertThat(found).isEmpty(); + } + + public static Stream topicsMetricsFilteringData() { + return Stream.of( + arguments( + "testTopic1", "testTopic2", "{\"query\": {}}", asList("testTopic1", "testTopic2")), + arguments( + "testTopic3", "testTopic4", "{\"query\": {\"published\": {\"gt\": \"20\"}}}", asList()), + arguments( + "testTopic5", + "testTopic6", + "{\"query\": {\"published\": {\"gt\": \"3\"}}}", + asList("testTopic6")), + arguments( + "testTopic7", + "testTopic8", + "{\"query\": {\"published\": {\"lt\": \"3\"}}}", + asList("testTopic7"))); + } + + @ParameterizedTest + @MethodSource("topicsMetricsFilteringData") + public void shouldQueryTopicsMetrics( + String topicName1, String topicName2, String query, List topicNames) { + // given + Group group = groupWithRandomName().build(); + hermes.initHelper().createGroup(group); + hermes + .initHelper() + .createTopic( + topic(group.getGroupName(), topicName1) .withContentType(JSON) .withTrackingEnabled(false) - .build(); - Topic topic3 = topicWithRandomName() - .withContentType(AVRO) - .withTrackingEnabled(true) - .build(); - hermes.initHelper().createTopicWithSchema(topicWithSchema(topic1, SCHEMA)); - hermes.initHelper().createTopicWithSchema(topicWithSchema(topic3, SCHEMA)); - hermes.initHelper().createTopic(topic2); - - // when - List found = hermes.api().queryTopics("{\"query\": {\"and\": [{\"trackingEnabled\": \"true\"}, {\"contentType\": \"AVRO\"}]}}") - .expectStatus().isOk() - .expectBodyList(Topic.class).returnResult().getResponseBody(); - - // then - List topicsWithAvroAndTracking = listAllTopics().stream() - .filter(topic -> topic.getContentType() == AVRO && topic.isTrackingEnabled()) - .toList(); - assertThat(found) - .containsExactlyInAnyOrderElementsOf(topicsWithAvroAndTracking); - } - - @Test - public void shouldReturnTopicsWithAtLeastOneMatchingProperty() { - // given - Topic topic1 = topicWithRandomName() - .withContentType(AVRO) - .withTrackingEnabled(false) - .build(); - Topic topic2 = topicWithRandomName() + .build()); + hermes + .initHelper() + .createTopic( + topic(group.getGroupName(), topicName2) .withContentType(JSON) .withTrackingEnabled(false) - .build(); - Topic topic3 = topicWithRandomName() - .withContentType(AVRO) - .withTrackingEnabled(true) - .build(); - hermes.initHelper().createTopicWithSchema(topicWithSchema(topic1, SCHEMA)); - hermes.initHelper().createTopicWithSchema(topicWithSchema(topic3, SCHEMA)); - hermes.initHelper().createTopic(topic2); - - // when - List found = hermes.api().queryTopics("{\"query\": {\"or\": [{\"trackingEnabled\": \"true\"}, {\"contentType\": \"AVRO\"}]}}") - .expectStatus().isOk() - .expectBodyList(Topic.class).returnResult().getResponseBody(); - - // then - List topicsWithAvroOrTracking = listAllTopics().stream() - .filter(topic -> topic.getContentType() == AVRO || topic.isTrackingEnabled()) - .toList(); - assertThat(found) - .containsExactlyInAnyOrderElementsOf(topicsWithAvroOrTracking); - } - - @Test - public void shouldReturnTopicsWithExactOwnerId() { - // given - Topic topic = topicWithRandomName() - .withContentType(JSON) - .withTrackingEnabled(true) - .withOwner(new OwnerId("Plaintext", "Team Alpha")) - .build(); - hermes.initHelper().createTopic(topic); - createTopicWithRandomName(); - - // when - List found = hermes.api().queryTopics("{\"query\": {\"owner.id\": \"Team Alpha\"}}") - .expectStatus().isOk() - .expectBodyList(Topic.class).returnResult().getResponseBody(); - - // then - List topicsOwnerId = listAllTopics().stream() - .filter(t -> t.getOwner().getId().equals("Team Alpha")) - .toList(); - assertThat(found) - .containsExactlyInAnyOrderElementsOf(topicsOwnerId); - } - - @Test - public void shouldReturnTopicsWithOwnerIdContainingString() { - // given - Topic topic = topicWithRandomName() - .withContentType(JSON) - .withTrackingEnabled(true) - .withOwner(new OwnerId("Plaintext", "Team Alpha")) - .build(); - hermes.initHelper().createTopic(topic); - createTopicWithRandomName(); - - // when - List found = hermes.api().queryTopics("{\"query\": {\"owner.id\": {\"like\": \".*Alph.*\"}}}") - .expectStatus().isOk() - .expectBodyList(Topic.class).returnResult().getResponseBody(); - - // then - List topicsOwnerId = listAllTopics().stream() - .filter(t -> t.getOwner().getId().contains("Alph")) - .toList(); - assertThat(found) - .containsExactlyInAnyOrderElementsOf(topicsOwnerId); - } - - private List listAllTopics() { - List groups = hermes.api().getGroups(); - List topics = new ArrayList<>(); - for (String groupName : groups) { - String[] topicNames = hermes.api().listTopics(groupName) - .expectStatus() - .isOk() - .expectBody(String[].class).returnResult().getResponseBody(); - for (String topicName : topicNames) { - TopicWithSchema topic = hermes.api().getTopicResponse(topicName) - .expectBody(TopicWithSchema.class).returnResult().getResponseBody(); - topics.add(topic.getTopic()); - } - } - return topics; + .build()); + + hermes.api().publishUntilSuccess(group.getGroupName() + "." + topicName1, "testMessage1"); + hermes.api().publishUntilSuccess(group.getGroupName() + "." + topicName2, "testMessage2"); + hermes.api().publishUntilSuccess(group.getGroupName() + "." + topicName2, "testMessage3"); + hermes.api().publishUntilSuccess(group.getGroupName() + "." + topicName2, "testMessage4"); + hermes.api().publishUntilSuccess(group.getGroupName() + "." + topicName2, "testMessage5"); + + List qualifiedNames = + topicNames.stream() + .map(topicName -> group.getGroupName() + "." + topicName) + .collect(toList()); + + waitAtMost(adjust(Duration.ofMinutes(1))) + .untilAsserted( + () -> { + // when + List found = + hermes + .api() + .queryTopicMetrics(query) + .expectStatus() + .isOk() + .expectBodyList(TopicNameWithMetrics.class) + .returnResult() + .getResponseBody(); + + // then + assertTopicMetricsMatchesToNames(found, qualifiedNames); + + found.forEach(it -> Assertions.assertThat(it.getVolume()).isGreaterThanOrEqualTo(0)); + }); + } + + @Test + public void shouldQuerySubscriptionsMetrics() { + // given + Topic topic1 = hermes.initHelper().createTopic(topicWithRandomName().build()); + Topic topic2 = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription1 = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic1.getName(), "http://endpoint1").build()); + Subscription subscription2 = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic2.getName(), "http://endpoint2").build()); + + final String queryGetAllSubscriptionsMetrics = "{\"query\": {}}"; + final String queryGetSubscriptionsMetricsWithPositiveThroughput = + "{\"query\": {\"throughput\": {\"gt\": 0}}}"; + final String queryGetSubscriptionsMetricsWithRateInRange = + "{\"query\": {\"or\": [{\"rate\": {\"gt\": 10}}, {\"rate\": {\"lt\": 50}}]}}"; + final String queryGetSubscriptionsMetricsWithLagNegative = + "{\"query\": {\"lag\": {\"lt\": 0}}}"; + final String queryGetSubscriptionsMetricsWithVolume = "{\"query\": {\"volume\": {\"gt\": -1}}}"; + prometheus.stubSubscriptionMetrics( + subscriptionMetrics(subscription1.getQualifiedName()) + .withRate(100) + .withThroughput(0) + .build()); + prometheus.stubSubscriptionMetrics( + subscriptionMetrics(subscription2.getQualifiedName()) + .withRate(40) + .withThroughput(10) + .build()); + + waitAtMost(adjust(Duration.ofMinutes(1))) + .untilAsserted( + () -> { + // when + final List allSubscriptions = + hermes + .api() + .querySubscriptionMetrics(queryGetAllSubscriptionsMetrics) + .expectStatus() + .isOk() + .expectBodyList(SubscriptionNameWithMetrics.class) + .returnResult() + .getResponseBody(); + final List subscriptionsWithPositiveThroughput = + hermes + .api() + .querySubscriptionMetrics(queryGetSubscriptionsMetricsWithPositiveThroughput) + .expectStatus() + .isOk() + .expectBodyList(SubscriptionNameWithMetrics.class) + .returnResult() + .getResponseBody(); + final List subscriptionsWithRateInRange = + hermes + .api() + .querySubscriptionMetrics(queryGetSubscriptionsMetricsWithRateInRange) + .expectStatus() + .isOk() + .expectBodyList(SubscriptionNameWithMetrics.class) + .returnResult() + .getResponseBody(); + final List subscriptionsWithNegativeLag = + hermes + .api() + .querySubscriptionMetrics(queryGetSubscriptionsMetricsWithLagNegative) + .expectStatus() + .isOk() + .expectBodyList(SubscriptionNameWithMetrics.class) + .returnResult() + .getResponseBody(); + final List subscriptionsWithVolume = + hermes + .api() + .querySubscriptionMetrics(queryGetSubscriptionsMetricsWithVolume) + .expectStatus() + .isOk() + .expectBodyList(SubscriptionNameWithMetrics.class) + .returnResult() + .getResponseBody(); + + // then + subscriptionsMatchesToNamesAndTheirTopicsNames( + allSubscriptions, subscription1, subscription2); + subscriptionsMatchesToNamesAndTheirTopicsNames( + subscriptionsWithPositiveThroughput, subscription2); + subscriptionsMatchesToNamesAndTheirTopicsNames( + subscriptionsWithRateInRange, subscription2); + subscriptionsMatchesToNamesAndTheirTopicsNames( + subscriptionsWithNegativeLag, subscription1, subscription2); + subscriptionsMatchesToNamesAndTheirTopicsNames( + subscriptionsWithVolume, subscription1, subscription2); + }); + } + + @Test + public void shouldHandleUnavailableSubscriptionsMetrics() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), "http://endpoint1").build()); + String queryGetAllSubscriptionsMetrics = "{\"query\": {}}"; + String queryGetSubscriptionsMetricsWithPositiveRate = "{\"query\": {\"rate\": {\"gt\": 0}}}"; + prometheus.stubDelay(Duration.ofMillis(3000)); + + waitAtMost(adjust(Duration.ofMinutes(1))) + .untilAsserted( + () -> { + // when + List allSubscriptions = + hermes + .api() + .querySubscriptionMetrics(queryGetAllSubscriptionsMetrics) + .expectStatus() + .isOk() + .expectBodyList(SubscriptionNameWithMetrics.class) + .returnResult() + .getResponseBody(); + List subscriptionsWithPositiveRate = + hermes + .api() + .querySubscriptionMetrics(queryGetSubscriptionsMetricsWithPositiveRate) + .expectStatus() + .isOk() + .expectBodyList(SubscriptionNameWithMetrics.class) + .returnResult() + .getResponseBody(); + + // then + assertThatRateIsUnavailable(allSubscriptions, subscription); + assertThatRateIsUnavailable(subscriptionsWithPositiveRate, subscription); + }); + } + + private static void assertThatRateIsUnavailable( + List allSubscriptions, Subscription... subscriptions) { + subscriptionsMatchesToNamesAndTheirTopicsNames(allSubscriptions, subscriptions); + for (SubscriptionNameWithMetrics metrics : allSubscriptions) { + assertThat(metrics.getRate().asString()).isEqualTo("unavailable"); } - - public static Stream subscriptionData() { - return Stream.of( - arguments("{\"query\": {}}", asList(1, 2, 3, 4)), - arguments("{\"query\": {\"name\": \"subscription1\"}}", asList(1)), - arguments("{\"query\": {\"name\": {\"like\": \".*cription1\"}}}", asList(1)), - arguments("{\"query\": {\"name\": {\"like\": \"subscript.*\"}}}", asList(1, 2, 4)), - arguments("{\"query\": {\"name\": \"subscription1\", \"endpoint\": \"http://endpoint1\"}}", asList(1)), - arguments("{\"query\": {\"and\": [{\"name\": \"subscription1\"}, {\"endpoint\": \"http://endpoint1\"}]}}", asList(1)), - arguments("{\"query\": {\"or\": [{\"name\": \"subscription1\"}, {\"endpoint\": \"http://endpoint1\"}]}}", asList(1, 3)), - arguments("{\"query\": {\"owner.id\": \"Team Alpha\"}}", asList(4)), - arguments("{\"query\": {\"owner.id\": {\"like\": \".*Alph.*\"}}}", asList(4)), - arguments("{\"query\": {\"endpoint\": \".*password.*\"}}", asList()) - ); - } - - @ParameterizedTest - @MethodSource("subscriptionData") - public void shouldQuerySubscription(String query, List positions) { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - - Subscription subscription1 = hermes.initHelper().createSubscription( - enrichSubscription(subscription(topic.getName(), "subscription1"), "http://endpoint1") - ); - - Subscription subscription2 = hermes.initHelper().createSubscription( - enrichSubscription(subscription(topic.getName(), "subscription2"), "http://endpoint2") - ); - - Subscription subscription3 = hermes.initHelper().createSubscription( - enrichSubscription(subscription(topic.getName(), "subTestScription3"), "http://login:password@endpoint1") - ); - - Subscription subscription4 = hermes.initHelper().createSubscription(enrichSubscription(subscription(topic.getName(), "subscription4") - .withOwner(new OwnerId("Plaintext", "Team Alpha")), "http://endpoint2") - ); - - List subscriptions = asList(subscription1, subscription2, subscription3.anonymize(), subscription4); - - // when - List found = hermes.api().querySubscriptions(query) - .expectStatus().isOk() - .expectBodyList(Subscription.class).returnResult().getResponseBody(); - - // then - assertListMatches(subscriptions, found, positions); - } - - @Test - public void shouldSkipEntitiesNotContainingQueriedField() { - // given - hermes.initHelper().createGroup(groupWithRandomName().build()); - - // when - List found = hermes.api().queryGroups("{\"query\": {\"missingField\": \"xxx\"}}") - .expectStatus().isOk() - .expectBodyList(Group.class).returnResult().getResponseBody(); - - // then - Assertions.assertThat(found).isEmpty(); - } - - @Test - public void shouldSkipEntitiesNotContainingQueriedNestedField() { - // given - hermes.initHelper().createGroup(groupWithRandomName().build()); - - // when - List found = hermes.api().queryGroups("{\"query\": {\"missing.nested.field\": \"xxx\"}}") - .expectStatus().isOk() - .expectBodyList(Group.class).returnResult().getResponseBody(); - - // then - Assertions.assertThat(found).isEmpty(); - } - - public static Stream topicsMetricsFilteringData() { - return Stream.of( - arguments("testTopic1", "testTopic2", "{\"query\": {}}", asList("testTopic1", "testTopic2")), - arguments("testTopic3", "testTopic4", "{\"query\": {\"published\": {\"gt\": \"20\"}}}", asList()), - arguments("testTopic5", "testTopic6", "{\"query\": {\"published\": {\"gt\": \"3\"}}}", asList("testTopic6")), - arguments("testTopic7", "testTopic8", "{\"query\": {\"published\": {\"lt\": \"3\"}}}", asList("testTopic7"))); - } - - @ParameterizedTest - @MethodSource("topicsMetricsFilteringData") - public void shouldQueryTopicsMetrics(String topicName1, String topicName2, String query, List topicNames) { - // given - Group group = groupWithRandomName().build(); - hermes.initHelper().createGroup(group); - hermes.initHelper().createTopic(topic(group.getGroupName(), topicName1).withContentType(JSON).withTrackingEnabled(false).build()); - hermes.initHelper().createTopic(topic(group.getGroupName(), topicName2).withContentType(JSON).withTrackingEnabled(false).build()); - - hermes.api().publishUntilSuccess(group.getGroupName() + "." + topicName1, "testMessage1"); - hermes.api().publishUntilSuccess(group.getGroupName() + "." + topicName2, "testMessage2"); - hermes.api().publishUntilSuccess(group.getGroupName() + "." + topicName2, "testMessage3"); - hermes.api().publishUntilSuccess(group.getGroupName() + "." + topicName2, "testMessage4"); - hermes.api().publishUntilSuccess(group.getGroupName() + "." + topicName2, "testMessage5"); - - List qualifiedNames = topicNames.stream() - .map(topicName -> group.getGroupName() + "." + topicName) - .collect(toList()); - - waitAtMost(adjust(Duration.ofMinutes(1))).untilAsserted(() -> { - // when - List found = hermes.api().queryTopicMetrics(query) - .expectStatus().isOk() - .expectBodyList(TopicNameWithMetrics.class).returnResult().getResponseBody(); - - // then - assertTopicMetricsMatchesToNames(found, qualifiedNames); - - found.forEach(it -> Assertions.assertThat(it.getVolume()).isGreaterThanOrEqualTo(0)); - } - ); - } - - @Test - public void shouldQuerySubscriptionsMetrics() { - // given - Topic topic1 = hermes.initHelper().createTopic(topicWithRandomName().build()); - Topic topic2 = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription1 = hermes.initHelper().createSubscription( - subscriptionWithRandomName(topic1.getName(), "http://endpoint1").build() - ); - Subscription subscription2 = hermes.initHelper().createSubscription( - subscriptionWithRandomName(topic2.getName(), "http://endpoint2").build() - ); - - final String queryGetAllSubscriptionsMetrics = "{\"query\": {}}"; - final String queryGetSubscriptionsMetricsWithPositiveThroughput = "{\"query\": {\"throughput\": {\"gt\": 0}}}"; - final String queryGetSubscriptionsMetricsWithRateInRange = "{\"query\": {\"or\": [{\"rate\": {\"gt\": 10}}, {\"rate\": {\"lt\": 50}}]}}"; - final String queryGetSubscriptionsMetricsWithLagNegative = "{\"query\": {\"lag\": {\"lt\": 0}}}"; - final String queryGetSubscriptionsMetricsWithVolume = "{\"query\": {\"volume\": {\"gt\": -1}}}"; - prometheus.stubSubscriptionMetrics( - subscriptionMetrics(subscription1.getQualifiedName()) - .withRate(100) - .withThroughput(0) - .build() - ); - prometheus.stubSubscriptionMetrics( - subscriptionMetrics(subscription2.getQualifiedName()) - .withRate(40) - .withThroughput(10) - .build() - ); - - waitAtMost(adjust(Duration.ofMinutes(1))).untilAsserted(() -> { - // when - final List allSubscriptions = hermes.api() - .querySubscriptionMetrics(queryGetAllSubscriptionsMetrics) - .expectStatus().isOk() - .expectBodyList(SubscriptionNameWithMetrics.class).returnResult().getResponseBody(); - final List subscriptionsWithPositiveThroughput = hermes.api() - .querySubscriptionMetrics(queryGetSubscriptionsMetricsWithPositiveThroughput) - .expectStatus().isOk() - .expectBodyList(SubscriptionNameWithMetrics.class).returnResult().getResponseBody(); - final List subscriptionsWithRateInRange = hermes.api() - .querySubscriptionMetrics(queryGetSubscriptionsMetricsWithRateInRange) - .expectStatus().isOk() - .expectBodyList(SubscriptionNameWithMetrics.class).returnResult().getResponseBody(); - final List subscriptionsWithNegativeLag = hermes.api() - .querySubscriptionMetrics(queryGetSubscriptionsMetricsWithLagNegative) - .expectStatus().isOk() - .expectBodyList(SubscriptionNameWithMetrics.class).returnResult().getResponseBody(); - final List subscriptionsWithVolume = hermes.api() - .querySubscriptionMetrics(queryGetSubscriptionsMetricsWithVolume) - .expectStatus().isOk() - .expectBodyList(SubscriptionNameWithMetrics.class).returnResult().getResponseBody(); - - // then - subscriptionsMatchesToNamesAndTheirTopicsNames(allSubscriptions, subscription1, subscription2); - subscriptionsMatchesToNamesAndTheirTopicsNames(subscriptionsWithPositiveThroughput, subscription2); - subscriptionsMatchesToNamesAndTheirTopicsNames(subscriptionsWithRateInRange, subscription2); - subscriptionsMatchesToNamesAndTheirTopicsNames(subscriptionsWithNegativeLag, subscription1, subscription2); - subscriptionsMatchesToNamesAndTheirTopicsNames(subscriptionsWithVolume, subscription1, subscription2); - }); - } - - @Test - public void shouldHandleUnavailableSubscriptionsMetrics() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription( - subscriptionWithRandomName(topic.getName(), "http://endpoint1").build() - ); - String queryGetAllSubscriptionsMetrics = "{\"query\": {}}"; - String queryGetSubscriptionsMetricsWithPositiveRate = "{\"query\": {\"rate\": {\"gt\": 0}}}"; - prometheus.stubDelay(Duration.ofMillis(3000)); - - waitAtMost(adjust(Duration.ofMinutes(1))).untilAsserted(() -> { - // when - List allSubscriptions = hermes.api() - .querySubscriptionMetrics(queryGetAllSubscriptionsMetrics) - .expectStatus().isOk() - .expectBodyList(SubscriptionNameWithMetrics.class).returnResult().getResponseBody(); - List subscriptionsWithPositiveRate = hermes.api() - .querySubscriptionMetrics(queryGetSubscriptionsMetricsWithPositiveRate) - .expectStatus().isOk() - .expectBodyList(SubscriptionNameWithMetrics.class).returnResult().getResponseBody(); - - // then - assertThatRateIsUnavailable(allSubscriptions, subscription); - assertThatRateIsUnavailable(subscriptionsWithPositiveRate, subscription); - }); - } - - private static void assertThatRateIsUnavailable(List allSubscriptions, Subscription... subscriptions) { - subscriptionsMatchesToNamesAndTheirTopicsNames(allSubscriptions, subscriptions); - for (SubscriptionNameWithMetrics metrics : allSubscriptions) { - assertThat(metrics.getRate().asString()).isEqualTo("unavailable"); - } - } - - private static void subscriptionsMatchesToNamesAndTheirTopicsNames(List found, - Subscription... expectedSubscriptions) { - assertThat(found).isNotNull(); - Map foundSubscriptionsAndTheirTopicNames = found.stream() - .collect(Collectors.toMap(SubscriptionNameWithMetrics::getName, SubscriptionNameWithMetrics::getTopicName)); - for (Subscription subscription : expectedSubscriptions) { - assertThat(foundSubscriptionsAndTheirTopicNames).containsKeys(subscription.getName()); - assertThat(foundSubscriptionsAndTheirTopicNames.get(subscription.getName())).isEqualTo(subscription.getQualifiedTopicName()); - } - } - - private Subscription enrichSubscription(SubscriptionBuilder subscription, String endpoint) { - return subscription - .withTrackingMode(TrackingMode.TRACK_ALL) - .withSubscriptionPolicy(subscriptionPolicy().applyDefaults().build()) - .withEndpoint(EndpointAddress.of(endpoint)) - .build(); - } - - private void assertListMatches(List elements, List found, List positions) { - found.removeIf(o -> !elements.contains(o)); - List expected = positions.stream().map(i -> elements.get(i - 1)).collect(toList()); - Assertions.assertThat(found).isSubsetOf(expected); - } - - private void assertTopicMetricsMatchesToNames(List found, List expectedQualifiedNames) { - List foundQualifiedNames = found.stream() - .map(TopicNameWithMetrics::getName) - .collect(toList()); - - Assertions.assertThat(foundQualifiedNames).containsAll(expectedQualifiedNames); - } - - private Topic createTopicWithRandomName() { - return hermes.initHelper().createTopic(topicWithRandomName().build()); - } - - private Group createGroupWithRandomName() { - return hermes.initHelper().createGroup(groupWithRandomName().build()); + } + + private static void subscriptionsMatchesToNamesAndTheirTopicsNames( + List found, Subscription... expectedSubscriptions) { + assertThat(found).isNotNull(); + Map foundSubscriptionsAndTheirTopicNames = + found.stream() + .collect( + Collectors.toMap( + SubscriptionNameWithMetrics::getName, + SubscriptionNameWithMetrics::getTopicName)); + for (Subscription subscription : expectedSubscriptions) { + assertThat(foundSubscriptionsAndTheirTopicNames).containsKeys(subscription.getName()); + assertThat(foundSubscriptionsAndTheirTopicNames.get(subscription.getName())) + .isEqualTo(subscription.getQualifiedTopicName()); } + } + + private Subscription enrichSubscription(SubscriptionBuilder subscription, String endpoint) { + return subscription + .withTrackingMode(TrackingMode.TRACK_ALL) + .withSubscriptionPolicy(subscriptionPolicy().applyDefaults().build()) + .withEndpoint(EndpointAddress.of(endpoint)) + .build(); + } + + private void assertListMatches(List elements, List found, List positions) { + found.removeIf(o -> !elements.contains(o)); + List expected = positions.stream().map(i -> elements.get(i - 1)).collect(toList()); + Assertions.assertThat(found).isSubsetOf(expected); + } + + private void assertTopicMetricsMatchesToNames( + List found, List expectedQualifiedNames) { + List foundQualifiedNames = + found.stream().map(TopicNameWithMetrics::getName).collect(toList()); + + Assertions.assertThat(foundQualifiedNames).containsAll(expectedQualifiedNames); + } + + private Topic createTopicWithRandomName() { + return hermes.initHelper().createTopic(topicWithRandomName().build()); + } + + private Group createGroupWithRandomName() { + return hermes.initHelper().createGroup(groupWithRandomName().build()); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/ReadOnlyModeTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/ReadOnlyModeTest.java index ac5059a78c..831be62c29 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/ReadOnlyModeTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/ReadOnlyModeTest.java @@ -1,5 +1,7 @@ package pl.allegro.tech.hermes.integrationtests.management; +import static pl.allegro.tech.hermes.test.helper.builder.GroupBuilder.groupWithRandomName; + import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -9,84 +11,81 @@ import pl.allegro.tech.hermes.management.TestSecurityProvider; import pl.allegro.tech.hermes.management.domain.mode.ModeService; -import static pl.allegro.tech.hermes.test.helper.builder.GroupBuilder.groupWithRandomName; - public class ReadOnlyModeTest { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - @BeforeEach - public void initialize() { - TestSecurityProvider.setUserIsAdmin(true); - hermes.api().setMode(ModeService.READ_WRITE); - } - - @AfterEach - public void cleanup() { - TestSecurityProvider.setUserIsAdmin(true); - } - - @Test - public void shouldAllowModifyingOperations() { - // given - hermes.api().setMode(ModeService.READ_WRITE); - TestSecurityProvider.setUserIsAdmin(false); - - // when - WebTestClient.ResponseSpec response = hermes.api().createGroup(groupWithRandomName().build()); - - // then - response.expectStatus().isCreated(); - } - - @Test - public void shouldRestrictModifyingOperationsForNonAdminUsers() { - // given - hermes.api().setMode(ModeService.READ_ONLY); - TestSecurityProvider.setUserIsAdmin(false); - - // when - WebTestClient.ResponseSpec response = hermes.api().createGroup(groupWithRandomName().build()); - - // then - response.expectStatus().isEqualTo(503); - } - - @Test - public void shouldNotRestrictModifyingOperationsForAdminUsers() { - // given - hermes.api().setMode(ModeService.READ_ONLY); - TestSecurityProvider.setUserIsAdmin(true); - - // when - WebTestClient.ResponseSpec response = hermes.api().createGroup(groupWithRandomName().build()); - - // then - response.expectStatus().isCreated(); - } - - @Test - public void shouldSwitchModeBack() { - // given - hermes.api().setMode(ModeService.READ_ONLY); - TestSecurityProvider.setUserIsAdmin(false); - - // when - WebTestClient.ResponseSpec response = hermes.api().createGroup(groupWithRandomName().build()); - - // then - response.expectStatus().isEqualTo(503); - - // and - TestSecurityProvider.setUserIsAdmin(true); - hermes.api().setMode(ModeService.READ_WRITE).expectStatus().isOk(); - TestSecurityProvider.setUserIsAdmin(false); - - // when - response = hermes.api().createGroup(groupWithRandomName().build()); - - // then - response.expectStatus().isCreated(); - } + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); + + @BeforeEach + public void initialize() { + TestSecurityProvider.setUserIsAdmin(true); + hermes.api().setMode(ModeService.READ_WRITE); + } + + @AfterEach + public void cleanup() { + TestSecurityProvider.setUserIsAdmin(true); + } + + @Test + public void shouldAllowModifyingOperations() { + // given + hermes.api().setMode(ModeService.READ_WRITE); + TestSecurityProvider.setUserIsAdmin(false); + + // when + WebTestClient.ResponseSpec response = hermes.api().createGroup(groupWithRandomName().build()); + + // then + response.expectStatus().isCreated(); + } + + @Test + public void shouldRestrictModifyingOperationsForNonAdminUsers() { + // given + hermes.api().setMode(ModeService.READ_ONLY); + TestSecurityProvider.setUserIsAdmin(false); + + // when + WebTestClient.ResponseSpec response = hermes.api().createGroup(groupWithRandomName().build()); + + // then + response.expectStatus().isEqualTo(503); + } + + @Test + public void shouldNotRestrictModifyingOperationsForAdminUsers() { + // given + hermes.api().setMode(ModeService.READ_ONLY); + TestSecurityProvider.setUserIsAdmin(true); + + // when + WebTestClient.ResponseSpec response = hermes.api().createGroup(groupWithRandomName().build()); + + // then + response.expectStatus().isCreated(); + } + + @Test + public void shouldSwitchModeBack() { + // given + hermes.api().setMode(ModeService.READ_ONLY); + TestSecurityProvider.setUserIsAdmin(false); + + // when + WebTestClient.ResponseSpec response = hermes.api().createGroup(groupWithRandomName().build()); + + // then + response.expectStatus().isEqualTo(503); + + // and + TestSecurityProvider.setUserIsAdmin(true); + hermes.api().setMode(ModeService.READ_WRITE).expectStatus().isOk(); + TestSecurityProvider.setUserIsAdmin(false); + + // when + response = hermes.api().createGroup(groupWithRandomName().build()); + + // then + response.expectStatus().isCreated(); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/ReadinessManagementTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/ReadinessManagementTest.java index 133f7ff0fb..6fe38d2905 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/ReadinessManagementTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/ReadinessManagementTest.java @@ -1,31 +1,34 @@ package pl.allegro.tech.hermes.integrationtests.management; +import static pl.allegro.tech.hermes.api.DatacenterReadiness.ReadinessStatus.READY; +import static pl.allegro.tech.hermes.infrastructure.dc.DefaultDatacenterNameProvider.DEFAULT_DC_NAME; + import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import pl.allegro.tech.hermes.api.DatacenterReadiness; import pl.allegro.tech.hermes.integrationtests.setup.HermesExtension; -import static pl.allegro.tech.hermes.api.DatacenterReadiness.ReadinessStatus.READY; -import static pl.allegro.tech.hermes.infrastructure.dc.DefaultDatacenterNameProvider.DEFAULT_DC_NAME; - public class ReadinessManagementTest { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); - @Test - public void shouldNotFailWhileSettingReadinessStatusForTheFirstTime() { - //when - hermes.api().setReadiness("unhealthy-dc", false); + @Test + public void shouldNotFailWhileSettingReadinessStatusForTheFirstTime() { + // when + hermes.api().setReadiness("unhealthy-dc", false); - //then - hermes.api().getReadiness() - .expectStatus() - .isOk() - .expectBodyList(DatacenterReadiness.class) - // 'unhealthy-dc' should not be returned here, since it doesn't exist in management configuration - // In this test, we are just verifying if setting readiness status for the first time doesn't break anything. - .hasSize(1) - .contains(new DatacenterReadiness(DEFAULT_DC_NAME, READY)); - } + // then + hermes + .api() + .getReadiness() + .expectStatus() + .isOk() + .expectBodyList(DatacenterReadiness.class) + // 'unhealthy-dc' should not be returned here, since it doesn't exist in management + // configuration + // In this test, we are just verifying if setting readiness status for the first time + // doesn't break anything. + .hasSize(1) + .contains(new DatacenterReadiness(DEFAULT_DC_NAME, READY)); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/SchemaManagementTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/SchemaManagementTest.java index 463beee22b..667dafd955 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/SchemaManagementTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/SchemaManagementTest.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.integrationtests.management; +import static pl.allegro.tech.hermes.api.ContentType.AVRO; +import static pl.allegro.tech.hermes.api.ContentType.JSON; +import static pl.allegro.tech.hermes.api.TopicWithSchema.topicWithSchema; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.springframework.test.web.reactive.server.WebTestClient; @@ -8,118 +13,109 @@ import pl.allegro.tech.hermes.integrationtests.setup.HermesExtension; import pl.allegro.tech.hermes.test.helper.avro.AvroUserSchemaLoader; -import static pl.allegro.tech.hermes.api.ContentType.AVRO; -import static pl.allegro.tech.hermes.api.ContentType.JSON; -import static pl.allegro.tech.hermes.api.TopicWithSchema.topicWithSchema; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class SchemaManagementTest { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); - private static final String SCHEMA_V1 = AvroUserSchemaLoader.load().toString(); + private static final String SCHEMA_V1 = AvroUserSchemaLoader.load().toString(); - private static final String SCHEMA_V2 = AvroUserSchemaLoader.load("/schema/user_v2.avsc").toString(); + private static final String SCHEMA_V2 = + AvroUserSchemaLoader.load("/schema/user_v2.avsc").toString(); - @Test - public void shouldNotSaveSchemaForInvalidTopic() { - // given && when - Topic topic = topicWithRandomName().build(); - WebTestClient.ResponseSpec response = hermes.api().saveSchema(topic.getQualifiedName(), "{}"); + @Test + public void shouldNotSaveSchemaForInvalidTopic() { + // given && when + Topic topic = topicWithRandomName().build(); + WebTestClient.ResponseSpec response = hermes.api().saveSchema(topic.getQualifiedName(), "{}"); - // then - response.expectStatus().isNotFound(); - } + // then + response.expectStatus().isNotFound(); + } - @Test - public void shouldSaveSchemaForExistingTopic() { - // given - TopicWithSchema topicWithSchema = topicWithSchema(topicWithRandomName() - .withContentType(AVRO) - .build(), SCHEMA_V1); + @Test + public void shouldSaveSchemaForExistingTopic() { + // given + TopicWithSchema topicWithSchema = + topicWithSchema(topicWithRandomName().withContentType(AVRO).build(), SCHEMA_V1); - Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); + Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); - // when - WebTestClient.ResponseSpec response = hermes.api().saveSchema(topic.getQualifiedName(), SCHEMA_V2); + // when + WebTestClient.ResponseSpec response = + hermes.api().saveSchema(topic.getQualifiedName(), SCHEMA_V2); - // then - response.expectStatus().isCreated(); - } + // then + response.expectStatus().isCreated(); + } - @Test - public void shouldReturnSchemaForTopic() { - // given - TopicWithSchema topicWithSchema = topicWithSchema(topicWithRandomName() - .withContentType(AVRO) - .build(), SCHEMA_V1); + @Test + public void shouldReturnSchemaForTopic() { + // given + TopicWithSchema topicWithSchema = + topicWithSchema(topicWithRandomName().withContentType(AVRO).build(), SCHEMA_V1); - Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); + Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); - // when - WebTestClient.ResponseSpec response = hermes.api().getSchema(topic.getQualifiedName()); + // when + WebTestClient.ResponseSpec response = hermes.api().getSchema(topic.getQualifiedName()); - // then - response.expectStatus().isOk() - .expectBody(String.class).isEqualTo(SCHEMA_V1); - } + // then + response.expectStatus().isOk().expectBody(String.class).isEqualTo(SCHEMA_V1); + } - @Test - public void shouldRespondWithNoContentOnMissingSchema() { - // when - Topic topic = topicWithRandomName().build(); + @Test + public void shouldRespondWithNoContentOnMissingSchema() { + // when + Topic topic = topicWithRandomName().build(); - WebTestClient.ResponseSpec response = hermes.api().getSchema(topic.getQualifiedName()); + WebTestClient.ResponseSpec response = hermes.api().getSchema(topic.getQualifiedName()); - // then - response.expectStatus().isNoContent(); - } + // then + response.expectStatus().isNoContent(); + } - @Test - public void shouldSuccessfullyRemoveSchemaWhenSchemaRemovingIsEnabled() { - // given - TopicWithSchema topicWithSchema = topicWithSchema(topicWithRandomName() - .withContentType(AVRO) - .build(), SCHEMA_V1); + @Test + public void shouldSuccessfullyRemoveSchemaWhenSchemaRemovingIsEnabled() { + // given + TopicWithSchema topicWithSchema = + topicWithSchema(topicWithRandomName().withContentType(AVRO).build(), SCHEMA_V1); - Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); + Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); - // when - WebTestClient.ResponseSpec response = hermes.api().deleteSchema(topic.getQualifiedName()); + // when + WebTestClient.ResponseSpec response = hermes.api().deleteSchema(topic.getQualifiedName()); - // then - response.expectStatus().isOk(); - } + // then + response.expectStatus().isOk(); + } - @Test - public void shouldNotSaveInvalidAvroSchema() { - // given - TopicWithSchema topicWithSchema = topicWithSchema(topicWithRandomName() - .withContentType(AVRO) - .build(), SCHEMA_V1); + @Test + public void shouldNotSaveInvalidAvroSchema() { + // given + TopicWithSchema topicWithSchema = + topicWithSchema(topicWithRandomName().withContentType(AVRO).build(), SCHEMA_V1); - Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); + Topic topic = hermes.initHelper().createTopicWithSchema(topicWithSchema); - // when - WebTestClient.ResponseSpec response = hermes.api().saveSchema(topic.getQualifiedName(), "{"); + // when + WebTestClient.ResponseSpec response = hermes.api().saveSchema(topic.getQualifiedName(), "{"); - // then - response.expectStatus().isBadRequest(); - } + // then + response.expectStatus().isBadRequest(); + } - @Test - public void shouldReturnBadRequestDueToNoSchemaValidatorForJsonTopic() { - // given - Topic topic = topicWithRandomName().withContentType(JSON).build(); + @Test + public void shouldReturnBadRequestDueToNoSchemaValidatorForJsonTopic() { + // given + Topic topic = topicWithRandomName().withContentType(JSON).build(); - hermes.initHelper().createTopic(topic); + hermes.initHelper().createTopic(topic); - // when - WebTestClient.ResponseSpec response = hermes.api().saveSchema(topic.getQualifiedName(), true, SCHEMA_V1); + // when + WebTestClient.ResponseSpec response = + hermes.api().saveSchema(topic.getQualifiedName(), true, SCHEMA_V1); - // then - response.expectStatus().isBadRequest(); - } + // then + response.expectStatus().isBadRequest(); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/StatsTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/StatsTest.java index d83df905be..f75182b372 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/StatsTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/StatsTest.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.integrationtests.management; +import static org.assertj.core.api.Assertions.assertThat; +import static pl.allegro.tech.hermes.api.TopicWithSchema.topicWithSchema; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + +import java.util.UUID; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import pl.allegro.tech.hermes.api.ContentType; @@ -12,74 +17,84 @@ import pl.allegro.tech.hermes.test.helper.avro.AvroUserSchemaLoader; import pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder; -import java.util.UUID; - -import static org.assertj.core.api.Assertions.assertThat; -import static pl.allegro.tech.hermes.api.TopicWithSchema.topicWithSchema; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class StatsTest { - private static final String SCHEMA = AvroUserSchemaLoader.load().toString(); - - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - @Test - public void shouldGetStats() { - // given - Stats initalStats = getStats(); - - TopicWithSchema topic1 = topicWithSchema(topic(Topic.Ack.ALL, ContentType.AVRO, true), SCHEMA); - Topic topic2 = topic(Topic.Ack.LEADER, ContentType.JSON, false); - - hermes.initHelper().createTopicWithSchema(topic1); - hermes.initHelper().createTopic(topic2); - hermes.initHelper().createTopic(topic(Topic.Ack.ALL, ContentType.JSON, true)); - - hermes.initHelper().createSubscription( - subscription(topic1.getQualifiedName(), ContentType.AVRO, TrackingMode.TRACKING_OFF) - ); - - hermes.initHelper().createSubscription( - subscription(topic2.getQualifiedName(), ContentType.JSON, TrackingMode.TRACK_ALL) - ); - - hermes.initHelper().createSubscription( - subscription(topic2.getQualifiedName(), ContentType.AVRO, TrackingMode.TRACKING_OFF) - ); - - // when - Stats stats = getStats(); - - // then - assertThat(stats.getTopicStats().getTopicCount()).isEqualTo(initalStats.getTopicStats().getTopicCount() + 3); - assertThat(stats.getTopicStats().getAckAllTopicCount()).isEqualTo(initalStats.getTopicStats().getAckAllTopicCount() + 2); - assertThat(stats.getTopicStats().getTrackingEnabledTopicCount()).isEqualTo(initalStats.getTopicStats().getTrackingEnabledTopicCount() + 2); - assertThat(stats.getTopicStats().getAvroTopicCount()).isEqualTo(initalStats.getTopicStats().getAvroTopicCount() + 1); - - assertThat(stats.getSubscriptionStats().getSubscriptionCount()).isEqualTo(initalStats.getSubscriptionStats().getSubscriptionCount() + 3); - assertThat(stats.getSubscriptionStats().getTrackingEnabledSubscriptionCount()).isEqualTo(initalStats.getSubscriptionStats().getTrackingEnabledSubscriptionCount() + 1); - assertThat(stats.getSubscriptionStats().getAvroSubscriptionCount()).isEqualTo(initalStats.getSubscriptionStats().getAvroSubscriptionCount() + 2); - } - - private Topic topic(Topic.Ack ack, ContentType contentType, boolean trackingEnabled) { - return topicWithRandomName() - .withAck(ack) - .withContentType(contentType) - .withTrackingEnabled(trackingEnabled) - .build(); - } - - private Subscription subscription(String topicQualifiedName, ContentType contentType, TrackingMode trackingMode) { - return SubscriptionBuilder.subscription(topicQualifiedName, UUID.randomUUID().toString()) - .withContentType(contentType) - .withTrackingMode(trackingMode) - .build(); - } - - private Stats getStats() { - return hermes.api().getManagementStats().expectStatus().isOk().expectBody(Stats.class).returnResult().getResponseBody(); - } - + private static final String SCHEMA = AvroUserSchemaLoader.load().toString(); + + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); + + @Test + public void shouldGetStats() { + // given + Stats initalStats = getStats(); + + TopicWithSchema topic1 = topicWithSchema(topic(Topic.Ack.ALL, ContentType.AVRO, true), SCHEMA); + Topic topic2 = topic(Topic.Ack.LEADER, ContentType.JSON, false); + + hermes.initHelper().createTopicWithSchema(topic1); + hermes.initHelper().createTopic(topic2); + hermes.initHelper().createTopic(topic(Topic.Ack.ALL, ContentType.JSON, true)); + + hermes + .initHelper() + .createSubscription( + subscription(topic1.getQualifiedName(), ContentType.AVRO, TrackingMode.TRACKING_OFF)); + + hermes + .initHelper() + .createSubscription( + subscription(topic2.getQualifiedName(), ContentType.JSON, TrackingMode.TRACK_ALL)); + + hermes + .initHelper() + .createSubscription( + subscription(topic2.getQualifiedName(), ContentType.AVRO, TrackingMode.TRACKING_OFF)); + + // when + Stats stats = getStats(); + + // then + assertThat(stats.getTopicStats().getTopicCount()) + .isEqualTo(initalStats.getTopicStats().getTopicCount() + 3); + assertThat(stats.getTopicStats().getAckAllTopicCount()) + .isEqualTo(initalStats.getTopicStats().getAckAllTopicCount() + 2); + assertThat(stats.getTopicStats().getTrackingEnabledTopicCount()) + .isEqualTo(initalStats.getTopicStats().getTrackingEnabledTopicCount() + 2); + assertThat(stats.getTopicStats().getAvroTopicCount()) + .isEqualTo(initalStats.getTopicStats().getAvroTopicCount() + 1); + + assertThat(stats.getSubscriptionStats().getSubscriptionCount()) + .isEqualTo(initalStats.getSubscriptionStats().getSubscriptionCount() + 3); + assertThat(stats.getSubscriptionStats().getTrackingEnabledSubscriptionCount()) + .isEqualTo(initalStats.getSubscriptionStats().getTrackingEnabledSubscriptionCount() + 1); + assertThat(stats.getSubscriptionStats().getAvroSubscriptionCount()) + .isEqualTo(initalStats.getSubscriptionStats().getAvroSubscriptionCount() + 2); + } + + private Topic topic(Topic.Ack ack, ContentType contentType, boolean trackingEnabled) { + return topicWithRandomName() + .withAck(ack) + .withContentType(contentType) + .withTrackingEnabled(trackingEnabled) + .build(); + } + + private Subscription subscription( + String topicQualifiedName, ContentType contentType, TrackingMode trackingMode) { + return SubscriptionBuilder.subscription(topicQualifiedName, UUID.randomUUID().toString()) + .withContentType(contentType) + .withTrackingMode(trackingMode) + .build(); + } + + private Stats getStats() { + return hermes + .api() + .getManagementStats() + .expectStatus() + .isOk() + .expectBody(Stats.class) + .returnResult() + .getResponseBody(); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/SubscriptionManagementTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/SubscriptionManagementTest.java index a1e3f1d475..3cf0ff68b5 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/SubscriptionManagementTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/SubscriptionManagementTest.java @@ -1,6 +1,26 @@ package pl.allegro.tech.hermes.integrationtests.management; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.waitAtMost; +import static pl.allegro.tech.hermes.api.PatchData.patchData; +import static pl.allegro.tech.hermes.api.SubscriptionHealth.Status.NO_DATA; +import static pl.allegro.tech.hermes.api.SubscriptionHealth.Status.UNHEALTHY; +import static pl.allegro.tech.hermes.api.SubscriptionHealthProblem.malfunctioning; +import static pl.allegro.tech.hermes.integrationtests.prometheus.SubscriptionMetrics.subscriptionMetrics; +import static pl.allegro.tech.hermes.integrationtests.prometheus.TopicMetrics.topicMetrics; +import static pl.allegro.tech.hermes.integrationtests.setup.HermesExtension.auditEvents; +import static pl.allegro.tech.hermes.integrationtests.setup.HermesExtension.brokerOperations; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + import com.google.common.collect.ImmutableMap; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Stream; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; @@ -27,654 +47,765 @@ import pl.allegro.tech.hermes.management.TestSecurityProvider; import pl.allegro.tech.hermes.test.helper.message.TestMessage; -import java.time.Duration; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Stream; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.waitAtMost; -import static pl.allegro.tech.hermes.api.PatchData.patchData; -import static pl.allegro.tech.hermes.api.SubscriptionHealth.Status.NO_DATA; -import static pl.allegro.tech.hermes.api.SubscriptionHealth.Status.UNHEALTHY; -import static pl.allegro.tech.hermes.api.SubscriptionHealthProblem.malfunctioning; -import static pl.allegro.tech.hermes.integrationtests.prometheus.SubscriptionMetrics.subscriptionMetrics; -import static pl.allegro.tech.hermes.integrationtests.prometheus.TopicMetrics.topicMetrics; -import static pl.allegro.tech.hermes.integrationtests.setup.HermesExtension.auditEvents; -import static pl.allegro.tech.hermes.integrationtests.setup.HermesExtension.brokerOperations; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class SubscriptionManagementTest { - @Order(0) - @RegisterExtension - public static final PrometheusExtension prometheus = new PrometheusExtension(); + @Order(0) + @RegisterExtension + public static final PrometheusExtension prometheus = new PrometheusExtension(); - @Order(1) - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension() - .withPrometheus(prometheus); + @Order(1) + @RegisterExtension + public static final HermesExtension hermes = new HermesExtension().withPrometheus(prometheus); - @RegisterExtension - public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); + @RegisterExtension + public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); - public static final TestMessage MESSAGE = TestMessage.of("hello", "world"); + public static final TestMessage MESSAGE = TestMessage.of("hello", "world"); - public static final PatchData PATCH_DATA = patchData().set("endpoint", EndpointAddress.of("http://localhost:7777/topics/test-topic")).build(); + public static final PatchData PATCH_DATA = + patchData() + .set("endpoint", EndpointAddress.of("http://localhost:7777/topics/test-topic")) + .build(); - @AfterEach - public void cleanup() { - TestSecurityProvider.reset(); - } + @AfterEach + public void cleanup() { + TestSecurityProvider.reset(); + } - @Test - public void shouldEmitAuditEventWhenSubscriptionCreated() { - //given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + @Test + public void shouldEmitAuditEventWhenSubscriptionCreated() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - //when - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); + // when + Subscription subscription = + hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); - //then - assertThat( - auditEvents.getLastReceivedRequest().getBodyAsString() - ).contains("CREATED", subscription.getName()); - } + // then + assertThat(auditEvents.getLastReceivedRequest().getBodyAsString()) + .contains("CREATED", subscription.getName()); + } - @Test - public void shouldReturnSubscription() { - //given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); + @Test + public void shouldReturnSubscription() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); - //when - Subscription fetchedSubscription = hermes.api().getSubscription(topic.getQualifiedName(), subscription.getName()); + // when + Subscription fetchedSubscription = + hermes.api().getSubscription(topic.getQualifiedName(), subscription.getName()); + + // then + assertThat(fetchedSubscription.getName()).isEqualTo(subscription.getName()); + assertThat(fetchedSubscription.isAutoDeleteWithTopicEnabled()) + .isEqualTo(subscription.isAutoDeleteWithTopicEnabled()); + assertThat(fetchedSubscription.getQualifiedTopicName()).isEqualTo(topic.getQualifiedName()); + assertThat(fetchedSubscription.isTrackingEnabled()).isEqualTo(subscription.isTrackingEnabled()); + } + + @Test + public void shouldEmitAuditEventWhenSubscriptionRemoved() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); - //then - assertThat(fetchedSubscription.getName()).isEqualTo(subscription.getName()); - assertThat(fetchedSubscription.isAutoDeleteWithTopicEnabled()).isEqualTo(subscription.isAutoDeleteWithTopicEnabled()); - assertThat(fetchedSubscription.getQualifiedTopicName()).isEqualTo(topic.getQualifiedName()); - assertThat(fetchedSubscription.isTrackingEnabled()).isEqualTo(subscription.isTrackingEnabled()); - } + // when + hermes.api().deleteSubscription(topic.getQualifiedName(), subscription.getName()); - @Test - public void shouldEmitAuditEventWhenSubscriptionRemoved() { - //given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); + // then + assertThat(auditEvents.getLastReceivedRequest().getBodyAsString()) + .contains("REMOVED", subscription.getName()); + } - //when - hermes.api().deleteSubscription(topic.getQualifiedName(), subscription.getName()); + @Test + public void shouldEmitAuditEventWhenSubscriptionEndpointUpdated() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); - //then - assertThat( - auditEvents.getLastReceivedRequest().getBodyAsString() - ).contains("REMOVED", subscription.getName()); - } - - @Test - public void shouldEmitAuditEventWhenSubscriptionEndpointUpdated() { - //given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); - - //when - hermes.api().updateSubscription(topic, subscription.getName(), patchData().set("endpoint", EndpointAddress.of("http://another-service")).build()); - - //then - assertThat( - auditEvents.getLastReceivedRequest().getBodyAsString() - ).contains("UPDATED", subscription.getName()); - } - - @Test - public void shouldCreateSubscriptionWithActiveStatus() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = subscriptionWithRandomName(topic.getName()).build(); - - // when - WebTestClient.ResponseSpec response = hermes.api().createSubscription(subscription); - - // then - response.expectStatus().isCreated(); - hermes.api().waitUntilSubscriptionActivated(topic.getQualifiedName(), subscription.getName()); - } - - @Test - public void shouldNotRemoveSubscriptionAfterItsRecreation() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); - - // when - WebTestClient.ResponseSpec response = hermes.api().createSubscription(subscription); - - // then - response.expectStatus().isBadRequest(); - - assertThat( - hermes.api().getSubscription(topic.getQualifiedName(), subscription.getName()).getName() - ).isEqualTo(subscription.getName()); - } - - @Test - public void shouldNotCreateSubscriptionWithoutTopic() { - // given - Subscription subscription = subscriptionWithRandomName(TopicName.fromQualifiedName("pl.group.non-existing")).build(); - - // when - WebTestClient.ResponseSpec response = hermes.api().createSubscription(subscription); - - // then - response.expectStatus().isNotFound(); - } - - @Test - public void shouldSuspendSubscription() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); - - // when - WebTestClient.ResponseSpec response = hermes.api().suspendSubscription(topic, subscription.getName()); - - // then - response.expectStatus().isOk(); - hermes.api().waitUntilSubscriptionSuspended(topic.getQualifiedName(), subscription.getName()); - } - - @Test - public void shouldUpdateSubscriptionEndpoint() { - //given - EndpointAddress updatedEndpoint = EndpointAddress.of("http://another-service-endpoint"); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); - - // when - WebTestClient.ResponseSpec response = hermes.api().updateSubscription(topic, subscription.getName(), patchData().set("endpoint", updatedEndpoint).build()); - - // then - response.expectStatus().isOk(); - waitAtMost(Duration.ofSeconds(10)) - .until(() -> hermes.api().getSubscriptionResponse(topic.getQualifiedName(), subscription.getName()) - .expectStatus() - .is2xxSuccessful() - .expectBody(Subscription.class) - .returnResult().getResponseBody().getEndpoint().equals(updatedEndpoint) - ); - } - - @Test - public void shouldUpdateSubscriptionPolicy() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); - PatchData patchData = patchData().set("subscriptionPolicy", ImmutableMap.builder() - .put("inflightSize", 100) - .put("messageBackoff", 100) - .put("messageTtl", 3600) - .put("rate", 300) - .put("requestTimeout", 1000) - .put("socketTimeout", 3000) - .put("retryClientErrors", false) - .put("sendingDelay", 1000) - .build() - ).build(); - - // when - WebTestClient.ResponseSpec response = hermes.api().updateSubscription(topic, subscription.getName(), patchData); - - - // then - response.expectStatus().isOk(); - SubscriptionPolicy policy = hermes.api().getSubscription(topic.getQualifiedName(), subscription.getName()).getSerialSubscriptionPolicy(); - assertThat(policy.getInflightSize()).isEqualTo(100); - assertThat(policy.getMessageBackoff()).isEqualTo(100); - assertThat(policy.getMessageTtl()).isEqualTo(3600); - assertThat(policy.getRate()).isEqualTo(300); - assertThat(policy.getRequestTimeout()).isEqualTo(1000); - assertThat(policy.getSocketTimeout()).isEqualTo(3000); - assertThat(policy.isRetryClientErrors()).isFalse(); - assertThat(policy.getSendingDelay()).isEqualTo(1000); - } - - @Test - public void shouldRemoveSubscription() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); - - // when - WebTestClient.ResponseSpec response = hermes.api().deleteSubscription(topic.getQualifiedName(), subscription.getName()); - - // then - response.expectStatus().isOk(); - hermes.api().getSubscriptionResponse(topic.getQualifiedName(), subscription.getName()).expectStatus().isBadRequest(); - } - - @Test - public void shouldReturnSubscriptionsThatAreCurrentlyTrackedForGivenTopic() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription trackedSubscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).withTrackingMode(TrackingMode.TRACK_ALL).build()); + // when + hermes + .api() + .updateSubscription( + topic, + subscription.getName(), + patchData().set("endpoint", EndpointAddress.of("http://another-service")).build()); + + // then + assertThat(auditEvents.getLastReceivedRequest().getBodyAsString()) + .contains("UPDATED", subscription.getName()); + } + + @Test + public void shouldCreateSubscriptionWithActiveStatus() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = subscriptionWithRandomName(topic.getName()).build(); + + // when + WebTestClient.ResponseSpec response = hermes.api().createSubscription(subscription); + + // then + response.expectStatus().isCreated(); + hermes.api().waitUntilSubscriptionActivated(topic.getQualifiedName(), subscription.getName()); + } + + @Test + public void shouldNotRemoveSubscriptionAfterItsRecreation() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); - // when - WebTestClient.ResponseSpec response = hermes.api().listTrackedSubscriptions(topic.getQualifiedName()); - - // then - response.expectStatus().isOk(); - assertThat(response - .expectBody(String[].class) - .returnResult() - .getResponseBody() - ).containsExactly(trackedSubscription.getName()); - } - - @Test - public void shouldReturnsTrackedAndNotSuspendedSubscriptionsForGivenTopic() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription trackedSubscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).withTrackingMode(TrackingMode.TRACK_ALL).build()); - Subscription trackedSubscriptionSuspended = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).withTrackingMode(TrackingMode.TRACK_ALL).build()); + // when + WebTestClient.ResponseSpec response = hermes.api().createSubscription(subscription); + + // then + response.expectStatus().isBadRequest(); + + assertThat( + hermes + .api() + .getSubscription(topic.getQualifiedName(), subscription.getName()) + .getName()) + .isEqualTo(subscription.getName()); + } + + @Test + public void shouldNotCreateSubscriptionWithoutTopic() { + // given + Subscription subscription = + subscriptionWithRandomName(TopicName.fromQualifiedName("pl.group.non-existing")).build(); + + // when + WebTestClient.ResponseSpec response = hermes.api().createSubscription(subscription); + + // then + response.expectStatus().isNotFound(); + } + + @Test + public void shouldSuspendSubscription() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); - hermes.api().suspendSubscription(topic, trackedSubscriptionSuspended.getName()); + // when + WebTestClient.ResponseSpec response = + hermes.api().suspendSubscription(topic, subscription.getName()); + + // then + response.expectStatus().isOk(); + hermes.api().waitUntilSubscriptionSuspended(topic.getQualifiedName(), subscription.getName()); + } + + @Test + public void shouldUpdateSubscriptionEndpoint() { + // given + EndpointAddress updatedEndpoint = EndpointAddress.of("http://another-service-endpoint"); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); - // and - String query = "{\"query\": {\"trackingEnabled\": \"true\", \"state\": {\"ne\": \"SUSPENDED\"}}}"; + // when + WebTestClient.ResponseSpec response = + hermes + .api() + .updateSubscription( + topic, + subscription.getName(), + patchData().set("endpoint", updatedEndpoint).build()); + + // then + response.expectStatus().isOk(); + waitAtMost(Duration.ofSeconds(10)) + .until( + () -> + hermes + .api() + .getSubscriptionResponse(topic.getQualifiedName(), subscription.getName()) + .expectStatus() + .is2xxSuccessful() + .expectBody(Subscription.class) + .returnResult() + .getResponseBody() + .getEndpoint() + .equals(updatedEndpoint)); + } + + @Test + public void shouldUpdateSubscriptionPolicy() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); + PatchData patchData = + patchData() + .set( + "subscriptionPolicy", + ImmutableMap.builder() + .put("inflightSize", 100) + .put("messageBackoff", 100) + .put("messageTtl", 3600) + .put("rate", 300) + .put("requestTimeout", 1000) + .put("socketTimeout", 3000) + .put("retryClientErrors", false) + .put("sendingDelay", 1000) + .build()) + .build(); + + // when + WebTestClient.ResponseSpec response = + hermes.api().updateSubscription(topic, subscription.getName(), patchData); + + // then + response.expectStatus().isOk(); + SubscriptionPolicy policy = + hermes + .api() + .getSubscription(topic.getQualifiedName(), subscription.getName()) + .getSerialSubscriptionPolicy(); + assertThat(policy.getInflightSize()).isEqualTo(100); + assertThat(policy.getMessageBackoff()).isEqualTo(100); + assertThat(policy.getMessageTtl()).isEqualTo(3600); + assertThat(policy.getRate()).isEqualTo(300); + assertThat(policy.getRequestTimeout()).isEqualTo(1000); + assertThat(policy.getSocketTimeout()).isEqualTo(3000); + assertThat(policy.isRetryClientErrors()).isFalse(); + assertThat(policy.getSendingDelay()).isEqualTo(1000); + } + + @Test + public void shouldRemoveSubscription() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); - // when - WebTestClient.ResponseSpec response = hermes.api().querySubscriptions(topic.getQualifiedName(), query); + // when + WebTestClient.ResponseSpec response = + hermes.api().deleteSubscription(topic.getQualifiedName(), subscription.getName()); - // then - assertThat(Arrays.stream(Objects.requireNonNull(response.expectBody(String[].class).returnResult().getResponseBody())).toList()) - .containsExactly(trackedSubscription.getName()); - } + // then + response.expectStatus().isOk(); + hermes + .api() + .getSubscriptionResponse(topic.getQualifiedName(), subscription.getName()) + .expectStatus() + .isBadRequest(); + } + + @Test + public void shouldReturnSubscriptionsThatAreCurrentlyTrackedForGivenTopic() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription trackedSubscription = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName()) + .withTrackingMode(TrackingMode.TRACK_ALL) + .build()); + hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); + + // when + WebTestClient.ResponseSpec response = + hermes.api().listTrackedSubscriptions(topic.getQualifiedName()); + + // then + response.expectStatus().isOk(); + assertThat(response.expectBody(String[].class).returnResult().getResponseBody()) + .containsExactly(trackedSubscription.getName()); + } + + @Test + public void shouldReturnsTrackedAndNotSuspendedSubscriptionsForGivenTopic() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription trackedSubscription = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName()) + .withTrackingMode(TrackingMode.TRACK_ALL) + .build()); + Subscription trackedSubscriptionSuspended = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName()) + .withTrackingMode(TrackingMode.TRACK_ALL) + .build()); + hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); + + hermes.api().suspendSubscription(topic, trackedSubscriptionSuspended.getName()); + + // and + String query = + "{\"query\": {\"trackingEnabled\": \"true\", \"state\": {\"ne\": \"SUSPENDED\"}}}"; + + // when + WebTestClient.ResponseSpec response = + hermes.api().querySubscriptions(topic.getQualifiedName(), query); + + // then + assertThat( + Arrays.stream( + Objects.requireNonNull( + response.expectBody(String[].class).returnResult().getResponseBody())) + .toList()) + .containsExactly(trackedSubscription.getName()); + } + + @Test + public void shouldNotAllowSubscriptionNameToContainDollarSign() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + + Stream.of("$name", "na$me", "name$") + .forEach( + name -> { + // when + WebTestClient.ResponseSpec response = + hermes.api().createSubscription(subscription(topic, name).build()); + + // then + response.expectStatus().isBadRequest(); + }); + } + + @Test + public void shouldReturnHealthyStatusForAHealthySubscription() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); - @Test - public void shouldNotAllowSubscriptionNameToContainDollarSign() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + // and + prometheus.stubTopicMetrics( + topicMetrics(topic.getName()) + .withDeliveryRate(100) + .withRate(100) + .withThroughput(0) + .build()); + prometheus.stubSubscriptionMetrics( + subscriptionMetrics(subscription.getQualifiedName()).withRate(100).build()); + + // when + WebTestClient.ResponseSpec response = + hermes.api().getSubscriptionHealth(topic.getQualifiedName(), subscription.getName()); + + // then + assertThat(response.expectBody(SubscriptionHealth.class).returnResult().getResponseBody()) + .isEqualTo(SubscriptionHealth.HEALTHY); + } + + @Test + public void shouldReturnUnhealthyStatusWithAProblemForMalfunctioningSubscription() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); - Stream.of("$name", "na$me", "name$").forEach(name -> { - // when - WebTestClient.ResponseSpec response = hermes.api().createSubscription(subscription(topic, name).build()); + // and + prometheus.stubTopicMetrics( + topicMetrics(topic.getName()).withDeliveryRate(100).withRate(50).withThroughput(0).build()); + prometheus.stubSubscriptionMetrics( + subscriptionMetrics(subscription.getQualifiedName()).withRate(50).with500Rate(11).build()); + + // when + WebTestClient.ResponseSpec response = + hermes.api().getSubscriptionHealth(topic.getQualifiedName(), subscription.getName()); + + // then + SubscriptionHealth subscriptionHealth = + response.expectBody(SubscriptionHealth.class).returnResult().getResponseBody(); + assertThat(subscriptionHealth.getStatus()).isEqualTo(UNHEALTHY); + assertThat(subscriptionHealth.getProblems()) + .containsOnly(malfunctioning(11, topic.getQualifiedName() + "$" + subscription.getName())); + } + + @Test + public void shouldReturnNoDataStatusWhenPrometheusRespondsWithAnError() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); - // then - response.expectStatus().isBadRequest(); - }); - } - - @Test - public void shouldReturnHealthyStatusForAHealthySubscription() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); - - // and - prometheus.stubTopicMetrics( - topicMetrics(topic.getName()) - .withDeliveryRate(100) - .withRate(100) - .withThroughput(0).build()); - prometheus.stubSubscriptionMetrics( - subscriptionMetrics(subscription.getQualifiedName()) - .withRate(100) - .build()); - - // when - WebTestClient.ResponseSpec response = hermes.api().getSubscriptionHealth(topic.getQualifiedName(), subscription.getName()); - - // then - assertThat(response.expectBody(SubscriptionHealth.class).returnResult().getResponseBody()) - .isEqualTo(SubscriptionHealth.HEALTHY); - } - - @Test - public void shouldReturnUnhealthyStatusWithAProblemForMalfunctioningSubscription() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); - - // and - prometheus.stubTopicMetrics( - topicMetrics(topic.getName()) - .withDeliveryRate(100) - .withRate(50) - .withThroughput(0).build()); - prometheus.stubSubscriptionMetrics( - subscriptionMetrics(subscription.getQualifiedName()) - .withRate(50) - .with500Rate(11) - .build()); - - // when - WebTestClient.ResponseSpec response = hermes.api().getSubscriptionHealth(topic.getQualifiedName(), subscription.getName()); - - // then - SubscriptionHealth subscriptionHealth = response.expectBody(SubscriptionHealth.class).returnResult().getResponseBody(); - assertThat(subscriptionHealth.getStatus()).isEqualTo(UNHEALTHY); - assertThat(subscriptionHealth.getProblems()).containsOnly(malfunctioning(11, topic.getQualifiedName() + "$" + subscription.getName())); - } - - @Test - public void shouldReturnNoDataStatusWhenPrometheusRespondsWithAnError() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); - - // and - prometheus.stub500Error(); - - // when - WebTestClient.ResponseSpec response = hermes.api().getSubscriptionHealth(topic.getQualifiedName(), subscription.getName()); - - // then - SubscriptionHealth subscriptionHealth = response.expectBody(SubscriptionHealth.class).returnResult().getResponseBody(); - assertThat(subscriptionHealth.getStatus()).isEqualTo(NO_DATA); - } - - @Test - public void shouldNotAllowSubscriptionWithBatchDeliveryAndAvroContentType() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = subscriptionWithRandomName(topic.getName()) - .withDeliveryType(DeliveryType.BATCH) - .withContentType(ContentType.AVRO) - .build(); - - // when - WebTestClient.ResponseSpec response = hermes.api().createSubscription(subscription); - - // then - response.expectStatus().isBadRequest(); - } - - @Test - public void shouldReturnConsumerGroupDescription() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()).build()); - hermes.api().publishUntilStatus(topic.getQualifiedName(), MESSAGE.body(), 201); - subscriber.waitUntilAnyMessageReceived(); - - // when - WebTestClient.ResponseSpec response = hermes.api().getConsumerGroupsDescription(topic.getQualifiedName(), subscription.getName()); - - // then - response.expectStatus().isOk(); - List consumerGroups = response.expectBodyList(ConsumerGroup.class).returnResult().getResponseBody(); - assertThat(consumerGroups.size()).isGreaterThan(0); - assertThat(consumerGroups) - .flatExtracting("members") - .flatExtracting("partitions") - .usingRecursiveFieldByFieldElementComparatorIgnoringFields("partition", "topic", "offsetMetadata", "contentType") - .containsOnlyOnce(new TopicPartition(-1, "any", 0, 1, "any", topic.getContentType())); - } - - @Test - public void shouldNotCreateSubscriptionNotOwnedByTheUser() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = subscription(topic.getQualifiedName(), "subscription") - .withOwner(new OwnerId("Plaintext", "subscriptionOwner")) - .build(); - TestSecurityProvider.setUserIsAdmin(false); - - // when - WebTestClient.ResponseSpec response = hermes.api().createSubscription(subscription); - - // then - response.expectStatus().isBadRequest(); - assertThat(response.expectBody(String.class).returnResult().getResponseBody()) - .contains("Provide an owner that includes you, you would not be able to manage this subscription later"); - } - - @Test - public void shouldNotUpdateSubscriptionNotOwnedByTheUser() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); - TestSecurityProvider.setUserIsAdmin(false); - - // when - WebTestClient.ResponseSpec response = hermes.api().updateSubscription(topic, subscription.getName(), PATCH_DATA); - - // then - response.expectStatus().isBadRequest(); - assertThat(response.expectBody(String.class).returnResult().getResponseBody()) - .contains("Provide an owner that includes you, you would not be able to manage this subscription later"); - } - - @Test - public void shouldAllowAdminToBypassSubscribingRestrictions() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().withSubscribingRestricted().build()); - Subscription subscription = subscription(topic.getQualifiedName(), "subscription") - .withEndpoint("http://localhost:8081/topics/test-topic") - .build(); - TestSecurityProvider.setUserIsAdmin(true); - - // when - WebTestClient.ResponseSpec response = hermes.api().createSubscription(subscription); - - // then - response.expectStatus().isCreated(); - - // when - WebTestClient.ResponseSpec updateResponse = hermes.api().updateSubscription(topic, subscription.getName(), PATCH_DATA); - - // then - updateResponse.expectStatus().isOk(); - } - - @Test - public void shouldAllowTopicOwnerToBypassSubscribingRestrictions() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().withSubscribingRestricted().build()); - Subscription subscription = subscription(topic.getQualifiedName(), "subscription") - .withEndpoint("http://localhost:8081/topics/test-topic") - .build(); - TestSecurityProvider.setUserIsAdmin(false); - TestSecurityProvider.setUserAsOwner(topic.getOwner(), subscription.getOwner()); - - // when - WebTestClient.ResponseSpec response = hermes.api().createSubscription(subscription); - - // then - response.expectStatus().isCreated(); - - // when - WebTestClient.ResponseSpec updateResponse = hermes.api().updateSubscription(topic, subscription.getName(), PATCH_DATA); - - // then - updateResponse.expectStatus().isOk(); - } - - @Test - public void shouldAllowPrivilegedSubscriberToBypassSubscribingRestrictions() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().withSubscribingRestricted().build()); - Subscription subscription = subscription(topic.getQualifiedName(), "subscription") - .withOwner(new OwnerId("Plaintext", "subscriberAllowedToAccessAnyTopic")) - .withEndpoint("http://localhost:8081/topics/test-topic") - .build(); - TestSecurityProvider.setUserIsAdmin(false); - TestSecurityProvider.setUserAsOwner(subscription.getOwner()); - - // when - WebTestClient.ResponseSpec response = hermes.api().createSubscription(subscription); - - // then - response.expectStatus().isCreated(); - - // when - WebTestClient.ResponseSpec updateResponse = hermes.api().updateSubscription(topic, subscription.getName(), PATCH_DATA); - - // then - updateResponse.expectStatus().isOk(); - } - - @Test - public void shouldRespectPrivilegedSubscriberProtocolsWhileCreatingSubscription() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().withSubscribingRestricted().build()); - Subscription subscription = subscription(topic.getQualifiedName(), "subscription") - .withOwner(new OwnerId("Plaintext", "subscriberAllowedToAccessAnyTopic")) - .withEndpoint("googlepubsub://pubsub.googleapis.com:443/projects/test-project/topics/test-topic") - .build(); - TestSecurityProvider.setUserIsAdmin(false); - TestSecurityProvider.setUserAsOwner(subscription.getOwner()); - - // when - WebTestClient.ResponseSpec response = hermes.api().createSubscription(subscription); - - // then - response.expectStatus().isForbidden(); - assertThat(response.expectBody(String.class).returnResult().getResponseBody()) - .contains("Subscribing to this topic has been restricted. Contact the topic owner to create a new subscription."); - } - - @Test - public void shouldRespectPrivilegedSubscriberProtocolsWhileUpdatingEndpoint() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().withSubscribingRestricted().build()); - Subscription subscription = subscription(topic.getQualifiedName(), "subscription") - .withOwner(new OwnerId("Plaintext", "subscriberAllowedToAccessAnyTopic")) - .withEndpoint("http://localhost:8081/topics/test-topic") - .build(); - hermes.initHelper().createSubscription(subscription); - TestSecurityProvider.setUserIsAdmin(false); - TestSecurityProvider.setUserAsOwner(subscription.getOwner()); - - // when - WebTestClient.ResponseSpec response = hermes.api().updateSubscription( + // and + prometheus.stub500Error(); + + // when + WebTestClient.ResponseSpec response = + hermes.api().getSubscriptionHealth(topic.getQualifiedName(), subscription.getName()); + + // then + SubscriptionHealth subscriptionHealth = + response.expectBody(SubscriptionHealth.class).returnResult().getResponseBody(); + assertThat(subscriptionHealth.getStatus()).isEqualTo(NO_DATA); + } + + @Test + public void shouldNotAllowSubscriptionWithBatchDeliveryAndAvroContentType() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + subscriptionWithRandomName(topic.getName()) + .withDeliveryType(DeliveryType.BATCH) + .withContentType(ContentType.AVRO) + .build(); + + // when + WebTestClient.ResponseSpec response = hermes.api().createSubscription(subscription); + + // then + response.expectStatus().isBadRequest(); + } + + @Test + public void shouldReturnConsumerGroupDescription() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()).build()); + hermes.api().publishUntilStatus(topic.getQualifiedName(), MESSAGE.body(), 201); + subscriber.waitUntilAnyMessageReceived(); + + // when + WebTestClient.ResponseSpec response = + hermes.api().getConsumerGroupsDescription(topic.getQualifiedName(), subscription.getName()); + + // then + response.expectStatus().isOk(); + List consumerGroups = + response.expectBodyList(ConsumerGroup.class).returnResult().getResponseBody(); + assertThat(consumerGroups.size()).isGreaterThan(0); + assertThat(consumerGroups) + .flatExtracting("members") + .flatExtracting("partitions") + .usingRecursiveFieldByFieldElementComparatorIgnoringFields( + "partition", "topic", "offsetMetadata", "contentType") + .containsOnlyOnce(new TopicPartition(-1, "any", 0, 1, "any", topic.getContentType())); + } + + @Test + public void shouldNotCreateSubscriptionNotOwnedByTheUser() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + subscription(topic.getQualifiedName(), "subscription") + .withOwner(new OwnerId("Plaintext", "subscriptionOwner")) + .build(); + TestSecurityProvider.setUserIsAdmin(false); + + // when + WebTestClient.ResponseSpec response = hermes.api().createSubscription(subscription); + + // then + response.expectStatus().isBadRequest(); + assertThat(response.expectBody(String.class).returnResult().getResponseBody()) + .contains( + "Provide an owner that includes you, you would not be able to manage this subscription later"); + } + + @Test + public void shouldNotUpdateSubscriptionNotOwnedByTheUser() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); + TestSecurityProvider.setUserIsAdmin(false); + + // when + WebTestClient.ResponseSpec response = + hermes.api().updateSubscription(topic, subscription.getName(), PATCH_DATA); + + // then + response.expectStatus().isBadRequest(); + assertThat(response.expectBody(String.class).returnResult().getResponseBody()) + .contains( + "Provide an owner that includes you, you would not be able to manage this subscription later"); + } + + @Test + public void shouldAllowAdminToBypassSubscribingRestrictions() { + // given + Topic topic = + hermes.initHelper().createTopic(topicWithRandomName().withSubscribingRestricted().build()); + Subscription subscription = + subscription(topic.getQualifiedName(), "subscription") + .withEndpoint("http://localhost:8081/topics/test-topic") + .build(); + TestSecurityProvider.setUserIsAdmin(true); + + // when + WebTestClient.ResponseSpec response = hermes.api().createSubscription(subscription); + + // then + response.expectStatus().isCreated(); + + // when + WebTestClient.ResponseSpec updateResponse = + hermes.api().updateSubscription(topic, subscription.getName(), PATCH_DATA); + + // then + updateResponse.expectStatus().isOk(); + } + + @Test + public void shouldAllowTopicOwnerToBypassSubscribingRestrictions() { + // given + Topic topic = + hermes.initHelper().createTopic(topicWithRandomName().withSubscribingRestricted().build()); + Subscription subscription = + subscription(topic.getQualifiedName(), "subscription") + .withEndpoint("http://localhost:8081/topics/test-topic") + .build(); + TestSecurityProvider.setUserIsAdmin(false); + TestSecurityProvider.setUserAsOwner(topic.getOwner(), subscription.getOwner()); + + // when + WebTestClient.ResponseSpec response = hermes.api().createSubscription(subscription); + + // then + response.expectStatus().isCreated(); + + // when + WebTestClient.ResponseSpec updateResponse = + hermes.api().updateSubscription(topic, subscription.getName(), PATCH_DATA); + + // then + updateResponse.expectStatus().isOk(); + } + + @Test + public void shouldAllowPrivilegedSubscriberToBypassSubscribingRestrictions() { + // given + Topic topic = + hermes.initHelper().createTopic(topicWithRandomName().withSubscribingRestricted().build()); + Subscription subscription = + subscription(topic.getQualifiedName(), "subscription") + .withOwner(new OwnerId("Plaintext", "subscriberAllowedToAccessAnyTopic")) + .withEndpoint("http://localhost:8081/topics/test-topic") + .build(); + TestSecurityProvider.setUserIsAdmin(false); + TestSecurityProvider.setUserAsOwner(subscription.getOwner()); + + // when + WebTestClient.ResponseSpec response = hermes.api().createSubscription(subscription); + + // then + response.expectStatus().isCreated(); + + // when + WebTestClient.ResponseSpec updateResponse = + hermes.api().updateSubscription(topic, subscription.getName(), PATCH_DATA); + + // then + updateResponse.expectStatus().isOk(); + } + + @Test + public void shouldRespectPrivilegedSubscriberProtocolsWhileCreatingSubscription() { + // given + Topic topic = + hermes.initHelper().createTopic(topicWithRandomName().withSubscribingRestricted().build()); + Subscription subscription = + subscription(topic.getQualifiedName(), "subscription") + .withOwner(new OwnerId("Plaintext", "subscriberAllowedToAccessAnyTopic")) + .withEndpoint( + "googlepubsub://pubsub.googleapis.com:443/projects/test-project/topics/test-topic") + .build(); + TestSecurityProvider.setUserIsAdmin(false); + TestSecurityProvider.setUserAsOwner(subscription.getOwner()); + + // when + WebTestClient.ResponseSpec response = hermes.api().createSubscription(subscription); + + // then + response.expectStatus().isForbidden(); + assertThat(response.expectBody(String.class).returnResult().getResponseBody()) + .contains( + "Subscribing to this topic has been restricted. Contact the topic owner to create a new subscription."); + } + + @Test + public void shouldRespectPrivilegedSubscriberProtocolsWhileUpdatingEndpoint() { + // given + Topic topic = + hermes.initHelper().createTopic(topicWithRandomName().withSubscribingRestricted().build()); + Subscription subscription = + subscription(topic.getQualifiedName(), "subscription") + .withOwner(new OwnerId("Plaintext", "subscriberAllowedToAccessAnyTopic")) + .withEndpoint("http://localhost:8081/topics/test-topic") + .build(); + hermes.initHelper().createSubscription(subscription); + TestSecurityProvider.setUserIsAdmin(false); + TestSecurityProvider.setUserAsOwner(subscription.getOwner()); + + // when + WebTestClient.ResponseSpec response = + hermes + .api() + .updateSubscription( topic, subscription.getName(), - patchData().set( - "endpoint", EndpointAddress.of("googlepubsub://pubsub.googleapis.com:443/projects/test-project/topics/test-topic") - ).build() - ); - - // then - response.expectStatus().isForbidden(); - assertThat(response.expectBody(String.class).returnResult().getResponseBody()) - .contains("Subscribing to this topic has been restricted. Contact the topic owner to modify the endpoint of this subscription."); - } - - @Test - public void shouldNotAllowUnprivilegedUserToCreateSubscriptionWhenSubscribingIsRestricted() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().withSubscribingRestricted().build()); - Subscription subscription = subscription(topic.getQualifiedName(), "subscription") - .withOwner(new OwnerId("Plaintext", "subscriptionOwner")) - .build(); - TestSecurityProvider.setUserIsAdmin(false); - TestSecurityProvider.setUserAsOwner(subscription.getOwner()); - - // when - WebTestClient.ResponseSpec response = hermes.api().createSubscription(subscription); - - // then - response.expectStatus().isForbidden(); - assertThat(response.expectBody(String.class).returnResult().getResponseBody()) - .contains("Subscribing to this topic has been restricted. Contact the topic owner to create a new subscription."); - } - - @Test - public void shouldNotAllowUnprivilegedUserToUpdateEndpointWhenSubscribingIsRestricted() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().withSubscribingRestricted().build()); - Subscription subscription = subscription(topic.getQualifiedName(), "subscription") - .withOwner(new OwnerId("Plaintext", "subscriptionOwner")) - .withEndpoint("http://localhost:8081/topics/test-topic") - .build(); - hermes.initHelper().createSubscription(subscription); - TestSecurityProvider.setUserIsAdmin(false); - TestSecurityProvider.setUserAsOwner(subscription.getOwner()); - - // when - WebTestClient.ResponseSpec response = hermes.api().updateSubscription(topic, subscription.getName(), PATCH_DATA); - - - // then - response.expectStatus().isForbidden(); - assertThat(response.expectBody(String.class).returnResult().getResponseBody()) - .contains("Subscribing to this topic has been restricted. Contact the topic owner to modify the endpoint of this subscription."); - } - - @Test - public void shouldSetInflightSizeToNullByDefault() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); - TestSecurityProvider.setUserIsAdmin(false); - - // when - Subscription response = hermes.api().getSubscription(topic.getQualifiedName(), subscription.getName()); - - // then - assertThat(response.getSerialSubscriptionPolicy().getInflightSize()).isNull(); - } - - @Test - public void shouldReturnInflightSizeWhenSetToNonNullValue() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()) - .withSubscriptionPolicy( + patchData() + .set( + "endpoint", + EndpointAddress.of( + "googlepubsub://pubsub.googleapis.com:443/projects/test-project/topics/test-topic")) + .build()); + + // then + response.expectStatus().isForbidden(); + assertThat(response.expectBody(String.class).returnResult().getResponseBody()) + .contains( + "Subscribing to this topic has been restricted. Contact the topic owner to modify the endpoint of this subscription."); + } + + @Test + public void shouldNotAllowUnprivilegedUserToCreateSubscriptionWhenSubscribingIsRestricted() { + // given + Topic topic = + hermes.initHelper().createTopic(topicWithRandomName().withSubscribingRestricted().build()); + Subscription subscription = + subscription(topic.getQualifiedName(), "subscription") + .withOwner(new OwnerId("Plaintext", "subscriptionOwner")) + .build(); + TestSecurityProvider.setUserIsAdmin(false); + TestSecurityProvider.setUserAsOwner(subscription.getOwner()); + + // when + WebTestClient.ResponseSpec response = hermes.api().createSubscription(subscription); + + // then + response.expectStatus().isForbidden(); + assertThat(response.expectBody(String.class).returnResult().getResponseBody()) + .contains( + "Subscribing to this topic has been restricted. Contact the topic owner to create a new subscription."); + } + + @Test + public void shouldNotAllowUnprivilegedUserToUpdateEndpointWhenSubscribingIsRestricted() { + // given + Topic topic = + hermes.initHelper().createTopic(topicWithRandomName().withSubscribingRestricted().build()); + Subscription subscription = + subscription(topic.getQualifiedName(), "subscription") + .withOwner(new OwnerId("Plaintext", "subscriptionOwner")) + .withEndpoint("http://localhost:8081/topics/test-topic") + .build(); + hermes.initHelper().createSubscription(subscription); + TestSecurityProvider.setUserIsAdmin(false); + TestSecurityProvider.setUserAsOwner(subscription.getOwner()); + + // when + WebTestClient.ResponseSpec response = + hermes.api().updateSubscription(topic, subscription.getName(), PATCH_DATA); + + // then + response.expectStatus().isForbidden(); + assertThat(response.expectBody(String.class).returnResult().getResponseBody()) + .contains( + "Subscribing to this topic has been restricted. Contact the topic owner to modify the endpoint of this subscription."); + } + + @Test + public void shouldSetInflightSizeToNullByDefault() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); + TestSecurityProvider.setUserIsAdmin(false); + + // when + Subscription response = + hermes.api().getSubscription(topic.getQualifiedName(), subscription.getName()); + + // then + assertThat(response.getSerialSubscriptionPolicy().getInflightSize()).isNull(); + } + + @Test + public void shouldReturnInflightSizeWhenSetToNonNullValue() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName()) + .withSubscriptionPolicy( SubscriptionPolicy.Builder.subscriptionPolicy() - .withInflightSize(42) - .build() - ).build()); - TestSecurityProvider.setUserIsAdmin(false); - - // when - Subscription response = hermes.api().getSubscription(topic.getQualifiedName(), subscription.getName()); - - // then - assertThat(response.getSerialSubscriptionPolicy().getInflightSize()).isEqualTo(42); - } - - @Test - public void shouldMoveOffsetsToTheEnd() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(503); - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()) - .withSubscriptionPolicy(SubscriptionPolicy.create(Map.of("messageTtl", 3600))) - .build()); - List messages = List.of(MESSAGE.body(), MESSAGE.body(), MESSAGE.body(), MESSAGE.body()); - - // prevents from moving offsets during messages sending - messages.forEach(message -> { - hermes.api().publishUntilSuccess(topic.getQualifiedName(), message); - subscriber.waitUntilReceived(message); + .withInflightSize(42) + .build()) + .build()); + TestSecurityProvider.setUserIsAdmin(false); + + // when + Subscription response = + hermes.api().getSubscription(topic.getQualifiedName(), subscription.getName()); + + // then + assertThat(response.getSerialSubscriptionPolicy().getInflightSize()).isEqualTo(42); + } + + @Test + public void shouldMoveOffsetsToTheEnd() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(503); + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName(), subscriber.getEndpoint()) + .withSubscriptionPolicy(SubscriptionPolicy.create(Map.of("messageTtl", 3600))) + .build()); + List messages = List.of(MESSAGE.body(), MESSAGE.body(), MESSAGE.body(), MESSAGE.body()); + + // prevents from moving offsets during messages sending + messages.forEach( + message -> { + hermes.api().publishUntilSuccess(topic.getQualifiedName(), message); + subscriber.waitUntilReceived(message); }); - assertThat(allConsumerGroupOffsetsMovedToTheEnd(subscription)).isFalse(); - - hermes.api().deleteSubscription(topic.getQualifiedName(), subscription.getName()); - - // when - waitAtMost(Duration.ofSeconds(10)) - .untilAsserted(() -> hermes.api() - .moveOffsetsToTheEnd(topic.getQualifiedName(), subscription.getName()).expectStatus().isOk()); - - // then - waitAtMost(Duration.ofSeconds(10)) - .untilAsserted(() -> assertThat(allConsumerGroupOffsetsMovedToTheEnd(subscription)).isTrue()); - } - - private boolean allConsumerGroupOffsetsMovedToTheEnd(Subscription subscription) { - List partitionsOffsets = brokerOperations.getTopicPartitionsOffsets(subscription.getQualifiedName()); - return !partitionsOffsets.isEmpty() && partitionsOffsets.stream().allMatch(BrokerOperations.ConsumerGroupOffset::movedToEnd); - } + assertThat(allConsumerGroupOffsetsMovedToTheEnd(subscription)).isFalse(); + + hermes.api().deleteSubscription(topic.getQualifiedName(), subscription.getName()); + + // when + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> + hermes + .api() + .moveOffsetsToTheEnd(topic.getQualifiedName(), subscription.getName()) + .expectStatus() + .isOk()); + + // then + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> assertThat(allConsumerGroupOffsetsMovedToTheEnd(subscription)).isTrue()); + } + + private boolean allConsumerGroupOffsetsMovedToTheEnd(Subscription subscription) { + List partitionsOffsets = + brokerOperations.getTopicPartitionsOffsets(subscription.getQualifiedName()); + return !partitionsOffsets.isEmpty() + && partitionsOffsets.stream().allMatch(BrokerOperations.ConsumerGroupOffset::movedToEnd); + } } diff --git a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/TopicManagementTest.java b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/TopicManagementTest.java index 461492ccd9..df422b6c2f 100644 --- a/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/TopicManagementTest.java +++ b/integration-tests/src/integrationTest/java/pl/allegro/tech/hermes/integrationtests/management/TopicManagementTest.java @@ -1,7 +1,27 @@ package pl.allegro.tech.hermes.integrationtests.management; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.waitAtMost; +import static pl.allegro.tech.hermes.api.ContentType.AVRO; +import static pl.allegro.tech.hermes.api.ContentType.JSON; +import static pl.allegro.tech.hermes.api.PatchData.patchData; +import static pl.allegro.tech.hermes.api.PublishingChaosPolicy.ChaosMode.DATACENTER; +import static pl.allegro.tech.hermes.api.TopicWithSchema.topicWithSchema; +import static pl.allegro.tech.hermes.integrationtests.setup.HermesExtension.auditEvents; +import static pl.allegro.tech.hermes.integrationtests.setup.HermesExtension.brokerOperations; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topic; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Stream; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -21,827 +41,1012 @@ import pl.allegro.tech.hermes.management.TestSecurityProvider; import pl.allegro.tech.hermes.test.helper.avro.AvroUserSchemaLoader; -import java.time.Duration; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Stream; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.waitAtMost; -import static pl.allegro.tech.hermes.api.ContentType.AVRO; -import static pl.allegro.tech.hermes.api.ContentType.JSON; -import static pl.allegro.tech.hermes.api.PatchData.patchData; -import static pl.allegro.tech.hermes.api.PublishingChaosPolicy.ChaosMode.DATACENTER; -import static pl.allegro.tech.hermes.api.TopicWithSchema.topicWithSchema; -import static pl.allegro.tech.hermes.integrationtests.setup.HermesExtension.auditEvents; -import static pl.allegro.tech.hermes.integrationtests.setup.HermesExtension.brokerOperations; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscriptionWithRandomName; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topic; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class TopicManagementTest { - @RegisterExtension - public static final HermesExtension hermes = new HermesExtension(); - - private static final String SCHEMA = AvroUserSchemaLoader.load().toString(); - - @AfterEach - public void cleanup() { - TestSecurityProvider.reset(); - } - - @Test - public void shouldEmitAuditEventWhenTopicCreated() { - //when - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - - //then - assertThat( - auditEvents.getLastReceivedRequest().getBodyAsString() - ).contains("CREATED", "Topic", topic.getQualifiedName()); - } - - @Test - public void shouldEmitAuditEventWhenTopicRemoved() { - //given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - - //when - hermes.api().deleteTopic(topic.getQualifiedName()).expectStatus().isOk(); - - //then - assertThat( - auditEvents.getLastReceivedRequest().getBodyAsString() - ).contains("REMOVED", topic.getQualifiedName()); - } - - @Test - public void shouldEmitAuditEventWhenTopicUpdated() { - //given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - PatchData patchData = PatchData.from(ImmutableMap.of("maxMessageSize", 2048)); + @RegisterExtension public static final HermesExtension hermes = new HermesExtension(); + + private static final String SCHEMA = AvroUserSchemaLoader.load().toString(); + + @AfterEach + public void cleanup() { + TestSecurityProvider.reset(); + } + + @Test + public void shouldEmitAuditEventWhenTopicCreated() { + // when + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + + // then + assertThat(auditEvents.getLastReceivedRequest().getBodyAsString()) + .contains("CREATED", "Topic", topic.getQualifiedName()); + } + + @Test + public void shouldEmitAuditEventWhenTopicRemoved() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + + // when + hermes.api().deleteTopic(topic.getQualifiedName()).expectStatus().isOk(); + + // then + assertThat(auditEvents.getLastReceivedRequest().getBodyAsString()) + .contains("REMOVED", topic.getQualifiedName()); + } + + @Test + public void shouldEmitAuditEventWhenTopicUpdated() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + PatchData patchData = PatchData.from(ImmutableMap.of("maxMessageSize", 2048)); + + // when + hermes.api().updateTopic(topic.getQualifiedName(), patchData); + + // then + assertThat(auditEvents.getLastReceivedRequest().getBodyAsString()) + .contains("UPDATED", topic.getQualifiedName()); + } + + @Test + public void shouldEmitAuditEventBeforeUpdateWhenWrongPatchDataKeyProvided() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + PatchData patchData = PatchData.from(ImmutableMap.of("someValue", 2048)); + + // when + hermes.api().updateTopic(topic.getQualifiedName(), patchData); + + // then + assertThat(auditEvents.getLastReceivedRequest().getBodyAsString()) + .contains("BEFORE_UPDATE", topic.getQualifiedName(), "someValue", "2048"); + } + + @Test + public void shouldCreateTopic() { + // given + TopicWithSchema topic = TopicWithSchema.topicWithSchema(topicWithRandomName().build()); + hermes.initHelper().createGroup(Group.from(topic.getName().getGroupName())); + + // when + WebTestClient.ResponseSpec response = hermes.api().createTopic(topic); + + // then + response.expectStatus().isCreated(); + hermes.api().getTopicResponse(topic.getQualifiedName()).expectStatus().isOk().expectBody(); + } + + @Test + public void shouldListTopics() { + // given + hermes.initHelper().createTopic(topic("listTopicsGroup.topic1").build()); + hermes.api().createTopic(new TopicWithSchema(topic("listTopicsGroup.topic2").build(), null)); + + // when then + assertThat(getGroupTopicsList("listTopicsGroup")) + .containsExactly("listTopicsGroup.topic1", "listTopicsGroup.topic2"); + } + + @Test + public void shouldRemoveTopic() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + + // when + WebTestClient.ResponseSpec response = hermes.api().deleteTopic(topic.getQualifiedName()); + + // then + response.expectStatus().isOk(); + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> assertThat(getGroupTopicsList(topic.getName().getGroupName())).isEmpty()); + } + + @Test + public void shouldUnblacklistTopicWhileDeleting() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + hermes.api().blacklistTopic(topic.getQualifiedName()); + + // when + WebTestClient.ResponseSpec response = hermes.api().deleteTopic(topic.getQualifiedName()); + + // then + response.expectStatus().isOk(); + waitAtMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> assertThat(getGroupTopicsList(topic.getName().getGroupName())).isEmpty()); + assertThat(hermes.api().isTopicBlacklisted(topic.getQualifiedName()).isBlacklisted()).isFalse(); + } + + @Test + public void shouldNotAllowOnDeletingTopicWithSubscriptions() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); + + // when + WebTestClient.ResponseSpec response = hermes.api().deleteTopic(topic.getQualifiedName()); + + // then + response.expectStatus().isForbidden(); + assertThat(getErrorCode(response)).isEqualTo(ErrorCode.TOPIC_NOT_EMPTY); + } + + @Test + public void shouldRemoveTopicWithRelatedSubscriptionsWhenAutoRemoveEnabled() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + Subscription subscription = + hermes + .initHelper() + .createSubscription( + subscriptionWithRandomName(topic.getName()) + .withAutoDeleteWithTopicEnabled(true) + .build()); + + // when + WebTestClient.ResponseSpec response = hermes.api().deleteTopic(topic.getQualifiedName()); + + // then + response.expectStatus().isOk(); + + // and + hermes + .api() + .getSubscriptionResponse(topic.getQualifiedName(), subscription.getName()) + .expectStatus() + .isBadRequest(); + } + + @Test + public void shouldNotCreateInvalidTopic() { + // given + String groupName = "invalidTopicGroup"; + hermes.initHelper().createGroup(Group.from(groupName)); + + // when + WebTestClient.ResponseSpec response = + hermes + .api() + .createTopic( + topicWithSchema( + topic(groupName, "shouldNotCreateInvalidTopic") + .withMaxMessageSize(Topic.MAX_MESSAGE_SIZE + 1) + .build())); + + // then + response.expectStatus().isBadRequest(); + assertThat(getErrorCode(response)).isEqualTo(ErrorCode.VALIDATION_ERROR); + assertThat(getGroupTopicsList(groupName)).isEmpty(); + } + + @Test + public void shouldNotCreateTopicWithMissingGroup() { + // given no group + + // when + TopicWithSchema topicWithSchema = + topicWithSchema( + topic("nonExistingGroup", "topic") + .withContentType(AVRO) + .withTrackingEnabled(false) + .build(), + SCHEMA); + WebTestClient.ResponseSpec createTopicResponse = hermes.api().createTopic(topicWithSchema); + WebTestClient.ResponseSpec schemaResponse = + hermes.api().getSchema(topicWithSchema.getQualifiedName()); + + // then + createTopicResponse.expectStatus().isNotFound(); + assertThat(getErrorCode(createTopicResponse)).isEqualTo(ErrorCode.GROUP_NOT_EXISTS); + schemaResponse.expectStatus().isNoContent(); + } + + @Test + public void shouldNotAllowOnCreatingSameTopicTwice() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + + // when + WebTestClient.ResponseSpec response = + hermes.api().createTopic(new TopicWithSchema(topic, null)); + + // then + response.expectStatus().isBadRequest(); + assertThat(getErrorCode(response)).isEqualTo(ErrorCode.TOPIC_ALREADY_EXISTS); + } + + @Test + public void shouldAllowMigratingTopicFromJsonToAvroByExtendingTopicWithAvroSchema() { + // given + Topic topic = + hermes.initHelper().createTopic(topicWithRandomName().withContentType(JSON).build()); + + // when + PatchData patch = + patchData() + .set("contentType", ContentType.AVRO) + .set("migratedFromJsonType", true) + .set("schema", AvroUserSchemaLoader.load().toString()) + .build(); + WebTestClient.ResponseSpec response = hermes.api().updateTopic(topic.getQualifiedName(), patch); + + // when + response.expectStatus().isOk(); + } + + @Test + public void shouldNotAllowMigratingTopicFromJsonToAvroWhenProvidingInvalidSchema() { + // given + Topic topic = + hermes.initHelper().createTopic(topicWithRandomName().withContentType(JSON).build()); + + // when + PatchData patch = + patchData() + .set("contentType", ContentType.AVRO) + .set("migratedFromJsonType", true) + .set("schema", "invalid...") + .build(); + WebTestClient.ResponseSpec response = hermes.api().updateTopic(topic.getQualifiedName(), patch); + + // when + response.expectStatus().isBadRequest(); + assertThat(getErrorCode(response)).isEqualTo(ErrorCode.SCHEMA_BAD_REQUEST); + } + + @Test + public void + shouldAllowMigratingTopicFromJsonToAvroWithoutProvidingAvroSchemaWhenItsAlreadyAvailableInRegistry() { + // given + Topic topic = + hermes.initHelper().createTopic(topicWithRandomName().withContentType(JSON).build()); + hermes + .api() + .saveSchema(topic.getQualifiedName(), false, AvroUserSchemaLoader.load().toString()); + + // when + PatchData patch = + patchData().set("contentType", ContentType.AVRO).set("migratedFromJsonType", true).build(); + WebTestClient.ResponseSpec response = hermes.api().updateTopic(topic.getQualifiedName(), patch); + + // when + response.expectStatus().isOk(); + } + + @Test + public void shouldReturnTopicsThatAreCurrentlyTracked() { + // given + Topic trackedTopic = + hermes.initHelper().createTopic(topicWithRandomName().withTrackingEnabled(true).build()); + Topic untrackedTopic = + hermes.initHelper().createTopic(topicWithRandomName().withTrackingEnabled(false).build()); - //when + // when + WebTestClient.ResponseSpec response = hermes.api().listTrackedTopics(""); + + // then + response.expectStatus().isOk(); + assertThat( + Arrays.stream( + Objects.requireNonNull( + response.expectBody(String[].class).returnResult().getResponseBody())) + .toList()) + .contains(trackedTopic.getQualifiedName()) + .doesNotContain(untrackedTopic.getQualifiedName()); + } + + @Test + public void shouldReturnTopicsThatAreCurrentlyTrackedForGivenGroup() { + // given + String groupName = "mixedTrackedGroup1"; + Topic trackedTopic = + hermes + .initHelper() + .createTopic(topic(groupName, "trackedTopic1").withTrackingEnabled(true).build()); + hermes.initHelper().createTopic(topicWithRandomName().withTrackingEnabled(false).build()); + + // when + WebTestClient.ResponseSpec response = hermes.api().listTrackedTopics(groupName); + + // then + response.expectStatus().isOk(); + assertThat( + Arrays.stream( + Objects.requireNonNull( + response.expectBody(String[].class).returnResult().getResponseBody())) + .toList()) + .containsExactly(trackedTopic.getQualifiedName()); + } + + @Test + public void shouldReturnTrackedTopicsWithAvroContentType() { + // given + String group = "mixedTrackedGroup2"; + Topic trackedTopicAvro = + hermes + .initHelper() + .createTopicWithSchema( + new TopicWithSchema( + topic(group, "trackedAvroTopic2") + .withTrackingEnabled(true) + .withContentType(AVRO) + .build(), + SCHEMA)); + Topic untrackedAvroTopic = + hermes + .initHelper() + .createTopicWithSchema( + new TopicWithSchema( + topicWithRandomName().withTrackingEnabled(false).withContentType(AVRO).build(), + SCHEMA)); + Topic untrackedJsonTopic = + hermes + .initHelper() + .createTopic( + topicWithRandomName().withTrackingEnabled(false).withContentType(JSON).build()); + Topic trackedJsonTopic = + hermes + .initHelper() + .createTopic( + topicWithRandomName().withTrackingEnabled(true).withContentType(JSON).build()); + + // and + String query = "{\"query\": {\"trackingEnabled\": \"true\", \"contentType\": \"AVRO\"}}"; + + // when + WebTestClient.ResponseSpec response = hermes.api().queryTopics("", query); + + // then + assertThat( + Arrays.stream( + Objects.requireNonNull( + response.expectBody(String[].class).returnResult().getResponseBody())) + .toList()) + .contains(trackedTopicAvro.getQualifiedName()) + .doesNotContain( + untrackedJsonTopic.getQualifiedName(), + untrackedAvroTopic.getQualifiedName(), + trackedJsonTopic.getQualifiedName()); + } + + @Test + public void shouldReturnTrackedTopicsWithAvroContentTypeForGivenGroup() { + // given + String group = "mixedTrackedGroup3"; + Topic trackedTopicAvro = + hermes + .initHelper() + .createTopicWithSchema( + new TopicWithSchema( + topic(group, "trackedAvroTopic3") + .withTrackingEnabled(true) + .withContentType(AVRO) + .build(), + SCHEMA)); + hermes + .api() + .createTopic( + new TopicWithSchema( + topic(group, "untrackedAvroTopic") + .withTrackingEnabled(false) + .withContentType(AVRO) + .build(), + SCHEMA)); + hermes + .api() + .createTopic( + new TopicWithSchema( + topic(group, "untrackedJsonTopic") + .withTrackingEnabled(false) + .withContentType(JSON) + .build(), + null)); + hermes + .api() + .createTopic( + new TopicWithSchema( + topic(group, "trackedJsonTopic") + .withTrackingEnabled(true) + .withContentType(JSON) + .build(), + null)); + + // and + String query = "{\"query\": {\"trackingEnabled\": \"true\", \"contentType\": \"AVRO\"}}"; + + // when + WebTestClient.ResponseSpec response = hermes.api().queryTopics(group, query); + + // then + assertThat( + Arrays.stream( + Objects.requireNonNull( + response.expectBody(String[].class).returnResult().getResponseBody())) + .toList()) + .containsExactly(trackedTopicAvro.getQualifiedName()) + .doesNotContain( + group + "." + "untrackedAvroTopic", + group + "." + "untrackedJsonTopic", + group + "." + "trackedJsonTopic"); + } + + @Test + public void shouldNotAllowDollarSign() { + // given + String group = "dollar"; + hermes.initHelper().createGroup(Group.from(group)); + + Stream.of("$name", "na$me", "name$") + .forEach( + topicName -> { + // when + WebTestClient.ResponseSpec response = + hermes + .api() + .createTopic(new TopicWithSchema(topic(group, topicName).build(), null)); + + // then + response.expectStatus().isBadRequest(); + }); + } + + @Test + public void shouldCreateTopicWithMaxMessageSize() { + // given + TopicWithSchema topic = + new TopicWithSchema(topicWithRandomName().withMaxMessageSize(2048).build(), null); + hermes.initHelper().createGroup(Group.from(topic.getName().getGroupName())); + + // when + WebTestClient.ResponseSpec response = hermes.api().createTopic(topic); + + // then + response.expectStatus().isCreated(); + int fetchedMessageSize = + hermes + .api() + .getTopicResponse(topic.getQualifiedName()) + .expectBody(TopicWithSchema.class) + .returnResult() + .getResponseBody() + .getMaxMessageSize(); + assertThat(fetchedMessageSize).isEqualTo(2048); + } + + @Test + public void shouldUpdateTopicWithMaxMessageSize() { + // given + Topic topic = + hermes.initHelper().createTopic(topicWithRandomName().withMaxMessageSize(2048).build()); + PatchData patchData = PatchData.from(ImmutableMap.of("maxMessageSize", 1024)); + + // when + WebTestClient.ResponseSpec response = hermes.api().updateTopic(topic.getQualifiedName(), patchData); - //then - assertThat( - auditEvents.getLastReceivedRequest().getBodyAsString() - ).contains("UPDATED", topic.getQualifiedName()); - } - - @Test - public void shouldEmitAuditEventBeforeUpdateWhenWrongPatchDataKeyProvided() { - //given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - PatchData patchData = PatchData.from(ImmutableMap.of("someValue", 2048)); - - //when + // then + response.expectStatus().isOk(); + int fetchedMessageSize = + hermes + .api() + .getTopicResponse(topic.getQualifiedName()) + .expectBody(TopicWithSchema.class) + .returnResult() + .getResponseBody() + .getMaxMessageSize(); + assertThat(fetchedMessageSize).isEqualTo(1024); + } + + @Test + public void shouldCreateTopicWithRestrictedSubscribing() { + // given + TopicWithSchema topic = + new TopicWithSchema(topicWithRandomName().withSubscribingRestricted().build(), null); + hermes.initHelper().createGroup(Group.from(topic.getName().getGroupName())); + + // when + WebTestClient.ResponseSpec response = hermes.api().createTopic(topic); + + // then + response.expectStatus().isCreated(); + boolean fetchedIsSubscribingRestricted = + hermes + .api() + .getTopicResponse(topic.getQualifiedName()) + .expectBody(TopicWithSchema.class) + .returnResult() + .getResponseBody() + .isSubscribingRestricted(); + assertThat(fetchedIsSubscribingRestricted).isTrue(); + } + + @Test + public void shouldUpdateTopicWithRestrictedSubscribing() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + PatchData patchData = PatchData.from(ImmutableMap.of("subscribingRestricted", true)); + + // when + WebTestClient.ResponseSpec response = hermes.api().updateTopic(topic.getQualifiedName(), patchData); - //then - assertThat( - auditEvents.getLastReceivedRequest().getBodyAsString() - ).contains("BEFORE_UPDATE", topic.getQualifiedName(), "someValue", "2048"); - } - - @Test - public void shouldCreateTopic() { - //given - TopicWithSchema topic = TopicWithSchema.topicWithSchema(topicWithRandomName().build()); - hermes.initHelper().createGroup(Group.from(topic.getName().getGroupName())); - - // when - WebTestClient.ResponseSpec response = hermes.api().createTopic(topic); - - // then - response.expectStatus().isCreated(); - hermes.api().getTopicResponse(topic.getQualifiedName()).expectStatus().isOk().expectBody(); - } - - @Test - public void shouldListTopics() { - // given - hermes.initHelper().createTopic(topic("listTopicsGroup.topic1").build()); - hermes.api().createTopic(new TopicWithSchema(topic("listTopicsGroup.topic2").build(), null)); - - // when then - assertThat(getGroupTopicsList("listTopicsGroup")).containsExactly("listTopicsGroup.topic1", "listTopicsGroup.topic2"); - } - - @Test - public void shouldRemoveTopic() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - - // when - WebTestClient.ResponseSpec response = hermes.api().deleteTopic(topic.getQualifiedName()); - - // then - response.expectStatus().isOk(); - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> assertThat(getGroupTopicsList(topic.getName().getGroupName())).isEmpty()); - } - - @Test - public void shouldUnblacklistTopicWhileDeleting() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - hermes.api().blacklistTopic(topic.getQualifiedName()); - - // when - WebTestClient.ResponseSpec response = hermes.api().deleteTopic(topic.getQualifiedName()); - - // then - response.expectStatus().isOk(); - waitAtMost(Duration.ofSeconds(10)).untilAsserted(() -> assertThat(getGroupTopicsList(topic.getName().getGroupName())).isEmpty()); - assertThat(hermes.api().isTopicBlacklisted(topic.getQualifiedName()).isBlacklisted()).isFalse(); - } - - @Test - public void shouldNotAllowOnDeletingTopicWithSubscriptions() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()).build()); - - // when - WebTestClient.ResponseSpec response = hermes.api().deleteTopic(topic.getQualifiedName()); - - // then - response.expectStatus().isForbidden(); - assertThat(getErrorCode(response)).isEqualTo(ErrorCode.TOPIC_NOT_EMPTY); - } - - @Test - public void shouldRemoveTopicWithRelatedSubscriptionsWhenAutoRemoveEnabled() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - Subscription subscription = hermes.initHelper().createSubscription(subscriptionWithRandomName(topic.getName()) - .withAutoDeleteWithTopicEnabled(true).build()); - - // when - WebTestClient.ResponseSpec response = hermes.api().deleteTopic(topic.getQualifiedName()); - - // then - response.expectStatus().isOk(); - - // and - hermes.api().getSubscriptionResponse(topic.getQualifiedName(), subscription.getName()).expectStatus().isBadRequest(); - } - - @Test - public void shouldNotCreateInvalidTopic() { - // given - String groupName = "invalidTopicGroup"; - hermes.initHelper().createGroup(Group.from(groupName)); - - // when - WebTestClient.ResponseSpec response = hermes.api().createTopic( - topicWithSchema(topic(groupName, "shouldNotCreateInvalidTopic").withMaxMessageSize(Topic.MAX_MESSAGE_SIZE + 1).build())); - - // then - response.expectStatus().isBadRequest(); - assertThat(getErrorCode(response)).isEqualTo(ErrorCode.VALIDATION_ERROR); - assertThat(getGroupTopicsList(groupName)).isEmpty(); - } - - @Test - public void shouldNotCreateTopicWithMissingGroup() { - // given no group - - // when - TopicWithSchema topicWithSchema = topicWithSchema(topic("nonExistingGroup", "topic") - .withContentType(AVRO) - .withTrackingEnabled(false).build(), SCHEMA); - WebTestClient.ResponseSpec createTopicResponse = hermes.api().createTopic(topicWithSchema); - WebTestClient.ResponseSpec schemaResponse = hermes.api().getSchema(topicWithSchema.getQualifiedName()); - - // then - createTopicResponse.expectStatus().isNotFound(); - assertThat(getErrorCode(createTopicResponse)).isEqualTo(ErrorCode.GROUP_NOT_EXISTS); - schemaResponse.expectStatus().isNoContent(); - } - - @Test - public void shouldNotAllowOnCreatingSameTopicTwice() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - - // when - WebTestClient.ResponseSpec response = hermes.api().createTopic(new TopicWithSchema(topic, null)); - - // then - response.expectStatus().isBadRequest(); - assertThat(getErrorCode(response)).isEqualTo(ErrorCode.TOPIC_ALREADY_EXISTS); - } - - @Test - public void shouldAllowMigratingTopicFromJsonToAvroByExtendingTopicWithAvroSchema() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().withContentType(JSON).build()); - - // when - PatchData patch = patchData() - .set("contentType", ContentType.AVRO) - .set("migratedFromJsonType", true) - .set("schema", AvroUserSchemaLoader.load().toString()) - .build(); - WebTestClient.ResponseSpec response = hermes.api().updateTopic(topic.getQualifiedName(), patch); - - // when - response.expectStatus().isOk(); - } - - @Test - public void shouldNotAllowMigratingTopicFromJsonToAvroWhenProvidingInvalidSchema() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().withContentType(JSON).build()); - - // when - PatchData patch = patchData() - .set("contentType", ContentType.AVRO) - .set("migratedFromJsonType", true) - .set("schema", "invalid...") - .build(); - WebTestClient.ResponseSpec response = hermes.api().updateTopic(topic.getQualifiedName(), patch); - - // when - response.expectStatus().isBadRequest(); - assertThat(getErrorCode(response)).isEqualTo(ErrorCode.SCHEMA_BAD_REQUEST); - } - - @Test - public void shouldAllowMigratingTopicFromJsonToAvroWithoutProvidingAvroSchemaWhenItsAlreadyAvailableInRegistry() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().withContentType(JSON).build()); - hermes.api().saveSchema(topic.getQualifiedName(), false, AvroUserSchemaLoader.load().toString()); - - // when - PatchData patch = patchData() - .set("contentType", ContentType.AVRO) - .set("migratedFromJsonType", true) - .build(); - WebTestClient.ResponseSpec response = hermes.api().updateTopic(topic.getQualifiedName(), patch); - - // when - response.expectStatus().isOk(); - } - - @Test - public void shouldReturnTopicsThatAreCurrentlyTracked() { - // given - Topic trackedTopic = hermes.initHelper().createTopic(topicWithRandomName().withTrackingEnabled(true).build()); - Topic untrackedTopic = hermes.initHelper().createTopic(topicWithRandomName().withTrackingEnabled(false).build()); - - // when - WebTestClient.ResponseSpec response = hermes.api().listTrackedTopics(""); - - // then - response.expectStatus().isOk(); - assertThat( - Arrays.stream(Objects.requireNonNull(response.expectBody(String[].class).returnResult().getResponseBody())).toList() - ).contains(trackedTopic.getQualifiedName()).doesNotContain(untrackedTopic.getQualifiedName()); - } - - @Test - public void shouldReturnTopicsThatAreCurrentlyTrackedForGivenGroup() { - // given - String groupName = "mixedTrackedGroup1"; - Topic trackedTopic = hermes.initHelper().createTopic(topic(groupName, "trackedTopic1").withTrackingEnabled(true).build()); - hermes.initHelper().createTopic(topicWithRandomName().withTrackingEnabled(false).build()); + // then + response.expectStatus().isOk(); + boolean fetchedIsSubscribingRestricted = + hermes + .api() + .getTopicResponse(topic.getQualifiedName()) + .expectBody(TopicWithSchema.class) + .returnResult() + .getResponseBody() + .isSubscribingRestricted(); + assertThat(fetchedIsSubscribingRestricted).isTrue(); + } + + @Test + public void shouldCreateTopicWithOfflineStorageSettings() { + // given + TopicWithSchema topic = + new TopicWithSchema(topicWithRandomName().withOfflineStorage(2).build(), null); + hermes.initHelper().createGroup(Group.from(topic.getName().getGroupName())); + + // when + WebTestClient.ResponseSpec response = hermes.api().createTopic(topic); + + // then + response.expectStatus().isCreated(); + int fetchedOfflineStorageDurationTime = + hermes + .api() + .getTopicResponse(topic.getQualifiedName()) + .expectBody(TopicWithSchema.class) + .returnResult() + .getResponseBody() + .getOfflineStorage() + .getRetentionTime() + .getDuration(); + assertThat(fetchedOfflineStorageDurationTime).isEqualTo(2); + } + + @Test + public void shouldUpdateTopicWithOfflineStorageSettings() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + PatchData patchData = + PatchData.from(ImmutableMap.of("offlineStorage", ImmutableMap.of("enabled", true))); + + // when + WebTestClient.ResponseSpec response = + hermes.api().updateTopic(topic.getQualifiedName(), patchData); - // when - WebTestClient.ResponseSpec response = hermes.api().listTrackedTopics(groupName); - - // then - response.expectStatus().isOk(); - assertThat( - Arrays.stream(Objects.requireNonNull(response.expectBody(String[].class).returnResult().getResponseBody())).toList() - ).containsExactly(trackedTopic.getQualifiedName()); - } - - @Test - public void shouldReturnTrackedTopicsWithAvroContentType() { - // given - String group = "mixedTrackedGroup2"; - Topic trackedTopicAvro = hermes.initHelper().createTopicWithSchema(new TopicWithSchema(topic(group, "trackedAvroTopic2").withTrackingEnabled(true).withContentType(AVRO).build(), SCHEMA)); - Topic untrackedAvroTopic = hermes.initHelper().createTopicWithSchema(new TopicWithSchema(topicWithRandomName().withTrackingEnabled(false).withContentType(AVRO).build(), SCHEMA)); - Topic untrackedJsonTopic = hermes.initHelper().createTopic(topicWithRandomName().withTrackingEnabled(false).withContentType(JSON).build()); - Topic trackedJsonTopic = hermes.initHelper().createTopic(topicWithRandomName().withTrackingEnabled(true).withContentType(JSON).build()); - - // and - String query = "{\"query\": {\"trackingEnabled\": \"true\", \"contentType\": \"AVRO\"}}"; - - // when - WebTestClient.ResponseSpec response = hermes.api().queryTopics("", query); - - // then - assertThat(Arrays.stream(Objects.requireNonNull(response.expectBody(String[].class).returnResult().getResponseBody())).toList()) - .contains(trackedTopicAvro.getQualifiedName()) - .doesNotContain(untrackedJsonTopic.getQualifiedName(), untrackedAvroTopic.getQualifiedName(), trackedJsonTopic.getQualifiedName()); - } - - @Test - public void shouldReturnTrackedTopicsWithAvroContentTypeForGivenGroup() { - // given - String group = "mixedTrackedGroup3"; - Topic trackedTopicAvro = hermes.initHelper().createTopicWithSchema(new TopicWithSchema(topic(group, "trackedAvroTopic3").withTrackingEnabled(true).withContentType(AVRO).build(), SCHEMA)); - hermes.api().createTopic(new TopicWithSchema(topic(group, "untrackedAvroTopic").withTrackingEnabled(false).withContentType(AVRO).build(), SCHEMA)); - hermes.api().createTopic(new TopicWithSchema(topic(group, "untrackedJsonTopic").withTrackingEnabled(false).withContentType(JSON).build(), null)); - hermes.api().createTopic(new TopicWithSchema(topic(group, "trackedJsonTopic").withTrackingEnabled(true).withContentType(JSON).build(), null)); - - // and - String query = "{\"query\": {\"trackingEnabled\": \"true\", \"contentType\": \"AVRO\"}}"; - - // when - WebTestClient.ResponseSpec response = hermes.api().queryTopics(group, query); - - // then - assertThat(Arrays.stream(Objects.requireNonNull(response.expectBody(String[].class).returnResult().getResponseBody())).toList()) - .containsExactly(trackedTopicAvro.getQualifiedName()) - .doesNotContain(group + "." + "untrackedAvroTopic", group + "." + "untrackedJsonTopic", group + "." + "trackedJsonTopic"); - } - - @Test - public void shouldNotAllowDollarSign() { - // given - String group = "dollar"; - hermes.initHelper().createGroup(Group.from(group)); - - Stream.of("$name", "na$me", "name$").forEach(topicName -> { - // when - WebTestClient.ResponseSpec response = hermes.api().createTopic(new TopicWithSchema(topic(group, topicName).build(), null)); - - // then - response.expectStatus().isBadRequest(); - }); - } - - @Test - public void shouldCreateTopicWithMaxMessageSize() { - // given - TopicWithSchema topic = new TopicWithSchema(topicWithRandomName().withMaxMessageSize(2048).build(), null); - hermes.initHelper().createGroup(Group.from(topic.getName().getGroupName())); - - // when - WebTestClient.ResponseSpec response = hermes.api().createTopic(topic); - - // then - response.expectStatus().isCreated(); - int fetchedMessageSize = hermes.api().getTopicResponse(topic.getQualifiedName()).expectBody(TopicWithSchema.class).returnResult().getResponseBody().getMaxMessageSize(); - assertThat(fetchedMessageSize).isEqualTo(2048); - } - - @Test - public void shouldUpdateTopicWithMaxMessageSize() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().withMaxMessageSize(2048).build()); - PatchData patchData = PatchData.from(ImmutableMap.of("maxMessageSize", 1024)); - - // when - WebTestClient.ResponseSpec response = hermes.api().updateTopic(topic.getQualifiedName(), patchData); - - // then - response.expectStatus().isOk(); - int fetchedMessageSize = hermes.api().getTopicResponse(topic.getQualifiedName()).expectBody(TopicWithSchema.class).returnResult().getResponseBody().getMaxMessageSize(); - assertThat(fetchedMessageSize).isEqualTo(1024); - } - - @Test - public void shouldCreateTopicWithRestrictedSubscribing() { - // given - TopicWithSchema topic = new TopicWithSchema(topicWithRandomName().withSubscribingRestricted().build(), null); - hermes.initHelper().createGroup(Group.from(topic.getName().getGroupName())); - - // when - WebTestClient.ResponseSpec response = hermes.api().createTopic(topic); - - // then - response.expectStatus().isCreated(); - boolean fetchedIsSubscribingRestricted = hermes.api().getTopicResponse(topic.getQualifiedName()).expectBody(TopicWithSchema.class).returnResult().getResponseBody().isSubscribingRestricted(); - assertThat(fetchedIsSubscribingRestricted).isTrue(); - } - - @Test - public void shouldUpdateTopicWithRestrictedSubscribing() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - PatchData patchData = PatchData.from(ImmutableMap.of("subscribingRestricted", true)); - - // when - WebTestClient.ResponseSpec response = hermes.api().updateTopic(topic.getQualifiedName(), patchData); - - // then - response.expectStatus().isOk(); - boolean fetchedIsSubscribingRestricted = hermes.api().getTopicResponse(topic.getQualifiedName()).expectBody(TopicWithSchema.class).returnResult().getResponseBody().isSubscribingRestricted(); - assertThat(fetchedIsSubscribingRestricted).isTrue(); - } - - @Test - public void shouldCreateTopicWithOfflineStorageSettings() { - // given - TopicWithSchema topic = new TopicWithSchema(topicWithRandomName().withOfflineStorage(2).build(), null); - hermes.initHelper().createGroup(Group.from(topic.getName().getGroupName())); - - // when - WebTestClient.ResponseSpec response = hermes.api().createTopic(topic); - - // then - response.expectStatus().isCreated(); - int fetchedOfflineStorageDurationTime = hermes.api().getTopicResponse(topic.getQualifiedName()).expectBody(TopicWithSchema.class).returnResult().getResponseBody().getOfflineStorage().getRetentionTime().getDuration(); - assertThat(fetchedOfflineStorageDurationTime).isEqualTo(2); - } - - @Test - public void shouldUpdateTopicWithOfflineStorageSettings() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - PatchData patchData = PatchData.from(ImmutableMap.of("offlineStorage", ImmutableMap.of("enabled", true))); - - // when - WebTestClient.ResponseSpec response = hermes.api().updateTopic(topic.getQualifiedName(), patchData); - - // then - response.expectStatus().isOk(); - boolean fetchedIsOfflineStorageEnabled = hermes.api().getTopicResponse(topic.getQualifiedName()).expectBody(TopicWithSchema.class).returnResult().getResponseBody().getOfflineStorage().isEnabled(); - assertThat(fetchedIsOfflineStorageEnabled).isTrue(); - } - - @Test - public void shouldCreateTopicWithLabels() { - //given - TopicWithSchema topic = new TopicWithSchema(topicWithRandomName() - .withLabels(ImmutableSet.of( - new TopicLabel("label-1"), - new TopicLabel("label-2"), - new TopicLabel("label-3") - )) - .build(), null); - hermes.initHelper().createGroup(Group.from(topic.getName().getGroupName())); - - //when - WebTestClient.ResponseSpec response = hermes.api().createTopic(topic); - - //then - response.expectStatus().isCreated(); - Set fetchedLabels = hermes.api().getTopicResponse(topic.getQualifiedName()).expectBody(TopicWithSchema.class).returnResult().getResponseBody().getLabels(); - assertThat(fetchedLabels).containsAll( - ImmutableSet.of( + // then + response.expectStatus().isOk(); + boolean fetchedIsOfflineStorageEnabled = + hermes + .api() + .getTopicResponse(topic.getQualifiedName()) + .expectBody(TopicWithSchema.class) + .returnResult() + .getResponseBody() + .getOfflineStorage() + .isEnabled(); + assertThat(fetchedIsOfflineStorageEnabled).isTrue(); + } + + @Test + public void shouldCreateTopicWithLabels() { + // given + TopicWithSchema topic = + new TopicWithSchema( + topicWithRandomName() + .withLabels( + ImmutableSet.of( new TopicLabel("label-1"), new TopicLabel("label-2"), - new TopicLabel("label-3") - ) - ); - } - - @Test - public void shouldUpdateTopicWithLabels() { - //given - TopicWithSchema topic = new TopicWithSchema(topicWithRandomName() - .withLabels(ImmutableSet.of( - new TopicLabel("label-1"), - new TopicLabel("label-3") - )) - .build(), null); - hermes.initHelper().createTopic(topic); - - //when - PatchData patchData = PatchData.from(ImmutableMap.of( - "labels", ImmutableSet.of( - new TopicLabel("label-1"), - new TopicLabel("label-2"), - new TopicLabel("label-3") - )) - ); - WebTestClient.ResponseSpec response = hermes.api().updateTopic(topic.getQualifiedName(), patchData); - - //then - response.expectStatus().isOk(); - Set fetchedLabels = hermes.api().getTopicResponse(topic.getQualifiedName()).expectBody(TopicWithSchema.class).returnResult().getResponseBody().getLabels(); - assertThat(fetchedLabels).containsAll( + new TopicLabel("label-3"))) + .build(), + null); + hermes.initHelper().createGroup(Group.from(topic.getName().getGroupName())); + + // when + WebTestClient.ResponseSpec response = hermes.api().createTopic(topic); + + // then + response.expectStatus().isCreated(); + Set fetchedLabels = + hermes + .api() + .getTopicResponse(topic.getQualifiedName()) + .expectBody(TopicWithSchema.class) + .returnResult() + .getResponseBody() + .getLabels(); + assertThat(fetchedLabels) + .containsAll( + ImmutableSet.of( + new TopicLabel("label-1"), new TopicLabel("label-2"), new TopicLabel("label-3"))); + } + + @Test + public void shouldUpdateTopicWithLabels() { + // given + TopicWithSchema topic = + new TopicWithSchema( + topicWithRandomName() + .withLabels(ImmutableSet.of(new TopicLabel("label-1"), new TopicLabel("label-3"))) + .build(), + null); + hermes.initHelper().createTopic(topic); + + // when + PatchData patchData = + PatchData.from( + ImmutableMap.of( + "labels", ImmutableSet.of( - new TopicLabel("label-1"), - new TopicLabel("label-2"), - new TopicLabel("label-3") - ) - ); - } - - @Test - public void shouldNotCreateTopicWithDisallowedLabels() { - // given - TopicWithSchema topic = new TopicWithSchema(topicWithRandomName() - .withLabels(ImmutableSet.of( - new TopicLabel("some-random-label"), - new TopicLabel("label-2"), - new TopicLabel("label-3") - )) - .build(), null); - hermes.initHelper().createGroup(Group.from(topic.getName().getGroupName())); - - //when - WebTestClient.ResponseSpec response = hermes.api().createTopic(topic); - - //then - response.expectStatus().isBadRequest(); - assertThat(getErrorCode(response)).isEqualTo(ErrorCode.VALIDATION_ERROR); - hermes.api().getTopicResponse(topic.getQualifiedName()).expectStatus().isNotFound(); - } - - @Test - public void shouldNotUpdateTopicWithDisallowedLabels() { - //given - TopicWithSchema topic = new TopicWithSchema(topicWithRandomName() - .withLabels(ImmutableSet.of( - new TopicLabel("label-1"), - new TopicLabel("label-3") - )) - .build(), null); - hermes.initHelper().createTopic(topic); - - //when - PatchData patchData = PatchData.from(ImmutableMap.of( - "labels", ImmutableSet.of( + new TopicLabel("label-1"), + new TopicLabel("label-2"), + new TopicLabel("label-3")))); + WebTestClient.ResponseSpec response = + hermes.api().updateTopic(topic.getQualifiedName(), patchData); + + // then + response.expectStatus().isOk(); + Set fetchedLabels = + hermes + .api() + .getTopicResponse(topic.getQualifiedName()) + .expectBody(TopicWithSchema.class) + .returnResult() + .getResponseBody() + .getLabels(); + assertThat(fetchedLabels) + .containsAll( + ImmutableSet.of( + new TopicLabel("label-1"), new TopicLabel("label-2"), new TopicLabel("label-3"))); + } + + @Test + public void shouldNotCreateTopicWithDisallowedLabels() { + // given + TopicWithSchema topic = + new TopicWithSchema( + topicWithRandomName() + .withLabels( + ImmutableSet.of( new TopicLabel("some-random-label"), new TopicLabel("label-2"), - new TopicLabel("label-3") - )) - ); - WebTestClient.ResponseSpec response = hermes.api().updateTopic(topic.getQualifiedName(), patchData); - - //then - response.expectStatus().isBadRequest(); - assertThat(getErrorCode(response)).isEqualTo(ErrorCode.VALIDATION_ERROR); - Set fetchedLabels = hermes.api().getTopicResponse(topic.getQualifiedName()).expectBody(TopicWithSchema.class).returnResult().getResponseBody().getLabels(); - assertThat(fetchedLabels).containsAll( + new TopicLabel("label-3"))) + .build(), + null); + hermes.initHelper().createGroup(Group.from(topic.getName().getGroupName())); + + // when + WebTestClient.ResponseSpec response = hermes.api().createTopic(topic); + + // then + response.expectStatus().isBadRequest(); + assertThat(getErrorCode(response)).isEqualTo(ErrorCode.VALIDATION_ERROR); + hermes.api().getTopicResponse(topic.getQualifiedName()).expectStatus().isNotFound(); + } + + @Test + public void shouldNotUpdateTopicWithDisallowedLabels() { + // given + TopicWithSchema topic = + new TopicWithSchema( + topicWithRandomName() + .withLabels(ImmutableSet.of(new TopicLabel("label-1"), new TopicLabel("label-3"))) + .build(), + null); + hermes.initHelper().createTopic(topic); + + // when + PatchData patchData = + PatchData.from( + ImmutableMap.of( + "labels", ImmutableSet.of( - new TopicLabel("label-1"), - new TopicLabel("label-3") - ) - ); - } - - @Test - public void shouldCreateTopicEvenIfExistsInBrokers() { - // given - String groupName = "existingTopicFromExternalBroker"; - String topicName = "topic"; - String qualifiedTopicName = groupName + "." + topicName; - hermes.initHelper().createGroup(Group.from(groupName)); - - brokerOperations.createTopic(qualifiedTopicName); - - // when - WebTestClient.ResponseSpec response = hermes.api().createTopic((topicWithSchema(topic(groupName, topicName).build()))); - - // then - response.expectStatus().isCreated(); - hermes.api().getTopicResponse(qualifiedTopicName).expectStatus().isOk(); - } - - @Test - public void shouldNotAllowNonAdminUserCreateTopicWithNonDefaultFallbackToRemoteDatacenter() { - // given - TestSecurityProvider.setUserIsAdmin(false); - TopicWithSchema topic = topicWithSchema( - topicWithRandomName() - .withFallbackToRemoteDatacenterEnabled() - .build() - ); - hermes.initHelper().createGroup(Group.from(topic.getName().getGroupName())); - - // when - WebTestClient.ResponseSpec response = hermes.api().createTopic(topic); - - //then - response.expectStatus().isBadRequest(); - assertThat(response.expectBody(String.class).returnResult().getResponseBody()) - .contains("User is not allowed to set non-default fallback to remote datacenter"); - } - - @Test - public void shouldAllowAdminUserCreateTopicWithNonDefaultFallbackToRemoteDatacenterEnabled() { - // given - TestSecurityProvider.setUserIsAdmin(true); - TopicWithSchema topic = topicWithSchema( - topicWithRandomName() - .withFallbackToRemoteDatacenterEnabled() - .build() - ); - hermes.initHelper().createGroup(Group.from(topic.getName().getGroupName())); - - // when - WebTestClient.ResponseSpec response = hermes.api().createTopic(topic); - - //then - response.expectStatus().isCreated(); - } - - @Test - public void shouldNotAllowNonAdminUserToChangeFallbackToRemoteDatacenter() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName() - .withFallbackToRemoteDatacenterEnabled().build()); - TestSecurityProvider.setUserIsAdmin(false); - PatchData patchData = PatchData.from(ImmutableMap.of("fallbackToRemoteDatacenterEnabled", false)); - - // when - WebTestClient.ResponseSpec response = hermes.api().updateTopic(topic.getQualifiedName(), patchData); - - //then - response.expectStatus().isBadRequest(); - assertThat(response.expectBody(String.class).returnResult().getResponseBody()) - .contains("User is not allowed to update fallback to remote datacenter for this topic"); - } - - @Test - public void shouldAllowAdminUserToChangeFallbackToRemoteDatacenter() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName() - .withFallbackToRemoteDatacenterEnabled() - .build()); - TestSecurityProvider.setUserIsAdmin(true); - PatchData patchData = PatchData.from(ImmutableMap.of("fallbackToRemoteDatacenterEnabled", false)); + new TopicLabel("some-random-label"), + new TopicLabel("label-2"), + new TopicLabel("label-3")))); + WebTestClient.ResponseSpec response = + hermes.api().updateTopic(topic.getQualifiedName(), patchData); - // when - WebTestClient.ResponseSpec response = hermes.api().updateTopic(topic.getQualifiedName(), patchData); + // then + response.expectStatus().isBadRequest(); + assertThat(getErrorCode(response)).isEqualTo(ErrorCode.VALIDATION_ERROR); + Set fetchedLabels = + hermes + .api() + .getTopicResponse(topic.getQualifiedName()) + .expectBody(TopicWithSchema.class) + .returnResult() + .getResponseBody() + .getLabels(); + assertThat(fetchedLabels) + .containsAll(ImmutableSet.of(new TopicLabel("label-1"), new TopicLabel("label-3"))); + } + + @Test + public void shouldCreateTopicEvenIfExistsInBrokers() { + // given + String groupName = "existingTopicFromExternalBroker"; + String topicName = "topic"; + String qualifiedTopicName = groupName + "." + topicName; + hermes.initHelper().createGroup(Group.from(groupName)); + + brokerOperations.createTopic(qualifiedTopicName); + + // when + WebTestClient.ResponseSpec response = + hermes.api().createTopic((topicWithSchema(topic(groupName, topicName).build()))); + + // then + response.expectStatus().isCreated(); + hermes.api().getTopicResponse(qualifiedTopicName).expectStatus().isOk(); + } + + @Test + public void shouldNotAllowNonAdminUserCreateTopicWithNonDefaultFallbackToRemoteDatacenter() { + // given + TestSecurityProvider.setUserIsAdmin(false); + TopicWithSchema topic = + topicWithSchema(topicWithRandomName().withFallbackToRemoteDatacenterEnabled().build()); + hermes.initHelper().createGroup(Group.from(topic.getName().getGroupName())); + + // when + WebTestClient.ResponseSpec response = hermes.api().createTopic(topic); + + // then + response.expectStatus().isBadRequest(); + assertThat(response.expectBody(String.class).returnResult().getResponseBody()) + .contains("User is not allowed to set non-default fallback to remote datacenter"); + } + + @Test + public void shouldAllowAdminUserCreateTopicWithNonDefaultFallbackToRemoteDatacenterEnabled() { + // given + TestSecurityProvider.setUserIsAdmin(true); + TopicWithSchema topic = + topicWithSchema(topicWithRandomName().withFallbackToRemoteDatacenterEnabled().build()); + hermes.initHelper().createGroup(Group.from(topic.getName().getGroupName())); + + // when + WebTestClient.ResponseSpec response = hermes.api().createTopic(topic); + + // then + response.expectStatus().isCreated(); + } + + @Test + public void shouldNotAllowNonAdminUserToChangeFallbackToRemoteDatacenter() { + // given + Topic topic = + hermes + .initHelper() + .createTopic(topicWithRandomName().withFallbackToRemoteDatacenterEnabled().build()); + TestSecurityProvider.setUserIsAdmin(false); + PatchData patchData = + PatchData.from(ImmutableMap.of("fallbackToRemoteDatacenterEnabled", false)); + + // when + WebTestClient.ResponseSpec response = + hermes.api().updateTopic(topic.getQualifiedName(), patchData); - //then - response.expectStatus().isOk(); - } + // then + response.expectStatus().isBadRequest(); + assertThat(response.expectBody(String.class).returnResult().getResponseBody()) + .contains("User is not allowed to update fallback to remote datacenter for this topic"); + } + + @Test + public void shouldAllowAdminUserToChangeFallbackToRemoteDatacenter() { + // given + Topic topic = + hermes + .initHelper() + .createTopic(topicWithRandomName().withFallbackToRemoteDatacenterEnabled().build()); + TestSecurityProvider.setUserIsAdmin(true); + PatchData patchData = + PatchData.from(ImmutableMap.of("fallbackToRemoteDatacenterEnabled", false)); + + // when + WebTestClient.ResponseSpec response = + hermes.api().updateTopic(topic.getQualifiedName(), patchData); - @Test - public void shouldAllowNonAdminUserToModifyTopicWithFallbackToRemoteDatacenterEnabled() { - // given - TestSecurityProvider.setUserIsAdmin(true); - Topic topic = hermes.initHelper().createTopic( - topicWithRandomName() - .withFallbackToRemoteDatacenterEnabled() - .build() - ); - TestSecurityProvider.setUserIsAdmin(false); - PatchData patchData = PatchData.from(ImmutableMap.of("description", "new description")); - - // when - WebTestClient.ResponseSpec response = hermes.api().updateTopic(topic.getQualifiedName(), patchData); - - //then - response.expectStatus().isOk(); - } - - @Test - public void shouldNotAllowNonAdminUserCreateTopicWithChaosEnabled() { - // given - TestSecurityProvider.setUserIsAdmin(false); - TopicWithSchema topic = topicWithSchema( - topicWithRandomName() - .withPublishingChaosPolicy(new PublishingChaosPolicy(DATACENTER, null, Map.of())) - .build() - ); - hermes.initHelper().createGroup(Group.from(topic.getName().getGroupName())); - - // when - WebTestClient.ResponseSpec response = hermes.api().createTopic(topic); - - //then - response.expectStatus().isBadRequest(); - assertThat(response.expectBody(String.class).returnResult().getResponseBody()) - .contains("User is not allowed to set chaos policy for this topic"); - } - - @Test - public void shouldAllowAdminUserCreateTopicWithChaosEnabled() { - // given - TestSecurityProvider.setUserIsAdmin(true); - TopicWithSchema topic = topicWithSchema( - topicWithRandomName() - .withPublishingChaosPolicy(new PublishingChaosPolicy(DATACENTER, null, Map.of())) - .build() - ); - hermes.initHelper().createGroup(Group.from(topic.getName().getGroupName())); - - // when - WebTestClient.ResponseSpec response = hermes.api().createTopic(topic); - - //then - response.expectStatus().isCreated(); - } - - @Test - public void shouldNotCreateTopicWithInvalidChaosPolicy() { - // given - TestSecurityProvider.setUserIsAdmin(true); - TopicWithSchema topic = topicWithSchema( - topicWithRandomName() - .withPublishingChaosPolicy( - new PublishingChaosPolicy(DATACENTER, null, Map.of("dc1", new ChaosPolicy(100, 100, 99, false))) - ) - .build() - ); - hermes.initHelper().createGroup(Group.from(topic.getName().getGroupName())); - - // when - WebTestClient.ResponseSpec response = hermes.api().createTopic(topic); - - //then - response.expectStatus().isBadRequest(); - assertThat(response.expectBody(String.class).returnResult().getResponseBody()) - .contains("Invalid chaos policy: 'delayFrom' and 'delayTo' must be >= 0, and 'delayFrom' <= 'delayTo'."); - } - - @Test - public void shouldNotAllowNonAdminUserToEnableChaos() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestSecurityProvider.setUserIsAdmin(false); - PatchData patchData = PatchData.from(ImmutableMap.of("chaos", ImmutableMap.of("mode", DATACENTER))); - - // when - WebTestClient.ResponseSpec response = hermes.api().updateTopic(topic.getQualifiedName(), patchData); - - //then - response.expectStatus().isBadRequest(); - assertThat(response.expectBody(String.class).returnResult().getResponseBody()) - .contains("User is not allowed to update chaos policy for this topic"); - } - - @Test - public void shouldAllowAdminUserToEnableChaos() { - // given - Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); - TestSecurityProvider.setUserIsAdmin(true); - PatchData patchData = PatchData.from(ImmutableMap.of("chaos", ImmutableMap.of("mode", DATACENTER))); - - // when - WebTestClient.ResponseSpec response = hermes.api().updateTopic(topic.getQualifiedName(), patchData); - - //then - response.expectStatus().isOk(); - } - - @Test - public void shouldAllowNonAdminUserToModifyTopicWithChaosEnabled() { - // given - TestSecurityProvider.setUserIsAdmin(true); - Topic topic = hermes.initHelper().createTopic( + // then + response.expectStatus().isOk(); + } + + @Test + public void shouldAllowNonAdminUserToModifyTopicWithFallbackToRemoteDatacenterEnabled() { + // given + TestSecurityProvider.setUserIsAdmin(true); + Topic topic = + hermes + .initHelper() + .createTopic(topicWithRandomName().withFallbackToRemoteDatacenterEnabled().build()); + TestSecurityProvider.setUserIsAdmin(false); + PatchData patchData = PatchData.from(ImmutableMap.of("description", "new description")); + + // when + WebTestClient.ResponseSpec response = + hermes.api().updateTopic(topic.getQualifiedName(), patchData); + + // then + response.expectStatus().isOk(); + } + + @Test + public void shouldNotAllowNonAdminUserCreateTopicWithChaosEnabled() { + // given + TestSecurityProvider.setUserIsAdmin(false); + TopicWithSchema topic = + topicWithSchema( + topicWithRandomName() + .withPublishingChaosPolicy(new PublishingChaosPolicy(DATACENTER, null, Map.of())) + .build()); + hermes.initHelper().createGroup(Group.from(topic.getName().getGroupName())); + + // when + WebTestClient.ResponseSpec response = hermes.api().createTopic(topic); + + // then + response.expectStatus().isBadRequest(); + assertThat(response.expectBody(String.class).returnResult().getResponseBody()) + .contains("User is not allowed to set chaos policy for this topic"); + } + + @Test + public void shouldAllowAdminUserCreateTopicWithChaosEnabled() { + // given + TestSecurityProvider.setUserIsAdmin(true); + TopicWithSchema topic = + topicWithSchema( + topicWithRandomName() + .withPublishingChaosPolicy(new PublishingChaosPolicy(DATACENTER, null, Map.of())) + .build()); + hermes.initHelper().createGroup(Group.from(topic.getName().getGroupName())); + + // when + WebTestClient.ResponseSpec response = hermes.api().createTopic(topic); + + // then + response.expectStatus().isCreated(); + } + + @Test + public void shouldNotCreateTopicWithInvalidChaosPolicy() { + // given + TestSecurityProvider.setUserIsAdmin(true); + TopicWithSchema topic = + topicWithSchema( + topicWithRandomName() + .withPublishingChaosPolicy( + new PublishingChaosPolicy( + DATACENTER, null, Map.of("dc1", new ChaosPolicy(100, 100, 99, false)))) + .build()); + hermes.initHelper().createGroup(Group.from(topic.getName().getGroupName())); + + // when + WebTestClient.ResponseSpec response = hermes.api().createTopic(topic); + + // then + response.expectStatus().isBadRequest(); + assertThat(response.expectBody(String.class).returnResult().getResponseBody()) + .contains( + "Invalid chaos policy: 'delayFrom' and 'delayTo' must be >= 0, and 'delayFrom' <= 'delayTo'."); + } + + @Test + public void shouldNotAllowNonAdminUserToEnableChaos() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestSecurityProvider.setUserIsAdmin(false); + PatchData patchData = + PatchData.from(ImmutableMap.of("chaos", ImmutableMap.of("mode", DATACENTER))); + + // when + WebTestClient.ResponseSpec response = + hermes.api().updateTopic(topic.getQualifiedName(), patchData); + + // then + response.expectStatus().isBadRequest(); + assertThat(response.expectBody(String.class).returnResult().getResponseBody()) + .contains("User is not allowed to update chaos policy for this topic"); + } + + @Test + public void shouldAllowAdminUserToEnableChaos() { + // given + Topic topic = hermes.initHelper().createTopic(topicWithRandomName().build()); + TestSecurityProvider.setUserIsAdmin(true); + PatchData patchData = + PatchData.from(ImmutableMap.of("chaos", ImmutableMap.of("mode", DATACENTER))); + + // when + WebTestClient.ResponseSpec response = + hermes.api().updateTopic(topic.getQualifiedName(), patchData); + + // then + response.expectStatus().isOk(); + } + + @Test + public void shouldAllowNonAdminUserToModifyTopicWithChaosEnabled() { + // given + TestSecurityProvider.setUserIsAdmin(true); + Topic topic = + hermes + .initHelper() + .createTopic( topicWithRandomName() - .withPublishingChaosPolicy(new PublishingChaosPolicy(DATACENTER, null, Map.of())) - .build() - ); - TestSecurityProvider.setUserIsAdmin(false); - PatchData patchData = PatchData.from(ImmutableMap.of("description", "new description")); - - // when - WebTestClient.ResponseSpec response = hermes.api().updateTopic(topic.getQualifiedName(), patchData); - - //then - response.expectStatus().isOk(); - } - - @Test - public void shouldNotUpdateTopicWithInvalidChaosPolicy() { - // given - TestSecurityProvider.setUserIsAdmin(true); - Topic topic = hermes.initHelper().createTopic( + .withPublishingChaosPolicy( + new PublishingChaosPolicy(DATACENTER, null, Map.of())) + .build()); + TestSecurityProvider.setUserIsAdmin(false); + PatchData patchData = PatchData.from(ImmutableMap.of("description", "new description")); + + // when + WebTestClient.ResponseSpec response = + hermes.api().updateTopic(topic.getQualifiedName(), patchData); + + // then + response.expectStatus().isOk(); + } + + @Test + public void shouldNotUpdateTopicWithInvalidChaosPolicy() { + // given + TestSecurityProvider.setUserIsAdmin(true); + Topic topic = + hermes + .initHelper() + .createTopic( topicWithRandomName() - .withPublishingChaosPolicy( - new PublishingChaosPolicy(DATACENTER, null, Map.of("dc1", new ChaosPolicy(100, 100, 100, false))) - ) - .build() - ); - PatchData patchData = PatchData.from( + .withPublishingChaosPolicy( + new PublishingChaosPolicy( + DATACENTER, null, Map.of("dc1", new ChaosPolicy(100, 100, 100, false)))) + .build()); + PatchData patchData = + PatchData.from( + ImmutableMap.of( + "chaos", ImmutableMap.of( - "chaos", - ImmutableMap.of( - "datacenterPolicies", - ImmutableMap.of("dc1", ImmutableMap.of("delayTo", 99)) - ) - ) - ); - - // when - WebTestClient.ResponseSpec response = hermes.api().updateTopic(topic.getQualifiedName(), patchData); - - //then - response.expectStatus().isBadRequest(); - assertThat(response.expectBody(String.class).returnResult().getResponseBody()) - .contains("Invalid chaos policy: 'delayFrom' and 'delayTo' must be >= 0, and 'delayFrom' <= 'delayTo'."); - } - - private static List getGroupTopicsList(String groupName) { - return Arrays.stream(Objects.requireNonNull(hermes.api().listTopics(groupName) - .expectStatus() - .isOk() - .expectBody(String[].class) - .returnResult() - .getResponseBody())) - .toList(); - } - - public static ErrorCode getErrorCode(WebTestClient.ResponseSpec createTopicResponse) { - return Objects.requireNonNull(createTopicResponse.expectBody(ErrorDescription.class).returnResult().getResponseBody()).getCode(); - } + "datacenterPolicies", ImmutableMap.of("dc1", ImmutableMap.of("delayTo", 99))))); + + // when + WebTestClient.ResponseSpec response = + hermes.api().updateTopic(topic.getQualifiedName(), patchData); + + // then + response.expectStatus().isBadRequest(); + assertThat(response.expectBody(String.class).returnResult().getResponseBody()) + .contains( + "Invalid chaos policy: 'delayFrom' and 'delayTo' must be >= 0, and 'delayFrom' <= 'delayTo'."); + } + + private static List getGroupTopicsList(String groupName) { + return Arrays.stream( + Objects.requireNonNull( + hermes + .api() + .listTopics(groupName) + .expectStatus() + .isOk() + .expectBody(String[].class) + .returnResult() + .getResponseBody())) + .toList(); + } + + public static ErrorCode getErrorCode(WebTestClient.ResponseSpec createTopicResponse) { + return Objects.requireNonNull( + createTopicResponse.expectBody(ErrorDescription.class).returnResult().getResponseBody()) + .getCode(); + } } diff --git a/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/AttachingKeepAliveHeaderTest.java b/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/AttachingKeepAliveHeaderTest.java index b93258b5bc..60593ff0d1 100644 --- a/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/AttachingKeepAliveHeaderTest.java +++ b/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/AttachingKeepAliveHeaderTest.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.integrationtests; +import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_KEEP_ALIVE_HEADER_ENABLED; +import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_KEEP_ALIVE_HEADER_TIMEOUT; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + +import java.util.function.Consumer; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -11,71 +16,70 @@ import pl.allegro.tech.hermes.test.helper.client.integration.FrontendTestClient; import pl.allegro.tech.hermes.test.helper.message.TestMessage; -import java.util.function.Consumer; - -import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_KEEP_ALIVE_HEADER_ENABLED; -import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_KEEP_ALIVE_HEADER_TIMEOUT; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class AttachingKeepAliveHeaderTest { - @Order(0) - @RegisterExtension - public static InfrastructureExtension infra = new InfrastructureExtension(); + @Order(0) + @RegisterExtension + public static InfrastructureExtension infra = new InfrastructureExtension(); - @Order(1) - @RegisterExtension - public static HermesManagementExtension management = new HermesManagementExtension(infra); + @Order(1) + @RegisterExtension + public static HermesManagementExtension management = new HermesManagementExtension(infra); - private static final String MESSAGE = TestMessage.of("hello", "world").body(); + private static final String MESSAGE = TestMessage.of("hello", "world").body(); - @Test - public void shouldAttachKeepAliveHeaderWhenEnabled() { - //given - HermesFrontendTestApp frontend = startFrontend(f -> { - f.withProperty(FRONTEND_KEEP_ALIVE_HEADER_ENABLED, true); - f.withProperty(FRONTEND_KEEP_ALIVE_HEADER_TIMEOUT, "2s"); - }); + @Test + public void shouldAttachKeepAliveHeaderWhenEnabled() { + // given + HermesFrontendTestApp frontend = + startFrontend( + f -> { + f.withProperty(FRONTEND_KEEP_ALIVE_HEADER_ENABLED, true); + f.withProperty(FRONTEND_KEEP_ALIVE_HEADER_TIMEOUT, "2s"); + }); - Topic topic = management.initHelper().createTopic(topicWithRandomName().build()); + Topic topic = management.initHelper().createTopic(topicWithRandomName().build()); - FrontendTestClient publisher = new FrontendTestClient(frontend.getPort()); + FrontendTestClient publisher = new FrontendTestClient(frontend.getPort()); - try { - //when - WebTestClient.ResponseSpec response = publisher.publish(topic.getQualifiedName(), MESSAGE); + try { + // when + WebTestClient.ResponseSpec response = publisher.publish(topic.getQualifiedName(), MESSAGE); - //then - response.expectHeader().valueEquals("Keep-Alive", "timeout=2"); - } finally { - frontend.stop(); - } + // then + response.expectHeader().valueEquals("Keep-Alive", "timeout=2"); + } finally { + frontend.stop(); } + } - @Test - public void shouldNotAttachKeepAliveHeaderWhenDisabled() { - //given - HermesFrontendTestApp frontend = startFrontend(f -> f.withProperty(FRONTEND_KEEP_ALIVE_HEADER_ENABLED, false)); - - Topic topic = management.initHelper().createTopic(topicWithRandomName().build()); + @Test + public void shouldNotAttachKeepAliveHeaderWhenDisabled() { + // given + HermesFrontendTestApp frontend = + startFrontend(f -> f.withProperty(FRONTEND_KEEP_ALIVE_HEADER_ENABLED, false)); - FrontendTestClient publisher = new FrontendTestClient(frontend.getPort()); + Topic topic = management.initHelper().createTopic(topicWithRandomName().build()); - try { - //when - WebTestClient.ResponseSpec response = publisher.publish(topic.getQualifiedName(), MESSAGE); + FrontendTestClient publisher = new FrontendTestClient(frontend.getPort()); - //then - response.expectHeader().doesNotExist("Keep-Alive"); - } finally { - frontend.stop(); - } - } + try { + // when + WebTestClient.ResponseSpec response = publisher.publish(topic.getQualifiedName(), MESSAGE); - private HermesFrontendTestApp startFrontend(Consumer frontendConfigUpdater) { - HermesFrontendTestApp frontend = new HermesFrontendTestApp(infra.hermesZookeeper(), infra.kafka(), infra.schemaRegistry()); - frontendConfigUpdater.accept(frontend); - frontend.start(); - return frontend; + // then + response.expectHeader().doesNotExist("Keep-Alive"); + } finally { + frontend.stop(); } + } + + private HermesFrontendTestApp startFrontend( + Consumer frontendConfigUpdater) { + HermesFrontendTestApp frontend = + new HermesFrontendTestApp(infra.hermesZookeeper(), infra.kafka(), infra.schemaRegistry()); + frontendConfigUpdater.accept(frontend); + frontend.start(); + return frontend; + } } diff --git a/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/BrokerLatencyReportingTest.java b/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/BrokerLatencyReportingTest.java index e53cf88ea7..13f878f8d7 100644 --- a/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/BrokerLatencyReportingTest.java +++ b/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/BrokerLatencyReportingTest.java @@ -1,5 +1,10 @@ package pl.allegro.tech.hermes.integrationtests; +import static org.awaitility.Awaitility.waitAtMost; +import static pl.allegro.tech.hermes.integrationtests.assertions.HermesAssertions.assertThatMetrics; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + +import java.time.Duration; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Order; @@ -13,63 +18,61 @@ import pl.allegro.tech.hermes.test.helper.client.integration.FrontendTestClient; import pl.allegro.tech.hermes.test.helper.message.TestMessage; -import java.time.Duration; - -import static org.awaitility.Awaitility.waitAtMost; -import static pl.allegro.tech.hermes.integrationtests.assertions.HermesAssertions.assertThatMetrics; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class BrokerLatencyReportingTest { - @Order(0) - @RegisterExtension - public static InfrastructureExtension infra = new InfrastructureExtension(); + @Order(0) + @RegisterExtension + public static InfrastructureExtension infra = new InfrastructureExtension(); - @Order(1) - @RegisterExtension - public static HermesManagementExtension management = new HermesManagementExtension(infra); + @Order(1) + @RegisterExtension + public static HermesManagementExtension management = new HermesManagementExtension(infra); - private static HermesFrontendTestApp frontend; + private static HermesFrontendTestApp frontend; - private static FrontendTestClient frontendTestClient; + private static FrontendTestClient frontendTestClient; - @BeforeAll - public static void setup() { - frontend = new HermesFrontendTestApp(infra.hermesZookeeper(), infra.kafka(), infra.schemaRegistry()); - frontend.withProperty(FrontendConfigurationProperties.BROKER_LATENCY_REPORTER_ENABLED, true); - frontend.start(); - frontendTestClient = new FrontendTestClient(frontend.getPort()); - } + @BeforeAll + public static void setup() { + frontend = + new HermesFrontendTestApp(infra.hermesZookeeper(), infra.kafka(), infra.schemaRegistry()); + frontend.withProperty(FrontendConfigurationProperties.BROKER_LATENCY_REPORTER_ENABLED, true); + frontend.start(); + frontendTestClient = new FrontendTestClient(frontend.getPort()); + } - @AfterAll - public static void tearDown() { - frontend.stop(); - } + @AfterAll + public static void tearDown() { + frontend.stop(); + } - @Test - public void shouldReportBrokerLatencyMetrics() { - // given - Topic topic = management.initHelper().createTopic(topicWithRandomName().build()); + @Test + public void shouldReportBrokerLatencyMetrics() { + // given + Topic topic = management.initHelper().createTopic(topicWithRandomName().build()); - TestMessage message = TestMessage.of("hello", "world"); + TestMessage message = TestMessage.of("hello", "world"); - // when - frontendTestClient.publishUntilSuccess(topic.getQualifiedName(), message.body()); + // when + frontendTestClient.publishUntilSuccess(topic.getQualifiedName(), message.body()); - // then - waitAtMost(Duration.ofSeconds(5)).untilAsserted(() -> { - frontendTestClient.getMetrics() - .expectStatus() - .isOk() - .expectBody(String.class) - .value((body) -> assertThatMetrics(body) - .contains("hermes_frontend_broker_latency_seconds_count") - .withLabels( - "ack", "LEADER", - "broker", "localhost" - ) - .withValueGreaterThan(0.0d) - ); - }); - } -} \ No newline at end of file + // then + waitAtMost(Duration.ofSeconds(5)) + .untilAsserted( + () -> { + frontendTestClient + .getMetrics() + .expectStatus() + .isOk() + .expectBody(String.class) + .value( + (body) -> + assertThatMetrics(body) + .contains("hermes_frontend_broker_latency_seconds_count") + .withLabels( + "ack", "LEADER", + "broker", "localhost") + .withValueGreaterThan(0.0d)); + }); + } +} diff --git a/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/HermesClientPublishingHttpsTest.java b/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/HermesClientPublishingHttpsTest.java index 35bacd5d11..a4848660f8 100644 --- a/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/HermesClientPublishingHttpsTest.java +++ b/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/HermesClientPublishingHttpsTest.java @@ -1,5 +1,17 @@ package pl.allegro.tech.hermes.integrationtests; +import static org.assertj.core.api.Assertions.assertThat; +import static pl.allegro.tech.hermes.client.HermesClientBuilder.hermesClient; +import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_HTTP2_ENABLED; +import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_SSL_ENABLED; +import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_SSL_KEYSTORE_SOURCE; +import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_SSL_PORT; +import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_SSL_TRUSTSTORE_SOURCE; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + +import java.net.URI; +import java.time.Duration; +import javax.net.ssl.X509TrustManager; import okhttp3.OkHttpClient; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -20,89 +32,78 @@ import pl.allegro.tech.hermes.integrationtests.setup.InfrastructureExtension; import pl.allegro.tech.hermes.test.helper.message.TestMessage; -import java.net.URI; -import java.time.Duration; -import javax.net.ssl.X509TrustManager; - -import static org.assertj.core.api.Assertions.assertThat; -import static pl.allegro.tech.hermes.client.HermesClientBuilder.hermesClient; -import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_HTTP2_ENABLED; -import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_SSL_ENABLED; -import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_SSL_KEYSTORE_SOURCE; -import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_SSL_PORT; -import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.FRONTEND_SSL_TRUSTSTORE_SOURCE; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class HermesClientPublishingHttpsTest { - @Order(0) - @RegisterExtension - public static InfrastructureExtension infra = new InfrastructureExtension(); - - @Order(1) - @RegisterExtension - public static HermesManagementExtension management = new HermesManagementExtension(infra); - - private static HermesFrontendTestApp frontend; - - @BeforeAll - public static void setup() { - frontend = new HermesFrontendTestApp(infra.hermesZookeeper(), infra.kafka(), infra.schemaRegistry()); - frontend.withProperty(FRONTEND_SSL_ENABLED, true); - frontend.withProperty(FRONTEND_HTTP2_ENABLED, true); - frontend.withProperty(FRONTEND_SSL_PORT, 0); - frontend.withProperty(FRONTEND_SSL_KEYSTORE_SOURCE, "provided"); - frontend.withProperty(FRONTEND_SSL_TRUSTSTORE_SOURCE, "provided"); - - frontend.start(); - } - - @AfterAll - public static void clean() { - frontend.stop(); - } - - @Test - public void shouldCommunicateWithHermesUsingHttp2() { - // given - Topic topic = management.initHelper().createTopic(topicWithRandomName().build()); - String message = TestMessage.of("hello", "world").body(); - - OkHttpHermesSender okHttpHermesSender = new OkHttpHermesSender(getOkHttpClientWithSslContextConfigured()); - HermesClient client = hermesClient(okHttpHermesSender) - .withRetries(5) - .withRetrySleep(Duration.ofSeconds(5).toMillis(), Duration.ofSeconds(10).toMillis()) - .withURI(URI.create("https://localhost:" + frontend.getSSLPort())) - .build(); - - // when - HermesResponse response = client.publish(topic.getQualifiedName(), message).join(); - - // then - assertThat(response.getProtocol()).isEqualTo("h2"); - assertThat(response.isSuccess()).isTrue(); - } - - private OkHttpClient getOkHttpClientWithSslContextConfigured() { - DefaultSslContextFactory sslContextFactory = getDefaultSslContextFactory(); - SSLContextHolder sslContextHolder = sslContextFactory.create(); - return new OkHttpClient.Builder() - .sslSocketFactory( - sslContextHolder.getSslContext().getSocketFactory(), - (X509TrustManager) sslContextHolder.getTrustManagers()[0] - ) - .build(); - } - - private static DefaultSslContextFactory getDefaultSslContextFactory() { - String protocol = "TLS"; - KeystoreProperties keystoreProperties = new KeystoreProperties("classpath:client.keystore", "JKS", "password"); - KeystoreProperties truststoreProperties = new KeystoreProperties("classpath:client.truststore", "JKS", "password"); - return new DefaultSslContextFactory( - protocol, - new ProvidedKeyManagersProvider(keystoreProperties), - new ProvidedTrustManagersProvider(truststoreProperties) - ); - } - + @Order(0) + @RegisterExtension + public static InfrastructureExtension infra = new InfrastructureExtension(); + + @Order(1) + @RegisterExtension + public static HermesManagementExtension management = new HermesManagementExtension(infra); + + private static HermesFrontendTestApp frontend; + + @BeforeAll + public static void setup() { + frontend = + new HermesFrontendTestApp(infra.hermesZookeeper(), infra.kafka(), infra.schemaRegistry()); + frontend.withProperty(FRONTEND_SSL_ENABLED, true); + frontend.withProperty(FRONTEND_HTTP2_ENABLED, true); + frontend.withProperty(FRONTEND_SSL_PORT, 0); + frontend.withProperty(FRONTEND_SSL_KEYSTORE_SOURCE, "provided"); + frontend.withProperty(FRONTEND_SSL_TRUSTSTORE_SOURCE, "provided"); + + frontend.start(); + } + + @AfterAll + public static void clean() { + frontend.stop(); + } + + @Test + public void shouldCommunicateWithHermesUsingHttp2() { + // given + Topic topic = management.initHelper().createTopic(topicWithRandomName().build()); + String message = TestMessage.of("hello", "world").body(); + + OkHttpHermesSender okHttpHermesSender = + new OkHttpHermesSender(getOkHttpClientWithSslContextConfigured()); + HermesClient client = + hermesClient(okHttpHermesSender) + .withRetries(5) + .withRetrySleep(Duration.ofSeconds(5).toMillis(), Duration.ofSeconds(10).toMillis()) + .withURI(URI.create("https://localhost:" + frontend.getSSLPort())) + .build(); + + // when + HermesResponse response = client.publish(topic.getQualifiedName(), message).join(); + + // then + assertThat(response.getProtocol()).isEqualTo("h2"); + assertThat(response.isSuccess()).isTrue(); + } + + private OkHttpClient getOkHttpClientWithSslContextConfigured() { + DefaultSslContextFactory sslContextFactory = getDefaultSslContextFactory(); + SSLContextHolder sslContextHolder = sslContextFactory.create(); + return new OkHttpClient.Builder() + .sslSocketFactory( + sslContextHolder.getSslContext().getSocketFactory(), + (X509TrustManager) sslContextHolder.getTrustManagers()[0]) + .build(); + } + + private static DefaultSslContextFactory getDefaultSslContextFactory() { + String protocol = "TLS"; + KeystoreProperties keystoreProperties = + new KeystoreProperties("classpath:client.keystore", "JKS", "password"); + KeystoreProperties truststoreProperties = + new KeystoreProperties("classpath:client.truststore", "JKS", "password"); + return new DefaultSslContextFactory( + protocol, + new ProvidedKeyManagersProvider(keystoreProperties), + new ProvidedTrustManagersProvider(truststoreProperties)); + } } diff --git a/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/HermesServerGracefulShutdownTest.java b/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/HermesServerGracefulShutdownTest.java index 7a790ffa0a..1e538e6ddc 100644 --- a/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/HermesServerGracefulShutdownTest.java +++ b/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/HermesServerGracefulShutdownTest.java @@ -1,5 +1,8 @@ package pl.allegro.tech.hermes.integrationtests; +import static org.awaitility.Awaitility.waitAtMost; + +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -12,52 +15,53 @@ import pl.allegro.tech.hermes.test.helper.client.integration.FrontendTestClient; import pl.allegro.tech.hermes.test.helper.message.TestMessage; -import java.util.concurrent.TimeUnit; - -import static org.awaitility.Awaitility.waitAtMost; - -public class HermesServerGracefulShutdownTest { - - @RegisterExtension - public static InfrastructureExtension infra = new InfrastructureExtension(); +public class HermesServerGracefulShutdownTest { - private HermesFrontendTestApp frontend; - private HermesServer hermesServer; - FrontendTestClient frontendClient; + @RegisterExtension public static InfrastructureExtension infra = new InfrastructureExtension(); - @BeforeEach - public void beforeEach() { - frontend = new HermesFrontendTestApp(infra.hermesZookeeper(), infra.kafka(), infra.schemaRegistry()); - frontend.start(); - hermesServer = frontend.getBean(HermesServer.class); - frontendClient = new FrontendTestClient(frontend.getPort()); - } + private HermesFrontendTestApp frontend; + private HermesServer hermesServer; + FrontendTestClient frontendClient; - @AfterEach - public void afterEach() { - frontend.stop(); - } + @BeforeEach + public void beforeEach() { + frontend = + new HermesFrontendTestApp(infra.hermesZookeeper(), infra.kafka(), infra.schemaRegistry()); + frontend.start(); + hermesServer = frontend.getBean(HermesServer.class); + frontendClient = new FrontendTestClient(frontend.getPort()); + } - @Test - public void shouldShutdownGracefully() throws Throwable { - //given - hermesServer.prepareForGracefulShutdown(); + @AfterEach + public void afterEach() { + frontend.stop(); + } - //when - WebTestClient.ResponseSpec response = frontendClient.publish("topic", TestMessage.of("hello", "world").body()); + @Test + public void shouldShutdownGracefully() throws Throwable { + // given + hermesServer.prepareForGracefulShutdown(); - //then - response.expectStatus().isEqualTo(HttpStatus.SERVICE_UNAVAILABLE); - } + // when + WebTestClient.ResponseSpec response = + frontendClient.publish("topic", TestMessage.of("hello", "world").body()); - @Test - public void shouldReturnCorrectHealthStatus() throws InterruptedException { - // when - hermesServer.prepareForGracefulShutdown(); + // then + response.expectStatus().isEqualTo(HttpStatus.SERVICE_UNAVAILABLE); + } - // then - waitAtMost(5, TimeUnit.SECONDS) - .untilAsserted(() -> frontendClient.getStatusPing().expectStatus().isEqualTo(HttpStatus.SERVICE_UNAVAILABLE)); - } + @Test + public void shouldReturnCorrectHealthStatus() throws InterruptedException { + // when + hermesServer.prepareForGracefulShutdown(); + // then + waitAtMost(5, TimeUnit.SECONDS) + .untilAsserted( + () -> + frontendClient + .getStatusPing() + .expectStatus() + .isEqualTo(HttpStatus.SERVICE_UNAVAILABLE)); + } } diff --git a/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/KafkaReadinessCheckTest.java b/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/KafkaReadinessCheckTest.java index 25873bdf84..d2cff160b2 100644 --- a/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/KafkaReadinessCheckTest.java +++ b/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/KafkaReadinessCheckTest.java @@ -1,5 +1,15 @@ package pl.allegro.tech.hermes.integrationtests; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.awaitility.Awaitility.await; +import static org.springframework.http.HttpStatus.SERVICE_UNAVAILABLE; +import static pl.allegro.tech.hermes.api.Topic.Ack.ALL; +import static pl.allegro.tech.hermes.integrationtests.assertions.HermesAssertions.assertThat; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -16,193 +26,188 @@ import pl.allegro.tech.hermes.test.helper.containers.ZookeeperContainer; import pl.allegro.tech.hermes.test.helper.environment.HermesTestApp; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.awaitility.Awaitility.await; -import static org.springframework.http.HttpStatus.SERVICE_UNAVAILABLE; -import static pl.allegro.tech.hermes.api.Topic.Ack.ALL; -import static pl.allegro.tech.hermes.integrationtests.assertions.HermesAssertions.assertThat; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class KafkaReadinessCheckTest { - private static final ZookeeperContainer hermesZookeeper = new ZookeeperContainer("HermesZookeeper"); - private static final KafkaContainerCluster kafka = new KafkaContainerCluster(3); - private static final ConfluentSchemaRegistryContainer schemaRegistry = new ConfluentSchemaRegistryContainer() - .withKafkaCluster(kafka); - private static Topic topic; - - @BeforeAll - public static void setup() { - Stream.of(hermesZookeeper, kafka).parallel().forEach(Startable::start); - schemaRegistry.start(); - HermesTestApp management = new HermesManagementTestApp(hermesZookeeper, kafka, schemaRegistry) - .start(); - - HermesInitHelper hermesInitHelper = new HermesInitHelper(management.getPort()); - topic = hermesInitHelper.createTopic( - topicWithRandomName() - .withAck(ALL) - .build() - ); - management.stop(); - } - - @AfterAll - public static void clean() { - Stream.of(hermesZookeeper, kafka, schemaRegistry).parallel().forEach(Startable::stop); - } - - @Test - public void shouldNotBeReadyUntilKafkaClusterIsUp() { - // given - kafka.cutOffConnectionsBetweenBrokersAndClients(); - - // when - HermesTestApp hermesFrontend = new HermesFrontendTestApp(hermesZookeeper, kafka, schemaRegistry) - .metadataMaxAgeInSeconds(1) - .readinessCheckIntervalInSeconds(1) - .kafkaCheckEnabled() - .start(); - - // then - getStatusReady(hermesFrontend).expectStatus().isEqualTo(SERVICE_UNAVAILABLE); - getStatusHealth(hermesFrontend).expectStatus().is2xxSuccessful(); - - // when - kafka.restoreConnectionsBetweenBrokersAndClients(); - - // then - await().atMost(5, SECONDS) - .untilAsserted(() -> getStatusReady(hermesFrontend).expectStatus().is2xxSuccessful()); - getStatusHealth(hermesFrontend).expectStatus().is2xxSuccessful(); - - // cleanup - hermesFrontend.stop(); - } - - @Test - public void shouldRespectKafkaCheckEnabledFlag() { - // given - kafka.cutOffConnectionsBetweenBrokersAndClients(); - - // when - HermesTestApp hermesFrontend = new HermesFrontendTestApp(hermesZookeeper, kafka, schemaRegistry) - .metadataMaxAgeInSeconds(1) - .readinessCheckIntervalInSeconds(1) - .kafkaCheckDisabled() - .start(); - - // then - getStatusReady(hermesFrontend).expectStatus().is2xxSuccessful(); - getStatusHealth(hermesFrontend).expectStatus().is2xxSuccessful(); - - // cleanup - kafka.restoreConnectionsBetweenBrokersAndClients(); - hermesFrontend.stop(); - } - - @Test - public void shouldNotBeReadyUntilThereAreNoUnderReplicatedPartitions() throws Exception { - // given - List brokers = kafka.getAllBrokers(); - List brokersToStop = brokers.subList(kafka.getMinInSyncReplicas() - 1, brokers.size()); - kafka.stop(brokersToStop); - assertThat(kafka.countUnderReplicatedPartitions() > 0).isTrue(); - - // when - HermesTestApp hermesFrontend = new HermesFrontendTestApp(hermesZookeeper, kafka, schemaRegistry) - .metadataMaxAgeInSeconds(1) - .readinessCheckIntervalInSeconds(1) - .kafkaCheckEnabled() - .start(); - - // then - getStatusReady(hermesFrontend).expectStatus().isEqualTo(SERVICE_UNAVAILABLE); - getStatusHealth(hermesFrontend).expectStatus().is2xxSuccessful(); - - // when - kafka.start(selectOne(brokersToStop)); - - // then - await().atMost(5, SECONDS) - .untilAsserted(() -> getStatusReady(hermesFrontend).expectStatus().is2xxSuccessful()); - getStatusHealth(hermesFrontend).expectStatus().is2xxSuccessful(); - - // cleanup - kafka.startAllStoppedBrokers(); - hermesFrontend.stop(); - } - - @Test - public void shouldNotBeReadyUntilThereAreNoOfflinePartitions() throws Exception { - // given: stop one of the brokers - List brokers = kafka.getAllBrokers(); - List brokersToStop = selectOne(brokers); - kafka.stop(brokersToStop); - - // and: send a message to remove stopped broker from in-sync replicas - publishSampleMessage(topic); - - // and: stop all in-sync replicas - List isr = selectAllOtherThan(brokers, brokersToStop); - kafka.stop(isr); - - // and: start the broker that is not in in-sync replicas - kafka.start(brokersToStop); - - // and: check if there is at least one offline partition - assertThat(kafka.countOfflinePartitions() > 0).isTrue(); - - // when - HermesTestApp hermesFrontend = new HermesFrontendTestApp(hermesZookeeper, kafka, schemaRegistry) - .metadataMaxAgeInSeconds(1) - .readinessCheckIntervalInSeconds(1) - .kafkaCheckEnabled() - .start(); - - // then - getStatusReady(hermesFrontend).expectStatus().isEqualTo(SERVICE_UNAVAILABLE); - getStatusHealth(hermesFrontend).expectStatus().is2xxSuccessful(); - - // when - kafka.start(isr); - - // then - await().atMost(5, SECONDS) - .untilAsserted(() -> getStatusReady(hermesFrontend).expectStatus().is2xxSuccessful()); - getStatusHealth(hermesFrontend).expectStatus().is2xxSuccessful(); - - // cleanup - hermesFrontend.stop(); - } - - private static List selectOne(List brokerIds) { - return brokerIds.subList(0, 1); - } - - private static List selectAllOtherThan(List brokerIds, List toExclude) { - return brokerIds.stream().filter(b -> !toExclude.contains(b)).collect(Collectors.toList()); - } - - private void publishSampleMessage(Topic topic) { - HermesTestApp hermesFrontend = new HermesFrontendTestApp(hermesZookeeper, kafka, schemaRegistry) - .start(); - FrontendTestClient client = new FrontendTestClient(hermesFrontend.getPort()); - client.publishUntilSuccess(topic.getQualifiedName(), "message"); - hermesFrontend.stop(); - } - - private WebTestClient.ResponseSpec getStatusHealth(HermesTestApp hermesFrontend) { - FrontendTestClient client = new FrontendTestClient(hermesFrontend.getPort()); - return client.getStatusHealth(); - } - - private WebTestClient.ResponseSpec getStatusReady(HermesTestApp hermesFrontend) { - FrontendTestClient client = new FrontendTestClient(hermesFrontend.getPort()); - return client.getStatusReady(); - } + private static final ZookeeperContainer hermesZookeeper = + new ZookeeperContainer("HermesZookeeper"); + private static final KafkaContainerCluster kafka = new KafkaContainerCluster(3); + private static final ConfluentSchemaRegistryContainer schemaRegistry = + new ConfluentSchemaRegistryContainer().withKafkaCluster(kafka); + private static Topic topic; + + @BeforeAll + public static void setup() { + Stream.of(hermesZookeeper, kafka).parallel().forEach(Startable::start); + schemaRegistry.start(); + HermesTestApp management = + new HermesManagementTestApp(hermesZookeeper, kafka, schemaRegistry).start(); + + HermesInitHelper hermesInitHelper = new HermesInitHelper(management.getPort()); + topic = hermesInitHelper.createTopic(topicWithRandomName().withAck(ALL).build()); + management.stop(); + } + + @AfterAll + public static void clean() { + Stream.of(hermesZookeeper, kafka, schemaRegistry).parallel().forEach(Startable::stop); + } + + @Test + public void shouldNotBeReadyUntilKafkaClusterIsUp() { + // given + kafka.cutOffConnectionsBetweenBrokersAndClients(); + + // when + HermesTestApp hermesFrontend = + new HermesFrontendTestApp(hermesZookeeper, kafka, schemaRegistry) + .metadataMaxAgeInSeconds(1) + .readinessCheckIntervalInSeconds(1) + .kafkaCheckEnabled() + .start(); + + // then + getStatusReady(hermesFrontend).expectStatus().isEqualTo(SERVICE_UNAVAILABLE); + getStatusHealth(hermesFrontend).expectStatus().is2xxSuccessful(); + + // when + kafka.restoreConnectionsBetweenBrokersAndClients(); + + // then + await() + .atMost(5, SECONDS) + .untilAsserted(() -> getStatusReady(hermesFrontend).expectStatus().is2xxSuccessful()); + getStatusHealth(hermesFrontend).expectStatus().is2xxSuccessful(); + + // cleanup + hermesFrontend.stop(); + } + + @Test + public void shouldRespectKafkaCheckEnabledFlag() { + // given + kafka.cutOffConnectionsBetweenBrokersAndClients(); + + // when + HermesTestApp hermesFrontend = + new HermesFrontendTestApp(hermesZookeeper, kafka, schemaRegistry) + .metadataMaxAgeInSeconds(1) + .readinessCheckIntervalInSeconds(1) + .kafkaCheckDisabled() + .start(); + + // then + getStatusReady(hermesFrontend).expectStatus().is2xxSuccessful(); + getStatusHealth(hermesFrontend).expectStatus().is2xxSuccessful(); + + // cleanup + kafka.restoreConnectionsBetweenBrokersAndClients(); + hermesFrontend.stop(); + } + + @Test + public void shouldNotBeReadyUntilThereAreNoUnderReplicatedPartitions() throws Exception { + // given + List brokers = kafka.getAllBrokers(); + List brokersToStop = + brokers.subList(kafka.getMinInSyncReplicas() - 1, brokers.size()); + kafka.stop(brokersToStop); + assertThat(kafka.countUnderReplicatedPartitions() > 0).isTrue(); + + // when + HermesTestApp hermesFrontend = + new HermesFrontendTestApp(hermesZookeeper, kafka, schemaRegistry) + .metadataMaxAgeInSeconds(1) + .readinessCheckIntervalInSeconds(1) + .kafkaCheckEnabled() + .start(); + + // then + getStatusReady(hermesFrontend).expectStatus().isEqualTo(SERVICE_UNAVAILABLE); + getStatusHealth(hermesFrontend).expectStatus().is2xxSuccessful(); + + // when + kafka.start(selectOne(brokersToStop)); + + // then + await() + .atMost(5, SECONDS) + .untilAsserted(() -> getStatusReady(hermesFrontend).expectStatus().is2xxSuccessful()); + getStatusHealth(hermesFrontend).expectStatus().is2xxSuccessful(); + + // cleanup + kafka.startAllStoppedBrokers(); + hermesFrontend.stop(); + } + + @Test + public void shouldNotBeReadyUntilThereAreNoOfflinePartitions() throws Exception { + // given: stop one of the brokers + List brokers = kafka.getAllBrokers(); + List brokersToStop = selectOne(brokers); + kafka.stop(brokersToStop); + + // and: send a message to remove stopped broker from in-sync replicas + publishSampleMessage(topic); + + // and: stop all in-sync replicas + List isr = selectAllOtherThan(brokers, brokersToStop); + kafka.stop(isr); + + // and: start the broker that is not in in-sync replicas + kafka.start(brokersToStop); + + // and: check if there is at least one offline partition + assertThat(kafka.countOfflinePartitions() > 0).isTrue(); + + // when + HermesTestApp hermesFrontend = + new HermesFrontendTestApp(hermesZookeeper, kafka, schemaRegistry) + .metadataMaxAgeInSeconds(1) + .readinessCheckIntervalInSeconds(1) + .kafkaCheckEnabled() + .start(); + + // then + getStatusReady(hermesFrontend).expectStatus().isEqualTo(SERVICE_UNAVAILABLE); + getStatusHealth(hermesFrontend).expectStatus().is2xxSuccessful(); + + // when + kafka.start(isr); + + // then + await() + .atMost(5, SECONDS) + .untilAsserted(() -> getStatusReady(hermesFrontend).expectStatus().is2xxSuccessful()); + getStatusHealth(hermesFrontend).expectStatus().is2xxSuccessful(); + + // cleanup + hermesFrontend.stop(); + } + + private static List selectOne(List brokerIds) { + return brokerIds.subList(0, 1); + } + + private static List selectAllOtherThan( + List brokerIds, List toExclude) { + return brokerIds.stream().filter(b -> !toExclude.contains(b)).collect(Collectors.toList()); + } + + private void publishSampleMessage(Topic topic) { + HermesTestApp hermesFrontend = + new HermesFrontendTestApp(hermesZookeeper, kafka, schemaRegistry).start(); + FrontendTestClient client = new FrontendTestClient(hermesFrontend.getPort()); + client.publishUntilSuccess(topic.getQualifiedName(), "message"); + hermesFrontend.stop(); + } + + private WebTestClient.ResponseSpec getStatusHealth(HermesTestApp hermesFrontend) { + FrontendTestClient client = new FrontendTestClient(hermesFrontend.getPort()); + return client.getStatusHealth(); + } + + private WebTestClient.ResponseSpec getStatusReady(HermesTestApp hermesFrontend) { + FrontendTestClient client = new FrontendTestClient(hermesFrontend.getPort()); + return client.getStatusReady(); + } } diff --git a/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/MessageBufferLoadingTest.java b/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/MessageBufferLoadingTest.java index 7d679f80dc..77d547af2d 100644 --- a/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/MessageBufferLoadingTest.java +++ b/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/MessageBufferLoadingTest.java @@ -1,7 +1,22 @@ package pl.allegro.tech.hermes.integrationtests; +import static jakarta.ws.rs.core.Response.Status.ACCEPTED; +import static java.nio.charset.Charset.defaultCharset; +import static java.time.Instant.now; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static pl.allegro.tech.hermes.api.ContentType.JSON; +import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.MESSAGES_LOCAL_STORAGE_DIRECTORY; +import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.MESSAGES_LOCAL_STORAGE_ENABLED; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.io.Files; +import java.io.File; +import java.time.Clock; +import java.util.Collections; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Order; @@ -23,133 +38,129 @@ import pl.allegro.tech.hermes.integrationtests.subscriber.TestSubscribersExtension; import pl.allegro.tech.hermes.test.helper.client.integration.FrontendTestClient; -import java.io.File; -import java.time.Clock; -import java.util.Collections; - -import static jakarta.ws.rs.core.Response.Status.ACCEPTED; -import static java.nio.charset.Charset.defaultCharset; -import static java.time.Instant.now; -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; -import static pl.allegro.tech.hermes.api.ContentType.JSON; -import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.MESSAGES_LOCAL_STORAGE_DIRECTORY; -import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.MESSAGES_LOCAL_STORAGE_ENABLED; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class MessageBufferLoadingTest { - private static final int ENTRIES = 100; - private static final int AVERAGE_MESSAGE_SIZE = 600; + private static final int ENTRIES = 100; + private static final int AVERAGE_MESSAGE_SIZE = 600; - @Order(0) - @RegisterExtension - public static InfrastructureExtension infra = new InfrastructureExtension(); + @Order(0) + @RegisterExtension + public static InfrastructureExtension infra = new InfrastructureExtension(); - @Order(1) - @RegisterExtension - public static HermesManagementExtension management = new HermesManagementExtension(infra); + @Order(1) + @RegisterExtension + public static HermesManagementExtension management = new HermesManagementExtension(infra); - private static final HermesConsumersTestApp consumers = new HermesConsumersTestApp(infra.hermesZookeeper(), infra.kafka(), infra.schemaRegistry()); + private static final HermesConsumersTestApp consumers = + new HermesConsumersTestApp(infra.hermesZookeeper(), infra.kafka(), infra.schemaRegistry()); - @BeforeAll - public static void setup() { - consumers.start(); - } + @BeforeAll + public static void setup() { + consumers.start(); + } - @AfterAll - public static void clean() { - consumers.stop(); - } - - @RegisterExtension - public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); - - @Test - public void shouldBackupMessage() { - // setup - String backupStorageDir = Files.createTempDir().getAbsolutePath(); - HermesFrontendTestApp frontend = new HermesFrontendTestApp(infra.hermesZookeeper(), infra.kafka(), infra.schemaRegistry()); - frontend.withProperty(MESSAGES_LOCAL_STORAGE_DIRECTORY, backupStorageDir); - frontend.withProperty(MESSAGES_LOCAL_STORAGE_ENABLED, true); - frontend.start(); + @AfterAll + public static void clean() { + consumers.stop(); + } - FrontendTestClient publisher = new FrontendTestClient(frontend.getPort()); + @RegisterExtension + public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); - Topic topic = management.initHelper().createTopic(topicWithRandomName().build()); + @Test + public void shouldBackupMessage() { + // setup + String backupStorageDir = Files.createTempDir().getAbsolutePath(); + HermesFrontendTestApp frontend = + new HermesFrontendTestApp(infra.hermesZookeeper(), infra.kafka(), infra.schemaRegistry()); + frontend.withProperty(MESSAGES_LOCAL_STORAGE_DIRECTORY, backupStorageDir); + frontend.withProperty(MESSAGES_LOCAL_STORAGE_ENABLED, true); + frontend.start(); - try { - //given - final ChronicleMapMessageRepository backupRepository = createBackupRepository(backupStorageDir); + FrontendTestClient publisher = new FrontendTestClient(frontend.getPort()); - publisher.publishUntilSuccess(topic.getQualifiedName(), "message"); + Topic topic = management.initHelper().createTopic(topicWithRandomName().build()); - // when - infra.kafka().cutOffConnectionsBetweenBrokersAndClients(); + try { + // given + final ChronicleMapMessageRepository backupRepository = + createBackupRepository(backupStorageDir); - publisher.publishUntilStatus(topic.getQualifiedName(), "message", ACCEPTED.getStatusCode()); - - // then - await().atMost(10, SECONDS).untilAsserted(() -> assertThat(backupRepository.findAll()).hasSize(1)); - - } finally { - // after - infra.kafka().restoreConnectionsBetweenBrokersAndClients(); - frontend.stop(); - } - } + publisher.publishUntilSuccess(topic.getQualifiedName(), "message"); - @Test - public void shouldLoadMessageFromBackupStorage() { - // given - String tempDirPath = Files.createTempDir().getAbsolutePath(); - Topic topic = management.initHelper().createTopic(topicWithRandomName().withContentType(JSON).build()); - backupFileWithOneMessage(tempDirPath, topic); + // when + infra.kafka().cutOffConnectionsBetweenBrokersAndClients(); - TestSubscriber subscriber = subscribers.createSubscriber(); + publisher.publishUntilStatus(topic.getQualifiedName(), "message", ACCEPTED.getStatusCode()); - management.initHelper().createSubscription( - subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()).build() - ); - - HermesFrontendTestApp frontend = new HermesFrontendTestApp(infra.hermesZookeeper(), infra.kafka(), infra.schemaRegistry()); - frontend.withProperty(MESSAGES_LOCAL_STORAGE_DIRECTORY, tempDirPath); - - // when - frontend.start(); - - // then - subscriber.waitUntilReceived("message"); - - // after - frontend.stop(); - } - - private void backupFileWithOneMessage(String tempDirPath, Topic topic) { - File backup = new File(tempDirPath, "hermes-buffer-v3.dat"); - - MessageRepository messageRepository = new ChronicleMapMessageRepository(backup, ENTRIES, AVERAGE_MESSAGE_SIZE); - JsonMessageContentWrapper contentWrapper = new JsonMessageContentWrapper("message", "metadata", new ObjectMapper()); - - CompositeMessageContentWrapper wrapper = new CompositeMessageContentWrapper(contentWrapper, null, null, null, null, null); - - String messageId = MessageIdGenerator.generate(); - long timestamp = now().toEpochMilli(); - byte[] content = wrapper.wrapJson("message".getBytes(defaultCharset()), - messageId, timestamp, Collections.emptyMap()); - - messageRepository.save(new JsonMessage(messageId, content, timestamp, null, Collections.emptyMap()), topic); - messageRepository.close(); - - } + // then + await() + .atMost(10, SECONDS) + .untilAsserted(() -> assertThat(backupRepository.findAll()).hasSize(1)); - private ChronicleMapMessageRepository createBackupRepository(String storageDirPath) { - return new ChronicleMapMessageRepository( - new BackupFilesManager(storageDirPath, Clock.systemUTC()).getCurrentBackupFile(), - ENTRIES, - AVERAGE_MESSAGE_SIZE - ); + } finally { + // after + infra.kafka().restoreConnectionsBetweenBrokersAndClients(); + frontend.stop(); } + } + + @Test + public void shouldLoadMessageFromBackupStorage() { + // given + String tempDirPath = Files.createTempDir().getAbsolutePath(); + Topic topic = + management.initHelper().createTopic(topicWithRandomName().withContentType(JSON).build()); + backupFileWithOneMessage(tempDirPath, topic); + + TestSubscriber subscriber = subscribers.createSubscriber(); + + management + .initHelper() + .createSubscription( + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()) + .build()); + + HermesFrontendTestApp frontend = + new HermesFrontendTestApp(infra.hermesZookeeper(), infra.kafka(), infra.schemaRegistry()); + frontend.withProperty(MESSAGES_LOCAL_STORAGE_DIRECTORY, tempDirPath); + + // when + frontend.start(); + + // then + subscriber.waitUntilReceived("message"); + + // after + frontend.stop(); + } + + private void backupFileWithOneMessage(String tempDirPath, Topic topic) { + File backup = new File(tempDirPath, "hermes-buffer-v3.dat"); + + MessageRepository messageRepository = + new ChronicleMapMessageRepository(backup, ENTRIES, AVERAGE_MESSAGE_SIZE); + JsonMessageContentWrapper contentWrapper = + new JsonMessageContentWrapper("message", "metadata", new ObjectMapper()); + + CompositeMessageContentWrapper wrapper = + new CompositeMessageContentWrapper(contentWrapper, null, null, null, null, null); + + String messageId = MessageIdGenerator.generate(); + long timestamp = now().toEpochMilli(); + byte[] content = + wrapper.wrapJson( + "message".getBytes(defaultCharset()), messageId, timestamp, Collections.emptyMap()); + + messageRepository.save( + new JsonMessage(messageId, content, timestamp, null, Collections.emptyMap()), topic); + messageRepository.close(); + } + + private ChronicleMapMessageRepository createBackupRepository(String storageDirPath) { + return new ChronicleMapMessageRepository( + new BackupFilesManager(storageDirPath, Clock.systemUTC()).getCurrentBackupFile(), + ENTRIES, + AVERAGE_MESSAGE_SIZE); + } } diff --git a/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/MultiDatacenterPublishingAndSubscribingTest.java b/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/MultiDatacenterPublishingAndSubscribingTest.java index 8c8093ad5d..b792c230f9 100644 --- a/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/MultiDatacenterPublishingAndSubscribingTest.java +++ b/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/MultiDatacenterPublishingAndSubscribingTest.java @@ -1,5 +1,11 @@ package pl.allegro.tech.hermes.integrationtests; +import static pl.allegro.tech.hermes.infrastructure.dc.DefaultDatacenterNameProvider.DEFAULT_DC_NAME; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + +import java.util.Map; +import java.util.stream.Stream; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -19,106 +25,91 @@ import pl.allegro.tech.hermes.test.helper.environment.HermesTestApp; import pl.allegro.tech.hermes.test.helper.message.TestMessage; -import java.util.Map; -import java.util.stream.Stream; - -import static pl.allegro.tech.hermes.infrastructure.dc.DefaultDatacenterNameProvider.DEFAULT_DC_NAME; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class MultiDatacenterPublishingAndSubscribingTest { - @RegisterExtension - public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); - - private static final ConfluentSchemaRegistryContainer schemaRegistry = new ConfluentSchemaRegistryContainer(); - private static final HermesDatacenter dc1 = new HermesDatacenter(); - private static final HermesDatacenter dc2 = new HermesDatacenter(); - - private static HermesManagementTestApp management; - private static HermesInitHelper initHelper; - - @BeforeAll - public static void setup() { - Stream.of(dc1, dc2) - .parallel() - .forEach(HermesDatacenter::startKafkaAndZookeeper); - schemaRegistry.start(); - management = new HermesManagementTestApp( - Map.of(DEFAULT_DC_NAME, dc1.hermesZookeeper, "dc2", dc2.hermesZookeeper), - Map.of(DEFAULT_DC_NAME, dc1.kafka, "dc2", dc2.kafka), - schemaRegistry - ); - management.start(); - initHelper = new HermesInitHelper(management.getPort()); - dc1.startConsumersAndFrontend(); - dc2.startConsumersAndFrontend(); + @RegisterExtension + public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); + + private static final ConfluentSchemaRegistryContainer schemaRegistry = + new ConfluentSchemaRegistryContainer(); + private static final HermesDatacenter dc1 = new HermesDatacenter(); + private static final HermesDatacenter dc2 = new HermesDatacenter(); + + private static HermesManagementTestApp management; + private static HermesInitHelper initHelper; + + @BeforeAll + public static void setup() { + Stream.of(dc1, dc2).parallel().forEach(HermesDatacenter::startKafkaAndZookeeper); + schemaRegistry.start(); + management = + new HermesManagementTestApp( + Map.of(DEFAULT_DC_NAME, dc1.hermesZookeeper, "dc2", dc2.hermesZookeeper), + Map.of(DEFAULT_DC_NAME, dc1.kafka, "dc2", dc2.kafka), + schemaRegistry); + management.start(); + initHelper = new HermesInitHelper(management.getPort()); + dc1.startConsumersAndFrontend(); + dc2.startConsumersAndFrontend(); + } + + @AfterAll + public static void clean() { + management.stop(); + Stream.of(dc1, dc2).parallel().forEach(HermesDatacenter::stop); + schemaRegistry.stop(); + } + + @Test + public void shouldPublishAndConsumeThroughMultipleDatacenters() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + TestMessage messageDc1 = TestMessage.of("key1", "value1"); + TestMessage messageDc2 = TestMessage.of("key2", "value2"); + Topic topic = initHelper.createTopic(topicWithRandomName().build()); + initHelper.createSubscription( + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()).build()); + + // when + dc1.api().publishUntilSuccess(topic.getQualifiedName(), messageDc1.body()); + dc2.api().publishUntilSuccess(topic.getQualifiedName(), messageDc2.body()); + + // then + subscriber.waitUntilReceived(messageDc1.body()); + subscriber.waitUntilReceived(messageDc2.body()); + } + + private static class HermesDatacenter { + + private final ZookeeperContainer hermesZookeeper = new ZookeeperContainer("HermesZookeeper"); + private final KafkaContainerCluster kafka = new KafkaContainerCluster(1); + private final HermesConsumersTestApp consumers = + new HermesConsumersTestApp(hermesZookeeper, kafka, schemaRegistry); + private final HermesFrontendTestApp frontend = + new HermesFrontendTestApp(hermesZookeeper, kafka, schemaRegistry); + private FrontendTestClient frontendClient; + + public HermesDatacenter() { + schemaRegistry.withKafkaCluster(kafka); + } + + void startKafkaAndZookeeper() { + Stream.of(hermesZookeeper, kafka).parallel().forEach(Startable::start); } - @AfterAll - public static void clean() { - management.stop(); - Stream.of(dc1, dc2) - .parallel() - .forEach(HermesDatacenter::stop); - schemaRegistry.stop(); + void startConsumersAndFrontend() { + frontend.start(); + consumers.start(); + frontendClient = new FrontendTestClient(frontend.getPort()); } - @Test - public void shouldPublishAndConsumeThroughMultipleDatacenters() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - TestMessage messageDc1 = TestMessage.of("key1", "value1"); - TestMessage messageDc2 = TestMessage.of("key2", "value2"); - Topic topic = initHelper.createTopic(topicWithRandomName().build()); - initHelper.createSubscription( - subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()).build() - ); - - // when - dc1.api().publishUntilSuccess(topic.getQualifiedName(), messageDc1.body()); - dc2.api().publishUntilSuccess(topic.getQualifiedName(), messageDc2.body()); - - // then - subscriber.waitUntilReceived(messageDc1.body()); - subscriber.waitUntilReceived(messageDc2.body()); + void stop() { + Stream.of(consumers, frontend).parallel().forEach(HermesTestApp::stop); + Stream.of(hermesZookeeper, kafka).parallel().forEach(Startable::stop); } - private static class HermesDatacenter { - - private final ZookeeperContainer hermesZookeeper = new ZookeeperContainer("HermesZookeeper"); - private final KafkaContainerCluster kafka = new KafkaContainerCluster(1); - private final HermesConsumersTestApp consumers = new HermesConsumersTestApp(hermesZookeeper, kafka, schemaRegistry); - private final HermesFrontendTestApp frontend = new HermesFrontendTestApp(hermesZookeeper, kafka, schemaRegistry); - private FrontendTestClient frontendClient; - - public HermesDatacenter() { - schemaRegistry.withKafkaCluster(kafka); - } - - void startKafkaAndZookeeper() { - Stream.of(hermesZookeeper, kafka) - .parallel() - .forEach(Startable::start); - } - - void startConsumersAndFrontend() { - frontend.start(); - consumers.start(); - frontendClient = new FrontendTestClient(frontend.getPort()); - } - - void stop() { - Stream.of(consumers, frontend) - .parallel() - .forEach(HermesTestApp::stop); - Stream.of(hermesZookeeper, kafka) - .parallel() - .forEach(Startable::stop); - } - - FrontendTestClient api() { - return frontendClient; - } + FrontendTestClient api() { + return frontendClient; } + } } diff --git a/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingAvroOnTopicWithoutSchemaTest.java b/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingAvroOnTopicWithoutSchemaTest.java index 3084513b02..ab79995124 100644 --- a/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingAvroOnTopicWithoutSchemaTest.java +++ b/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/PublishingAvroOnTopicWithoutSchemaTest.java @@ -1,5 +1,11 @@ package pl.allegro.tech.hermes.integrationtests; +import static pl.allegro.tech.hermes.api.ContentType.AVRO; +import static pl.allegro.tech.hermes.api.ErrorCode.SCHEMA_COULD_NOT_BE_LOADED; +import static pl.allegro.tech.hermes.api.TopicWithSchema.topicWithSchema; +import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.SCHEMA_REPOSITORY_SERVER_URL; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + import com.github.tomakehurst.wiremock.WireMockServer; import com.github.tomakehurst.wiremock.core.Options; import org.assertj.core.api.Assertions; @@ -19,62 +25,61 @@ import pl.allegro.tech.hermes.test.helper.avro.AvroUserSchemaLoader; import pl.allegro.tech.hermes.test.helper.client.integration.FrontendTestClient; -import static pl.allegro.tech.hermes.api.ContentType.AVRO; -import static pl.allegro.tech.hermes.api.ErrorCode.SCHEMA_COULD_NOT_BE_LOADED; -import static pl.allegro.tech.hermes.api.TopicWithSchema.topicWithSchema; -import static pl.allegro.tech.hermes.frontend.FrontendConfigurationProperties.SCHEMA_REPOSITORY_SERVER_URL; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class PublishingAvroOnTopicWithoutSchemaTest { - @Order(0) - @RegisterExtension - public static InfrastructureExtension infra = new InfrastructureExtension(); - - @Order(1) - @RegisterExtension - public static HermesManagementExtension management = new HermesManagementExtension(infra); - - private static FrontendTestClient publisher; - private static HermesFrontendTestApp frontend; - - private static final WireMockServer emptySchemaRegistryMock = new WireMockServer(Options.DYNAMIC_PORT); - - @BeforeAll - public static void setup() { - emptySchemaRegistryMock.start(); - - frontend = new HermesFrontendTestApp(infra.hermesZookeeper(), infra.kafka(), infra.schemaRegistry()); - frontend.withProperty(SCHEMA_REPOSITORY_SERVER_URL, "http://localhost:" + emptySchemaRegistryMock.port()); - frontend.start(); - - publisher = new FrontendTestClient(frontend.getPort()); - - } - - @AfterAll - public static void clean() { - emptySchemaRegistryMock.stop(); - frontend.stop(); - } - - @Test - public void shouldReturnServerInternalErrorResponseOnMissingSchema() { - // given - TopicWithSchema topicWithSchema = topicWithSchema(topicWithRandomName() - .withContentType(AVRO) - .build(), AvroUserSchemaLoader.load().toString()); - - Topic topic = management.initHelper().createTopicWithSchema(topicWithSchema); - - // when - String message = new AvroUser("Bob", 50, "blue").asJson(); - // ensure topic is created - publisher.publishUntilStatus(topic.getQualifiedName(), message, 500); - WebTestClient.ResponseSpec response = publisher.publish(topic.getQualifiedName(), message); - - // then - response.expectStatus().isEqualTo(500); - Assertions.assertThat(response.expectBody(ErrorDescription.class).returnResult().getResponseBody().getCode()).isEqualTo(SCHEMA_COULD_NOT_BE_LOADED); - } + @Order(0) + @RegisterExtension + public static InfrastructureExtension infra = new InfrastructureExtension(); + + @Order(1) + @RegisterExtension + public static HermesManagementExtension management = new HermesManagementExtension(infra); + + private static FrontendTestClient publisher; + private static HermesFrontendTestApp frontend; + + private static final WireMockServer emptySchemaRegistryMock = + new WireMockServer(Options.DYNAMIC_PORT); + + @BeforeAll + public static void setup() { + emptySchemaRegistryMock.start(); + + frontend = + new HermesFrontendTestApp(infra.hermesZookeeper(), infra.kafka(), infra.schemaRegistry()); + frontend.withProperty( + SCHEMA_REPOSITORY_SERVER_URL, "http://localhost:" + emptySchemaRegistryMock.port()); + frontend.start(); + + publisher = new FrontendTestClient(frontend.getPort()); + } + + @AfterAll + public static void clean() { + emptySchemaRegistryMock.stop(); + frontend.stop(); + } + + @Test + public void shouldReturnServerInternalErrorResponseOnMissingSchema() { + // given + TopicWithSchema topicWithSchema = + topicWithSchema( + topicWithRandomName().withContentType(AVRO).build(), + AvroUserSchemaLoader.load().toString()); + + Topic topic = management.initHelper().createTopicWithSchema(topicWithSchema); + + // when + String message = new AvroUser("Bob", 50, "blue").asJson(); + // ensure topic is created + publisher.publishUntilStatus(topic.getQualifiedName(), message, 500); + WebTestClient.ResponseSpec response = publisher.publish(topic.getQualifiedName(), message); + + // then + response.expectStatus().isEqualTo(500); + Assertions.assertThat( + response.expectBody(ErrorDescription.class).returnResult().getResponseBody().getCode()) + .isEqualTo(SCHEMA_COULD_NOT_BE_LOADED); + } } diff --git a/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/RemoteDatacenterProduceFallbackTest.java b/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/RemoteDatacenterProduceFallbackTest.java index 360b6ecc13..94fc0d80e1 100644 --- a/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/RemoteDatacenterProduceFallbackTest.java +++ b/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/RemoteDatacenterProduceFallbackTest.java @@ -1,5 +1,12 @@ package pl.allegro.tech.hermes.integrationtests; +import static pl.allegro.tech.hermes.infrastructure.dc.DefaultDatacenterNameProvider.DEFAULT_DC_NAME; +import static pl.allegro.tech.hermes.integrationtests.assertions.HermesAssertions.assertThatMetrics; +import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; + +import java.util.Map; +import java.util.stream.Stream; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -23,277 +30,261 @@ import pl.allegro.tech.hermes.test.helper.containers.ZookeeperContainer; import pl.allegro.tech.hermes.test.helper.message.TestMessage; -import java.util.Map; -import java.util.stream.Stream; - -import static pl.allegro.tech.hermes.infrastructure.dc.DefaultDatacenterNameProvider.DEFAULT_DC_NAME; -import static pl.allegro.tech.hermes.integrationtests.assertions.HermesAssertions.assertThatMetrics; -import static pl.allegro.tech.hermes.test.helper.builder.SubscriptionBuilder.subscription; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topicWithRandomName; - public class RemoteDatacenterProduceFallbackTest { - @RegisterExtension - public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); - - private static final ConfluentSchemaRegistryContainer schemaRegistry = new ConfluentSchemaRegistryContainer(); - private static final HermesDatacenter dc1 = new HermesDatacenter(); - private static final HermesDatacenter dc2 = new HermesDatacenter(); - - private static HermesManagementTestApp management; - private static HermesInitHelper initHelper; - private static HermesFrontendTestApp frontendDC1; - private static HermesConsumersTestApp consumerDC1; - private static HermesConsumersTestApp consumerDC2; - - private static HermesTestClient DC1; - private static final String REMOTE_DC_NAME = "dc2"; - - @BeforeAll - public static void setup() { - Stream.of(dc1, dc2) - .parallel() - .forEach(HermesDatacenter::startKafkaAndZookeeper); - schemaRegistry.start(); - management = new HermesManagementTestApp( - Map.of(DEFAULT_DC_NAME, dc1.hermesZookeeper, REMOTE_DC_NAME, dc2.hermesZookeeper), - Map.of(DEFAULT_DC_NAME, dc1.kafka, REMOTE_DC_NAME, dc2.kafka), - schemaRegistry - ); - management.start(); - frontendDC1 = new HermesFrontendTestApp(dc1.hermesZookeeper, - Map.of("dc", dc1.kafka, REMOTE_DC_NAME, dc2.kafka), - schemaRegistry - ); - frontendDC1.start(); - - consumerDC1 = new HermesConsumersTestApp(dc1.hermesZookeeper, dc1.kafka, schemaRegistry); - consumerDC1.start(); - - consumerDC2 = new HermesConsumersTestApp(dc2.hermesZookeeper, dc2.kafka, schemaRegistry); - consumerDC2.start(); - - DC1 = new HermesTestClient(management.getPort(), frontendDC1.getPort(), consumerDC1.getPort()); - initHelper = new HermesInitHelper(management.getPort()); + @RegisterExtension + public static final TestSubscribersExtension subscribers = new TestSubscribersExtension(); + + private static final ConfluentSchemaRegistryContainer schemaRegistry = + new ConfluentSchemaRegistryContainer(); + private static final HermesDatacenter dc1 = new HermesDatacenter(); + private static final HermesDatacenter dc2 = new HermesDatacenter(); + + private static HermesManagementTestApp management; + private static HermesInitHelper initHelper; + private static HermesFrontendTestApp frontendDC1; + private static HermesConsumersTestApp consumerDC1; + private static HermesConsumersTestApp consumerDC2; + + private static HermesTestClient DC1; + private static final String REMOTE_DC_NAME = "dc2"; + + @BeforeAll + public static void setup() { + Stream.of(dc1, dc2).parallel().forEach(HermesDatacenter::startKafkaAndZookeeper); + schemaRegistry.start(); + management = + new HermesManagementTestApp( + Map.of(DEFAULT_DC_NAME, dc1.hermesZookeeper, REMOTE_DC_NAME, dc2.hermesZookeeper), + Map.of(DEFAULT_DC_NAME, dc1.kafka, REMOTE_DC_NAME, dc2.kafka), + schemaRegistry); + management.start(); + frontendDC1 = + new HermesFrontendTestApp( + dc1.hermesZookeeper, + Map.of("dc", dc1.kafka, REMOTE_DC_NAME, dc2.kafka), + schemaRegistry); + frontendDC1.start(); + + consumerDC1 = new HermesConsumersTestApp(dc1.hermesZookeeper, dc1.kafka, schemaRegistry); + consumerDC1.start(); + + consumerDC2 = new HermesConsumersTestApp(dc2.hermesZookeeper, dc2.kafka, schemaRegistry); + consumerDC2.start(); + + DC1 = new HermesTestClient(management.getPort(), frontendDC1.getPort(), consumerDC1.getPort()); + initHelper = new HermesInitHelper(management.getPort()); + } + + @AfterAll + public static void clean() { + management.stop(); + consumerDC2.stop(); + frontendDC1.stop(); + consumerDC1.stop(); + schemaRegistry.stop(); + Stream.of(dc1, dc2).parallel().forEach(HermesDatacenter::stop); + } + + @AfterEach + public void afterEach() { + Stream.of(dc1, dc2).forEach(dc -> dc.kafka.restoreConnectionsBetweenBrokersAndClients()); + DC1.setReadiness(DEFAULT_DC_NAME, true); + DC1.setReadiness(REMOTE_DC_NAME, true); + } + + @Test + public void shouldPublishAndConsumeViaRemoteDCWhenLocalKafkaIsUnavailable() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = + initHelper.createTopic( + topicWithRandomName().withFallbackToRemoteDatacenterEnabled().build()); + initHelper.createSubscription( + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()).build()); + + double remoteDCInitialSendTotal = assertRemoteDCSendTotalMetric().withInitialValue(); + + // when dc1 is not available + dc1.kafka.cutOffConnectionsBetweenBrokersAndClients(); + + // and message is published to dc1 + TestMessage message = TestMessage.of("key1", "value1"); + DC1.publishUntilSuccess(topic.getQualifiedName(), message.body()); + + // then message is received in dc2 + subscriber.waitUntilReceived(message.body()); + + // and metrics that message was published to remote dc is incremented + DC1.getFrontendMetrics() + .expectStatus() + .isOk() + .expectBody(String.class) + .value( + (body) -> { + assertThatMetrics(body) + .contains("hermes_frontend_topic_published_total") + .withLabels( + "group", topic.getName().getGroupName(), + "topic", topic.getName().getName(), + "storageDc", REMOTE_DC_NAME) + .withValue(1.0); + assertRemoteDCSendTotalMetric().withValueGreaterThan(remoteDCInitialSendTotal); + }); + } + + @Test + public void shouldReturn500whenBothDCsAreUnavailable() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = + initHelper.createTopic( + topicWithRandomName().withFallbackToRemoteDatacenterEnabled().build()); + initHelper.createSubscription( + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()).build()); + + // when both dcs are not available + dc1.kafka.cutOffConnectionsBetweenBrokersAndClients(); + dc2.kafka.cutOffConnectionsBetweenBrokersAndClients(); + + // and message is published + TestMessage message = TestMessage.of("key1", "value1"); + DC1.publishUntilStatus(topic.getQualifiedName(), message.body(), 503); + + // then no messages are received + subscriber.noMessagesReceived(); + } + + @Test + public void shouldNotFallBackToNotReadyDatacenter() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + Topic topic = + initHelper.createTopic( + topicWithRandomName().withFallbackToRemoteDatacenterEnabled().build()); + initHelper.createSubscription( + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()).build()); + + // when local datacenter is not available and remote is not ready + dc1.kafka.cutOffConnectionsBetweenBrokersAndClients(); + DC1.setReadiness(REMOTE_DC_NAME, false); + + // and message is published + TestMessage message = TestMessage.of("key1", "value1"); + DC1.publishUntilStatus(topic.getQualifiedName(), message.body(), 503); + + // then no messages are received + subscriber.noMessagesReceived(); + } + + @Test + public void shouldPublishAndConsumeViaRemoteDCWhenChaosExperimentIsEnabledForLocalKafka() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + + Topic topic = + initHelper.createTopic( + topicWithRandomName() + .withFallbackToRemoteDatacenterEnabled() + .withPublishingChaosPolicy(completeWithErrorForDatacenter(DEFAULT_DC_NAME)) + .build()); + initHelper.createSubscription( + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()).build()); + + // and message is published to dc1 + TestMessage message = TestMessage.of("key1", "value1"); + DC1.publishUntilSuccess(topic.getQualifiedName(), message.body()); + + // then message is received in dc2 + subscriber.waitUntilReceived(message.body()); + + // and metrics that message was published to remote dc is incremented + DC1.getFrontendMetrics() + .expectStatus() + .isOk() + .expectBody(String.class) + .value( + (body) -> + assertThatMetrics(body) + .contains("hermes_frontend_topic_published_total") + .withLabels( + "group", topic.getName().getGroupName(), + "topic", topic.getName().getName(), + "storageDc", REMOTE_DC_NAME) + .withValue(1.0)); + } + + @Test + public void shouldReturnErrorWhenChaosExperimentIsEnabledForAllDatacenters() { + // given + TestSubscriber subscriber = subscribers.createSubscriber(); + + Topic topic = + initHelper.createTopic( + topicWithRandomName() + .withFallbackToRemoteDatacenterEnabled() + .withPublishingChaosPolicy(completeWithErrorForAllDatacenters()) + .build()); + initHelper.createSubscription( + subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()).build()); + TestMessage message = TestMessage.of("key1", "value1"); + + // when + DC1.publishUntilStatus(topic.getQualifiedName(), message.body(), 500); + + // then + subscriber.noMessagesReceived(); + } + + private static PublishingChaosPolicy completeWithErrorForAllDatacenters() { + int delayFrom = 100; + int delayTo = 200; + int probability = 100; + boolean completeWithError = true; + return new PublishingChaosPolicy( + ChaosMode.GLOBAL, + new ChaosPolicy(probability, delayFrom, delayTo, completeWithError), + null); + } + + private static PublishingChaosPolicy completeWithErrorForDatacenter(String datacenter) { + int delayFrom = 100; + int delayTo = 200; + int probability = 100; + boolean completeWithError = true; + return new PublishingChaosPolicy( + ChaosMode.DATACENTER, + null, + Map.of(datacenter, new ChaosPolicy(probability, delayFrom, delayTo, completeWithError))); + } + + private static class HermesDatacenter { + + private final ZookeeperContainer hermesZookeeper = new ZookeeperContainer("HermesZookeeper"); + private final KafkaContainerCluster kafka = new KafkaContainerCluster(1); + + public HermesDatacenter() { + schemaRegistry.withKafkaCluster(kafka); } - @AfterAll - public static void clean() { - management.stop(); - consumerDC2.stop(); - frontendDC1.stop(); - consumerDC1.stop(); - schemaRegistry.stop(); - Stream.of(dc1, dc2) - .parallel() - .forEach(HermesDatacenter::stop); + void startKafkaAndZookeeper() { + Stream.of(hermesZookeeper, kafka).parallel().forEach(Startable::start); } - @AfterEach - public void afterEach() { - Stream.of(dc1, dc2).forEach(dc -> dc.kafka.restoreConnectionsBetweenBrokersAndClients()); - DC1.setReadiness(DEFAULT_DC_NAME, true); - DC1.setReadiness(REMOTE_DC_NAME, true); + void stop() { + Stream.of(hermesZookeeper, kafka).parallel().forEach(Startable::stop); } + } - @Test - public void shouldPublishAndConsumeViaRemoteDCWhenLocalKafkaIsUnavailable() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = initHelper.createTopic(topicWithRandomName().withFallbackToRemoteDatacenterEnabled().build()); - initHelper.createSubscription( - subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()).build() - ); - - double remoteDCInitialSendTotal = assertRemoteDCSendTotalMetric().withInitialValue(); - - // when dc1 is not available - dc1.kafka.cutOffConnectionsBetweenBrokersAndClients(); - - // and message is published to dc1 - TestMessage message = TestMessage.of("key1", "value1"); - DC1.publishUntilSuccess(topic.getQualifiedName(), message.body()); - - // then message is received in dc2 - subscriber.waitUntilReceived(message.body()); - - // and metrics that message was published to remote dc is incremented - DC1.getFrontendMetrics() + PrometheusMetricsAssertion.PrometheusMetricAssertion assertRemoteDCSendTotalMetric() { + return assertThatMetrics( + DC1.getFrontendMetrics() .expectStatus() .isOk() .expectBody(String.class) - .value((body) -> { - assertThatMetrics(body) - .contains("hermes_frontend_topic_published_total") - .withLabels( - "group", topic.getName().getGroupName(), - "topic", topic.getName().getName(), - "storageDc", REMOTE_DC_NAME - ) - .withValue(1.0); - assertRemoteDCSendTotalMetric().withValueGreaterThan(remoteDCInitialSendTotal); - } - ); - } - - @Test - public void shouldReturn500whenBothDCsAreUnavailable() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = initHelper.createTopic(topicWithRandomName().withFallbackToRemoteDatacenterEnabled().build()); - initHelper.createSubscription( - subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()).build() - ); - - // when both dcs are not available - dc1.kafka.cutOffConnectionsBetweenBrokersAndClients(); - dc2.kafka.cutOffConnectionsBetweenBrokersAndClients(); - - // and message is published - TestMessage message = TestMessage.of("key1", "value1"); - DC1.publishUntilStatus(topic.getQualifiedName(), message.body(), 503); - - // then no messages are received - subscriber.noMessagesReceived(); - } - - @Test - public void shouldNotFallBackToNotReadyDatacenter() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - Topic topic = initHelper.createTopic(topicWithRandomName().withFallbackToRemoteDatacenterEnabled().build()); - initHelper.createSubscription( - subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()).build() - ); - - // when local datacenter is not available and remote is not ready - dc1.kafka.cutOffConnectionsBetweenBrokersAndClients(); - DC1.setReadiness(REMOTE_DC_NAME, false); - - // and message is published - TestMessage message = TestMessage.of("key1", "value1"); - DC1.publishUntilStatus(topic.getQualifiedName(), message.body(), 503); - - // then no messages are received - subscriber.noMessagesReceived(); - } - - @Test - public void shouldPublishAndConsumeViaRemoteDCWhenChaosExperimentIsEnabledForLocalKafka() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - - Topic topic = initHelper.createTopic( - topicWithRandomName() - .withFallbackToRemoteDatacenterEnabled() - .withPublishingChaosPolicy(completeWithErrorForDatacenter(DEFAULT_DC_NAME)) - .build() - ); - initHelper.createSubscription( - subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()).build() - ); - - // and message is published to dc1 - TestMessage message = TestMessage.of("key1", "value1"); - DC1.publishUntilSuccess(topic.getQualifiedName(), message.body()); - - // then message is received in dc2 - subscriber.waitUntilReceived(message.body()); - - // and metrics that message was published to remote dc is incremented - DC1.getFrontendMetrics() - .expectStatus() - .isOk() - .expectBody(String.class) - .value((body) -> assertThatMetrics(body) - .contains("hermes_frontend_topic_published_total") - .withLabels( - "group", topic.getName().getGroupName(), - "topic", topic.getName().getName(), - "storageDc", REMOTE_DC_NAME - ) - .withValue(1.0) - ); - } - - @Test - public void shouldReturnErrorWhenChaosExperimentIsEnabledForAllDatacenters() { - // given - TestSubscriber subscriber = subscribers.createSubscriber(); - - Topic topic = initHelper.createTopic( - topicWithRandomName() - .withFallbackToRemoteDatacenterEnabled() - .withPublishingChaosPolicy(completeWithErrorForAllDatacenters()) - .build() - ); - initHelper.createSubscription( - subscription(topic.getQualifiedName(), "subscription", subscriber.getEndpoint()).build() - ); - TestMessage message = TestMessage.of("key1", "value1"); - - // when - DC1.publishUntilStatus(topic.getQualifiedName(), message.body(), 500); - - // then - subscriber.noMessagesReceived(); - } - - private static PublishingChaosPolicy completeWithErrorForAllDatacenters() { - int delayFrom = 100; - int delayTo = 200; - int probability = 100; - boolean completeWithError = true; - return new PublishingChaosPolicy( - ChaosMode.GLOBAL, - new ChaosPolicy(probability, delayFrom, delayTo, completeWithError), - null - ); - } - - private static PublishingChaosPolicy completeWithErrorForDatacenter(String datacenter) { - int delayFrom = 100; - int delayTo = 200; - int probability = 100; - boolean completeWithError = true; - return new PublishingChaosPolicy( - ChaosMode.DATACENTER, - null, - Map.of(datacenter, new ChaosPolicy(probability, delayFrom, delayTo, completeWithError)) - ); - } - - private static class HermesDatacenter { - - private final ZookeeperContainer hermesZookeeper = new ZookeeperContainer("HermesZookeeper"); - private final KafkaContainerCluster kafka = new KafkaContainerCluster(1); - - public HermesDatacenter() { - schemaRegistry.withKafkaCluster(kafka); - } - - void startKafkaAndZookeeper() { - Stream.of(hermesZookeeper, kafka) - .parallel() - .forEach(Startable::start); - } - - - void stop() { - Stream.of(hermesZookeeper, kafka) - .parallel() - .forEach(Startable::stop); - } - } - - PrometheusMetricsAssertion.PrometheusMetricAssertion assertRemoteDCSendTotalMetric() { - return assertThatMetrics(DC1 - .getFrontendMetrics().expectStatus().isOk() - .expectBody(String.class).returnResult().getResponseBody()) - .contains("hermes_frontend_kafka_producer_ack_leader_record_send_total") - .withLabels( - "storageDc", REMOTE_DC_NAME, - "sender", "failFast" - ); - } + .returnResult() + .getResponseBody()) + .contains("hermes_frontend_kafka_producer_ack_leader_record_send_total") + .withLabels("storageDc", REMOTE_DC_NAME, "sender", "failFast"); + } } diff --git a/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/TopicCreationRollbackTest.java b/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/TopicCreationRollbackTest.java index ab8f26c935..94546e7b1d 100644 --- a/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/TopicCreationRollbackTest.java +++ b/integration-tests/src/slowIntegrationTest/java/pl/allegro/tech/hermes/integrationtests/TopicCreationRollbackTest.java @@ -1,5 +1,14 @@ package pl.allegro.tech.hermes.integrationtests; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.waitAtMost; +import static pl.allegro.tech.hermes.api.TopicWithSchema.topicWithSchema; +import static pl.allegro.tech.hermes.infrastructure.dc.DefaultDatacenterNameProvider.DEFAULT_DC_NAME; +import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topic; + +import java.time.Duration; +import java.util.Map; +import java.util.stream.Stream; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -12,72 +21,63 @@ import pl.allegro.tech.hermes.test.helper.containers.KafkaContainerCluster; import pl.allegro.tech.hermes.test.helper.containers.ZookeeperContainer; -import java.time.Duration; -import java.util.Map; -import java.util.stream.Stream; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.waitAtMost; -import static pl.allegro.tech.hermes.api.TopicWithSchema.topicWithSchema; -import static pl.allegro.tech.hermes.infrastructure.dc.DefaultDatacenterNameProvider.DEFAULT_DC_NAME; -import static pl.allegro.tech.hermes.test.helper.builder.TopicBuilder.topic; - public class TopicCreationRollbackTest { - private static HermesManagementTestApp management; - - private static final ZookeeperContainer hermesZookeeper = new ZookeeperContainer("HermesZookeeper"); - - private static final KafkaContainerCluster kafka1 = new KafkaContainerCluster(1); - - private static final KafkaContainerCluster kafka2 = new KafkaContainerCluster(1); - - private static final ConfluentSchemaRegistryContainer schemaRegistry = new ConfluentSchemaRegistryContainer() - .withKafkaCluster(kafka1); - private static HermesTestClient hermesApi; - - private static BrokerOperations brokerOperations1; - - @BeforeAll - public static void setup() { - Stream.of(hermesZookeeper, kafka1, kafka2) - .parallel() - .forEach(Startable::start); - schemaRegistry.start(); - management = new HermesManagementTestApp( - Map.of(DEFAULT_DC_NAME, hermesZookeeper), - Map.of(DEFAULT_DC_NAME, kafka1, "dc2", kafka2), - schemaRegistry - ); - management.start(); - hermesApi = new HermesTestClient(management.getPort(), management.getPort(), management.getPort()); - brokerOperations1 = new BrokerOperations(kafka1.getBootstrapServersForExternalClients(), "itTest"); - } - - @AfterAll - public static void clean() { - management.stop(); - Stream.of(hermesZookeeper, kafka1, kafka2) - .parallel() - .forEach(Startable::stop); - schemaRegistry.stop(); - } - - @Test - public void topicCreationRollbackShouldNotDeleteTopicOnBroker() { - // given - String groupName = "topicCreationRollbackShouldNotDeleteTopicOnBroker"; - String topicName = "topic"; - String qualifiedTopicName = groupName + "." + topicName; - hermesApi.createGroup(Group.from(groupName)); - - brokerOperations1.createTopic(qualifiedTopicName); - waitAtMost(Duration.ofMinutes(1)).untilAsserted(() -> assertThat(brokerOperations1.topicExists(qualifiedTopicName)).isTrue()); - - // when - hermesApi.createTopic((topicWithSchema(topic(groupName, topicName).build()))); - - // then - assertThat(brokerOperations1.topicExists(qualifiedTopicName)).isTrue(); - } + private static HermesManagementTestApp management; + + private static final ZookeeperContainer hermesZookeeper = + new ZookeeperContainer("HermesZookeeper"); + + private static final KafkaContainerCluster kafka1 = new KafkaContainerCluster(1); + + private static final KafkaContainerCluster kafka2 = new KafkaContainerCluster(1); + + private static final ConfluentSchemaRegistryContainer schemaRegistry = + new ConfluentSchemaRegistryContainer().withKafkaCluster(kafka1); + private static HermesTestClient hermesApi; + + private static BrokerOperations brokerOperations1; + + @BeforeAll + public static void setup() { + Stream.of(hermesZookeeper, kafka1, kafka2).parallel().forEach(Startable::start); + schemaRegistry.start(); + management = + new HermesManagementTestApp( + Map.of(DEFAULT_DC_NAME, hermesZookeeper), + Map.of(DEFAULT_DC_NAME, kafka1, "dc2", kafka2), + schemaRegistry); + management.start(); + hermesApi = + new HermesTestClient(management.getPort(), management.getPort(), management.getPort()); + brokerOperations1 = + new BrokerOperations(kafka1.getBootstrapServersForExternalClients(), "itTest"); + } + + @AfterAll + public static void clean() { + management.stop(); + Stream.of(hermesZookeeper, kafka1, kafka2).parallel().forEach(Startable::stop); + schemaRegistry.stop(); + } + + @Test + public void topicCreationRollbackShouldNotDeleteTopicOnBroker() { + // given + String groupName = "topicCreationRollbackShouldNotDeleteTopicOnBroker"; + String topicName = "topic"; + String qualifiedTopicName = groupName + "." + topicName; + hermesApi.createGroup(Group.from(groupName)); + + brokerOperations1.createTopic(qualifiedTopicName); + waitAtMost(Duration.ofMinutes(1)) + .untilAsserted( + () -> assertThat(brokerOperations1.topicExists(qualifiedTopicName)).isTrue()); + + // when + hermesApi.createTopic((topicWithSchema(topic(groupName, topicName).build()))); + + // then + assertThat(brokerOperations1.topicExists(qualifiedTopicName)).isTrue(); + } }