diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000..94ae0493f --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,29 @@ +name: Benchmark + +on: + schedule: + - cron: "15 0 * * *" + push: + branches: + - main + pull_request: + workflow_dispatch: + +env: + MAVEN_ARGS: "--no-transfer-progress --errors --fail-at-end --show-version" + +jobs: + build: + name: benchmark + runs-on: 'ubuntu-latest' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: JDK + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build + run: ./mvnw clean verify -Dtests.groups=BenchmarkTest diff --git a/jollyday-tests/pom.xml b/jollyday-tests/pom.xml index 25619d2b9..f79d67846 100644 --- a/jollyday-tests/pom.xml +++ b/jollyday-tests/pom.xml @@ -56,6 +56,16 @@ 0.1.2 test + + org.openjdk.jmh + jmh-core + test + + + org.openjdk.jmh + jmh-generator-annprocess + test + diff --git a/jollyday-tests/src/test/java/de/focus_shift/jollyday/tests/Benchmarks.java b/jollyday-tests/src/test/java/de/focus_shift/jollyday/tests/Benchmarks.java new file mode 100644 index 000000000..93eb1e44d --- /dev/null +++ b/jollyday-tests/src/test/java/de/focus_shift/jollyday/tests/Benchmarks.java @@ -0,0 +1,19 @@ +package de.focus_shift.jollyday.tests; + +import org.openjdk.jmh.results.RunResult; + +import java.text.DecimalFormat; + +import static org.assertj.core.api.Assertions.assertThat; + +public abstract class Benchmarks { + + protected static void assertDeviationWithin(final RunResult result, final double referenceScore, final double maxDeviationInPercent) { + final double score = result.getPrimaryResult().getScore(); + final double minimumValidScore = referenceScore - (referenceScore * maxDeviationInPercent); + + assertThat(score) + .withFailMessage("The score (%s) must be greater than the minimum valid score (%s)", score, minimumValidScore) + .isGreaterThan(minimumValidScore); + } +} diff --git a/jollyday-tests/src/test/java/de/focus_shift/jollyday/tests/HolidayManagerCreatingBenchmarkTest.java b/jollyday-tests/src/test/java/de/focus_shift/jollyday/tests/HolidayManagerCreatingBenchmarkTest.java new file mode 100644 index 000000000..2c1464bd3 --- /dev/null +++ b/jollyday-tests/src/test/java/de/focus_shift/jollyday/tests/HolidayManagerCreatingBenchmarkTest.java @@ -0,0 +1,78 @@ +package de.focus_shift.jollyday.tests; + +import de.focus_shift.jollyday.core.HolidayManager; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.results.RunResult; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static de.focus_shift.jollyday.core.ManagerParameters.create; +import static org.assertj.core.api.Assertions.assertThat; + +@BenchmarkMode(Mode.Throughput) +@Fork(3) +@Warmup(iterations = 5, time = 2000, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 5, time = 2000, timeUnit = TimeUnit.MILLISECONDS) +public class HolidayManagerCreatingBenchmarkTest extends Benchmarks { + + static final Map REFERENCE_SCORES = Map.of( + "benchmarkCreateHolidayManagerWithCachingEnabled", 54_000.00, + "benchmarkCreateHolidayManagerWithCachingDisabled", 1_200.00 + ); + + @Benchmark + public static HolidayManager benchmarkCreateHolidayManagerWithCachingEnabled() { + return HolidayManager.getInstance(create("test")); + } + + @State(Scope.Thread) + public static class HolidayManagerCacheDisabledState { + @Setup(Level.Trial) + public void doSetup() { + HolidayManager.setManagerCachingEnabled(false); + } + + @TearDown(Level.Trial) + public void doTearDown() { + HolidayManager.setManagerCachingEnabled(true); + } + } + + @Benchmark + public static HolidayManager benchmarkCreateHolidayManagerWithCachingDisabled(final HolidayManagerCacheDisabledState state) { + return HolidayManager.getInstance(create("test")); + } + + @Test + @Tag("BenchmarkTest") + void runJmhBenchmark() throws RunnerException { + final Options opt = new OptionsBuilder() + .include(HolidayManagerCreatingBenchmarkTest.class.getSimpleName()) + .build(); + + final Collection runResults = new Runner(opt).run(); + assertThat(runResults).isNotEmpty(); + + for (RunResult runResult : runResults) { + assertDeviationWithin(runResult, REFERENCE_SCORES.get(runResult.getPrimaryResult().getLabel()), 0.05); + } + } +} diff --git a/jollyday-tests/src/test/java/de/focus_shift/jollyday/tests/HolidayManagerGetHolidayBenchmarkTest.java b/jollyday-tests/src/test/java/de/focus_shift/jollyday/tests/HolidayManagerGetHolidayBenchmarkTest.java new file mode 100644 index 000000000..5c30ef0b5 --- /dev/null +++ b/jollyday-tests/src/test/java/de/focus_shift/jollyday/tests/HolidayManagerGetHolidayBenchmarkTest.java @@ -0,0 +1,60 @@ +package de.focus_shift.jollyday.tests; + +import de.focus_shift.jollyday.core.Holiday; +import de.focus_shift.jollyday.core.HolidayManager; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.results.RunResult; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import java.util.Collection; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import static de.focus_shift.jollyday.core.ManagerParameters.create; +import static org.assertj.core.api.Assertions.assertThat; + +@BenchmarkMode(Mode.Throughput) +@Fork(3) +@Warmup(iterations = 5, time = 2000, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 5, time = 2000, timeUnit = TimeUnit.MILLISECONDS) +public class HolidayManagerGetHolidayBenchmarkTest extends Benchmarks { + + private static final double REFERENCE_SCORE = 45_000.00; + + @State(Scope.Thread) + public static class HolidayManagerState { + public final HolidayManager holidayManager = HolidayManager.getInstance(create("test")); + } + + @Benchmark + public static Set benchmarkGetHolidays(final HolidayManagerState holidayManagerState) { + return holidayManagerState.holidayManager.getHolidays(2010); + } + + @Test + @Tag("BenchmarkTest") + void runJmhBenchmark() throws RunnerException { + final Options opt = new OptionsBuilder() + .include(HolidayManagerGetHolidayBenchmarkTest.class.getSimpleName()) + .build(); + + final Collection runResults = new Runner(opt).run(); + assertThat(runResults).isNotEmpty(); + + for (RunResult runResult : runResults) { + assertDeviationWithin(runResult, REFERENCE_SCORE, 0.05); + } + } +} diff --git a/jollyday-tests/src/test/java/de/focus_shift/jollyday/tests/HolidayManagerIsHolidayBenchmarkTest.java b/jollyday-tests/src/test/java/de/focus_shift/jollyday/tests/HolidayManagerIsHolidayBenchmarkTest.java new file mode 100644 index 000000000..3ace013f9 --- /dev/null +++ b/jollyday-tests/src/test/java/de/focus_shift/jollyday/tests/HolidayManagerIsHolidayBenchmarkTest.java @@ -0,0 +1,59 @@ +package de.focus_shift.jollyday.tests; + +import de.focus_shift.jollyday.core.HolidayManager; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.results.RunResult; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import java.time.LocalDate; +import java.util.Collection; +import java.util.concurrent.TimeUnit; + +import static de.focus_shift.jollyday.core.ManagerParameters.create; +import static org.assertj.core.api.Assertions.assertThat; + +@BenchmarkMode(Mode.Throughput) +@Fork(3) +@Warmup(iterations = 5, time = 2000, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 5, time = 2000, timeUnit = TimeUnit.MILLISECONDS) +public class HolidayManagerIsHolidayBenchmarkTest extends Benchmarks { + + private static final double REFERENCE_SCORE = 8_300_000.000; + + @State(Scope.Thread) + public static class HolidayManagerState { + public final HolidayManager holidayManager = HolidayManager.getInstance(create("test")); + } + + @Benchmark + public static boolean benchmarkIsHoliday(final HolidayManagerState holidayManagerState) { + return holidayManagerState.holidayManager.isHoliday(LocalDate.of(2010, 1, 1)); + } + + @Test + @Tag("BenchmarkTest") + void runJmhBenchmark() throws RunnerException { + final Options opt = new OptionsBuilder() + .include(HolidayManagerIsHolidayBenchmarkTest.class.getSimpleName()) + .build(); + + final Collection runResults = new Runner(opt).run(); + assertThat(runResults).isNotEmpty(); + + for (RunResult runResult : runResults) { + assertDeviationWithin(runResult, REFERENCE_SCORE, 0.05); + } + } +} diff --git a/jollyday-tests/src/test/java/de/focus_shift/jollyday/tests/HolidayManagerPerformanceTest.java b/jollyday-tests/src/test/java/de/focus_shift/jollyday/tests/HolidayManagerPerformanceTest.java deleted file mode 100644 index 8562790c7..000000000 --- a/jollyday-tests/src/test/java/de/focus_shift/jollyday/tests/HolidayManagerPerformanceTest.java +++ /dev/null @@ -1,47 +0,0 @@ -package de.focus_shift.jollyday.tests; - -import de.focus_shift.jollyday.core.HolidayManager; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.time.LocalDate; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicLong; - -import static de.focus_shift.jollyday.core.ManagerParameters.create; -import static java.util.concurrent.TimeUnit.SECONDS; - -class HolidayManagerPerformanceTest { - - private static final Logger LOG = LoggerFactory.getLogger(HolidayManagerPerformanceTest.class); - - @Test - void testIsHolidayPerformanceMultiThread() throws InterruptedException { - - LocalDate date = LocalDate.of(2010, 1, 1); - final AtomicLong count = new AtomicLong(0); - final AtomicLong sumDuration = new AtomicLong(0); - - final ExecutorService executorService = Executors.newCachedThreadPool(); - while (date.getYear() < 2013) { - final LocalDate localDate = date; - executorService.submit(() -> { - final long start = System.currentTimeMillis(); - - final HolidayManager holidayManager = HolidayManager.getInstance(create("test")); - holidayManager.isHoliday(localDate); - - final long duration = System.currentTimeMillis() - start; - count.incrementAndGet(); - sumDuration.addAndGet(duration); - }); - date = date.plusDays(1); - } - executorService.shutdown(); - executorService.awaitTermination(5, SECONDS); - - LOG.info("isHoliday took {} millis average tested with {} calls.", sumDuration.doubleValue() / count.doubleValue(), count.longValue()); - } -} diff --git a/pom.xml b/pom.xml index f7ccb7c0c..a17bbdae3 100644 --- a/pom.xml +++ b/pom.xml @@ -36,10 +36,13 @@ UTF-8 11 + !BenchmarkTest + 5.10.2 5.10.0 3.25.3 1.8.3 + 1.37 @@ -139,6 +142,20 @@ 1.5.0 test + + + + org.openjdk.jmh + jmh-core + ${jmh.version} + test + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + test + @@ -155,6 +172,9 @@ org.apache.maven.plugins maven-surefire-plugin 3.2.5 + + ${tests.groups} +