Merge pull request #473 from focus-shift/benchmark

Add benchmark tests via jmh

.github/workflows/benchmark.yml

@@ -0,0 +1,29 @@
+name: Benchmark
+
+on:
+  schedule:
+    - cron: "15 0 * * *"
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+env:
+  MAVEN_ARGS: "--no-transfer-progress --errors --fail-at-end --show-version"
+
+jobs:
+  build:
+    name: benchmark
+    runs-on: 'ubuntu-latest'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: JDK
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '11'
+          cache: 'maven'
+      - name: Build
+        run: ./mvnw clean verify -Dtests.groups=BenchmarkTest

jollyday-tests/pom.xml

@@ -56,6 +56,16 @@
       <version>0.1.2</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.openjdk.jmh</groupId>
+      <artifactId>jmh-core</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.openjdk.jmh</groupId>
+      <artifactId>jmh-generator-annprocess</artifactId>
+      <scope>test</scope>
+    </dependency>
 
     <!-- Logging -->
     <dependency>

jollyday-tests/src/test/java/de/focus_shift/jollyday/tests/Benchmarks.java

@@ -0,0 +1,19 @@
+package de.focus_shift.jollyday.tests;
+
+import org.openjdk.jmh.results.RunResult;
+
+import java.text.DecimalFormat;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public abstract class Benchmarks {
+
+  protected static void assertDeviationWithin(final RunResult result, final double referenceScore, final double maxDeviationInPercent) {
+    final double score = result.getPrimaryResult().getScore();
+    final double minimumValidScore = referenceScore - (referenceScore * maxDeviationInPercent);
+
+    assertThat(score)
+      .withFailMessage("The score (%s) must be greater than the minimum valid score (%s)", score, minimumValidScore)
+      .isGreaterThan(minimumValidScore);
+  }
+}

jollyday-tests/src/test/java/de/focus_shift/jollyday/tests/HolidayManagerCreatingBenchmarkTest.java

@@ -0,0 +1,78 @@
+package de.focus_shift.jollyday.tests;
+
+import de.focus_shift.jollyday.core.HolidayManager;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.results.RunResult;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static de.focus_shift.jollyday.core.ManagerParameters.create;
+import static org.assertj.core.api.Assertions.assertThat;
+
+@BenchmarkMode(Mode.Throughput)
+@Fork(3)
+@Warmup(iterations = 5, time = 2000, timeUnit = TimeUnit.MILLISECONDS)
+@Measurement(iterations = 5, time = 2000, timeUnit = TimeUnit.MILLISECONDS)
+public class HolidayManagerCreatingBenchmarkTest extends Benchmarks {
+
+  static final Map<String, Double> REFERENCE_SCORES = Map.of(
+    "benchmarkCreateHolidayManagerWithCachingEnabled", 54_000.00,
+    "benchmarkCreateHolidayManagerWithCachingDisabled", 1_200.00
+  );
+
+  @Benchmark
+  public static HolidayManager benchmarkCreateHolidayManagerWithCachingEnabled() {
+    return HolidayManager.getInstance(create("test"));
+  }
+
+  @State(Scope.Thread)
+  public static class HolidayManagerCacheDisabledState {
+    @Setup(Level.Trial)
+    public void doSetup() {
+      HolidayManager.setManagerCachingEnabled(false);
+    }
+
+    @TearDown(Level.Trial)
+    public void doTearDown() {
+      HolidayManager.setManagerCachingEnabled(true);
+    }
+  }
+
+  @Benchmark
+  public static HolidayManager benchmarkCreateHolidayManagerWithCachingDisabled(final HolidayManagerCacheDisabledState state) {
+    return HolidayManager.getInstance(create("test"));
+  }
+
+  @Test
+  @Tag("BenchmarkTest")
+  void runJmhBenchmark() throws RunnerException {
+    final Options opt = new OptionsBuilder()
+      .include(HolidayManagerCreatingBenchmarkTest.class.getSimpleName())
+      .build();
+
+    final Collection<RunResult> runResults = new Runner(opt).run();
+    assertThat(runResults).isNotEmpty();
+
+    for (RunResult runResult : runResults) {
+      assertDeviationWithin(runResult, REFERENCE_SCORES.get(runResult.getPrimaryResult().getLabel()), 0.05);
+    }
+  }
+}

jollyday-tests/src/test/java/de/focus_shift/jollyday/tests/HolidayManagerGetHolidayBenchmarkTest.java

@@ -0,0 +1,60 @@
+package de.focus_shift.jollyday.tests;
+
+import de.focus_shift.jollyday.core.Holiday;
+import de.focus_shift.jollyday.core.HolidayManager;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.results.RunResult;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.util.Collection;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import static de.focus_shift.jollyday.core.ManagerParameters.create;
+import static org.assertj.core.api.Assertions.assertThat;
+
+@BenchmarkMode(Mode.Throughput)
+@Fork(3)
+@Warmup(iterations = 5, time = 2000, timeUnit = TimeUnit.MILLISECONDS)
+@Measurement(iterations = 5, time = 2000, timeUnit = TimeUnit.MILLISECONDS)
+public class HolidayManagerGetHolidayBenchmarkTest extends Benchmarks {
+
+  private static final double REFERENCE_SCORE = 45_000.00;
+
+  @State(Scope.Thread)
+  public static class HolidayManagerState {
+    public final HolidayManager holidayManager = HolidayManager.getInstance(create("test"));
+  }
+
+  @Benchmark
+  public static Set<Holiday> benchmarkGetHolidays(final HolidayManagerState holidayManagerState) {
+    return holidayManagerState.holidayManager.getHolidays(2010);
+  }
+
+  @Test
+  @Tag("BenchmarkTest")
+  void runJmhBenchmark() throws RunnerException {
+    final Options opt = new OptionsBuilder()
+      .include(HolidayManagerGetHolidayBenchmarkTest.class.getSimpleName())
+      .build();
+
+    final Collection<RunResult> runResults = new Runner(opt).run();
+    assertThat(runResults).isNotEmpty();
+
+    for (RunResult runResult : runResults) {
+      assertDeviationWithin(runResult, REFERENCE_SCORE, 0.05);
+    }
+  }
+}

jollyday-tests/src/test/java/de/focus_shift/jollyday/tests/HolidayManagerIsHolidayBenchmarkTest.java

@@ -0,0 +1,59 @@
+package de.focus_shift.jollyday.tests;
+
+import de.focus_shift.jollyday.core.HolidayManager;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.results.RunResult;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.time.LocalDate;
+import java.util.Collection;
+import java.util.concurrent.TimeUnit;
+
+import static de.focus_shift.jollyday.core.ManagerParameters.create;
+import static org.assertj.core.api.Assertions.assertThat;
+
+@BenchmarkMode(Mode.Throughput)
+@Fork(3)
+@Warmup(iterations = 5, time = 2000, timeUnit = TimeUnit.MILLISECONDS)
+@Measurement(iterations = 5, time = 2000, timeUnit = TimeUnit.MILLISECONDS)
+public class HolidayManagerIsHolidayBenchmarkTest extends Benchmarks {
+
+  private static final double REFERENCE_SCORE = 8_300_000.000;
+
+  @State(Scope.Thread)
+  public static class HolidayManagerState {
+    public final HolidayManager holidayManager = HolidayManager.getInstance(create("test"));
+  }
+
+  @Benchmark
+  public static boolean benchmarkIsHoliday(final HolidayManagerState holidayManagerState) {
+    return holidayManagerState.holidayManager.isHoliday(LocalDate.of(2010, 1, 1));
+  }
+
+  @Test
+  @Tag("BenchmarkTest")
+  void runJmhBenchmark() throws RunnerException {
+    final Options opt = new OptionsBuilder()
+      .include(HolidayManagerIsHolidayBenchmarkTest.class.getSimpleName())
+      .build();
+
+    final Collection<RunResult> runResults = new Runner(opt).run();
+    assertThat(runResults).isNotEmpty();
+
+    for (RunResult runResult : runResults) {
+      assertDeviationWithin(runResult, REFERENCE_SCORE, 0.05);
+    }
+  }
+}

pom.xml

@@ -36,10 +36,13 @@
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <java.version>11</java.version>
 
+    <tests.groups>!BenchmarkTest</tests.groups>
+
     <junit.version>5.10.2</junit.version>
     <mockito.version>5.10.0</mockito.version>
     <assertj.version>3.25.3</assertj.version>
     <jqwik.version>1.8.3</jqwik.version>
+    <jmh.version>1.37</jmh.version>
 
     <!--suppress UnresolvedMavenProperty - for IDEA-->
     <sonar.coverage.jacoco.xmlReportPaths>
@@ -139,6 +142,20 @@
         <version>1.5.0</version>
         <scope>test</scope>
       </dependency>
+
+      <!-- jmh -->
+      <dependency>
+        <groupId>org.openjdk.jmh</groupId>
+        <artifactId>jmh-core</artifactId>
+        <version>${jmh.version}</version>
+        <scope>test</scope>
+      </dependency>
+      <dependency>
+        <groupId>org.openjdk.jmh</groupId>
+        <artifactId>jmh-generator-annprocess</artifactId>
+        <version>${jmh.version}</version>
+        <scope>test</scope>
+      </dependency>
     </dependencies>
   </dependencyManagement>
 
@@ -155,6 +172,9 @@
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-surefire-plugin</artifactId>
           <version>3.2.5</version>
+          <configuration>
+            <groups>${tests.groups}</groups>
+          </configuration>
         </plugin>
 
         <plugin>