JMH Benchmarks added. #178
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| package software.amazon.event.ruler.jmh; | ||
|
|
||
| import org.openjdk.jmh.annotations.Benchmark; | ||
| import org.openjdk.jmh.annotations.BenchmarkMode; | ||
| import org.openjdk.jmh.annotations.Fork; | ||
| import org.openjdk.jmh.annotations.Measurement; | ||
| import org.openjdk.jmh.annotations.Mode; | ||
| import org.openjdk.jmh.annotations.OperationsPerInvocation; | ||
| import org.openjdk.jmh.annotations.Timeout; | ||
| import org.openjdk.jmh.annotations.Warmup; | ||
| import org.openjdk.jmh.infra.Blackhole; | ||
| import software.amazon.event.ruler.Machine; | ||
|
|
||
| import java.util.Objects; | ||
|
|
||
| import static java.util.concurrent.TimeUnit.SECONDS; | ||
|
|
||
|
|
||
| @BenchmarkMode(Mode.Throughput) | ||
| @Fork(value = 2, jvmArgsAppend = { | ||
| "-Xmx2g", "-Xms2g", "-XX:+AlwaysPreTouch", "-XX:+UseTransparentHugePages", "-XX:+UseSerialGC", | ||
| "-XX:-BackgroundCompilation", "-XX:CompileCommand=dontinline,com/fasterxml/*.*", | ||
| }) | ||
| @Timeout(time = 90, timeUnit = SECONDS) | ||
| @OperationsPerInvocation(CityLots2State.DATASET_SIZE) | ||
| public class CityLots2JmhBenchmarks { | ||
|
|
||
| @Benchmark | ||
| @Warmup(iterations = 4, batchSize = 1, time = 10, timeUnit = SECONDS) | ||
| @Measurement(iterations = 5, batchSize = 1, time = 10, timeUnit = SECONDS) | ||
| public void group01Simple(MachineStateSimple machineState, CityLots2State cityLots2State, Blackhole blackhole) throws Exception { | ||
| run(machineState, cityLots2State, blackhole); | ||
| } | ||
|
|
||
| @Benchmark | ||
| @Warmup(iterations = 2, batchSize = 1, time = 60, timeUnit = SECONDS) | ||
| @Measurement(iterations = 3, batchSize = 1, time = 60, timeUnit = SECONDS) | ||
| public void group02Complex(MachineStateComplex machineState, CityLots2State cityLots2State, Blackhole blackhole) throws Exception { | ||
| run(machineState, cityLots2State, blackhole); | ||
| } | ||
|
|
||
|
Comment on lines
+30
to
+43
There was a problem hiding this comment. is it worth splitting this by each matcher to parallelize the runs and reduce the runtime even more ? There was a problem hiding this comment. If you go this route, you would be able to get away with building a HashMap of rule-matchines during setup() and then return pass the machine during load tests. It'll reduce the number of scaffolding classes for benchmarks. There was a problem hiding this comment. This grouping is done not for parallelism, parallelizing these benchmark on the same box is a bad idea because the results will be noisy. The reason for this grouping is to have different setup for different benchmarks: for simple ones it is 40 seconds warmup + 50 seconds measurement, for complex ones it is 2 minutes warmup + 3 minutes measurement just because every call for complex benchmarks takes much longer. We could split across multiple boxes for parallelism, but I don't think it is worth the effort and probably would be harder to collect results. What we actually want to do regarding parallelism is to run every benchmark by 2 threads (and check that throughput is ~2x of 1 thread) to make sure that we don't have any contention or false sharing. Right now these benchmarks are single threaded and thus we don't actually know if we have an issue. I will push this change. |
||
| private void run(MachineState machineState, CityLots2State cityLots2State, Blackhole blackhole) throws Exception { | ||
| Machine machine = Objects.requireNonNull(machineState.machine); | ||
|
|
||
| for (String event : cityLots2State.getCityLots2()) { | ||
| blackhole.consume(machine.rulesForJSONEvent(event)); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| package software.amazon.event.ruler.jmh; | ||
|
|
||
| import org.openjdk.jmh.annotations.Scope; | ||
| import org.openjdk.jmh.annotations.State; | ||
| import software.amazon.event.ruler.Benchmarks; | ||
|
|
||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
|
|
||
| @State(Scope.Benchmark) | ||
| public class CityLots2State { | ||
|
|
||
| public static final int DATASET_SIZE = 213068; | ||
|
|
||
| private static final List<String> citylots2 = new ArrayList<>(); | ||
|
|
||
| static { | ||
| Benchmarks.readCityLots2(citylots2); | ||
|
|
||
| if (citylots2.size() != DATASET_SIZE) { | ||
| throw new RuntimeException("Expected dataset size: " + DATASET_SIZE + ", actual: " + citylots2.size()); | ||
| } | ||
| } | ||
|
|
||
| public Iterable<String> getCityLots2() { | ||
| return citylots2; | ||
| } | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is kept because we haven't moved everything yet?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I have ported everything from
CL2Benchmarkbut notCL2NoCompileBenchmark. Not sure if it is actually an important use case.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
They are important so I'm happy to have "Run Benchmarks" step still until these tests that check for rule building, memory usage, et all.