From 4077b54e978145739fb2028e9a45865fd3fcd412 Mon Sep 17 00:00:00 2001 From: Thanh Le Date: Sun, 2 Jul 2023 13:09:06 +0200 Subject: [PATCH] Improve benchmarks - Reduce number of games and nodes for PerftBench - Change val => var - Move unrelated calculation from benchmarks - Use more of Blackhole - @Fork = 3 - Fix HashBench --- .../src/main/scala/benchmarks/HashBench.scala | 32 ++++-- .../InsufficientMaterialBench.scala | 9 +- .../main/scala/benchmarks/ParserBench.scala | 29 ++++-- .../main/scala/benchmarks/PerftBench.scala | 99 +++++++++++++------ .../src/main/scala/benchmarks/PlayBench.scala | 69 +++++++++---- 5 files changed, 162 insertions(+), 76 deletions(-) diff --git a/bench/src/main/scala/benchmarks/HashBench.scala b/bench/src/main/scala/benchmarks/HashBench.scala index 0cdaeb489..eb3bb3c29 100644 --- a/bench/src/main/scala/benchmarks/HashBench.scala +++ b/bench/src/main/scala/benchmarks/HashBench.scala @@ -1,27 +1,39 @@ package benchmarks import org.openjdk.jmh.annotations._ +import org.openjdk.jmh.infra.Blackhole +import java.util.concurrent.TimeUnit import cats.syntax.all.* - -import java.util.concurrent.TimeUnit -import chess.format.pgn.Fixtures -import chess.format.pgn.Reader -import chess.MoveOrDrop.move -import chess.Hash +import chess.format.pgn.{ Fixtures, Reader } +import chess.MoveOrDrop.situationAfter +import chess.{ Hash, Situation } @State(Scope.Thread) @BenchmarkMode(Array(Mode.Throughput)) @OutputTimeUnit(TimeUnit.SECONDS) @Measurement(iterations = 15, timeUnit = TimeUnit.SECONDS, time = 3) @Warmup(iterations = 15, timeUnit = TimeUnit.SECONDS, time = 3) +@Fork(value = 3) @Threads(value = 1) class HashBench: - var games = Fixtures.gamesForPerfTest.traverse(Reader.full(_)).toOption.get.traverse(_.valid).toOption.get + // the unit of CPU work per iteration + private[this] val Work: Long = 10 + + var situations: List[Situation] = _ - var situations = games.flatMap(_.moves).flatMap(_.move).map(_.situationAfter) + @Setup + def setup() = + var results = for + results <- Fixtures.gamesForPerfTest.traverse(Reader.full(_)) + replays <- results.traverse(_.valid) + yield replays.flatMap(_.moves).map(_.situationAfter) + situations = results.toOption.get @Benchmark - def hashes() = - situations.map(Hash(_)) + def hashes(bh: Blackhole) = + var result = situations.map: x => + Blackhole.consumeCPU(Work) + Hash(x) + bh.consume(result) diff --git a/bench/src/main/scala/benchmarks/InsufficientMaterialBench.scala b/bench/src/main/scala/benchmarks/InsufficientMaterialBench.scala index e4f9170f4..3bc4aebfa 100644 --- a/bench/src/main/scala/benchmarks/InsufficientMaterialBench.scala +++ b/bench/src/main/scala/benchmarks/InsufficientMaterialBench.scala @@ -1,14 +1,10 @@ package benchmarks -import org.openjdk.jmh.annotations._ +import org.openjdk.jmh.annotations.* import cats.syntax.all.* - import java.util.concurrent.TimeUnit -import chess.format.pgn.Fixtures -import chess.format.pgn.Parser -import chess.format.EpdFen -import chess.format.Fen +import chess.format.{ EpdFen, Fen } import chess.variant.Horde @State(Scope.Thread) @@ -16,6 +12,7 @@ import chess.variant.Horde @OutputTimeUnit(TimeUnit.SECONDS) @Measurement(iterations = 15, timeUnit = TimeUnit.SECONDS, time = 3) @Warmup(iterations = 15, timeUnit = TimeUnit.SECONDS, time = 3) +@Fork(value = 3) @Threads(value = 1) class InsufficientMaterialBench: diff --git a/bench/src/main/scala/benchmarks/ParserBench.scala b/bench/src/main/scala/benchmarks/ParserBench.scala index e16dc550d..b06fe7e22 100644 --- a/bench/src/main/scala/benchmarks/ParserBench.scala +++ b/bench/src/main/scala/benchmarks/ParserBench.scala @@ -1,22 +1,35 @@ package benchmarks -import org.openjdk.jmh.annotations._ +import org.openjdk.jmh.annotations.* +import org.openjdk.jmh.infra.Blackhole +import java.util.concurrent.TimeUnit import cats.syntax.all.* - -import java.util.concurrent.TimeUnit -import chess.format.pgn.Fixtures -import chess.format.pgn.Parser +import chess.format.pgn.{ Fixtures, Parser, PgnStr } @State(Scope.Thread) @BenchmarkMode(Array(Mode.Throughput)) @OutputTimeUnit(TimeUnit.SECONDS) @Measurement(iterations = 15, timeUnit = TimeUnit.SECONDS, time = 3) @Warmup(iterations = 15, timeUnit = TimeUnit.SECONDS, time = 3) +@Fork(value = 3) @Threads(value = 1) class ParserBench: - var games = Fixtures.gamesForPerfTest + // the unit of CPU work per iteration + private[this] val Work: Long = 10 + + var games: List[PgnStr] = _ + + @Setup + def setup() = + games = Fixtures.gamesForPerfTest + @Benchmark - def pgnParser(): Boolean = - games.traverse(Parser.full).isRight + def pgnParser(bh: Blackhole) = + var result = games.traverse { x => + Blackhole.consumeCPU(Work) + Parser.full(x) + } + bh.consume(result) + result diff --git a/bench/src/main/scala/benchmarks/PerftBench.scala b/bench/src/main/scala/benchmarks/PerftBench.scala index 048529628..16f9eb8a8 100644 --- a/bench/src/main/scala/benchmarks/PerftBench.scala +++ b/bench/src/main/scala/benchmarks/PerftBench.scala @@ -1,10 +1,11 @@ package benchmarks -import org.openjdk.jmh.annotations._ - -import chess.perft.Perft - +import org.openjdk.jmh.annotations.* +import org.openjdk.jmh.infra.Blackhole import java.util.concurrent.TimeUnit + +import chess.perft.{ Perft, Result } +import chess.format.Fen import chess.variant.* @State(Scope.Thread) @@ -12,51 +13,85 @@ import chess.variant.* @OutputTimeUnit(TimeUnit.SECONDS) @Measurement(iterations = 15, timeUnit = TimeUnit.SECONDS, time = 3) @Warmup(iterations = 15, timeUnit = TimeUnit.SECONDS, time = 3) +@Fork(value = 3) @Threads(value = 1) class PerftBench: - var threecheckPerfts = Perft.threeCheckPerfts - var nodeLimit = 10_000L - var gameLimit = 100 + // the unit of CPU work per iteration + private[this] val Work: Long = 10 + + @Param(Array("50")) + var games: Int = _ + + @Param(Array("10000")) + var nodes: Long = _ + + var threecheckPerfts: List[Perft] = _ + var antichessPerfts: List[Perft] = _ + var atomicPerfts: List[Perft] = _ + var crazyhousePerfts: List[Perft] = _ + var racingkingsPerfts: List[Perft] = _ + var hordePerfts: List[Perft] = _ + var randomPerfts: List[Perft] = _ + var trickyPerfts: List[Perft] = _ + + @Setup + def setup(): Unit = + threecheckPerfts = makePerft(Perft.threeCheckPerfts, games, nodes) + antichessPerfts = makePerft(Perft.antichessPerfts, games, nodes) + atomicPerfts = makePerft(Perft.atomicPerfts, games, nodes) + crazyhousePerfts = makePerft(Perft.crazyhousePerfts, games, nodes) + racingkingsPerfts = makePerft(Perft.racingkingsPerfts, games, nodes) + hordePerfts = makePerft(Perft.hordePerfts, games, nodes) + randomPerfts = makePerft(Perft.randomPerfts, games, nodes) + trickyPerfts = makePerft(Perft.trickyPerfts, games, nodes) @Benchmark - def threecheck() = - bench(threecheckPerfts, ThreeCheck, nodeLimit, gameLimit) + def threecheck(bh: Blackhole) = + bench(threecheckPerfts, ThreeCheck)(bh) - var antichessPerfts = Perft.antichessPerfts @Benchmark - def antichess() = - bench(antichessPerfts, Antichess, nodeLimit, gameLimit) + def antichess(bh: Blackhole) = + bench(antichessPerfts, Antichess)(bh) - var atomicPerfts = Perft.atomicPerfts @Benchmark - def atomic() = - bench(atomicPerfts, Atomic, nodeLimit, gameLimit) + def atomic(bh: Blackhole) = + bench(atomicPerfts, Atomic)(bh) - var crazyhousePerfts = Perft.crazyhousePerfts @Benchmark - def crazyhouse() = - bench(crazyhousePerfts, Crazyhouse, nodeLimit, gameLimit) + def crazyhouse(bh: Blackhole) = + bench(crazyhousePerfts, Crazyhouse)(bh) - var hordePerfts = Perft.hordePerfts @Benchmark - def horde() = - bench(hordePerfts, Horde, nodeLimit, gameLimit) + def horde(bh: Blackhole) = + bench(hordePerfts, Horde)(bh) - var racingkingsPerfts = Perft.racingkingsPerfts @Benchmark - def racingkings() = - bench(racingkingsPerfts, RacingKings, nodeLimit, gameLimit) + def racingkings(bh: Blackhole) = + bench(racingkingsPerfts, RacingKings)(bh) - var randomPerfts = Perft.randomPerfts.take(50) @Benchmark - def chess960() = - bench(randomPerfts, Chess960, nodeLimit, gameLimit) + def chess960(bh: Blackhole) = + bench(randomPerfts, Chess960)(bh) - var trickyPerfts = Perft.trickyPerfts @Benchmark - def tricky() = - bench(trickyPerfts, Chess960, nodeLimit, gameLimit) + def tricky(bh: Blackhole) = + bench(trickyPerfts, Chess960)(bh) + + private def makePerft(perfts: List[Perft], games: Int, nodes: Long) = + perfts.take(games).map(_.withLimit(nodes)) + + private def bench(perfts: List[Perft], variant: Variant)(bh: Blackhole) = + var x = perfts.map: + Blackhole.consumeCPU(Work) + _.calculate(variant) + bh.consume(x) + x - private def bench(perfts: List[Perft], variant: Variant, nodeLimit: Long, gameLimit: Int) = - perfts.take(gameLimit).map(_.withLimit(nodeLimit).calculate(variant)) + extension (perft: Perft) + def bench(variant: Variant): List[Result] = + var situation = Fen.read(variant, perft.epd).get + perft.cases.map: c => + import Perft.* + Blackhole.consumeCPU(Work) + Result(c.depth, situation.perft(c.depth), c.result) diff --git a/bench/src/main/scala/benchmarks/PlayBench.scala b/bench/src/main/scala/benchmarks/PlayBench.scala index 6139c4fd7..9c98ea093 100644 --- a/bench/src/main/scala/benchmarks/PlayBench.scala +++ b/bench/src/main/scala/benchmarks/PlayBench.scala @@ -1,13 +1,12 @@ package benchmarks -import org.openjdk.jmh.annotations._ +import org.openjdk.jmh.annotations.* +import org.openjdk.jmh.infra.Blackhole import java.util.concurrent.TimeUnit import cats.syntax.all.* - import chess.Square.* -import chess.format.pgn.Fixtures -import chess.format.pgn.SanStr +import chess.format.pgn.{ Fixtures, SanStr } import chess.variant.Standard import chess.{ Mode => _, * } @@ -16,29 +15,51 @@ import chess.{ Mode => _, * } @OutputTimeUnit(TimeUnit.SECONDS) @Measurement(iterations = 15, timeUnit = TimeUnit.SECONDS, time = 3) @Warmup(iterations = 15, timeUnit = TimeUnit.SECONDS, time = 3) +@Fork(value = 3) @Threads(value = 1) class PlayBench: - var standard = Game(Board init chess.variant.Standard, White) - var moves = Fixtures.fromProd2 - var gameReplay = Replay.boards(SanStr from moves.split(' ').toList, None, Standard).toOption.get + // the unit of CPU work per iteration + private[this] val Work: Long = 10 - @Benchmark - def divider() = - Divider(gameReplay) + var dividerGames: List[List[Board]] = _ + var gameMoves: List[List[SanStr]] = _ + var standard: Game = _ + + def gameReplay(sans: String) = + Replay.boards(SanStr.from(sans.split(' ')), None, Standard).toOption.get + + @Setup + def setup() = + dividerGames = Fixtures.prod500standard.map(gameReplay) - var nb = 50 - var games = Fixtures.prod500standard - var gameMoves = games.take(nb).map(g => SanStr from g.split(' ').toList) + var nb = 50 + var games = Fixtures.prod500standard + gameMoves = games.take(nb).map(g => SanStr from g.split(' ').toList) + + standard = Game(Board init chess.variant.Standard, White) + + @Benchmark + def divider(bh: Blackhole) = + var result = dividerGames.map { x => + Blackhole.consumeCPU(Work) + Divider(x) + } + bh.consume(result) + result @Benchmark - def replay() = - gameMoves.map: moves => + def replay(bh: Blackhole) = + var result = gameMoves.map: moves => + Blackhole.consumeCPU(Work) Replay.gameMoveWhileValid(moves, chess.format.Fen.initial, chess.variant.Standard) + bh.consume(result) + result @Benchmark - def play() = - standard.playMoves( + def play(bh: Blackhole) = + var result = standard.playMoves( + bh, E2 -> E4, D7 -> D5, E4 -> D5, @@ -67,12 +88,20 @@ class PlayBench: B7 -> C6, E2 -> A6 ) + bh.consume(result) + result extension (game: Game) def as(color: Color): Game = game.withPlayer(color) - def playMoves(moves: (Square, Square)*): Either[ErrorStr, Game] = playMoveList(moves) + def playMoves(bh: Blackhole, moves: (Square, Square)*): Either[ErrorStr, Game] = playMoveList(bh, moves) - def playMoveList(moves: Iterable[(Square, Square)]): Either[ErrorStr, Game] = + def playMoveList(bh: Blackhole, moves: Iterable[(Square, Square)]): Either[ErrorStr, Game] = moves.toList.foldM(game): - case (game, (o, d)) => game(o, d).map(_._1) + case (game, (o, d)) => + // because possible moves are asked for player highlight + // before the move is played (on initial situation) + Blackhole.consumeCPU(Work) + var result = game.situation.destinations + bh.consume(result) + game(o, d).map(_._1)