Merge pull request #435 from lenguyenthanh/bench-improve

Improve benchmarks

bench/src/main/scala/benchmarks/HashBench.scala

@@ -1,27 +1,39 @@
 package benchmarks
 
 import org.openjdk.jmh.annotations._
+import org.openjdk.jmh.infra.Blackhole
+import java.util.concurrent.TimeUnit
 
 import cats.syntax.all.*
-
-import java.util.concurrent.TimeUnit
-import chess.format.pgn.Fixtures
-import chess.format.pgn.Reader
-import chess.MoveOrDrop.move
-import chess.Hash
+import chess.format.pgn.{ Fixtures, Reader }
+import chess.MoveOrDrop.situationAfter
+import chess.{ Hash, Situation }
 
 @State(Scope.Thread)
 @BenchmarkMode(Array(Mode.Throughput))
 @OutputTimeUnit(TimeUnit.SECONDS)
 @Measurement(iterations = 15, timeUnit = TimeUnit.SECONDS, time = 3)
 @Warmup(iterations = 15, timeUnit = TimeUnit.SECONDS, time = 3)
+@Fork(value = 3)
 @Threads(value = 1)
 class HashBench:
 
-  var games = Fixtures.gamesForPerfTest.traverse(Reader.full(_)).toOption.get.traverse(_.valid).toOption.get
+  // the unit of CPU work per iteration
+  private[this] val Work: Long = 10
+
+  var situations: List[Situation] = _
 
-  var situations = games.flatMap(_.moves).flatMap(_.move).map(_.situationAfter)
+  @Setup
+  def setup() =
+    var results = for
+      results <- Fixtures.gamesForPerfTest.traverse(Reader.full(_))
+      replays <- results.traverse(_.valid)
+    yield replays.flatMap(_.moves).map(_.situationAfter)
+    situations = results.toOption.get
 
   @Benchmark
-  def hashes() =
-    situations.map(Hash(_))
+  def hashes(bh: Blackhole) =
+    var result = situations.map: x =>
+      Blackhole.consumeCPU(Work)
+      Hash(x)
+    bh.consume(result)

bench/src/main/scala/benchmarks/InsufficientMaterialBench.scala

@@ -1,21 +1,18 @@
 package benchmarks
 
-import org.openjdk.jmh.annotations._
+import org.openjdk.jmh.annotations.*
 
 import cats.syntax.all.*
-
 import java.util.concurrent.TimeUnit
-import chess.format.pgn.Fixtures
-import chess.format.pgn.Parser
-import chess.format.EpdFen
-import chess.format.Fen
+import chess.format.{ EpdFen, Fen }
 import chess.variant.Horde
 
 @State(Scope.Thread)
 @BenchmarkMode(Array(Mode.Throughput))
 @OutputTimeUnit(TimeUnit.SECONDS)
 @Measurement(iterations = 15, timeUnit = TimeUnit.SECONDS, time = 3)
 @Warmup(iterations = 15, timeUnit = TimeUnit.SECONDS, time = 3)
+@Fork(value = 3)
 @Threads(value = 1)
 class InsufficientMaterialBench:
 

bench/src/main/scala/benchmarks/ParserBench.scala

@@ -1,22 +1,35 @@
 package benchmarks
 
-import org.openjdk.jmh.annotations._
+import org.openjdk.jmh.annotations.*
+import org.openjdk.jmh.infra.Blackhole
+import java.util.concurrent.TimeUnit
 
 import cats.syntax.all.*
-
-import java.util.concurrent.TimeUnit
-import chess.format.pgn.Fixtures
-import chess.format.pgn.Parser
+import chess.format.pgn.{ Fixtures, Parser, PgnStr }
 
 @State(Scope.Thread)
 @BenchmarkMode(Array(Mode.Throughput))
 @OutputTimeUnit(TimeUnit.SECONDS)
 @Measurement(iterations = 15, timeUnit = TimeUnit.SECONDS, time = 3)
 @Warmup(iterations = 15, timeUnit = TimeUnit.SECONDS, time = 3)
+@Fork(value = 3)
 @Threads(value = 1)
 class ParserBench:
 
-  var games = Fixtures.gamesForPerfTest
+  // the unit of CPU work per iteration
+  private[this] val Work: Long = 10
+
+  var games: List[PgnStr] = _
+
+  @Setup
+  def setup() =
+    games = Fixtures.gamesForPerfTest
+
   @Benchmark
-  def pgnParser(): Boolean =
-    games.traverse(Parser.full).isRight
+  def pgnParser(bh: Blackhole) =
+    var result = games.traverse { x =>
+      Blackhole.consumeCPU(Work)
+      Parser.full(x)
+    }
+    bh.consume(result)
+    result

bench/src/main/scala/benchmarks/PerftBench.scala

@@ -1,62 +1,97 @@
 package benchmarks
 
-import org.openjdk.jmh.annotations._
-
-import chess.perft.Perft
-
+import org.openjdk.jmh.annotations.*
+import org.openjdk.jmh.infra.Blackhole
 import java.util.concurrent.TimeUnit
+
+import chess.perft.{ Perft, Result }
+import chess.format.Fen
 import chess.variant.*
 
 @State(Scope.Thread)
 @BenchmarkMode(Array(Mode.Throughput))
 @OutputTimeUnit(TimeUnit.SECONDS)
 @Measurement(iterations = 15, timeUnit = TimeUnit.SECONDS, time = 3)
 @Warmup(iterations = 15, timeUnit = TimeUnit.SECONDS, time = 3)
+@Fork(value = 3)
 @Threads(value = 1)
 class PerftBench:
 
-  var threecheckPerfts = Perft.threeCheckPerfts
-  var nodeLimit = 10_000L
-  var gameLimit = 100
+  // the unit of CPU work per iteration
+  private[this] val Work: Long = 10
+
+  @Param(Array("50"))
+  var games: Int = _
+
+  @Param(Array("10000"))
+  var nodes: Long = _
+
+  var threecheckPerfts: List[Perft]  = _
+  var antichessPerfts: List[Perft]   = _
+  var atomicPerfts: List[Perft]      = _
+  var crazyhousePerfts: List[Perft]  = _
+  var racingkingsPerfts: List[Perft] = _
+  var hordePerfts: List[Perft]       = _
+  var randomPerfts: List[Perft]      = _
+  var trickyPerfts: List[Perft]      = _
+
+  @Setup
+  def setup(): Unit =
+    threecheckPerfts = makePerft(Perft.threeCheckPerfts, games, nodes)
+    antichessPerfts = makePerft(Perft.antichessPerfts, games, nodes)
+    atomicPerfts = makePerft(Perft.atomicPerfts, games, nodes)
+    crazyhousePerfts = makePerft(Perft.crazyhousePerfts, games, nodes)
+    racingkingsPerfts = makePerft(Perft.racingkingsPerfts, games, nodes)
+    hordePerfts = makePerft(Perft.hordePerfts, games, nodes)
+    randomPerfts = makePerft(Perft.randomPerfts, games, nodes)
+    trickyPerfts = makePerft(Perft.trickyPerfts, games, nodes)
 
   @Benchmark
-  def threecheck() =
-    bench(threecheckPerfts, ThreeCheck, nodeLimit, gameLimit)
+  def threecheck(bh: Blackhole) =
+    bench(threecheckPerfts, ThreeCheck)(bh)
 
-  var antichessPerfts = Perft.antichessPerfts
   @Benchmark
-  def antichess() =
-    bench(antichessPerfts, Antichess, nodeLimit, gameLimit)
+  def antichess(bh: Blackhole) =
+    bench(antichessPerfts, Antichess)(bh)
 
-  var atomicPerfts = Perft.atomicPerfts
   @Benchmark
-  def atomic() =
-    bench(atomicPerfts, Atomic, nodeLimit, gameLimit)
+  def atomic(bh: Blackhole) =
+    bench(atomicPerfts, Atomic)(bh)
 
-  var crazyhousePerfts = Perft.crazyhousePerfts
   @Benchmark
-  def crazyhouse() =
-    bench(crazyhousePerfts, Crazyhouse, nodeLimit, gameLimit)
+  def crazyhouse(bh: Blackhole) =
+    bench(crazyhousePerfts, Crazyhouse)(bh)
 
-  var hordePerfts = Perft.hordePerfts
   @Benchmark
-  def horde() =
-    bench(hordePerfts, Horde, nodeLimit, gameLimit)
+  def horde(bh: Blackhole) =
+    bench(hordePerfts, Horde)(bh)
 
-  var racingkingsPerfts = Perft.racingkingsPerfts
   @Benchmark
-  def racingkings() =
-    bench(racingkingsPerfts, RacingKings, nodeLimit, gameLimit)
+  def racingkings(bh: Blackhole) =
+    bench(racingkingsPerfts, RacingKings)(bh)
 
-  var randomPerfts = Perft.randomPerfts.take(50)
   @Benchmark
-  def chess960() =
-    bench(randomPerfts, Chess960, nodeLimit, gameLimit)
+  def chess960(bh: Blackhole) =
+    bench(randomPerfts, Chess960)(bh)
 
-  var trickyPerfts = Perft.trickyPerfts
   @Benchmark
-  def tricky() =
-    bench(trickyPerfts, Chess960, nodeLimit, gameLimit)
+  def tricky(bh: Blackhole) =
+    bench(trickyPerfts, Chess960)(bh)
+
+  private def makePerft(perfts: List[Perft], games: Int, nodes: Long) =
+    perfts.take(games).map(_.withLimit(nodes))
+
+  private def bench(perfts: List[Perft], variant: Variant)(bh: Blackhole) =
+    var x = perfts.map:
+      Blackhole.consumeCPU(Work)
+      _.calculate(variant)
+    bh.consume(x)
+    x
 
-  private def bench(perfts: List[Perft], variant: Variant, nodeLimit: Long, gameLimit: Int) =
-    perfts.take(gameLimit).map(_.withLimit(nodeLimit).calculate(variant))
+  extension (perft: Perft)
+    def bench(variant: Variant): List[Result] =
+      var situation = Fen.read(variant, perft.epd).get
+      perft.cases.map: c =>
+        import Perft.*
+        Blackhole.consumeCPU(Work)
+        Result(c.depth, situation.perft(c.depth), c.result)

bench/src/main/scala/benchmarks/PlayBench.scala

@@ -1,13 +1,12 @@
 package benchmarks
 
-import org.openjdk.jmh.annotations._
+import org.openjdk.jmh.annotations.*
+import org.openjdk.jmh.infra.Blackhole
 import java.util.concurrent.TimeUnit
 
 import cats.syntax.all.*
-
 import chess.Square.*
-import chess.format.pgn.Fixtures
-import chess.format.pgn.SanStr
+import chess.format.pgn.{ Fixtures, SanStr }
 import chess.variant.Standard
 import chess.{ Mode => _, * }
 
@@ -16,29 +15,51 @@ import chess.{ Mode => _, * }
 @OutputTimeUnit(TimeUnit.SECONDS)
 @Measurement(iterations = 15, timeUnit = TimeUnit.SECONDS, time = 3)
 @Warmup(iterations = 15, timeUnit = TimeUnit.SECONDS, time = 3)
+@Fork(value = 3)
 @Threads(value = 1)
 class PlayBench:
 
-  var standard = Game(Board init chess.variant.Standard, White)
-  var moves      = Fixtures.fromProd2
-  var gameReplay = Replay.boards(SanStr from moves.split(' ').toList, None, Standard).toOption.get
+  // the unit of CPU work per iteration
+  private[this] val Work: Long = 10
 
-  @Benchmark
-  def divider() =
-    Divider(gameReplay)
+  var dividerGames: List[List[Board]] = _
+  var gameMoves: List[List[SanStr]]   = _
+  var standard: Game                  = _
+
+  def gameReplay(sans: String) =
+    Replay.boards(SanStr.from(sans.split(' ')), None, Standard).toOption.get
+
+  @Setup
+  def setup() =
+    dividerGames = Fixtures.prod500standard.map(gameReplay)
 
-  var nb    = 50
-  var games = Fixtures.prod500standard
-  var gameMoves = games.take(nb).map(g => SanStr from g.split(' ').toList)
+    var nb    = 50
+    var games = Fixtures.prod500standard
+    gameMoves = games.take(nb).map(g => SanStr from g.split(' ').toList)
+
+    standard = Game(Board init chess.variant.Standard, White)
+
+  @Benchmark
+  def divider(bh: Blackhole) =
+    var result = dividerGames.map { x =>
+      Blackhole.consumeCPU(Work)
+      Divider(x)
+    }
+    bh.consume(result)
+    result
 
   @Benchmark
-  def replay() =
-    gameMoves.map: moves =>
+  def replay(bh: Blackhole) =
+    var result = gameMoves.map: moves =>
+      Blackhole.consumeCPU(Work)
       Replay.gameMoveWhileValid(moves, chess.format.Fen.initial, chess.variant.Standard)
+    bh.consume(result)
+    result
 
   @Benchmark
-  def play() =
-    standard.playMoves(
+  def play(bh: Blackhole) =
+    var result = standard.playMoves(
+      bh,
       E2 -> E4,
       D7 -> D5,
       E4 -> D5,
@@ -67,12 +88,20 @@ class PlayBench:
       B7 -> C6,
       E2 -> A6
     )
+    bh.consume(result)
+    result
 
   extension (game: Game)
     def as(color: Color): Game = game.withPlayer(color)
 
-    def playMoves(moves: (Square, Square)*): Either[ErrorStr, Game] = playMoveList(moves)
+    def playMoves(bh: Blackhole, moves: (Square, Square)*): Either[ErrorStr, Game] = playMoveList(bh, moves)
 
-    def playMoveList(moves: Iterable[(Square, Square)]): Either[ErrorStr, Game] =
+    def playMoveList(bh: Blackhole, moves: Iterable[(Square, Square)]): Either[ErrorStr, Game] =
       moves.toList.foldM(game):
-        case (game, (o, d)) => game(o, d).map(_._1)
+        case (game, (o, d)) =>
+          // because possible moves are asked for player highlight
+          // before the move is played (on initial situation)
+          Blackhole.consumeCPU(Work)
+          var result = game.situation.destinations
+          bh.consume(result)
+          game(o, d).map(_._1)