PR benchmark stats (#138)

* run benchmarks with opcache enabled * output github friendly benchmarks * ci: run benchmarks with diff * ci: don't fail if baseline is not available * ci: fix github mardown template * simplify markdown table generation without hooking into phpbench * simplify benchmark in ci * fix ci * fix ci * remove phpbench from packages * fix ci * run main branch benchmarks with main branch dependencies * step summary instead of PR comment * ci: add retry threshold to benchmarks * ci: disable jit for opcache enabled benchmarks --------- Co-authored-by: jlabedo <jean@needelp.com> Co-authored-by: Dariusz Gafka <dgafka.mail@gmail.com>
ecotoneframework · Jun 6, 2023 · 35280d2 · 35280d2
1 parent 399d380
commit 35280d2
Show file tree

Hide file tree

Showing 11 changed files with 194 additions and 90 deletions.
diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml
@@ -0,0 +1,115 @@
+name: 'PR stats'
+
+on:
+  pull_request:
+
+jobs:
+  run:
+    name: "Benchmark"
+    runs-on: ${{ matrix.operating-system }}
+    strategy:
+      matrix:
+        operating-system: [ ubuntu-latest ]
+        php-versions: [ 8.2 ]
+        stability: [prefer-stable]
+    services:
+      rabbitmq:
+        image: rabbitmq:3.8-alpine
+        env:
+          RABBITMQ_DEFAULT_USER: guest
+          RABBITMQ_DEFAULT_PASS: guest
+        ports:
+          - 5672:5672
+      mysql:
+        image: mysql:8.0
+        env:
+          MYSQL_ROOT_PASSWORD: "secret"
+          MYSQL_USER: "ecotone"
+          MYSQL_PASSWORD: "secret"
+          MYSQL_DATABASE: "ecotone"
+          DB_PORT: 3306
+        options: >-
+          --health-cmd="mysqladmin ping"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
+        ports:
+          - 3306:3306
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_USER: ecotone
+          POSTGRES_PASSWORD: secret
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+      localstack:
+        image: localstack/localstack:0.8.10
+        env:
+          SERVICES: 'sqs,sns'
+        ports:
+          - '4576:4576'
+          - '4575:4575'
+      redis:
+        image: redis:7-alpine
+        ports:
+          - '6379:6379'
+    env:
+      RABBIT_HOST: amqp://127.0.0.1:5672
+      SQS_DSN: sqs:?key=key&secret=secret&region=us-east-1&endpoint=http://127.0.0.1:4576&version=latest
+      REDIS_DSN: redis://127.0.0.1:6379
+      DATABASE_DSN: pgsql://ecotone:secret@127.0.0.1:5432/ecotone
+      APP_DB_HOST: 127.0.0.1
+      APP_DB_PORT: 5432
+      APP_DB_DRIVER: pdo_pgsql
+    steps:
+      - name: PHP ${{ matrix.php-versions }} - ${{ matrix.stability }}
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: ${{ matrix.php-versions }}
+          coverage: none
+
+      - uses: actions/checkout@v3
+        with:
+          ref: main
+
+      - name: Install dependencies
+        run: composer update --${{ matrix.stability }} --prefer-dist --no-interaction
+
+      - name: Benchmark main as baseline
+        id: baseline
+        continue-on-error: true
+        run: |
+          vendor/bin/phpbench --profile=opcache_enabled --retry-threshold=5 --tag=main.opcache_enabled
+          vendor/bin/phpbench --profile=opcache_disabled --retry-threshold=5 --tag=main.opcache_disabled
+
+      - uses: actions/checkout@v3
+        with:
+          clean: false
+
+      - name: Install dependencies
+        run: composer update --${{ matrix.stability }} --prefer-dist --no-interaction
+
+      - name: Benchmark PR with opcache enabled
+        id: opcache_enabled
+        env:
+          PHPBENCH_REF_OPTION: "${{ steps.baseline.outcome == 'success' && '--ref=main.opcache_enabled' || '' }}"
+        run: |
+          vendor/bin/phpbench run --profile=opcache_enabled --retry-threshold=5 --report=github-report $PHPBENCH_REF_OPTION | bin/phpbench-to-md.sh > opcache_enabled.md
+          cat opcache_enabled.md
+          echo '## Opcache enabled' >> "$GITHUB_STEP_SUMMARY"
+          cat opcache_enabled.md >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Benchmark PR with opcache disabled
+        id: opcache_disabled
+        env:
+          PHPBENCH_REF_OPTION: "${{ steps.baseline.outcome == 'success' && '--ref=main.opcache_disabled' || '' }}"
+        run: |
+          vendor/bin/phpbench run --profile=opcache_disabled --retry-threshold=5 --report=github-report $PHPBENCH_REF_OPTION | bin/phpbench-to-md.sh > opcache_disabled.md
+          cat opcache_disabled.md
+          echo '## Opcache disabled' >> "$GITHUB_STEP_SUMMARY"
+          cat opcache_disabled.md >> "$GITHUB_STEP_SUMMARY"
diff --git a/.github/workflows/test-monorepo.yml b/.github/workflows/test-monorepo.yml
@@ -119,11 +119,3 @@ jobs:
         APP_DB_HOST: 127.0.0.1
         APP_DB_PORT: 3306
         APP_DB_DRIVER: pdo_mysql
-
-    - name: Benchmarks
-      run: composer tests:phpbench
-      env:
-        DATABASE_DSN: pgsql://ecotone:secret@127.0.0.1:5432/ecotone
-        APP_DB_HOST: 127.0.0.1
-        APP_DB_PORT: 5432
-        APP_DB_DRIVER: pdo_pgsql
diff --git a/.gitignore b/.gitignore
@@ -13,4 +13,5 @@ cache
 packages/cache
 auth.json
 ./config.json
-.env
+.env
+.phpbench/
diff --git a/...tone/tests/Benchmark/EcotoneBenchmark.php → Monorepo/Benchmark/EcotoneBenchmark.php b/...tone/tests/Benchmark/EcotoneBenchmark.php → Monorepo/Benchmark/EcotoneBenchmark.php
@@ -1,27 +1,24 @@
 <?php
 
-namespace Test\Ecotone\Benchmark;
+namespace Monorepo\Benchmark;
 
+use Assert\Assertion;
 use Ecotone\Lite\EcotoneLite;
 use Ecotone\Messaging\Config\ModulePackageList;
 use Ecotone\Messaging\Config\ServiceConfiguration;
-use PHPUnit\Framework\TestCase;
 use Test\Ecotone\Modelling\Fixture\CommandEventFlow\CreateMerchant;
 use Test\Ecotone\Modelling\Fixture\CommandEventFlow\Merchant;
 use Test\Ecotone\Modelling\Fixture\CommandEventFlow\MerchantSubscriber;
 use Test\Ecotone\Modelling\Fixture\CommandEventFlow\User;
 use Test\Ecotone\Modelling\Fixture\CommandHandler\Aggregate\InMemoryStandardRepository;
 
 /**
- * @internal
+ * @Revs(10)
+ * @Iterations(5)
+ * @Warmup(1)
  */
-class EcotoneBenchmark extends TestCase
+class EcotoneBenchmark
 {
-    /**
-     * @Revs(10)
-     * @Iterations(5)
-     * @Warmup(1)
-     */
     public function bench_running_ecotone_lite()
     {
         $this->execute(
@@ -32,11 +29,6 @@ public function bench_running_ecotone_lite()
         );
     }
 
-    /**
-     * @Revs(10)
-     * @Iterations(5)
-     * @Warmup(1)
-     */
     public function bench_running_ecotone_lite_with_fail_fast()
     {
         $this->execute(
@@ -47,11 +39,6 @@ public function bench_running_ecotone_lite_with_fail_fast()
         );
     }
 
-    /**
-     * @Revs(10)
-     * @Iterations(5)
-     * @Warmup(1)
-     */
     public function bench_running_ecotone_lite_with_cache()
     {
         $this->execute(
@@ -78,7 +65,7 @@ private function execute(ServiceConfiguration $serviceConfiguration, bool $useCa
         $merchantId = '123';
         $ecotoneApplication->getCommandBus()->send(new CreateMerchant($merchantId));
 
-        $this->assertTrue(
+        Assertion::true(
             $ecotoneApplication->getQueryBus()->sendWithRouting('user.get', metadata: ['aggregate.id' => $merchantId])
         );
     }

diff --git a/...pplication/Benchmark/EcotoneBenchmark.php → Monorepo/Benchmark/LaravelBenchmark.php b/...pplication/Benchmark/EcotoneBenchmark.php → Monorepo/Benchmark/LaravelBenchmark.php
@@ -1,56 +1,37 @@
 <?php
 
-namespace Test\Ecotone\Laravel\Application\Benchmark;
+namespace Monorepo\Benchmark;
 
 use Ecotone\Messaging\Config\ConfiguredMessagingSystem;
 use Illuminate\Foundation\Application;
 use Illuminate\Foundation\Http\Kernel;
-use PHPUnit\Framework\TestCase;
 
 /**
- * @internal
+ * @Revs(10)
+ * @Iterations(5)
+ * @Warmup(1)
  */
-class EcotoneBenchmark extends TestCase
+class LaravelBenchmark
 {
-    /**
-     * @Revs(10)
-     * @Iterations(5)
-     * @Warmup(1)
-     */
     public function bench_kernel_boot_on_prod()
     {
         putenv('APP_ENV=production');
         $this->createApplication();
     }
 
-    /**
-     * @Revs(10)
-     * @Iterations(5)
-     * @Warmup(1)
-     */
     public function bench_kernel_boot_on_dev()
     {
         putenv('APP_ENV=development');
         $this->createApplication();
     }
 
-    /**
-     * @Revs(10)
-     * @Iterations(5)
-     * @Warmup(1)
-     */
     public function bench_messaging_boot_on_prod()
     {
         putenv('APP_ENV=production');
         $app = $this->createApplication();
         $app->make(ConfiguredMessagingSystem::class);
     }
 
-    /**
-     * @Revs(10)
-     * @Iterations(5)
-     * @Warmup(1)
-     */
     public function bench_messaging_boot_on_dev()
     {
         putenv('APP_ENV=development');
@@ -60,7 +41,7 @@ public function bench_messaging_boot_on_dev()
 
     public function createApplication(): Application
     {
-        $app = require __DIR__ . '/../bootstrap/app.php';
+        $app = require \dirname(__DIR__, 2) . '/packages/Laravel/tests/Application/bootstrap/app.php';
 
         $app->make(Kernel::class)->bootstrap();
 

diff --git a/...fony/tests/Benchmark/EcotoneBenchmark.php → Monorepo/Benchmark/SymfonyBenchmark.php b/...fony/tests/Benchmark/EcotoneBenchmark.php → Monorepo/Benchmark/SymfonyBenchmark.php
@@ -1,51 +1,36 @@
 <?php
 
-namespace Benchmark;
+namespace Monorepo\Benchmark;
 
 use Ecotone\Messaging\Config\ConfiguredMessagingSystem;
 use Ecotone\SymfonyBundle\App\Kernel;
 
-class EcotoneBenchmark
+/**
+ * @Revs(10)
+ * @Iterations(5)
+ * @Warmup(1)
+ */
+class SymfonyBenchmark
 {
-    /**
-     * @Revs(10)
-     * @Iterations(5)
-     * @Warmup(1)
-     */
     public function bench_kernel_boot_on_prod()
     {
         $kernel = new Kernel('prod', false);
         $kernel->boot();
     }
 
-    /**
-     * @Revs(10)
-     * @Iterations(5)
-     * @Warmup(1)
-     */
     public function bench_kernel_boot_on_dev()
     {
         $kernel = new Kernel('dev', false);
         $kernel->boot();
     }
 
-    /**
-     * @Revs(10)
-     * @Iterations(5)
-     * @Warmup(1)
-     */
     public function bench_messaging_boot_on_prod()
     {
         $kernel = new Kernel('prod', false);
         $kernel->boot();
         $kernel->getContainer()->get(ConfiguredMessagingSystem::class);
     }
 
-    /**
-     * @Revs(10)
-     * @Iterations(5)
-     * @Warmup(1)
-     */
     public function bench_messaging_boot_on_dev()
     {
         $kernel = new Kernel('dev', false);

diff --git a/bin/phpbench-to-md.sh b/bin/phpbench-to-md.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+# Transform the table output of phpbench
+#  +------------------+-------------------------------------------+------+-----+----------+-----------+--------+
+#  | benchmark        | subject                                   | revs | its | mem_peak | mode      | rstdev |
+#  +------------------+-------------------------------------------+------+-----+----------+-----------+--------+
+#  | EcotoneBenchmark | bench_running_ecotone_lite_with_fail_fast | 1    | 1   | 3.186mb  | 151.997ms | ±0.00% |
+#  | EcotoneBenchmark | bench_running_ecotone_lite_with_cache     | 1    | 1   | 4.915mb  | 14.683ms  | ±0.00% |
+#  +------------------+-------------------------------------------+------+-----+----------+-----------+--------+
+#  (newline)
+#
+# into a valid markdown table:
+#  | benchmark        | subject                                   | revs | its | mem_peak | mode      | rstdev |
+#  | ---------------- | ----------------------------------------- | ---- | --- | -------- | --------- | ------ |
+#  | EcotoneBenchmark | bench_running_ecotone_lite_with_fail_fast | 1    | 1   | 3.186mb  | 150.799ms | ±0.00% |
+#  | EcotoneBenchmark | bench_running_ecotone_lite_with_cache     | 1    | 1   | 4.915mb  | 14.348ms  | ±0.00% |
+
+# Read the input from stdin
+# Remove the last two lines
+# Remove the first line
+# Replace the table borders with pipes
+# Print the result
+head -n -2 | sed -n -e '1d' -e 's/-+-/ | /g' -e 's/+-/| /g' -e 's/-+/ |/g' -e p
diff --git a/packages/Ecotone/composer.json b/packages/Ecotone/composer.json
@@ -44,8 +44,7 @@
         "phpunit/phpunit": "^9.5",
         "behat/behat": "^3.10",
         "phpstan/phpstan": "^1.8",
-        "symfony/expression-language": "^6.0",
-        "phpbench/phpbench": "^1.2"
+        "symfony/expression-language": "^6.0"
     },
     "scripts": {
         "tests:phpstan": "vendor/bin/phpstan",
@@ -55,9 +54,6 @@
         "tests:behat": [
             "vendor/bin/behat -vvv"
         ],
-        "tests:phpbench": [
-            "vendor/bin/phpbench run tests/Benchmark --bootstrap=./vendor/autoload.php"
-        ],
         "tests:ci": [
             "@tests:phpstan",
             "@tests:phpunit",

diff --git a/packages/Laravel/composer.json b/packages/Laravel/composer.json
@@ -43,8 +43,7 @@
         "phpstan/phpstan": "^1.8",
         "orchestra/testbench": "^7.6",
         "wikimedia/composer-merge-plugin": "^2.0",
-        "symfony/expression-language": "^6.0",
-        "phpbench/phpbench": "^1.2"
+        "symfony/expression-language": "^6.0"
     },
     "extra": {
         "laravel": {
@@ -69,9 +68,6 @@
         "tests:phpunit": [
             "vendor/bin/phpunit"
         ],
-        "tests:phpbench": [
-            "vendor/bin/phpbench run tests/Application/Benchmark --bootstrap=./vendor/autoload.php"
-        ],
         "tests:ci": [
             "@tests:phpstan",
             "@tests:phpunit"