diff --git a/native/hat/report/MAD.svg b/native/hat/report/MAD.svg index bbda3f0..353b467 100644 --- a/native/hat/report/MAD.svg +++ b/native/hat/report/MAD.svg @@ -41,101 +41,76 @@ - + 0 - - 0.01 + + 0.5 - - 0.02 + + 1 - - 0.03 + + 1.5 - - 0.04 + + 2 - - 0.05 + + 2.5 - - 0.06 + + 1.8 - - 0.07 + + 2 - - 0.08 + + 2.2 - - 5 + + 2.4 - - 10 + + 2.6 - - 15 - - - - - 20 - - - - - 25 - - - - - 30 - - - - - 35 - - - - - 40 + + 2.8 - + Density (a.u.) - + Average time (µs) @@ -150,69 +125,69 @@ - + Confidence interval @@ -225,57 +200,57 @@ - + @@ -288,16 +263,16 @@ - + - + - + sensor read: MAD diff --git a/native/hat/report/SD.svg b/native/hat/report/SD.svg index ce77e48..637bc4a 100644 --- a/native/hat/report/SD.svg +++ b/native/hat/report/SD.svg @@ -41,92 +41,97 @@ - + 0 - - 1 + + 0.05 - - 2 + + 0.1 - - 3 + + 0.15 - - 4 + + 0.2 - - 5 + + 0.25 - - 6 + + 0.3 - - 7 + + 2 - - 8 + + 3 - - 9 + + 4 - - 10 + + 5 + + + + + 6 - - 0.95 + + 7 - - 1 + + 8 - - 1.05 + + 9 - - 1.1 + + 10 - + Density (a.u.) - - Average time (ms) + + Average time (µs) @@ -140,69 +145,69 @@ - + Confidence interval @@ -215,57 +220,57 @@ - + @@ -278,16 +283,16 @@ - + - + - + sensor read: SD diff --git a/wasmtime/host/report/relative_iteration_times_small.svg b/native/hat/report/both/iteration_times.svg similarity index 52% rename from wasmtime/host/report/relative_iteration_times_small.svg rename to native/hat/report/both/iteration_times.svg index 12ec99d..749a41d 100644 --- a/wasmtime/host/report/relative_iteration_times_small.svg +++ b/native/hat/report/both/iteration_times.svg @@ -1,6 +1,6 @@ @@ -10,7 +10,7 @@ - + @@ -45,12 +45,12 @@ - + - - 78 + + 6.95 @@ -58,12 +58,12 @@ - + - - 79 + + 7 @@ -71,12 +71,38 @@ - + - - 80 + + 7.05 + + + + + + + + + + + + + 7.1 + + + + + + + + + + + + + 7.15 @@ -84,12 +110,12 @@ - + - - 81 + + 7.2 @@ -97,12 +123,12 @@ - + - - 82 + + 7.25 @@ -110,12 +136,12 @@ - + - - 83 + + 7.3 @@ -123,12 +149,12 @@ - + - - 84 + + 7.35 @@ -136,12 +162,12 @@ - + - - 85 + + 7.4 @@ -149,11 +175,11 @@ - + - + 0 @@ -162,11 +188,11 @@ - + - + 10 @@ -175,11 +201,11 @@ - + - + 20 @@ -188,11 +214,11 @@ - + - + 30 @@ -201,11 +227,11 @@ - + - + 40 @@ -214,11 +240,11 @@ - + - + 50 @@ -227,11 +253,11 @@ - + - + 60 @@ -240,11 +266,11 @@ - + - + 70 @@ -253,11 +279,11 @@ - + - + 80 @@ -266,11 +292,11 @@ - + - + 90 @@ -279,25 +305,25 @@ - + - + 100 - + - + Average Iteration Time (ms) - + Sample @@ -305,210 +331,222 @@ Base - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + Base + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Current - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + Current + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -518,7 +556,12 @@ - + + + + sensor read + + diff --git a/native/hat/report/both/pdf.svg b/native/hat/report/both/pdf.svg new file mode 100644 index 0000000..f5e5961 --- /dev/null +++ b/native/hat/report/both/pdf.svg @@ -0,0 +1,338 @@ + + + +Gnuplot +Produced by GNUPLOT 5.4 patchlevel 4 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + + + 20 + + + + + 40 + + + + + 60 + + + + + 80 + + + + + 100 + + + + + 120 + + + + + 6.9 + + + + + 6.95 + + + + + 7 + + + + + 7.05 + + + + + 7.1 + + + + + 7.15 + + + + + 7.2 + + + + + 7.25 + + + + + 7.3 + + + + + 7.35 + + + + + 7.4 + + + + + + + + + Density (a.u.) + + + + + Average time (ms) + + + + + Base PDF + + + Base PDF + + + + + + + + + + Base Mean + + + + + Base Mean + + + + + + New PDF + + + + + New PDF + + + + + + + + + + New Mean + + + + + New Mean + + + + + + + + + + + + + + sensor read + + + + + + + diff --git a/wasmtime/host/report/change/mean.svg b/native/hat/report/change/mean.svg similarity index 51% rename from wasmtime/host/report/change/mean.svg rename to native/hat/report/change/mean.svg index 1985c0a..5ba93a6 100644 --- a/wasmtime/host/report/change/mean.svg +++ b/native/hat/report/change/mean.svg @@ -46,78 +46,73 @@ - - 20 + + 100 - - 40 + + 200 - - 60 + + 300 - - 80 + + 400 - - 100 + + 500 - - 120 + + 600 - - 140 + + 700 - - 160 - - - - - 180 + + 800 - 200 + 900 - - -0.4 + + -0.05 - - -0.2 + + 0 - - 0 + + 0.05 - - 0.2 + + 0.1 - - 0.4 + + 0.15 @@ -145,69 +140,69 @@ - + Confidence interval @@ -220,32 +215,32 @@ - + Noise threshold diff --git a/native/hat/report/change/median.svg b/native/hat/report/change/median.svg new file mode 100644 index 0000000..6116dc3 --- /dev/null +++ b/native/hat/report/change/median.svg @@ -0,0 +1,325 @@ + + + +Gnuplot +Produced by GNUPLOT 5.4 patchlevel 4 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + + + 1000 + + + + + 2000 + + + + + 3000 + + + + + 4000 + + + + + 5000 + + + + + 6000 + + + + + 7000 + + + + + 8000 + + + + + 9000 + + + + + -0.06 + + + + + -0.055 + + + + + -0.05 + + + + + -0.045 + + + + + -0.04 + + + + + -0.035 + + + + + -0.03 + + + + + + + + + Density (a.u.) + + + + + Relative change (%) + + + + + Bootstrap distribution + + + + + Bootstrap distribution + + + + + + Confidence interval + + + + + Confidence interval + + + + + + + + + + Point estimate + + + + + Point estimate + + + + + + Noise threshold + + + + + Noise threshold + + + + + + + + + + + + + + + + + + + + sensor read: median + + + + + + + diff --git a/native/hat/report/change/t-test.svg b/native/hat/report/change/t-test.svg new file mode 100644 index 0000000..686c48b --- /dev/null +++ b/native/hat/report/change/t-test.svg @@ -0,0 +1,250 @@ + + + +Gnuplot +Produced by GNUPLOT 5.4 patchlevel 4 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + + + 0.05 + + + + + 0.1 + + + + + 0.15 + + + + + 0.2 + + + + + 0.25 + + + + + 0.3 + + + + + 0.35 + + + + + 0.4 + + + + + -4 + + + + + -3 + + + + + -2 + + + + + -1 + + + + + 0 + + + + + 1 + + + + + 2 + + + + + 3 + + + + + 4 + + + + + + + + + Density + + + + + t score + + + + + t distribution + + + t distribution + + + + + + + + + + t statistic + + + + + t statistic + + + + + + + + + + + + + + sensor read: Welch t test + + + + + + + diff --git a/native/hat/report/index.html b/native/hat/report/index.html index a728ece..766c7f9 100644 --- a/native/hat/report/index.html +++ b/native/hat/report/index.html @@ -118,33 +118,33 @@

Additional Statistics:

R² - 0.0179936 - 0.0186438 - 0.0179536 + 0.0001052 + 0.0001081 + 0.0001031 Mean - 5.2432 ms - 5.4459 ms - 5.6550 ms + 7.3148 ms + 7.3159 ms + 7.3174 ms Std. Dev. - 959.75 µs - 1.0617 ms - 1.1248 ms + 2.1435 µs + 6.5690 µs + 10.006 µs Median - 4.6886 ms - 4.6908 ms - 4.7069 ms + 7.3141 ms + 7.3147 ms + 7.3155 ms MAD - 6.9690 µs - 10.719 µs - 36.807 µs + 1.7722 µs + 2.1332 µs + 2.7432 µs @@ -182,6 +182,82 @@

Understanding this report:

documentation for more details on the additional statistics.

+
+

Change Since Previous Benchmark

+
+ + + + + + + +
+ + PDF Comparison + + + + Iteration Time Comparison + +
+
+
+
+
+

Additional Statistics:

+ + + + + + + + + + + + + + + + + + + +
Lower boundEstimateUpper bound
Change in time-0.0488%+0.0431%+0.1749%(p = 0.56 > + 0.05)
+ No change in performance detected. +
+
+

Additional Plots:

+ +
+
+
+

Understanding this report:

+

The plot on the left shows the probability of the function taking a certain amount of time. The red + curve represents the saved measurements from the last time this benchmark was run, while the blue curve + shows the measurements from this run. The lines represent the mean time per iteration. Click on the + plot for a larger view.

+

The plot on the right shows the iteration times for the two measurements. Again, the red dots represent + the previous measurement while the blue dots show the current measurement.

+

See the + documentation for more details on the additional statistics.

+