diff --git a/scripts/degradation-tester/degradation-check.sh b/scripts/degradation-tester/degradation-check.sh index 5b6fd97980..787abd3bf7 100644 --- a/scripts/degradation-tester/degradation-check.sh +++ b/scripts/degradation-tester/degradation-check.sh @@ -15,7 +15,7 @@ for pkgPath in "${packagePaths[@]}"; do go test -bench=. -count=10 -benchmem "$pkgPath" | tee "$outputFile" - benchstat "$oldBenchmarks" "$outputFile" | tee "${benchStatFile}" + benchstat -format csv "$oldBenchmarks" "$outputFile" | tee "${benchStatFile}" degradation-tester "${configFile}" "${benchStatFile}" if [ $? -ne 0 ]; then @@ -25,6 +25,5 @@ for pkgPath in "${packagePaths[@]}"; do fi echo "✅ Degradation tests have passed for ${packageName} package." - rm "${benchStatFile}" "${outputFile}" done diff --git a/scripts/degradation-tester/go.mod b/scripts/degradation-tester/go.mod index bd4bdfd0a6..73cb54d5d4 100644 --- a/scripts/degradation-tester/go.mod +++ b/scripts/degradation-tester/go.mod @@ -3,3 +3,14 @@ module github.com/bloxapp/ssv/scripts/degradation-tester go 1.20 require gopkg.in/yaml.v3 v3.0.1 + +require ( + github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 // indirect + golang.org/x/perf v0.0.0-20240305160248-5eefbfdba9dd // indirect +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/testify v1.9.0 +) diff --git a/scripts/degradation-tester/go.sum b/scripts/degradation-tester/go.sum index a62c313c5b..9090f60ef6 100644 --- a/scripts/degradation-tester/go.sum +++ b/scripts/degradation-tester/go.sum @@ -1,3 +1,13 @@ +github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 h1:xlwdaKcTNVW4PtpQb8aKA4Pjy0CdJHEqvFbAnvR5m2g= +github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794/go.mod h1:7e+I0LQFUI9AXWxOfsQROs9xPhoJtbsyWcjJqDd4KPY= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/perf v0.0.0-20240305160248-5eefbfdba9dd h1:rglj7j7GZzz4GcR21t9lmupN+8ALC8L//rdzE/50vLE= +golang.org/x/perf v0.0.0-20240305160248-5eefbfdba9dd/go.mod h1:9aZNLn0je8D5R0rbpRog/X1gTnJt4uajOXR4k1WpzXk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/scripts/degradation-tester/main.go b/scripts/degradation-tester/main.go index 445317d4d4..6ca8fcf47a 100644 --- a/scripts/degradation-tester/main.go +++ b/scripts/degradation-tester/main.go @@ -2,6 +2,7 @@ package main import ( "bufio" + "encoding/csv" "fmt" "math" "os" @@ -32,6 +33,21 @@ type TestCase struct { AllocDelta float64 `yaml:"AllocDelta"` } +type Benchmark struct { + TestName string + OldValue float64 + OldCI float64 + NewValue float64 + NewCI float64 + VsBase string + P string +} + +type BenchStatOutput struct { + OsData string + benchmarks []Benchmark +} + func main() { if len(os.Args) < 3 { fmt.Println("Usage: degradation-tester ") @@ -50,6 +66,14 @@ func main() { } defer file.Close() + totalErrors := checkFile(file, config) + + if totalErrors > 0 { + os.Exit(1) + } +} + +func checkFile(file *os.File, config *Config) int { var currentSection string scanner := bufio.NewScanner(file) @@ -60,10 +84,13 @@ func main() { switch { case strings.Contains(line, "sec/op"): currentSection = "sec/op" + continue case strings.Contains(line, "B/op"): currentSection = "B/op" + continue case strings.Contains(line, "allocs/op"): currentSection = "allocs/op" + continue } totalErrors += checkLine(config, line, currentSection) @@ -73,9 +100,84 @@ func main() { fmt.Printf("Error reading results file: %v\n", err) } - if totalErrors > 0 { + return totalErrors +} + +func checkLine( + config *Config, + line string, + section string, +) int { + if line == "" { + return 0 + } + csvRowReader := csv.NewReader(strings.NewReader(line)) + csvRowReader.Comment = '#' + csvRowReader.Comma = ',' + + row, err := csvRowReader.Read() + if err != nil { + fmt.Printf("failed parsing CSV line %s with erorr: %v\n", line, err) os.Exit(1) } + + if len(row) != 7 { + // ignore all except the becnhmark result lines with exactly 7 columns + return 0 + } + + // The "geomean" represents a statistical summary (geometric mean) of multiple test results, + // not an individual test result, hence we should just skip it + if row[0] == "geomean" { + return 0 + } + + normalizedTestName := normalizeTestName(row[0]) + + oldValue, err := strconv.ParseFloat(row[1], 64) + if err != nil { + fmt.Printf("⚠️ Error parsing float: %v\n", err) + return 1 + } + oldChangeStr := row[2] + newValue, err := strconv.ParseFloat(row[3], 64) + if err != nil { + fmt.Printf("⚠️ Error parsing float: %v\n", err) + return 1 + } + newChangeStr := row[4] + + oldChange, err := strconv.ParseFloat(strings.TrimSuffix(oldChangeStr, "%"), 64) + if err != nil { + fmt.Printf("⚠️ Error parsing float: %v\n", err) + return 1 + } + newChange, err := strconv.ParseFloat(strings.TrimSuffix(newChangeStr, "%"), 64) + if err != nil { + fmt.Printf("⚠️ Error parsing float: %v\n", err) + return 1 + } + + threshold := getThresholdForTestCase(config, normalizedTestName, section) + + if math.Abs(oldChange-newChange) > threshold { + fmt.Printf("❌ Change in section %s for test %s exceeds threshold: %s\n", section, normalizedTestName, newChangeStr) + return 1 + } + + b := &Benchmark{ + TestName: row[0], + OldValue: oldValue, + OldCI: oldChange, + NewValue: newValue, + NewCI: newChange, + VsBase: row[5], + P: row[6], + } + + fmt.Printf("Debug: bench [TestName %s], [OldValue %f], [OldCI %f], [NewValue %f], [NewCI %f], [VsBase %s], [P %s]\n", b.TestName, b.OldValue, b.OldCI, b.NewValue, b.NewCI, b.VsBase, b.P) + + return 0 } func loadConfig(filename string) *Config { @@ -107,7 +209,7 @@ func loadConfig(filename string) *Config { return config } -func checkLine( +func checkLine2( config *Config, line string, section string, diff --git a/scripts/degradation-tester/main_test.go b/scripts/degradation-tester/main_test.go new file mode 100644 index 0000000000..bbba5b65e4 --- /dev/null +++ b/scripts/degradation-tester/main_test.go @@ -0,0 +1,52 @@ +package main + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParseBenchStatFile(t *testing.T) { + benchstatCsvFileName := "benchstat.csv" + f, err := os.Create(benchstatCsvFileName) + require.NoError(t, err) + + _, err = f.WriteString(stubBenchStatCSVFileData) + require.NoError(t, err) + + cfg := &Config{ + DefaultAllocDelta: 10, + DefaultOpDelta: 10, + } + f, err = os.Open(benchstatCsvFileName) + require.NoError(t, err) + totalErrors := checkFile(f, cfg) + require.Equal(t, 0, totalErrors) + + // clear test artefacts + defer func() { + _ = os.Remove(benchstatCsvFileName) + }() +} + +const stubBenchStatCSVFileData = ` +goos: darwin +goarch: amd64 +pkg: github.com/bloxapp/ssv/message/validation +cpu: xxxx +,./scripts/degradation-tester/benchmarks/validation_results_old.txt,,./scripts/degradation-tester/benchmarks/validation_results_new.txt +,sec/op,CI,sec/op,CI,vs base,P +VerifyRSASignature-8,0.000266723,2%,0.0002658285,1%,~,p=0.123 n=10 +geomean,0.0002667229999999998,,0.0002658284999999999,,-0.34% + +,./scripts/degradation-tester/benchmarks/validation_results_old.txt,,./scripts/degradation-tester/benchmarks/validation_results_new.txt +,B/op,CI,B/op,CI,vs base,P +VerifyRSASignature-8,1904.5,22%,1892.5,22%,~,p=0.927 n=10 +geomean,1904.4999999999998,,1892.4999999999995,,-0.63% + +,./scripts/degradation-tester/benchmarks/validation_results_old.txt,,./scripts/degradation-tester/benchmarks/validation_results_new.txt +,allocs/op,CI,allocs/op,CI,vs base,P +VerifyRSASignature-8,9.5,16%,9.5,16%,~,p=0.878 n=10 +geomean,9.500000000000002,,9.500000000000002,,+0.00% +`