') - - -## Generate Overview Plot -overview_plot <- compare_runtime_ratio_of_suites_plot( - data_chg_slow |> filter(criterion == "total"), - slower_runtime_ratio, faster_runtime_ratio, - fast_color, slow_color, chg_colors) -ggsave('overview.svg', overview_plot$plot, "svg", output_dir, width = 4.5, height = overview_plot$height, units = "in") -ggsave('overview.png', overview_plot$plot, "png", output_dir, width = 4.5, height = overview_plot$height, units = "in") - -out('

Result Overview

') - -out('') - -out('
-
Number of Run Configurations
-
', stats_all_total$num_benchmarks, '
- -
Run time (geomean)
-
', round(stats_all_total$geomean, 3), ' (min. ', r2(stats_all_total$min), - ', max. ', r2(stats_all_total$max), ')
- -
GC time (geomean)
-
', round(stats_all_gctime$geomean, 3), ' (min. ', r2(stats_all_gctime$min), - ', max. ', r2(stats_all_gctime$max), ')
- -
Allocated bytes (geomean)
-
', round(stats_all_allocated$geomean, 3), ' (min. ', r2(stats_all_allocated$min), - ', max. ', r2(stats_all_allocated$max), ')
-
') - - - -if (nrow(not_in_both) > 0) { - nib <- not_in_both |> - select(commitid, exe, suite, bench, - varvalue, cores, inputsize, extraargs) - out("

Changes in Benchmark Set

") - library(xtable) - str <- print(xtable(nib), type="html", - print.results=FALSE, - html.table.attributes = getOption("xtable.html.table.attributes", "")) - out(str) -} - -out("

Benchmark Performance

") - -perf_diff_table_es <- function(data_es, stats_es, warmup_es, profiles_es, start_row_count, group, colors, colors_light, show_warmup) { - group_col <- enquo(group) - row_count <- start_row_count - - - - out('') - out(' - - - - - - - - - - -') - - # b <- "DeltaBlue" - # data_ea <- data_b - - for (b in levels(data_es$bench)) { data_b <- data_es |> filter(bench == b) |> droplevels() - for (v in levels(data_b$varvalue)) { data_v <- data_b |> filter(varvalue == v) |> droplevels() - for (c in levels(data_v$cores)) { data_c <- data_v |> filter(cores == c) |> droplevels() - for (i in levels(data_c$inputsize)) { data_i <- data_c |> filter(inputsize == i) |> droplevels() - for (ea in levels(data_i$extraargs)) { data_ea <- data_i |> filter(extraargs == ea) |> droplevels() - - for (en in levels(data_ea$envid)) { data_en <- data_ea |> filter(envid == en) |> droplevels() - - - - args <- "" - if (length(levels(data_b$varvalue)) > 1) { args <- paste0(args, v) } - if (length(levels(data_v$cores)) > 1) { args <- paste0(args, c) } - if (length(levels(data_c$inputsize)) > 1) { args <- paste0(args, i) } - if (length(levels(data_i$extraargs)) > 1) { args <- paste0(args, ea) } - - if (nchar(args) > 0) { - args <- paste0('', args, '') - } - - # capture the beginning of the path but leave the last element of it - # this regex is also used in render.js's renderBenchmark() function - cmdline <- str_replace_all(data_en$cmdline[[1]], "^([^\\s]*)\\/([^\\s]+\\s.*$)", "\\2") - - # format all environment information into a single string - if (!is.null(environments)) { - env <- environments |> - filter(envid == levels(data_en$envid)) |> - unique(incomparables = FALSE) - environment_str <- paste0(env$hostname, - " | ", env$ostype, " | ", as_human_mem(env$memory), - " | ", env$cpu, " | ", as_human_hz(env$clockspeed)) - } - - stats_b_total <- stats_es |> - ungroup() |> - filter(bench == b, varvalue == v, cores == c, inputsize == i, extraargs == ea, criterion == "total") - stats_b_gctime <- stats_es |> - ungroup() |> - filter(bench == b, varvalue == v, cores == c, inputsize == i, extraargs == ea, criterion == "GC time") - stats_b_allocated <- stats_es |> - ungroup() |> - filter(bench == b, varvalue == v, cores == c, inputsize == i, extraargs == ea, criterion == "Allocated") - - if ("commitid" %in% colnames(stats_b_total)) { - stats_b_total <- stats_b_total |> - filter(commitid == change_hash6) - } - stats_b_total <- stats_b_total |> droplevels() - - if ("commitid" %in% colnames(stats_b_gctime)) { - stats_b_gctime <- stats_b_gctime |> - filter(commitid == change_hash6) - } - stats_b_gctime <- stats_b_gctime |> droplevels() - - if ("commitid" %in% colnames(stats_b_allocated)) { - stats_b_allocated <- stats_b_allocated |> - filter(commitid == change_hash6) - } - stats_b_allocated <- stats_b_allocated |> droplevels() - - group_size <- (data_en |> - filter(criterion == "total") |> - select(!!group_col) |> - unique() |> - count())$n - - if (nrow(stats_b_total) > 0) { - out('') - out('') - out('\n') - - if (nrow(stats_b_total) == 1) { - out('\n') - out('\n') - out('\n') - out('\n') - out('\n') - out('\n') - out('\n') - } else { - exes <- levels(stats_b_total$exe) - common_start <- common_string_start(exes) - - out('\n') - - out('\n') - - - out('\n') - - out('\n') - - out('\n') - - out('\n') - - out('\n') - } - - out(''); - out('\n') - } else { - out('') - out('\n') - out('') - } - } } } } } - } - - out('
#Mmedian time
in ', levels((data_es |> filter(criterion == "total") |> droplevels())$unit), '		time diff %median GC
time in ', levels((data_es |> filter(criterion == "GC time") |> droplevels())$unit), '		GC diff %median
allocated ', levels((data_es |> filter(criterion == "Allocated") |> droplevels())$unit), '		Alloc diff %
', b, args, '') - p <- small_inline_comparison(data_en, !!group_col, colors, colors_light) - img_file <- paste0('inline-', row_count, '.svg') - ggsave(img_file, p, "svg", output_dir, width = 3.5, height = 0.12 + 0.14 * group_size, units = "in") - out('') - - row_count <- row_count + 1 - out('', stats_b_total$samples, '', r2(stats_b_total$median), '', pro(stats_b_total$change_m), '', r0(stats_b_gctime$median), '', pro(stats_b_gctime$change_m), '', as_human_mem(stats_b_allocated$median), '', pro(stats_b_allocated$change_m), '') - first <- TRUE - for (e in exes) { - if (first) { - first <- FALSE - } else { - out('
\n') - } - out(substring(e, common_start) , " ", filter(stats_b_total, exe == e)$samples) - } - out('		') - first <- TRUE - for (e in exes) { - if (first) { - first <- FALSE - } else { - out('
\n') - } - out(r2(filter(stats_b_total, exe == e)$median)) - } - out('		') - first <- TRUE - for (e in exes) { - if (first) { - first <- FALSE - } else { - out('
\n') - } - out('', pro(filter(stats_b_total, exe == e)$change_m), '') - } - out('		') - first <- TRUE - for (e in exes) { - if (first) { - first <- FALSE - } else { - out('
\n') - } - out(r0(filter(stats_b_gctime, exe == e)$median)) - } - out('		') - first <- TRUE - for (e in exes) { - if (first) { - first <- FALSE - } else { - out('
\n') - } - out('', pro(filter(stats_b_gctime, exe == e)$change_m), '') - } - out('		') - first <- TRUE - for (e in exes) { - if (first) { - first <- FALSE - } else { - out('
\n') - } - out(as_human_mem(filter(stats_b_allocated, exe == e)$median)) - } - out('		') - first <- TRUE - for (e in exes) { - if (first) { - first <- FALSE - } else { - out('
\n') - } - out('', pro(filter(stats_b_allocated, exe == e)$change_m), '') - } - out('		\n') - if (!is.null(environments)) { - out('') - } - - warmup_ea <- warmup_es |> - filter(bench == b, varvalue == v, cores == c, inputsize == i, extraargs == ea) - - if (nrow(warmup_ea) > 0 && show_warmup) { - img_file <- paste0('warmup-', row_count, '.svg') - p <- warmup_plot(warmup_ea, !!group_col, colors) - ggsave(img_file, p, "svg", output_dir, width = 6, height = 2.5, units = "in") - out('\n') - } - - if (!is.null(profiles_es)) { - profiles_for_bench <- profiles_es |> - filter(bench == b, varvalue == v, cores == c, inputsize == i, extraargs == ea) |> - select(commitid, runid, trialid) |> - unite("id", commitid, runid, trialid, sep = "/") - - if (nrow(profiles_for_bench) > 0) { - ids <- str_flatten(profiles_for_bench$id, ",") - out('\n') - } - } - - benchmark_id_json <- paste0( - '{"b":"', b, - '","e":"', levels(data_en$exe), - '","s":"', levels(data_en$suite), '"') - - if (length(levels(data_b$varvalue)) > 1) { benchmark_id_json <- paste0(benchmark_id_json, ',"v":"', v, '"') } - if (length(levels(data_v$cores)) > 1) { benchmark_id_json <- paste0(benchmark_id_json, ',"c":"', c, '"') } - if (length(levels(data_c$inputsize)) > 1) { benchmark_id_json <- paste0(benchmark_id_json, ',"i":"', i, '"') } - if (length(levels(data_i$extraargs)) > 1) { benchmark_id_json <- paste0(benchmark_id_json, ',"ea":"', ea, '"') } - - benchmark_id_json <- paste0(benchmark_id_json, '}') - - out('\n') - - out('
', b, 'missing in one of the data sets
') - row_count -} - -perf_diff_table <- function(norm, stats, start_row_count) { - # e <- "TruffleSOM-graal-bc" - - row_count <- start_row_count - - for (e in levels(norm$exe)) { data_e <- norm |> filter(exe == e) |> droplevels() - for (s in levels(data_e$suite)) { data_s <- data_e |> filter(suite == s) |> droplevels() - # e <- "TruffleSOM-graal" - # s <- "macro-steady" - out('
') - out('

', s, '

') - out('
Executor: ', e, "
") - - stats_es <- stats |> - ungroup() |> - filter(exe == e, suite == s) - - warmup_es <- warmup |> - ungroup() |> - filter(exe == e, suite == s) - - if (is.null(profiles)) { - profiles_es <- NULL - } else { - profiles_es <- profiles |> - filter(exe == e, suite == s) - } - - row_count <- perf_diff_table_es( - data_s, stats_es, warmup_es, profiles_es, - row_count, commitid, chg_colors, chg_colors_light, TRUE) - - out('
') # exe-suite-group - } - } - row_count -} - -row_count <- perf_diff_table(norm, stats, 0) - - -## Output the Executor Comparison - - -if (nrow(suites_for_comparison) > 0) { - out('

Executor Comparisons

\n') - - for (s in suites_for_comparison$suite) { - # s <- "macro-startup" - out('

', s ,'

\n') - - change_s <- change_data |> - filter(suite == s) |> - droplevels() - exes <- sort(levels(change_s$exe)) - baseline_exe <- exes[[1]] - - out("

Baseline: ", baseline_exe, "

") - - - warmup_s <- warmup |> - restrict_to_change_data() |> - filter(suite == s) - - peak_s <- peak |> - restrict_to_change_data() |> - filter(suite == s) - - base_s <- peak_s |> - filter(exe == baseline_exe) |> - group_by(bench, - varvalue, cores, inputsize, extraargs, criterion) |> - summarise(base_mean = mean(value), - base_median = median(value), - .groups = "drop") - - norm_s <- peak_s |> - left_join(base_s, by = c( - "bench", "varvalue", "cores", "inputsize", "extraargs", "criterion")) |> - group_by(bench, varvalue, cores, inputsize, extraargs, criterion) |> - transform(ratio_mean = value / base_mean, - ratio_median = value / base_median) - - - stats_s <- norm_s |> - group_by(exe, bench, - varvalue, cores, inputsize, extraargs, criterion) |> - filter(is.na(warmup) | iteration >= warmup) |> - calculate_stats() - - not_in_both_s <- stats_s |> - filter(is.na(ratio) & criterion == "total") - - stats_s <- stats_s |> - filter(!(is.na(ratio) & criterion == "total")) - - p <- ggplot(stats_s |> filter(criterion == "total"), aes(ratio, exe, fill=exe)) + - geom_vline(aes(xintercept=1), colour="#999999", linetype="solid") + - geom_vline(aes(xintercept=slower_runtime_ratio), colour="#cccccc", linetype="dashed") + - geom_vline(aes(xintercept=faster_runtime_ratio), colour="#cccccc", linetype="dashed") + - geom_boxplot(aes(colour = exe), - outlier.size = 0.9, - outlier.alpha = 0.6) + - stat_summary(fun = negative_geometric.mean, - size = 1, colour = "#503000", geom = "point") + - scale_x_log10() + - scale_y_discrete(limits = rev) + - ylab("") + - #coord_cartesian(xlim=c(0.5, 2.5)) + - theme_simple(8) + - scale_color_manual(values = exes_colors) + - scale_fill_manual(values = exes_colors_light) + - # scale_fill_manual(breaks=c("slower", "faster", "indeterminate"), - # values=c(slow_color, fast_color, NA)) + - theme(legend.position = "none") - - ggsave(paste0('overview.', s, '.svg'), p, "svg", output_dir, width = 4.5, height = 2.5, units = "in") - ggsave(paste0('overview.', s, '.png'), p, "png", output_dir, width = 4.5, height = 2.5, units = "in") - - out('') - - row_count <- perf_diff_table_es( - norm_s, stats_s, warmup_s, NULL, row_count + 1, exe, exes_colors, exes_colors_light, FALSE) - } - -} - -time <- timing.stop() - -time <- format(time, digits = 2, nsmall = 2) -out('
Run time of Report: ', time, '
') - -out('