Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

networking-bench: Update benchmarks payload #7056

Merged
merged 7 commits into from
Jan 9, 2025
Merged

networking-bench: Update benchmarks payload #7056

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
690e1bb
networking-bench: Update benchmarks payload
AndreiEres Jan 6, 2025
51d86cd
Add name to benches
AndreiEres Jan 6, 2025
269ae07
Preheat workers
AndreiEres Jan 7, 2025
48c99e8
Merge branch 'master' into AndreiEres/update-networking-charts
AndreiEres Jan 7, 2025
8517f0a
Turn on request benches
AndreiEres Jan 7, 2025
337a60f
Change sample size
AndreiEres Jan 7, 2025
cd51b73
Add preheating for request-response
AndreiEres Jan 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/networking-benchmarks.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ jobs:
uses: benchmark-action/github-action-benchmark@v1
with:
tool: "cargo"
name: ${{ env.BENCH }}
output-file-path: ./charts/${{ env.BENCH }}.txt
benchmark-data-dir-path: ./bench/${{ env.BENCH }}
github-token: ${{ steps.app-token.outputs.token }}
Expand All @@ -101,6 +102,7 @@ jobs:
uses: benchmark-action/github-action-benchmark@v1
with:
tool: "cargo"
name: ${{ env.BENCH }}
output-file-path: ./charts/${{ env.BENCH }}.txt
benchmark-data-dir-path: ./bench/${{ env.BENCH }}
github-token: ${{ steps.app-token.outputs.token }}
Expand Down
85 changes: 35 additions & 50 deletions substrate/client/network/benches/notifications_protocol.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,19 +36,16 @@ use std::{sync::Arc, time::Duration};
use substrate_test_runtime_client::runtime;
use tokio::{sync::Mutex, task::JoinHandle};

const SMALL_PAYLOAD: &[(u32, usize, &'static str)] = &[
// (Exponent of size, number of notifications, label)
(6, 100, "64B"),
(9, 100, "512B"),
(12, 100, "4KB"),
(15, 100, "64KB"),
];
const LARGE_PAYLOAD: &[(u32, usize, &'static str)] = &[
// (Exponent of size, number of notifications, label)
(18, 10, "256KB"),
(21, 10, "2MB"),
(24, 10, "16MB"),
(27, 10, "128MB"),
const NUMBER_OF_NOTIFICATIONS: usize = 10;
dmitry-markin marked this conversation as resolved.
Show resolved Hide resolved
const PAYLOAD: &[(u32, &'static str)] = &[
// (Exponent of size, label)
(6, "64B"),
(9, "512B"),
(12, "4KB"),
(15, "64KB"),
(18, "256KB"),
(21, "2MB"),
(24, "16MB"),
];
const MAX_SIZE: u64 = 2u64.pow(30);

Expand Down Expand Up @@ -255,64 +252,52 @@ async fn run_with_backpressure(setup: Arc<BenchSetup>, size: usize, limit: usize
let _ = tokio::join!(network1, network2);
}

fn run_benchmark(c: &mut Criterion, payload: &[(u32, usize, &'static str)], group: &str) {
fn run_benchmark(c: &mut Criterion) {
let rt = tokio::runtime::Runtime::new().unwrap();
let plot_config = PlotConfiguration::default().summary_scale(AxisScale::Logarithmic);
let mut group = c.benchmark_group(group);
let mut group = c.benchmark_group("notifications_protocol");
group.plot_config(plot_config);

let libp2p_setup = setup_workers::<runtime::Block, runtime::Hash, NetworkWorker<_, _>>(&rt);
for &(exponent, limit, label) in payload.iter() {
for &(exponent, label) in PAYLOAD.iter() {
let size = 2usize.pow(exponent);
group.throughput(Throughput::Bytes(limit as u64 * size as u64));
group.bench_with_input(
BenchmarkId::new("libp2p/serially", label),
&(size, limit),
|b, &(size, limit)| {
b.to_async(&rt).iter(|| run_serially(Arc::clone(&libp2p_setup), size, limit));
},
);
group.throughput(Throughput::Bytes(NUMBER_OF_NOTIFICATIONS as u64 * size as u64));
group.bench_with_input(BenchmarkId::new("libp2p/serially", label), &size, |b, &size| {
b.to_async(&rt)
.iter(|| run_serially(Arc::clone(&libp2p_setup), size, NUMBER_OF_NOTIFICATIONS));
});
group.bench_with_input(
BenchmarkId::new("libp2p/with_backpressure", label),
&(size, limit),
|b, &(size, limit)| {
b.to_async(&rt)
.iter(|| run_with_backpressure(Arc::clone(&libp2p_setup), size, limit));
&size,
|b, &size| {
b.to_async(&rt).iter(|| {
run_with_backpressure(Arc::clone(&libp2p_setup), size, NUMBER_OF_NOTIFICATIONS)
});
},
);
}
drop(libp2p_setup);

let litep2p_setup = setup_workers::<runtime::Block, runtime::Hash, Litep2pNetworkBackend>(&rt);
for &(exponent, limit, label) in payload.iter() {
for &(exponent, label) in PAYLOAD.iter() {
let size = 2usize.pow(exponent);
group.throughput(Throughput::Bytes(limit as u64 * size as u64));
group.bench_with_input(
BenchmarkId::new("litep2p/serially", label),
&(size, limit),
|b, &(size, limit)| {
b.to_async(&rt).iter(|| run_serially(Arc::clone(&litep2p_setup), size, limit));
},
);
group.throughput(Throughput::Bytes(NUMBER_OF_NOTIFICATIONS as u64 * size as u64));
group.bench_with_input(BenchmarkId::new("litep2p/serially", label), &size, |b, &size| {
b.to_async(&rt)
.iter(|| run_serially(Arc::clone(&litep2p_setup), size, NUMBER_OF_NOTIFICATIONS));
});
group.bench_with_input(
BenchmarkId::new("litep2p/with_backpressure", label),
&(size, limit),
|b, &(size, limit)| {
b.to_async(&rt)
.iter(|| run_with_backpressure(Arc::clone(&litep2p_setup), size, limit));
&size,
|b, &size| {
b.to_async(&rt).iter(|| {
run_with_backpressure(Arc::clone(&litep2p_setup), size, NUMBER_OF_NOTIFICATIONS)
});
},
);
}
drop(litep2p_setup);
}

fn run_benchmark_with_small_payload(c: &mut Criterion) {
run_benchmark(c, SMALL_PAYLOAD, "notifications_protocol/small_payload");
}

fn run_benchmark_with_large_payload(c: &mut Criterion) {
run_benchmark(c, LARGE_PAYLOAD, "notifications_protocol/large_payload");
}

criterion_group!(benches, run_benchmark_with_small_payload, run_benchmark_with_large_payload);
criterion_group!(benches, run_benchmark);
criterion_main!(benches);
67 changes: 25 additions & 42 deletions substrate/client/network/benches/request_response_protocol.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,19 +37,16 @@ use substrate_test_runtime_client::runtime;
use tokio::{sync::Mutex, task::JoinHandle};

const MAX_SIZE: u64 = 2u64.pow(30);
const SMALL_PAYLOAD: &[(u32, usize, &'static str)] = &[
// (Exponent of size, number of requests, label)
(6, 100, "64B"),
(9, 100, "512B"),
(12, 100, "4KB"),
(15, 100, "64KB"),
];
const LARGE_PAYLOAD: &[(u32, usize, &'static str)] = &[
// (Exponent of size, number of requests, label)
(18, 10, "256KB"),
(21, 10, "2MB"),
(24, 10, "16MB"),
(27, 10, "128MB"),
const NUMBER_OF_REQUESTS: usize = 10;
const PAYLOAD: &[(u32, &'static str)] = &[
// (Exponent of size, label)
(6, "64B"),
(9, "512B"),
(12, "4KB"),
(15, "64KB"),
(18, "256KB"),
(21, "2MB"),
(24, "16MB"),
];

pub fn create_network_worker<B, H, N>() -> (
Expand Down Expand Up @@ -269,49 +266,35 @@ async fn run_with_backpressure(setup: Arc<BenchSetup>, size: usize, limit: usize
let _ = tokio::join!(network1, network2);
}

fn run_benchmark(c: &mut Criterion, payload: &[(u32, usize, &'static str)], group: &str) {
fn run_benchmark(c: &mut Criterion) {
let rt = tokio::runtime::Runtime::new().unwrap();
let plot_config = PlotConfiguration::default().summary_scale(AxisScale::Logarithmic);
let mut group = c.benchmark_group(group);
let mut group = c.benchmark_group("request_response_protocol");
group.plot_config(plot_config);

let libp2p_setup = setup_workers::<runtime::Block, runtime::Hash, NetworkWorker<_, _>>(&rt);
for &(exponent, limit, label) in payload.iter() {
for &(exponent, label) in PAYLOAD.iter() {
let size = 2usize.pow(exponent);
group.throughput(Throughput::Bytes(limit as u64 * size as u64));
group.bench_with_input(
BenchmarkId::new("libp2p/serially", label),
&(size, limit),
|b, &(size, limit)| {
b.to_async(&rt).iter(|| run_serially(Arc::clone(&libp2p_setup), size, limit));
},
);
group.throughput(Throughput::Bytes(NUMBER_OF_REQUESTS as u64 * size as u64));
group.bench_with_input(BenchmarkId::new("libp2p/serially", label), &size, |b, &size| {
b.to_async(&rt)
.iter(|| run_serially(Arc::clone(&libp2p_setup), size, NUMBER_OF_REQUESTS));
});
}
drop(libp2p_setup);

// TODO: NetworkRequest::request should be implemented for Litep2pNetworkService
let litep2p_setup = setup_workers::<runtime::Block, runtime::Hash, Litep2pNetworkBackend>(&rt);
// for &(exponent, limit, label) in payload.iter() {
// for &(exponent, label) in PAYLOAD.iter() {
// let size = 2usize.pow(exponent);
// group.throughput(Throughput::Bytes(limit as u64 * size as u64));
// group.bench_with_input(
// BenchmarkId::new("litep2p/serially", label),
// &(size, limit),
// |b, &(size, limit)| {
// b.to_async(&rt).iter(|| run_serially(Arc::clone(&litep2p_setup), size, limit));
// },
// );
// group.throughput(Throughput::Bytes(NUMBER_OF_REQUESTS as u64 * size as u64));
// group.bench_with_input(BenchmarkId::new("litep2p/serially", label), &size, |b, &size| {
// b.to_async(&rt)
// .iter(|| run_serially(Arc::clone(&litep2p_setup), size, NUMBER_OF_REQUESTS));
// });
// }
drop(litep2p_setup);
}

fn run_benchmark_with_small_payload(c: &mut Criterion) {
run_benchmark(c, SMALL_PAYLOAD, "request_response_benchmark/small_payload");
}

fn run_benchmark_with_large_payload(c: &mut Criterion) {
run_benchmark(c, LARGE_PAYLOAD, "request_response_benchmark/large_payload");
}

criterion_group!(benches, run_benchmark_with_small_payload, run_benchmark_with_large_payload);
criterion_group!(benches, run_benchmark);
criterion_main!(benches);
Loading