diff --git a/.github/workflows/benches.yml b/.github/workflows/benches.yml
new file mode 100644
index 00000000000..0e90da7efa0
--- /dev/null
+++ b/.github/workflows/benches.yml
@@ -0,0 +1,40 @@
+name: benches
+
+on:
+  push:
+    branches:
+      - "main"
+  pull_request:
+  # `workflow_dispatch` allows CodSpeed to trigger backtest
+  # performance analysis in order to generate initial data.
+  workflow_dispatch:
+
+jobs:
+  benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+      - uses: dtolnay/rust-toolchain@stable
+
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            pyo3-benches/target
+            target
+          key: cargo-${{ runner.os }}-bench-${{ hashFiles('**/Cargo.toml') }}
+        continue-on-error: true
+
+      - name: Install cargo-codspeed
+        run: cargo install cargo-codspeed
+
+      - name: Install nox
+        run: pip install nox
+
+      - name: Run the benchmarks
+        uses: CodSpeedHQ/action@v1
+        with:
+          run: nox -s codspeed
+          token: ${{ secrets.CODSPEED_TOKEN }}
diff --git a/noxfile.py b/noxfile.py
index c14d0283e4e..e17a19b9623 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -137,6 +137,19 @@ def bench(session: nox.Session) -> bool:
     _run_cargo(session, "bench", _BENCHES, *session.posargs)
 
 
+@nox.session()
+def codspeed(session: nox.Session) -> bool:
+    # rust benchmarks
+    os.chdir(PYO3_DIR / "pyo3-benches")
+    _run_cargo(session, "codspeed", "build")
+    _run_cargo(session, "codspeed", "run")
+    # python benchmarks
+    os.chdir(PYO3_DIR / "pytests")
+    session.install("-r", "requirements-dev.txt", "pytest-codspeed")
+    session.install(".")
+    _run(session, "pytest", "--codspeed", external=True)
+
+
 @nox.session(name="clippy-all", venv_backend="none")
 def clippy_all(session: nox.Session) -> None:
     success = True
diff --git a/pyo3-benches/Cargo.toml b/pyo3-benches/Cargo.toml
index cc82fb00fec..ade2ae74362 100644
--- a/pyo3-benches/Cargo.toml
+++ b/pyo3-benches/Cargo.toml
@@ -7,11 +7,13 @@ edition = "2021"
 publish = false
 
 [dependencies]
-pyo3 = { path = "../", features = ["auto-initialize"] }
+pyo3 = { path = "../", features = ["auto-initialize", "full"] }
 
 [dev-dependencies]
 criterion = "0.5.1"
+codspeed-criterion-compat = "2.2"
 num-bigint = "0.4.3"
+rust_decimal = { version = "1.0.0", default-features = false }
 
 [[bench]]
 name = "bench_any"
@@ -32,17 +34,14 @@ harness = false
 [[bench]]
 name = "bench_decimal"
 harness = false
-required-features = ["pyo3/rust_decimal"]
 
 [[bench]]
 name = "bench_dict"
 harness = false
-required-features = ["pyo3/hashbrown"]
 
 [[bench]]
 name = "bench_frompyobject"
 harness = false
-required-features = ["pyo3/macros"]
 
 [[bench]]
 name = "bench_gil"
@@ -55,7 +54,6 @@ harness = false
 [[bench]]
 name = "bench_pyclass"
 harness = false
-required-features = ["pyo3/macros"]
 
 [[bench]]
 name = "bench_pyobject"
@@ -64,7 +62,6 @@ harness = false
 [[bench]]
 name = "bench_set"
 harness = false
-required-features = ["pyo3/hashbrown"]
 
 [[bench]]
 name = "bench_tuple"
@@ -81,6 +78,5 @@ harness = false
 [[bench]]
 name = "bench_bigint"
 harness = false
-required-features = ["pyo3/num-bigint"]
 
 [workspace]
diff --git a/pyo3-benches/benches/bench_any.rs b/pyo3-benches/benches/bench_any.rs
index 765497fa079..bfd010efd19 100644
--- a/pyo3-benches/benches/bench_any.rs
+++ b/pyo3-benches/benches/bench_any.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::{
     types::{
diff --git a/pyo3-benches/benches/bench_bigint.rs b/pyo3-benches/benches/bench_bigint.rs
index 3df24d210d4..d3c71629ba4 100644
--- a/pyo3-benches/benches/bench_bigint.rs
+++ b/pyo3-benches/benches/bench_bigint.rs
@@ -1,4 +1,4 @@
-use criterion::{black_box, criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::{types::PyDict, PyAny, Python};
 
diff --git a/pyo3-benches/benches/bench_call.rs b/pyo3-benches/benches/bench_call.rs
index dd98c15905f..50772097961 100644
--- a/pyo3-benches/benches/bench_call.rs
+++ b/pyo3-benches/benches/bench_call.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::prelude::*;
 
diff --git a/pyo3-benches/benches/bench_comparisons.rs b/pyo3-benches/benches/bench_comparisons.rs
index bfa4ac63fa4..ffd4c1a452f 100644
--- a/pyo3-benches/benches/bench_comparisons.rs
+++ b/pyo3-benches/benches/bench_comparisons.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::{prelude::*, pyclass::CompareOp, Python};
 
diff --git a/pyo3-benches/benches/bench_decimal.rs b/pyo3-benches/benches/bench_decimal.rs
index 7a370ac3505..41e0bf809ca 100644
--- a/pyo3-benches/benches/bench_decimal.rs
+++ b/pyo3-benches/benches/bench_decimal.rs
@@ -1,4 +1,4 @@
-use criterion::{black_box, criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::prelude::*;
 use pyo3::types::PyDict;
diff --git a/pyo3-benches/benches/bench_dict.rs b/pyo3-benches/benches/bench_dict.rs
index 64398a65e39..aa807e5b00d 100644
--- a/pyo3-benches/benches/bench_dict.rs
+++ b/pyo3-benches/benches/bench_dict.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::types::IntoPyDict;
 use pyo3::{prelude::*, types::PyMapping};
diff --git a/pyo3-benches/benches/bench_err.rs b/pyo3-benches/benches/bench_err.rs
index e31945122e6..998ed6975b0 100644
--- a/pyo3-benches/benches/bench_err.rs
+++ b/pyo3-benches/benches/bench_err.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::{exceptions::PyValueError, prelude::*};
 
diff --git a/pyo3-benches/benches/bench_extract.rs b/pyo3-benches/benches/bench_extract.rs
index af454930511..6ff22d3834d 100644
--- a/pyo3-benches/benches/bench_extract.rs
+++ b/pyo3-benches/benches/bench_extract.rs
@@ -1,4 +1,4 @@
-use criterion::{black_box, criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::{
     types::{PyDict, PyFloat, PyInt, PyString},
diff --git a/pyo3-benches/benches/bench_frompyobject.rs b/pyo3-benches/benches/bench_frompyobject.rs
index c2dfbc0ea85..1ad53c717f3 100644
--- a/pyo3-benches/benches/bench_frompyobject.rs
+++ b/pyo3-benches/benches/bench_frompyobject.rs
@@ -1,4 +1,4 @@
-use criterion::{black_box, criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::{
     prelude::*,
diff --git a/pyo3-benches/benches/bench_gil.rs b/pyo3-benches/benches/bench_gil.rs
index e7290644cd0..283e9149b00 100644
--- a/pyo3-benches/benches/bench_gil.rs
+++ b/pyo3-benches/benches/bench_gil.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, BatchSize, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, BatchSize, Bencher, Criterion};
 
 use pyo3::{prelude::*, GILPool};
 
diff --git a/pyo3-benches/benches/bench_intern.rs b/pyo3-benches/benches/bench_intern.rs
index b70edf927ca..d8dd1b8fd30 100644
--- a/pyo3-benches/benches/bench_intern.rs
+++ b/pyo3-benches/benches/bench_intern.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::prelude::*;
 
diff --git a/pyo3-benches/benches/bench_list.rs b/pyo3-benches/benches/bench_list.rs
index dd2e3db12ab..34bf9d855e5 100644
--- a/pyo3-benches/benches/bench_list.rs
+++ b/pyo3-benches/benches/bench_list.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::prelude::*;
 use pyo3::types::{PyList, PySequence};
diff --git a/pyo3-benches/benches/bench_pyclass.rs b/pyo3-benches/benches/bench_pyclass.rs
index 66cca044a75..dbb635765ae 100644
--- a/pyo3-benches/benches/bench_pyclass.rs
+++ b/pyo3-benches/benches/bench_pyclass.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Criterion};
 use pyo3::{impl_::pyclass::LazyTypeObject, prelude::*};
 
 /// This is a feature-rich class instance used to benchmark various parts of the pyclass lifecycle.
diff --git a/pyo3-benches/benches/bench_pyobject.rs b/pyo3-benches/benches/bench_pyobject.rs
index 3d98e07910c..af25d61ce6a 100644
--- a/pyo3-benches/benches/bench_pyobject.rs
+++ b/pyo3-benches/benches/bench_pyobject.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::prelude::*;
 
diff --git a/pyo3-benches/benches/bench_set.rs b/pyo3-benches/benches/bench_set.rs
index 1bc815997b9..0753a2f9979 100644
--- a/pyo3-benches/benches/bench_set.rs
+++ b/pyo3-benches/benches/bench_set.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::prelude::*;
 use pyo3::types::PySet;
diff --git a/pyo3-benches/benches/bench_tuple.rs b/pyo3-benches/benches/bench_tuple.rs
index a2988b15f5c..f224ee1bc4d 100644
--- a/pyo3-benches/benches/bench_tuple.rs
+++ b/pyo3-benches/benches/bench_tuple.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::prelude::*;
 use pyo3::types::{PyList, PySequence, PyTuple};