diff --git a/.Rbuildignore b/.Rbuildignore
index 689f0329a..da3c899e6 100644
--- a/.Rbuildignore
+++ b/.Rbuildignore
@@ -43,3 +43,4 @@ _files$
 ^compile_commands\.json$
 ^\.cache$
 ^tests/testthat/helper-rethrow\.R$
+^cleanup$
diff --git a/cleanup b/cleanup
new file mode 100755
index 000000000..1d53bd0bb
--- /dev/null
+++ b/cleanup
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -ex
+
+cd src
+tar cvJf duckdb.tar.xz duckdb
+rm -rf duckdb
diff --git a/configure b/configure
index 2d00e6308..3b9189ac9 100755
--- a/configure
+++ b/configure
@@ -27,3 +27,10 @@ if [ -f "${DUCKDB_R_PREBUILT_ARCHIVE}" ] && tar -xm -f ${DUCKDB_R_PREBUILT_ARCHI
 else
   cp include/to-tar.mk Makevars.duckdb
 fi
+
+# The duckdb sources are xz-compressed in the tarball to keep it under 5000000 bytes.
+# This happens in the cleanup script.
+
+if [ -f duckdb.tar.xz ]; then
+  tar xJf duckdb.tar.xz
+fi
diff --git a/configure.win b/configure.win
index 5e95002ac..c36eaa0cb 100755
--- a/configure.win
+++ b/configure.win
@@ -33,3 +33,10 @@ if [ -f "${DUCKDB_R_PREBUILT_ARCHIVE}" ] && tar -xm --force-local -f ${DUCKDB_R_
 else
   cp include/to-tar-win.mk Makevars.duckdb
 fi
+
+# The duckdb sources are xz-compressed in the tarball to keep it under 5000000 bytes.
+# This happens in the cleanup script.
+
+if [ -f duckdb.tar.xz ]; then
+  tar xJf duckdb.tar.xz
+fi