chore: Cache duckdb .o files for fast CI/CD

.github/workflows/custom/after-install/action.yml

@@ -0,0 +1,22 @@
+name: 'Custom steps to run after R packages are installed'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Compute ccache key
+      run: |
+        echo "GHA_CCACHE_KEY=${{ runner.os }}-$($(R CMD config CXX) --version | head -n 1 | sed -r 's/[, ]/-/g')-R-$(Rscript --vanilla -e 'cat(format(getRversion()))')" | tee -a $GITHUB_ENV
+      shell: bash
+
+    - name: Install persistent ccache
+      uses: hendrikmuhs/ccache-action@v1.2.10
+      with:
+        key: ${{ env.GHA_CCACHE_KEY }}
+        max-size: 200M
+        verbose: 1
+
+    - name: Install to avoid R CMD INSTALL --pre-clean run otherwise
+      run: |
+        mkdir ../lib
+        R CMD INSTALL --library=$(dirname $(pwd))/lib -d --no-byte-compile .
+      shell: bash

.github/workflows/custom/before-install/action.yml

@@ -7,3 +7,15 @@ runs:
       run: |
         echo '_R_CHECK_PKG_SIZES_=FALSE' | tee -a $GITHUB_ENV
       shell: bash
+
+    - name: Compute cache key
+      run: |
+        echo "DUCKDB_R_PREBUILT_ARCHIVE_GHA_CACHE_KEY=duckdb-tar-${{ runner.os }}-$($(R CMD config CXX) --version | head -n 1 | sed -r 's/[, ]/-/g')-${{ hashFiles('src/duckdb/**') }}" | tee -a $GITHUB_ENV
+        echo "DUCKDB_R_PREBUILT_ARCHIVE=${{ runner.temp }}/duckdb.tar" | tee -a $GITHUB_ENV
+      shell: bash
+
+    - name: Cache duckdb archive
+      uses: actions/cache@v3
+      with:
+        path: ${{ env.DUCKDB_R_PREBUILT_ARCHIVE }}
+        key: ${{ env.DUCKDB_R_PREBUILT_ARCHIVE_GHA_CACHE_KEY }}

.github/workflows/install/action.yml

@@ -106,8 +106,8 @@ runs:
         echo 'CCACHE_SLOPPINESS=locale,time_macros' | tee -a $GITHUB_ENV
 
         # echo 'CCACHE_DEBUG=true' | tee -a $GITHUB_ENV
-        # echo "CCACHE_DEBUGDIR=$(realpath $(pwd)/../ccache-debug" | tee -a $GITHUB_ENV
-        # mkdir -p $(pwd)/../.ccache-debug
+        # echo "CCACHE_DEBUGDIR=$(dirname $(pwd))/ccache-debug" | tee -a $GITHUB_ENV
+        # mkdir -p $(dirname $(pwd))/.ccache-debug
 
         echo 'PKG_BUILD_EXTRA_FLAGS=false' | tee -a $GITHUB_ENV
       shell: bash

.gitignore

@@ -97,12 +97,14 @@ $RECYCLE.BIN/
 *.lnk
 # End of https://www.toptal.com/developers/gitignore/api/windows
 
-/duckdb
 /.vscode
 /build
 *.tar.gz
 *.tgz
 src/**/*.o
 src/*.so
 /scripts/__pycache__
+/duckdb
+/src/Makevars.duckdb
 /.ccache
+/src/symbols.rds

configure

@@ -0,0 +1,29 @@
+#!/bin/sh
+
+set -ex
+
+cd $(dirname $0)/src
+
+# The purpose of this is to avoid rebuilding duckdb in every CI/CD run.
+# We set the DUCKDB_R_PREBUILT_ARCHIVE environment variable to a file path
+# that contains a .tar of all object files.
+# This .tar file is cached and restored in CI/CD, keyed by compiler, OS,
+# and hash of the duckdb/src subtree.
+#
+# Depending on the availability of the file, we either use the object files
+# (via the rules in include/from-tar.mk), or create them as usual
+# (via include/to-tar.mk). In the latter case, if the DUCKDB_R_PREBUILT_ARCHIVE
+# environment variable is set, the .tar file is created.
+#
+# For local installations, this is typically not needed
+# because ccache achieves the same purpose but better.
+# In CI/CD, this approach gives better results than ccache.
+#
+# This logic is identical for Windows or non-Windows builds,
+# only that we need to-tar-win.mk instead of to-tar.mk on Windows.
+
+if [ -f "${DUCKDB_R_PREBUILT_ARCHIVE}" ] && tar -xm -f ${DUCKDB_R_PREBUILT_ARCHIVE}; then
+  cp include/from-tar.mk Makevars.duckdb
+else
+  cp include/to-tar.mk Makevars.duckdb
+fi

configure.win

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+cd $(dirname $0)/src
+
+# The purpose of this is to avoid rebuilding duckdb in every CI/CD run.
+# We set the DUCKDB_R_PREBUILT_ARCHIVE environment variable to a file path
+# that contains a .tar of all object files.
+# This .tar file is cached and restored in CI/CD, keyed by compiler, OS,
+# and hash of the duckdb/src subtree.
+#
+# Depending on the availability of the file, we either use the object files
+# (via the rules in include/from-tar.mk), or create them as usual
+# (via include/to-tar.mk). In the latter case, if the DUCKDB_R_PREBUILT_ARCHIVE
+# environment variable is set, the .tar file is created.
+#
+# For local installations, this is typically not needed
+# because ccache achieves the same purpose but better.
+# In CI/CD, this approach gives better results than ccache.
+#
+# This logic is identical for Windows or non-Windows builds,
+# only that we need to-tar-win.mk instead of to-tar.mk on Windows.
+
+if [ -f "${DUCKDB_R_PREBUILT_ARCHIVE}" ] && tar -xm --force-local -f ${DUCKDB_R_PREBUILT_ARCHIVE}; then
+  cp include/from-tar.mk Makevars.duckdb
+else
+  cp include/to-tar-win.mk Makevars.duckdb
+fi

src/Makevars

@@ -1,4 +1,17 @@
+# This is required as a first target if we add additional targets
+# (and we do in Makevars.duckdb, which is either include/from-tar.mk
+# or include/to-tar.mk).
+#
+# The symbols.rds file helps troubleshoot issues with forbidden symbols
+# such as abort.
+all: duckdb.tar symbols.rds $(SHLIB)
+
 include include/sources.mk
+
+# This file is created by the configure or configure.win scripts
+# depending on the value of the DUCKDB_R_PREBUILT_ARCHIVE environment variable.
+include Makevars.duckdb
+
 CXX_STD = CXX17
 PKG_CPPFLAGS = -Iinclude -I../inst/include -DDUCKDB_DISABLE_PRINT -DDUCKDB_R_BUILD -Iduckdb/src/include -Iduckdb/third_party/fmt/include -Iduckdb/third_party/fsst -Iduckdb/third_party/re2 -Iduckdb/third_party/miniz -Iduckdb/third_party/utf8proc/include -Iduckdb/third_party/utf8proc -Iduckdb/third_party/hyperloglog -Iduckdb/third_party/fastpforlib -Iduckdb/third_party/tdigest -Iduckdb/third_party/libpg_query/include -Iduckdb/third_party/libpg_query -Iduckdb/third_party/concurrentqueue -Iduckdb/third_party/pcg -Iduckdb/third_party/httplib -Iduckdb/third_party/fast_float -Iduckdb/third_party/mbedtls -Iduckdb/third_party/mbedtls/include -Iduckdb/third_party/mbedtls/library -Iduckdb/third_party/jaro_winkler -Iduckdb/third_party/jaro_winkler/details -Iduckdb/extension/parquet/include -Iduckdb/third_party/parquet -Iduckdb/third_party/snappy -Iduckdb/third_party/thrift -Iduckdb/third_party/zstd/include -I../inst/include -Iduckdb -DDUCKDB_EXTENSION_PARQUET_LINKED -DDUCKDB_BUILD_LIBRARY
 OBJECTS=database.o connection.o statement.o register.o relational.o scan.o transform.o utils.o reltoaltrep.o types.o cpp11.o $(SOURCES)

src/Makevars.in

@@ -1,4 +1,17 @@
+# This is required as a first target if we add additional targets
+# (and we do in Makevars.duckdb, which is either include/from-tar.mk
+# or include/to-tar.mk).
+#
+# The symbols.rds file helps troubleshoot issues with forbidden symbols
+# such as abort.
+all: duckdb.tar symbols.rds $(SHLIB)
+
 include include/sources.mk
+
+# This file is created by the configure or configure.win scripts
+# depending on the value of the DUCKDB_R_PREBUILT_ARCHIVE environment variable.
+include Makevars.duckdb
+
 CXX_STD = CXX17
 PKG_CPPFLAGS = -Iinclude -I../inst/include -DDUCKDB_DISABLE_PRINT -DDUCKDB_R_BUILD {{ INCLUDES }}
 OBJECTS=database.o connection.o statement.o register.o relational.o scan.o transform.o utils.o reltoaltrep.o types.o cpp11.o $(SOURCES)

src/Makevars.win

@@ -1,4 +1,17 @@
+# This is required as a first target if we add additional targets
+# (and we do in Makevars.duckdb, which is either include/from-tar.mk
+# or include/to-tar.mk).
+#
+# The symbols.rds file helps troubleshoot issues with forbidden symbols
+# such as abort.
+all: duckdb.tar symbols.rds $(SHLIB)
+
 include include/sources.mk
+
+# This file is created by the configure or configure.win scripts
+# depending on the value of the DUCKDB_R_PREBUILT_ARCHIVE environment variable.
+include Makevars.duckdb
+
 CXX_STD = CXX17
 PKG_CPPFLAGS = -Iinclude -I../inst/include -DDUCKDB_DISABLE_PRINT -DDUCKDB_R_BUILD -Iduckdb/src/include -Iduckdb/third_party/fmt/include -Iduckdb/third_party/fsst -Iduckdb/third_party/re2 -Iduckdb/third_party/miniz -Iduckdb/third_party/utf8proc/include -Iduckdb/third_party/utf8proc -Iduckdb/third_party/hyperloglog -Iduckdb/third_party/fastpforlib -Iduckdb/third_party/tdigest -Iduckdb/third_party/libpg_query/include -Iduckdb/third_party/libpg_query -Iduckdb/third_party/concurrentqueue -Iduckdb/third_party/pcg -Iduckdb/third_party/httplib -Iduckdb/third_party/fast_float -Iduckdb/third_party/mbedtls -Iduckdb/third_party/mbedtls/include -Iduckdb/third_party/mbedtls/library -Iduckdb/third_party/jaro_winkler -Iduckdb/third_party/jaro_winkler/details -Iduckdb/extension/parquet/include -Iduckdb/third_party/parquet -Iduckdb/third_party/snappy -Iduckdb/third_party/thrift -Iduckdb/third_party/zstd/include -I../inst/include -Iduckdb -DDUCKDB_EXTENSION_PARQUET_LINKED -DDUCKDB_BUILD_LIBRARY -DDUCKDB_PLATFORM_RTOOLS=1
 OBJECTS=database.o connection.o statement.o register.o relational.o scan.o transform.o utils.o reltoaltrep.o types.o cpp11.o $(SOURCES)

src/include/from-tar.mk

@@ -0,0 +1,8 @@
+# This file is used from Makevars if $DUCKDB_R_PREBUILT_ARCHIVE points
+# to an existing file.
+# We print details on the object files already extracted in the configure script
+# for diagnostic purposes.
+duckdb.tar: $(DUCKDB_R_PREBUILT_ARCHIVE)
+	ls -lR
+	# Ensure the file is recreated in the next run
+	rm -f Makevars.duckdb

src/include/to-tar-win.mk

@@ -0,0 +1,6 @@
+# This file is used from Makevars if $DUCKDB_R_PREBUILT_ARCHIVE is empty
+# or does not point to an existing file.
+# In the latter case, we create the file after the object files have been built.
+duckdb.tar: $(SOURCES)
+	if [ -n "$(DUCKDB_R_PREBUILT_ARCHIVE)" ]; then tar -cv --force-local -f "$(DUCKDB_R_PREBUILT_ARCHIVE)" $(SOURCES); fi
+	rm -f Makevars.duckdb

src/include/to-tar.mk

@@ -0,0 +1,6 @@
+# This file is used from Makevars if $DUCKDB_R_PREBUILT_ARCHIVE is empty
+# or does not point to an existing file.
+# In the latter case, we create the file after the object files have been built.
+duckdb.tar: $(SOURCES)
+	if [ -n "$(DUCKDB_R_PREBUILT_ARCHIVE)" ]; then tar -cv -f "$(DUCKDB_R_PREBUILT_ARCHIVE)" $(SOURCES); fi
+	rm -f Makevars.duckdb