Merge pull request #44 from buildkite-plugins/toote_compression

Compression

README.md

@@ -62,6 +62,14 @@ You also need the agent to have access to the following defined environment vari
 
 Setting the `BUILDKITE_PLUGIN_S3_CACHE_ONLY_SHOW_ERRORS` environment variable will reduce logging of file operations towards S3.
 
+### `compression` (string, optional)
+
+Allows for the cached file/folder to be saved/restored as a single file. You will need to make sure to use the same compression when saving and restoring or it will cause a cache miss.
+
+Assuming the underlying executables are available, the allowed values are:
+* `tgz`: `tar` with gzip compression
+* `zip`: `(un)zip` compression
+
 ### `manifest` (string, required if using `file` caching level)
 
 A path to a file or folder that will be hashed to create file-level caches.

hooks/post-checkout

@@ -22,7 +22,23 @@ elif [ "${MAX_LEVEL}" = 'file' ] && [ -z "$(plugin_read_config MANIFEST)" ]; the
   exit 1
 fi
 
-build_key "${MAX_LEVEL}" "${RESTORE_PATH}" # to validate the level
+COMPRESS=$(plugin_read_config COMPRESSION 'none')
+if ! validate_compression "${COMPRESS}"; then
+  echo "+++ 🚨 Invalid value for compression option"
+  exit 1
+fi
+
+build_key "${MAX_LEVEL}" "${RESTORE_PATH}" >/dev/null # to validate the level
+
+ACTUAL_PATH=$(mktemp)
+
+if [ "${COMPRESS}" = 'tgz' ]; then
+  UNCOMPRESS_COMMAND=(tar xzf)
+elif [ "${COMPRESS}" = 'zip' ]; then
+  UNCOMPRESS_COMMAND=(unzip)
+else
+  ACTUAL_PATH="${RESTORE_PATH}"
+fi
 
 SORTED_LEVELS=(file step branch pipeline all)
 
@@ -31,10 +47,16 @@ for CURRENT_LEVEL in "${SORTED_LEVELS[@]}"; do
     continue
   fi
 
-  KEY=$(build_key "${CURRENT_LEVEL}" "${RESTORE_PATH}")
+  KEY=$(build_key "${CURRENT_LEVEL}" "${RESTORE_PATH}" "${COMPRESS}")
   if backend_exec exists "${KEY}"; then
     echo "Cache hit at ${CURRENT_LEVEL} level, restoring ${RESTORE_PATH}..."
-    backend_exec get "${KEY}" "${RESTORE_PATH}"
+    backend_exec get "${KEY}" "${ACTUAL_PATH}"
+
+    if [ "${COMPRESS}" != 'none' ]; then
+      echo "Cache is compressed, uncompressing..."
+      "${UNCOMPRESS_COMMAND[@]}" "${ACTUAL_PATH}" "${RESTORE_PATH}"
+    fi
+
     exit 0
   elif [ "${CURRENT_LEVEL}" = "${MAX_LEVEL}" ]; then
     echo "Cache miss up to ${CURRENT_LEVEL}-level, sorry"

hooks/post-command

@@ -22,7 +22,27 @@ elif [ "${LEVEL}" = 'file' ] && [ -z "$(plugin_read_config MANIFEST)" ]; then
   exit 1
 fi
 
-KEY=$(build_key "${LEVEL}" "${CACHE_PATH}")
+COMPRESS=$(plugin_read_config COMPRESSION 'none')
+if ! validate_compression "${COMPRESS}"; then
+  echo "+++ 🚨 Invalid value for compression option"
+  exit 1
+fi
+
+KEY=$(build_key "${LEVEL}" "${CACHE_PATH}" "${COMPRESS}")
+
+if [ "${COMPRESS}" = 'tgz' ]; then
+  COMPRESS_COMMAND=(tar czf)
+elif [ "${COMPRESS}" = 'zip' ]; then
+  COMPRESS_COMMAND=(zip)
+fi
+
+if [ "${COMPRESS}" != 'none' ]; then
+  echo "Compressing ${CACHE_PATH} with ${COMPRESS}..."
+  ACTUAL_PATH=$(mktemp)
+  "${COMPRESS_COMMAND[@]}" "${ACTUAL_PATH}" "${CACHE_PATH}"
+else
+  ACTUAL_PATH="${CACHE_PATH}"
+fi
 
 echo "Saving ${LEVEL}-level cache of ${CACHE_PATH}"
-backend_exec save "${KEY}" "${CACHE_PATH}"
+backend_exec save "${KEY}" "${ACTUAL_PATH}"

lib/shared.bash

@@ -36,7 +36,7 @@ hash_files() {
 build_key() {
   local LEVEL="$1"
   local CACHE_PATH="$2"
-  local BASE
+  local COMPRESSION="${3:-}"
 
   if [ "${LEVEL}" = 'file' ]; then
     BASE="$(hash_files "$(plugin_read_config MANIFEST)")"
@@ -53,12 +53,25 @@ build_key() {
     exit 1
   fi
 
-  echo "cache-${LEVEL}-${BASE}-${CACHE_PATH}" | "$(sha)" | cut -d\  -f1
+  echo "cache-${LEVEL}-${BASE}-${CACHE_PATH}-${COMPRESSION}" | "$(sha)" | cut -d\  -f1
 }
 
 backend_exec() {
   local BACKEND_NAME
   BACKEND_NAME=$(plugin_read_config BACKEND 'fs')
 
   PATH="${PATH}:${DIR}/../backends" "cache_${BACKEND_NAME}" "$@"
+}
+
+validate_compression() {
+  local COMPRESSION="$1"
+
+  VALID_COMPRESSIONS=(none tgz zip)
+  for VALID in "${VALID_COMPRESSIONS[@]}"; do
+    if [ "${COMPRESSION}" = "${VALID}" ]; then
+      return 0
+    fi
+  done
+
+  return 1
 }

plugin.yml

@@ -6,10 +6,16 @@ configuration:
   properties:
     backend:
       type: string
-    path:
+    compression:
       type: string
+      enum:
+        - zip
+        - tar
+        - tgz
     manifest:
       type: string
+    path:
+      type: string
     restore:
       type: string
       enum:

tests/post-checkout-tgz.bats

@@ -0,0 +1,157 @@
+#!/usr/bin/env bats
+
+# To debug stubs, uncomment these lines:
+# export CACHE_DUMMY_STUB_DEBUG=/dev/tty
+# export TAR_STUB_DEBUG=/dev/tty
+
+setup() {
+  load "${BATS_PLUGIN_PATH}/load.bash"
+
+  mkdir -p tests/data/my_files
+  echo "all the llamas" > "tests/data/my_files/llamas.txt"
+  echo "no alpacas" > "tests/data/my_files/alpacas.txt"
+
+  export BUILDKITE_PLUGIN_CACHE_BACKEND=dummy
+  export BUILDKITE_PLUGIN_CACHE_COMPRESSION=tgz
+  export BUILDKITE_PLUGIN_CACHE_PATH=tests/data/my_files
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST=tests/data/my_files/llamas.txt
+
+  # necessary for key-calculations
+  export BUILDKITE_LABEL="step-label"
+  export BUILDKITE_BRANCH="tests"
+  export BUILDKITE_ORGANIZATION_SLUG="bk-cache-test"
+  export BUILDKITE_PIPELINE_SLUG="cache-pipeline"
+
+  # stub is the same for all tests
+  stub tar \
+    "xzf \* \* : echo uncompressed \$2 into \$3"
+}
+
+teardown() {
+  rm -rf tests/data
+
+  unstub tar
+}
+
+@test 'Existing file-based restore' {
+  export BUILDKITE_PLUGIN_CACHE_RESTORE=file
+
+  stub cache_dummy \
+    'exists \* : exit 0' \
+    "get \* \* : echo restoring \$2 to \$3"
+
+  run "$PWD/hooks/post-checkout"
+
+  assert_success
+  assert_output --partial 'Cache hit at file level'
+  assert_output --partial 'Cache is compressed, uncompressing...'
+
+  unstub cache_dummy
+}
+
+@test 'Existing file-based restore even when max-level is higher' {
+  export BUILDKITE_PLUGIN_CACHE_RESTORE=all
+
+  stub cache_dummy \
+    'exists \* : exit 0' \
+    "get \* \* : echo restoring \$2 to \$3"
+
+  run "$PWD/hooks/post-checkout"
+
+  assert_success
+  assert_output --partial 'Cache hit at file level'
+  assert_output --partial 'Cache is compressed, uncompressing...'
+
+  unstub cache_dummy
+}
+
+@test 'Existing step-based restore' {
+  export BUILDKITE_PLUGIN_CACHE_RESTORE=step
+
+  stub cache_dummy \
+    'exists \* : exit 1' \
+    'exists \* : exit 0' \
+    "get \* \* : echo restoring \$2 to \$3"
+
+  run "$PWD/hooks/post-checkout"
+
+  assert_success
+  assert_output --partial 'Cache hit at step level'
+  assert_output --partial 'Cache is compressed, uncompressing...'
+
+  unstub cache_dummy
+}
+
+@test 'Existing branch-based restore' {
+  export BUILDKITE_PLUGIN_CACHE_RESTORE=branch
+
+  stub cache_dummy \
+    'exists \* : exit 1' \
+    'exists \* : exit 1' \
+    'exists \* : exit 0' \
+    "get \* \* : echo restoring \$2 to \$3"
+
+  run "$PWD/hooks/post-checkout"
+
+  assert_success
+  assert_output --partial 'Cache hit at branch level'
+  assert_output --partial 'Cache is compressed, uncompressing...'
+
+  unstub cache_dummy
+}
+@test 'Existing pipeline-based restore' {
+  export BUILDKITE_PLUGIN_CACHE_RESTORE=pipeline
+
+  stub cache_dummy \
+    'exists \* : exit 1' \
+    'exists \* : exit 1' \
+    'exists \* : exit 1' \
+    'exists \* : exit 0' \
+    "get \* \* : echo restoring \$2 to \$3"
+
+  run "$PWD/hooks/post-checkout"
+
+  assert_success
+  assert_output --partial 'Cache hit at pipeline level'
+  assert_output --partial 'Cache is compressed, uncompressing...'
+
+  unstub cache_dummy
+}
+
+@test 'Existing all-based restore' {
+  export BUILDKITE_PLUGIN_CACHE_RESTORE=all
+
+  stub cache_dummy \
+    'exists \* : exit 1' \
+    'exists \* : exit 1' \
+    'exists \* : exit 1' \
+    'exists \* : exit 1' \
+    'exists \* : exit 0' \
+    "get \* \* : echo restoring \$2 to \$3"
+
+  run "$PWD/hooks/post-checkout"
+
+  assert_success
+  assert_output --partial 'Cache hit at all level'
+  assert_output --partial 'Cache is compressed, uncompressing...'
+
+  unstub cache_dummy
+}
+
+@test 'Existing lower level restore works' {
+  export BUILDKITE_PLUGIN_CACHE_RESTORE=all
+
+  stub cache_dummy \
+    'exists \* : exit 1' \
+    'exists \* : exit 1' \
+    'exists \* : exit 0' \
+    "get \* \* : echo restoring \$2 to \$3"
+
+  run "$PWD/hooks/post-checkout"
+
+  assert_success
+  assert_output --partial 'Cache hit at branch level'
+  assert_output --partial 'Cache is compressed, uncompressing...'
+
+  unstub cache_dummy
+}