Merge pull request #45 from buildkite-plugins/toote_race_condition

FS backend race condition

backends/cache_fs

@@ -1,21 +1,40 @@
 #!/bin/bash
+set -euo pipefail
 
 CACHE_FOLDER="${BUILDKITE_PLUGIN_FS_CACHE_FOLDER:-/var/cache/buildkite}"
 
+DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)"
+
+# shellcheck source=lib/lock.bash
+. "${DIR}/../lib/lock.bash"
+
 restore_cache() {
   local from="${CACHE_FOLDER}/$1"
   local to="$2"
+
+  wait_and_lock "${from}"
+  # shellcheck disable=2064  # actually want variable interpolated here
+  trap "release_lock_folder '${from}'" SIGINT SIGTERM SIGQUIT
+
   cp -a "$from" "$to"
+
+  release_lock "${from}"
 }
 
 save_cache() {
   local to="${CACHE_FOLDER}/$1"
   local from="$2"
 
+  wait_and_lock "${to}"
+  # shellcheck disable=2064  # actually want variable interpolated here
+  trap "release_lock '${to}'" SIGINT SIGTERM SIGQUIT
+
   if [ -n "$1" ] && [ -e "$to" ]; then
     rm -rf "$to"
   fi
   cp -a "$from" "$to"
+
+  release_lock "${to}"
 }
 
 exists_cache() {

lib/lock.bash

@@ -0,0 +1,31 @@
+#!/bin/bash
+set -euo pipefail
+
+lock () {
+  local file="${1}.lock"
+  (
+    set -o noclobber
+    date > "${file}"
+  ) 2>/dev/null
+}
+
+wait_and_lock () {
+  local file="${1}.lock"
+  local max_attempts="${2:-5}"
+
+  for ATTEMPT in $(seq 1 "${max_attempts}"); do
+    if ! lock "${1}"; then
+      echo 'Waiting for folder lock'
+      sleep "${ATTEMPT}"
+    else
+      return 0
+    fi
+  done
+
+  return 1
+}
+
+release_lock () {
+  local file="${1}.lock"
+  rm -f "${file}"
+}

tests/lock.bats

@@ -0,0 +1,52 @@
+#!/usr/bin/env bats
+
+setup() {
+  load "${BATS_PLUGIN_PATH}/load.bash"
+
+  source "${PWD}/lib/lock.bash"
+}
+
+@test 'basic locking works as expected' {
+  LOCKFILE="${BATS_TEST_TMPDIR}/lock.file"
+
+  run lock "${LOCKFILE}"
+  assert_success
+  assert_output ''
+
+  run lock "${LOCKFILE}"
+  assert_failure
+  assert_output ''
+
+  run release_lock "${LOCKFILE}"
+  assert_success
+  assert_output ''
+
+  run lock "${LOCKFILE}"
+  assert_success
+  assert_output ''
+
+  rm -f "${LOCKFILE}"
+}
+
+
+@test 'lock times out' {
+  LOCKFILE="${BATS_TEST_TMPDIR}/lock.file"
+
+  run wait_and_lock "${LOCKFILE}"
+  assert_success
+  assert_output ''
+
+  run wait_and_lock "${LOCKFILE}" 1
+  assert_failure
+  assert_equal "$(echo "${output}" | wc -l)" "1"
+
+  run wait_and_lock "${LOCKFILE}"
+  assert_failure
+  assert_equal "$(echo "${output}" | wc -l)" "5"
+
+  run release_lock "${LOCKFILE}"
+  assert_success
+  assert_output ''
+
+  rm -f "${LOCKFILE}"
+}