From 7cf6e90520d909104b3f59dc4964ff3a34a507b6 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Sat, 4 May 2024 19:08:00 +0300 Subject: [PATCH] Time based cache refresh Previously to update the cache you had to clear the cache and fetch. This leaves a window where a build may use uncached value and fail. A better way is to refresh the cache periodically *without* clearing the cache, ensuring that builds will always use cached values. To support time based refresh, we fetch data only if the cached item it too old or missing. Upon successful fetch, the item age is set to zero, so the next access to the cache will skip the fetch. We use 2 threshold for refreshing the cache: - REFRESH_THRSHOLD: 12 hours - FETCH_THRESHOLD: 48 hours This allows refreshing the cache once per day (e.g. from a cron job). On errors, we can rerun the refresh until the refresh succeeds. Fresh items are skipped during the refresh. If refreshing the cache failed, jobs will trigger a refresh after 48 hours. The longer threshold ensures that this will never happen unless the refresh job is broken. The `fetch` command was renamed to `cache`. This describe better the purpose of the command. The hook was update as well to match the command name. The `drenv.cache` module provides now 2 operations: - cache.refresh() - rebuild the cache if older than cache.REFRESH_THRESHOLD (12 hours). Will is used in the `cache` hook. - cache.get() - rebuild the cache if older than cache.FETCH_THRESHOLD (48 hours) and return the path to the cached value. Will is used in `start` hooks to get a cached value. Signed-off-by: Nir Soffer --- test/addons/cdi/{fetch => cache} | 4 +-- test/addons/cdi/start | 4 +-- .../{rook-cephfs/fetch => csi-addons/cache} | 2 +- test/addons/csi-addons/start | 2 +- test/addons/kubevirt/{fetch => cache} | 4 +-- test/addons/kubevirt/start | 4 +-- test/addons/ocm-controller/cache | 10 +++++++ test/addons/ocm-controller/start | 2 +- test/addons/recipe/{fetch => cache} | 2 +- test/addons/recipe/start | 2 +- .../{csi-addons/fetch => rook-cephfs/cache} | 2 +- test/addons/rook-cephfs/start | 2 +- .../fetch => rook-cluster/cache} | 2 +- test/addons/rook-cluster/fetch | 10 ------- test/addons/rook-cluster/start | 2 +- test/addons/rook-operator/cache | 10 +++++++ test/addons/rook-operator/fetch | 10 ------- test/addons/rook-operator/start | 2 +- test/addons/rook-toolbox/cache | 10 +++++++ test/addons/rook-toolbox/fetch | 10 ------- test/addons/rook-toolbox/start | 2 +- test/drenv/__main__.py | 16 +++++----- test/drenv/cache.py | 29 ++++++++++++++++--- 23 files changed, 82 insertions(+), 61 deletions(-) rename test/addons/cdi/{fetch => cache} (65%) rename test/addons/{rook-cephfs/fetch => csi-addons/cache} (80%) rename test/addons/kubevirt/{fetch => cache} (62%) create mode 100755 test/addons/ocm-controller/cache rename test/addons/recipe/{fetch => cache} (81%) rename test/addons/{csi-addons/fetch => rook-cephfs/cache} (79%) rename test/addons/{ocm-controller/fetch => rook-cluster/cache} (79%) delete mode 100755 test/addons/rook-cluster/fetch create mode 100755 test/addons/rook-operator/cache delete mode 100755 test/addons/rook-operator/fetch create mode 100755 test/addons/rook-toolbox/cache delete mode 100755 test/addons/rook-toolbox/fetch diff --git a/test/addons/cdi/fetch b/test/addons/cdi/cache similarity index 65% rename from test/addons/cdi/fetch rename to test/addons/cdi/cache index 9661e386f..d45b306b9 100755 --- a/test/addons/cdi/fetch +++ b/test/addons/cdi/cache @@ -7,5 +7,5 @@ import os from drenv import cache os.chdir(os.path.dirname(__file__)) -cache.fetch("operator", "addons/cdi-operator.yaml") -cache.fetch("cr", "addons/cdi-cr.yaml") +cache.refresh("operator", "addons/cdi-operator.yaml") +cache.refresh("cr", "addons/cdi-cr.yaml") diff --git a/test/addons/cdi/start b/test/addons/cdi/start index 336c9ad1f..625beae2f 100755 --- a/test/addons/cdi/start +++ b/test/addons/cdi/start @@ -14,7 +14,7 @@ NAMESPACE = "cdi" def deploy(cluster): print("Deploying cdi operator") - path = cache.fetch("operator", "addons/cdi-operator.yaml") + path = cache.get("operator", "addons/cdi-operator.yaml") kubectl.apply("--filename", path, context=cluster) print("Waiting until cdi-operator is rolled out") @@ -27,7 +27,7 @@ def deploy(cluster): ) print("Deploying cdi cr") - path = cache.fetch("cr", "addons/cdi-cr.yaml") + path = cache.get("cr", "addons/cdi-cr.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/addons/rook-cephfs/fetch b/test/addons/csi-addons/cache similarity index 80% rename from test/addons/rook-cephfs/fetch rename to test/addons/csi-addons/cache index dd0141dc8..7b37a5978 100755 --- a/test/addons/rook-cephfs/fetch +++ b/test/addons/csi-addons/cache @@ -7,4 +7,4 @@ import os from drenv import cache os.chdir(os.path.dirname(__file__)) -cache.fetch(".", "addons/rook-cephfs.yaml") +cache.refresh(".", "addons/csi-addons.yaml") diff --git a/test/addons/csi-addons/start b/test/addons/csi-addons/start index d206387b2..7c1ffa788 100755 --- a/test/addons/csi-addons/start +++ b/test/addons/csi-addons/start @@ -12,7 +12,7 @@ from drenv import cache def deploy(cluster): print("Deploying csi addon for volume replication") - path = cache.fetch(".", "addons/csi-addons.yaml") + path = cache.get(".", "addons/csi-addons.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/addons/kubevirt/fetch b/test/addons/kubevirt/cache similarity index 62% rename from test/addons/kubevirt/fetch rename to test/addons/kubevirt/cache index 304058b84..9f3c23506 100755 --- a/test/addons/kubevirt/fetch +++ b/test/addons/kubevirt/cache @@ -7,5 +7,5 @@ import os from drenv import cache os.chdir(os.path.dirname(__file__)) -cache.fetch("operator", "addons/kubevirt-operator.yaml") -cache.fetch("cr", "addons/kubevirt-cr.yaml") +cache.refresh("operator", "addons/kubevirt-operator.yaml") +cache.refresh("cr", "addons/kubevirt-cr.yaml") diff --git a/test/addons/kubevirt/start b/test/addons/kubevirt/start index 14a022ab5..5cd6f32a0 100755 --- a/test/addons/kubevirt/start +++ b/test/addons/kubevirt/start @@ -14,7 +14,7 @@ NAMESPACE = "kubevirt" def deploy(cluster): print("Deploying kubevirt operator") - path = cache.fetch("operator", "addons/kubevirt-operator.yaml") + path = cache.get("operator", "addons/kubevirt-operator.yaml") kubectl.apply("--filename", path, context=cluster) print("Waiting until virt-operator is rolled out") @@ -27,7 +27,7 @@ def deploy(cluster): ) print("Deploying kubevirt cr") - path = cache.fetch("cr", "addons/kubevirt-cr.yaml") + path = cache.get("cr", "addons/kubevirt-cr.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/addons/ocm-controller/cache b/test/addons/ocm-controller/cache new file mode 100755 index 000000000..bc4004e18 --- /dev/null +++ b/test/addons/ocm-controller/cache @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +import os +from drenv import cache + +os.chdir(os.path.dirname(__file__)) +cache.refresh(".", "addons/ocm-controller.yaml") diff --git a/test/addons/ocm-controller/start b/test/addons/ocm-controller/start index 431c5eca8..6b30c74f6 100755 --- a/test/addons/ocm-controller/start +++ b/test/addons/ocm-controller/start @@ -12,7 +12,7 @@ from drenv import cache def deploy(cluster): print("Deploying ocm controller") - path = cache.fetch(".", "addons/ocm-controller.yaml") + path = cache.get(".", "addons/ocm-controller.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/addons/recipe/fetch b/test/addons/recipe/cache similarity index 81% rename from test/addons/recipe/fetch rename to test/addons/recipe/cache index 221500f14..a094d54c5 100755 --- a/test/addons/recipe/fetch +++ b/test/addons/recipe/cache @@ -7,4 +7,4 @@ import os from drenv import cache os.chdir(os.path.dirname(__file__)) -cache.fetch(".", "addons/recipe.yaml") +cache.refresh(".", "addons/recipe.yaml") diff --git a/test/addons/recipe/start b/test/addons/recipe/start index b1bb697cc..bd0bdbec7 100755 --- a/test/addons/recipe/start +++ b/test/addons/recipe/start @@ -16,5 +16,5 @@ os.chdir(os.path.dirname(__file__)) cluster = sys.argv[1] print("Deploying recipe crd") -path = cache.fetch(".", "addons/recipe.yaml") +path = cache.get(".", "addons/recipe.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/addons/csi-addons/fetch b/test/addons/rook-cephfs/cache similarity index 79% rename from test/addons/csi-addons/fetch rename to test/addons/rook-cephfs/cache index 71c61cc52..56205b17e 100755 --- a/test/addons/csi-addons/fetch +++ b/test/addons/rook-cephfs/cache @@ -7,4 +7,4 @@ import os from drenv import cache os.chdir(os.path.dirname(__file__)) -cache.fetch(".", "addons/csi-addons.yaml") +cache.refresh(".", "addons/rook-cephfs.yaml") diff --git a/test/addons/rook-cephfs/start b/test/addons/rook-cephfs/start index d8dcbaceb..489b1b1e7 100755 --- a/test/addons/rook-cephfs/start +++ b/test/addons/rook-cephfs/start @@ -13,7 +13,7 @@ from drenv import kubectl def deploy(cluster): print("Creating CephFS instance and storage/snapshot classes") - path = cache.fetch(".", "addons/rook-cephfs.yaml") + path = cache.get(".", "addons/rook-cephfs.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/addons/ocm-controller/fetch b/test/addons/rook-cluster/cache similarity index 79% rename from test/addons/ocm-controller/fetch rename to test/addons/rook-cluster/cache index 37c55444f..1da61bb7f 100755 --- a/test/addons/ocm-controller/fetch +++ b/test/addons/rook-cluster/cache @@ -7,4 +7,4 @@ import os from drenv import cache os.chdir(os.path.dirname(__file__)) -cache.fetch(".", "addons/ocm-controller.yaml") +cache.refresh(".", "addons/rook-cluster.yaml") diff --git a/test/addons/rook-cluster/fetch b/test/addons/rook-cluster/fetch deleted file mode 100755 index ddf1c53d3..000000000 --- a/test/addons/rook-cluster/fetch +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-FileCopyrightText: The RamenDR authors -# SPDX-License-Identifier: Apache-2.0 - -import os -from drenv import cache - -os.chdir(os.path.dirname(__file__)) -cache.fetch(".", "addons/rook-cluster.yaml") diff --git a/test/addons/rook-cluster/start b/test/addons/rook-cluster/start index 530d9507a..333b6b920 100755 --- a/test/addons/rook-cluster/start +++ b/test/addons/rook-cluster/start @@ -20,7 +20,7 @@ TIMEOUT = 600 def deploy(cluster): print("Deploying rook ceph cluster") - path = cache.fetch(".", "addons/rook-cluster.yaml") + path = cache.get(".", "addons/rook-cluster.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/addons/rook-operator/cache b/test/addons/rook-operator/cache new file mode 100755 index 000000000..d328f5804 --- /dev/null +++ b/test/addons/rook-operator/cache @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +import os +from drenv import cache + +os.chdir(os.path.dirname(__file__)) +cache.refresh(".", "addons/rook-operator.yaml") diff --git a/test/addons/rook-operator/fetch b/test/addons/rook-operator/fetch deleted file mode 100755 index 4b0d8bae9..000000000 --- a/test/addons/rook-operator/fetch +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-FileCopyrightText: The RamenDR authors -# SPDX-License-Identifier: Apache-2.0 - -import os -from drenv import cache - -os.chdir(os.path.dirname(__file__)) -cache.fetch(".", "addons/rook-operator.yaml") diff --git a/test/addons/rook-operator/start b/test/addons/rook-operator/start index 62cf3e7a4..48275ca70 100755 --- a/test/addons/rook-operator/start +++ b/test/addons/rook-operator/start @@ -12,7 +12,7 @@ from drenv import cache def deploy(cluster): print("Deploying rook ceph operator") - path = cache.fetch(".", "addons/rook-operator.yaml") + path = cache.get(".", "addons/rook-operator.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/addons/rook-toolbox/cache b/test/addons/rook-toolbox/cache new file mode 100755 index 000000000..faf716ebe --- /dev/null +++ b/test/addons/rook-toolbox/cache @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +import os +from drenv import cache + +os.chdir(os.path.dirname(__file__)) +cache.refresh(".", "addons/rook-toolbox.yaml") diff --git a/test/addons/rook-toolbox/fetch b/test/addons/rook-toolbox/fetch deleted file mode 100755 index 572695efa..000000000 --- a/test/addons/rook-toolbox/fetch +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-FileCopyrightText: The RamenDR authors -# SPDX-License-Identifier: Apache-2.0 - -import os -from drenv import cache - -os.chdir(os.path.dirname(__file__)) -cache.fetch(".", "addons/rook-toolbox.yaml") diff --git a/test/addons/rook-toolbox/start b/test/addons/rook-toolbox/start index 8ebe28710..161d9ee9c 100755 --- a/test/addons/rook-toolbox/start +++ b/test/addons/rook-toolbox/start @@ -13,7 +13,7 @@ from drenv import cache def deploy(cluster): print("Deploying rook ceph toolbox") - path = cache.fetch(".", "addons/rook-toolbox.yaml") + path = cache.get(".", "addons/rook-toolbox.yaml") kubectl.apply("--filename", path, context=cluster) diff --git a/test/drenv/__main__.py b/test/drenv/__main__.py index c18277406..52e78216e 100644 --- a/test/drenv/__main__.py +++ b/test/drenv/__main__.py @@ -85,7 +85,7 @@ def parse_args(): add_command(sp, "suspend", do_suspend, help="suspend virtual machines") add_command(sp, "resume", do_resume, help="resume virtual machines") add_command(sp, "dump", do_dump, help="dump an environment yaml") - add_command(sp, "fetch", do_fetch, help="cache environment resources") + add_command(sp, "cache", do_cache, help="cache environment resources") add_command(sp, "clear", do_clear, help="cleared cached resources", envfile=False) add_command(sp, "setup", do_setup, help="setup minikube for drenv", envfile=False) @@ -177,20 +177,20 @@ def do_clear(args): cache.clear() -def do_fetch(args): +def do_cache(args): env = load_env(args) start = time.monotonic() - logging.info("[%s] Fetching", env["name"]) + logging.info("[%s] Refreshing cached addons", env["name"]) addons = collect_addons(env) execute( - fetch_addon, + cache_addon, addons, - "fetch", + "cache", max_workers=args.max_workers, ctx=env["name"], ) logging.info( - "[%s] Fetching finishied in %.2f seconds", + "[%s] Cached addons refreshed in %.2f seconds", env["name"], time.monotonic() - start, ) @@ -491,13 +491,13 @@ def run_worker(worker, hooks=(), reverse=False, allow_failure=False): run_addon(addon, worker["name"], hooks=hooks, allow_failure=allow_failure) -def fetch_addon(addon, ctx="global"): +def cache_addon(addon, ctx="global"): addon_dir = os.path.join(ADDONS_DIR, addon["name"]) if not os.path.isdir(addon_dir): skip_addon(addon, ctx) return - hook = os.path.join(addon_dir, "fetch") + hook = os.path.join(addon_dir, "cache") if os.path.isfile(hook): run_hook(hook, (), ctx) diff --git a/test/drenv/cache.py b/test/drenv/cache.py index f4576bde8..25e1341ae 100644 --- a/test/drenv/cache.py +++ b/test/drenv/cache.py @@ -4,9 +4,13 @@ import os import shutil import subprocess +import time from . import commands +REFRESH_THRESHOLD = 12 * 3600 +FETCH_THRESHOLD = 48 * 3600 + def clear(key=""): """ @@ -19,17 +23,34 @@ def clear(key=""): pass -def fetch(kustomization_dir, key, log=print): +def refresh(kustomization_dir, key, log=print): """ - Build kustomization and cache the output yaml. Retrun the path to the - cached yaml. + Rebuild the cache if cached item age is bigger than REFRESH_THRESHOLD. """ dest = _path(key) - if not os.path.exists(dest): + if _age(dest) > REFRESH_THRESHOLD: + _fetch(kustomization_dir, dest, log=log) + + +def get(kustomization_dir, key, log=print): + """ + Rebuild the cache if cached item age is bigger than FETCH_THRESHOLD. + Return the path to cached kustomization yaml. + """ + dest = _path(key) + if _age(dest) > FETCH_THRESHOLD: _fetch(kustomization_dir, dest, log=log) return dest +def _age(path): + try: + mtime = os.path.getmtime(path) + except FileNotFoundError: + mtime = 0 + return time.time() - mtime + + def _fetch(kustomization_dir, dest, log=print): log(f"Fetching {dest}") dest_dir = os.path.dirname(dest)