Time based cache refresh

Previously to update the cache you had to clear the cache and fetch.
This leaves a window where a build may use uncached value and fail.  A
better way is to refresh the cache periodically *without* clearing the
cache, ensuring that builds will always use cached values.

To support time based refresh, we fetch data only if the cached item it
too old or missing. Upon successful fetch, the item age is set to zero,
so the next access to the cache will skip the fetch.

We use 2 threshold for refreshing the cache:

- REFRESH_THRSHOLD: 12 hours
- FETCH_THRESHOLD: 48 hours

This allows refreshing the cache once per day (e.g. from a cron job). On
errors, we can rerun the refresh until the refresh succeeds. Fresh items
are skipped during the refresh.

If refreshing the cache failed, jobs will trigger a refresh after 48
hours. The longer threshold ensures that this will never happen unless
the refresh job is broken.

The `fetch` command was renamed to `cache`. This describe better the
purpose of the command. The hook was update as well to match the command
name.

The `drenv.cache` module provides now 2 operations:

- cache.refresh() - rebuild the cache if older than
  cache.REFRESH_THRESHOLD (12 hours). Will is used in the `cache` hook.

- cache.get() - rebuild the cache if older than cache.FETCH_THRESHOLD
  (48 hours) and return the path to the cached value. Will is used in
  `start` hooks to get a cached value.

Signed-off-by: Nir Soffer <nsoffer@redhat.com>

test/addons/cdi/fetch → test/addons/cdi/cache

@@ -7,5 +7,5 @@ import os
 from drenv import cache
 
 os.chdir(os.path.dirname(__file__))
-cache.fetch("operator", "addons/cdi-operator.yaml")
-cache.fetch("cr", "addons/cdi-cr.yaml")
+cache.refresh("operator", "addons/cdi-operator.yaml")
+cache.refresh("cr", "addons/cdi-cr.yaml")

test/addons/cdi/start

@@ -14,7 +14,7 @@ NAMESPACE = "cdi"
 
 def deploy(cluster):
     print("Deploying cdi operator")
-    path = cache.fetch("operator", "addons/cdi-operator.yaml")
+    path = cache.get("operator", "addons/cdi-operator.yaml")
     kubectl.apply("--filename", path, context=cluster)
 
     print("Waiting until cdi-operator is rolled out")
@@ -27,7 +27,7 @@ def deploy(cluster):
     )
 
     print("Deploying cdi cr")
-    path = cache.fetch("cr", "addons/cdi-cr.yaml")
+    path = cache.get("cr", "addons/cdi-cr.yaml")
     kubectl.apply("--filename", path, context=cluster)
 
 

test/addons/rook-cephfs/fetch → test/addons/csi-addons/cache

@@ -7,4 +7,4 @@ import os
 from drenv import cache
 
 os.chdir(os.path.dirname(__file__))
-cache.fetch(".", "addons/rook-cephfs.yaml")
+cache.refresh(".", "addons/csi-addons.yaml")

test/addons/csi-addons/start

@@ -12,7 +12,7 @@ from drenv import cache
 
 def deploy(cluster):
     print("Deploying csi addon for volume replication")
-    path = cache.fetch(".", "addons/csi-addons.yaml")
+    path = cache.get(".", "addons/csi-addons.yaml")
     kubectl.apply("--filename", path, context=cluster)
 
 

test/addons/kubevirt/fetch → test/addons/kubevirt/cache

@@ -7,5 +7,5 @@ import os
 from drenv import cache
 
 os.chdir(os.path.dirname(__file__))
-cache.fetch("operator", "addons/kubevirt-operator.yaml")
-cache.fetch("cr", "addons/kubevirt-cr.yaml")
+cache.refresh("operator", "addons/kubevirt-operator.yaml")
+cache.refresh("cr", "addons/kubevirt-cr.yaml")

test/addons/kubevirt/start

@@ -14,7 +14,7 @@ NAMESPACE = "kubevirt"
 
 def deploy(cluster):
     print("Deploying kubevirt operator")
-    path = cache.fetch("operator", "addons/kubevirt-operator.yaml")
+    path = cache.get("operator", "addons/kubevirt-operator.yaml")
     kubectl.apply("--filename", path, context=cluster)
 
     print("Waiting until virt-operator is rolled out")
@@ -27,7 +27,7 @@ def deploy(cluster):
     )
 
     print("Deploying kubevirt cr")
-    path = cache.fetch("cr", "addons/kubevirt-cr.yaml")
+    path = cache.get("cr", "addons/kubevirt-cr.yaml")
     kubectl.apply("--filename", path, context=cluster)
 
 

test/addons/ocm-controller/cache

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: The RamenDR authors
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from drenv import cache
+
+os.chdir(os.path.dirname(__file__))
+cache.refresh(".", "addons/ocm-controller.yaml")

test/addons/ocm-controller/start

@@ -12,7 +12,7 @@ from drenv import cache
 
 def deploy(cluster):
     print("Deploying ocm controller")
-    path = cache.fetch(".", "addons/ocm-controller.yaml")
+    path = cache.get(".", "addons/ocm-controller.yaml")
     kubectl.apply("--filename", path, context=cluster)
 
 

test/addons/recipe/fetch → test/addons/recipe/cache

@@ -7,4 +7,4 @@ import os
 from drenv import cache
 
 os.chdir(os.path.dirname(__file__))
-cache.fetch(".", "addons/recipe.yaml")
+cache.refresh(".", "addons/recipe.yaml")

test/addons/recipe/start

@@ -16,5 +16,5 @@ os.chdir(os.path.dirname(__file__))
 cluster = sys.argv[1]
 
 print("Deploying recipe crd")
-path = cache.fetch(".", "addons/recipe.yaml")
+path = cache.get(".", "addons/recipe.yaml")
 kubectl.apply("--filename", path, context=cluster)

test/addons/csi-addons/fetch → test/addons/rook-cephfs/cache

@@ -7,4 +7,4 @@ import os
 from drenv import cache
 
 os.chdir(os.path.dirname(__file__))
-cache.fetch(".", "addons/csi-addons.yaml")
+cache.refresh(".", "addons/rook-cephfs.yaml")

test/addons/rook-cephfs/start

@@ -13,7 +13,7 @@ from drenv import kubectl
 
 def deploy(cluster):
     print("Creating CephFS instance and storage/snapshot classes")
-    path = cache.fetch(".", "addons/rook-cephfs.yaml")
+    path = cache.get(".", "addons/rook-cephfs.yaml")
     kubectl.apply("--filename", path, context=cluster)
 
 

test/addons/ocm-controller/fetch → test/addons/rook-cluster/cache

@@ -7,4 +7,4 @@ import os
 from drenv import cache
 
 os.chdir(os.path.dirname(__file__))
-cache.fetch(".", "addons/ocm-controller.yaml")
+cache.refresh(".", "addons/rook-cluster.yaml")

test/addons/rook-cluster/start

@@ -20,7 +20,7 @@ TIMEOUT = 600
 
 def deploy(cluster):
     print("Deploying rook ceph cluster")
-    path = cache.fetch(".", "addons/rook-cluster.yaml")
+    path = cache.get(".", "addons/rook-cluster.yaml")
     kubectl.apply("--filename", path, context=cluster)
 
 

test/addons/rook-operator/cache

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: The RamenDR authors
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from drenv import cache
+
+os.chdir(os.path.dirname(__file__))
+cache.refresh(".", "addons/rook-operator.yaml")

test/addons/rook-operator/start

@@ -12,7 +12,7 @@ from drenv import cache
 
 def deploy(cluster):
     print("Deploying rook ceph operator")
-    path = cache.fetch(".", "addons/rook-operator.yaml")
+    path = cache.get(".", "addons/rook-operator.yaml")
     kubectl.apply("--filename", path, context=cluster)
 
 

test/addons/rook-toolbox/cache

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: The RamenDR authors
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from drenv import cache
+
+os.chdir(os.path.dirname(__file__))
+cache.refresh(".", "addons/rook-toolbox.yaml")

test/addons/rook-toolbox/start

@@ -13,7 +13,7 @@ from drenv import cache
 
 def deploy(cluster):
     print("Deploying rook ceph toolbox")
-    path = cache.fetch(".", "addons/rook-toolbox.yaml")
+    path = cache.get(".", "addons/rook-toolbox.yaml")
     kubectl.apply("--filename", path, context=cluster)
 
 

test/drenv/__main__.py

@@ -85,7 +85,7 @@ def parse_args():
     add_command(sp, "suspend", do_suspend, help="suspend virtual machines")
     add_command(sp, "resume", do_resume, help="resume virtual machines")
     add_command(sp, "dump", do_dump, help="dump an environment yaml")
-    add_command(sp, "fetch", do_fetch, help="cache environment resources")
+    add_command(sp, "cache", do_cache, help="cache environment resources")
 
     add_command(sp, "clear", do_clear, help="cleared cached resources", envfile=False)
     add_command(sp, "setup", do_setup, help="setup minikube for drenv", envfile=False)
@@ -177,20 +177,20 @@ def do_clear(args):
     cache.clear()
 
 
-def do_fetch(args):
+def do_cache(args):
     env = load_env(args)
     start = time.monotonic()
-    logging.info("[%s] Fetching", env["name"])
+    logging.info("[%s] Refreshing cached addons", env["name"])
     addons = collect_addons(env)
     execute(
-        fetch_addon,
+        cache_addon,
         addons,
-        "fetch",
+        "cache",
         max_workers=args.max_workers,
         ctx=env["name"],
     )
     logging.info(
-        "[%s] Fetching finishied in %.2f seconds",
+        "[%s] Cached addons refreshed in %.2f seconds",
         env["name"],
         time.monotonic() - start,
     )
@@ -491,13 +491,13 @@ def run_worker(worker, hooks=(), reverse=False, allow_failure=False):
         run_addon(addon, worker["name"], hooks=hooks, allow_failure=allow_failure)
 
 
-def fetch_addon(addon, ctx="global"):
+def cache_addon(addon, ctx="global"):
     addon_dir = os.path.join(ADDONS_DIR, addon["name"])
     if not os.path.isdir(addon_dir):
         skip_addon(addon, ctx)
         return
 
-    hook = os.path.join(addon_dir, "fetch")
+    hook = os.path.join(addon_dir, "cache")
     if os.path.isfile(hook):
         run_hook(hook, (), ctx)
 

test/drenv/cache.py

@@ -4,9 +4,13 @@
 import os
 import shutil
 import subprocess
+import time
 
 from . import commands
 
+REFRESH_THRESHOLD = 12 * 3600
+FETCH_THRESHOLD = 48 * 3600
+
 
 def clear(key=""):
     """
@@ -19,17 +23,34 @@ def clear(key=""):
         pass
 
 
-def fetch(kustomization_dir, key, log=print):
+def refresh(kustomization_dir, key, log=print):
     """
-    Build kustomization and cache the output yaml. Retrun the path to the
-    cached yaml.
+    Rebuild the cache if cached item age is bigger than REFRESH_THRESHOLD.
     """
     dest = _path(key)
-    if not os.path.exists(dest):
+    if _age(dest) > REFRESH_THRESHOLD:
+        _fetch(kustomization_dir, dest, log=log)
+
+
+def get(kustomization_dir, key, log=print):
+    """
+    Rebuild the cache if cached item age is bigger than FETCH_THRESHOLD.
+    Return the path to cached kustomization yaml.
+    """
+    dest = _path(key)
+    if _age(dest) > FETCH_THRESHOLD:
         _fetch(kustomization_dir, dest, log=log)
     return dest
 
 
+def _age(path):
+    try:
+        mtime = os.path.getmtime(path)
+    except FileNotFoundError:
+        mtime = 0
+    return time.time() - mtime
+
+
 def _fetch(kustomization_dir, dest, log=print):
     log(f"Fetching {dest}")
     dest_dir = os.path.dirname(dest)