[LP#2067404] Check and log all events on failed resources (#7)

* Check and log all events on failed resources

Co-authored-by: Homayoon (Hue) Alimohammadi <homayoon.alimohammadi@canonical.com>

src/storage_manifests.py

@@ -2,13 +2,24 @@
 # See LICENSE file for licensing details.
 """Implementation of cinder-csi specific details of the kubernetes manifests."""
 
+import datetime
 import logging
 import pickle
 from hashlib import md5
-from typing import Dict, Optional
+from typing import Dict, List, Optional
 
+from lightkube import Client
 from lightkube.codecs import AnyResource, from_dict
-from ops.manifests import Addition, ConfigRegistry, ManifestLabel, Manifests, Patch
+from lightkube.resources.core_v1 import Event, Pod
+from ops.manifests import (
+    Addition,
+    ConfigRegistry,
+    HashableResource,
+    ManifestLabel,
+    Manifests,
+    Patch,
+)
+from ops.manifests.manipulations import AnyCondition
 
 log = logging.getLogger(__file__)
 NAMESPACE = "kube-system"
@@ -149,6 +160,56 @@ def evaluate(self) -> Optional[str]:
                 return f"Storage manifests waiting for definition of {prop}"
         return None
 
+    def is_ready(self, obj: HashableResource, cond: AnyCondition) -> Optional[bool]:
+        """Determine if the resource is ready."""
+        is_ready = super().is_ready(obj, cond)
+        if not is_ready:
+            try:
+                log_events(self.client, obj)
+            except Exception as e:
+                log.error("failed to log events: %s", e)
+
+        return is_ready
+
+
+def log_events(client: Client, obj: HashableResource) -> None:
+    """Log events for the object."""
+    object_events = collect_events(client, obj.resource)
+
+    if obj.kind in ["Deployment", "DaemonSet"]:
+        involved_pods = client.list(Pod, namespace=obj.namespace, labels={"app": obj.name})
+        object_events += [event for pod in involved_pods for event in collect_events(client, pod)]
+
+    for event in sorted(object_events, key=by_localtime):
+        log.info(
+            "Event %s/%s %s msg=%s",
+            event.involvedObject.kind,
+            event.involvedObject.name,
+            event.lastTimestamp and event.lastTimestamp.astimezone() or "Date not recorded",
+            event.message,
+        )
+
+
+def by_localtime(event: Event) -> datetime.datetime:
+    """Return the last timestamp of the event in local time."""
+    dt = event.lastTimestamp or datetime.datetime.now(datetime.timezone.utc)
+    return dt.astimezone()
+
+
+def collect_events(client: Client, resource: AnyResource) -> List[Event]:
+    """Collect events from the resource."""
+    kind: str = resource.kind or type(resource).__name__
+    return list(
+        client.list(
+            Event,
+            namespace=resource.metadata.namespace,
+            fields={
+                "involvedObject.kind": kind,
+                "involvedObject.name": resource.metadata.name,
+            },
+        )
+    )
+
 
 class UpdateControllerPlugin(Patch):
     """Update the controller args in Deployments."""

tox.ini

@@ -55,7 +55,7 @@ commands =
     codespell {toxinidir} --skip {toxinidir}/.git --skip {toxinidir}/.tox \
       --skip {toxinidir}/build --skip {toxinidir}/lib \
       --skip {toxinidir}/.mypy_cache --skip {toxinidir}/icon.svg \
-      --skip {[vars]cov_path} \
+      --skip {[vars]cov_path} --skip {toxinidir}/env \
       --skip "*.yaml"
     # pflake8 wrapper supports config from pyproject.toml
     pflake8 {[vars]all_path}