From d20ee414823cd90df1786e14da10ed7b78500a63 Mon Sep 17 00:00:00 2001
From: Meir Tseitlin <meir@imubit.com>
Date: Mon, 20 May 2024 16:39:24 -0500
Subject: [PATCH] temp

---
 src/k8s_job_scheduler/job_manager.py | 34 +++++++++++++++++++++++++---
 1 file changed, 31 insertions(+), 3 deletions(-)

diff --git a/src/k8s_job_scheduler/job_manager.py b/src/k8s_job_scheduler/job_manager.py
index 0dff1df..980885d 100644
--- a/src/k8s_job_scheduler/job_manager.py
+++ b/src/k8s_job_scheduler/job_manager.py
@@ -163,12 +163,40 @@ def job_logs(self, job_name):
         # Get pods
         pods = self.list_pods(job_name=job_name)
 
-        all_logs = [
-            self._core_api.read_namespaced_pod_log(pod, self._namespace) for pod in pods
+        all_status = [
+            self._core_api.read_namespaced_pod_status(pod, self._namespace)
+            for pod in pods
         ]
 
+        # Check status before pulling logs
+        active_pods = [
+            pod.metadata.name for pod in all_status if pod.status.phase == "active"
+        ]
+        inactive_pods = {
+            pod.metadata.name: pod for pod in all_status if pod.status.phase != "active"
+        }
+
+        inactive_statuses = {
+            pod: (
+                f"<b>Pod {pod} is inactive.</b> <br/> "
+                f"<b>Phase</b>: {inactive_pods[pod].status.phase} <br/> "
+                f'<b>Container state</b>: {inactive_pods[pod].status.container_statuses[0].state if inactive_pods[pod].status.container_statuses else "N/A"} <br/>'  # noqa: E501
+                f"<b>Conditions</b>: {inactive_pods[pod].status.conditions}"
+            )
+            for pod in inactive_pods
+        }
+
+        all_logs = {
+            pod: self._core_api.read_namespaced_pod_log_with_http_info(
+                pod, self._namespace
+            )
+            for pod in active_pods
+        }
+
+        all_logs.update(inactive_statuses)
+
         return (
-            all_logs[0]
+            next(iter(all_logs.values()))
             if len(all_logs) == 1
             else all_logs
             if len(all_logs) > 1