From 622193e097f60c7ceff8951ae5aeb82955e86e83 Mon Sep 17 00:00:00 2001
From: Petr Balogh <pbalogh@redhat.com>
Date: Fri, 4 Oct 2024 13:08:59 +0200
Subject: [PATCH] Release lock when exception hit in exec_cmd

This will prevent deadlocks when exception during subprocess.run hit.

Related issue:
https://github.com/red-hat-storage/ocs-ci/issues/10586

Hopefully it will fix this issue above.

Signed-off-by: Petr Balogh <pbalogh@redhat.com>
---
 ocs_ci/utility/utils.py | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/ocs_ci/utility/utils.py b/ocs_ci/utility/utils.py
index 6f13d4b9554..7d387de4402 100644
--- a/ocs_ci/utility/utils.py
+++ b/ocs_ci/utility/utils.py
@@ -597,6 +597,7 @@ def exec_cmd(
     silent=False,
     use_shell=False,
     cluster_config=None,
+    lock_timeout=7200,
     **kwargs,
 ):
     """
@@ -619,6 +620,7 @@ def exec_cmd(
         use_shell (bool): If True will pass the cmd without splitting
         cluster_config (MultiClusterConfig): In case of multicluster environment this object
                 will be non-null
+        lock_timeout (int): maximum timeout to wait for lock to prevent deadlocks (default 2 hours)
 
     Raises:
         CommandFailed: In case the command execution fails
@@ -668,18 +670,20 @@ def exec_cmd(
                 log.info(f"Found oc plugin {subcmd}")
         cmd = list_insert_at_position(cmd, kube_index, ["--kubeconfig"])
         cmd = list_insert_at_position(cmd, kube_index + 1, [kubepath])
-    if threading_lock and cmd[0] == "oc":
-        threading_lock.acquire()
-    completed_process = subprocess.run(
-        cmd,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        stdin=subprocess.PIPE,
-        timeout=timeout,
-        **kwargs,
-    )
-    if threading_lock and cmd[0] == "oc":
-        threading_lock.release()
+    try:
+        if threading_lock and cmd[0] == "oc":
+            threading_lock.acquire(timeout=lock_timeout)
+        completed_process = subprocess.run(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            stdin=subprocess.PIPE,
+            timeout=timeout,
+            **kwargs,
+        )
+    finally:
+        if threading_lock and cmd[0] == "oc":
+            threading_lock.release()
     masked_stdout = mask_secrets(completed_process.stdout.decode(), secrets)
     if len(completed_process.stdout) > 0:
         log.debug(f"Command stdout: {masked_stdout}")