Make kill process more robustness.

.github/workflows/test_pr.yml

@@ -5,7 +5,7 @@ on: [pull_request]
 jobs:
   is_source_changed:
     name: Check if sources has changed
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     outputs:
       is_source_changed: ${{steps.check_source_change.outputs.changed}}
     steps:
@@ -30,9 +30,9 @@ jobs:
             requirements.txt
 
   success:
-    name: PR Tets Completed
+    name: PR Test Completed
     needs: [is_source_changed, test]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     if: always() && needs.is_source_changed.result == 'success' && (needs.is_source_changed.outputs.is_source_changed == 'false' || needs.Test.result == 'success')
     steps:
       - run: echo ${{needs.is_source_changed.result}} ${{needs.is_source_changed.outputs.is_source_changed}} ${{needs.Test.result}}
@@ -41,7 +41,7 @@ jobs:
   lint_python:
     name: Run lint on python code
     needs: [is_source_changed]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     if: needs.is_source_changed.outputs.is_source_changed == 'true'
     env:
       BLACK_VERSION: "22.10.0"
@@ -64,7 +64,7 @@ jobs:
   build_ui:
     name: Build UI (with test and lint)
     needs: [is_source_changed]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     if: needs.is_source_changed.outputs.is_source_changed == 'true'
     env:
       REACT_APP_API_BASE_URL: "/fake/api/endpoint"
@@ -108,7 +108,7 @@ jobs:
       CI: ""
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest]
+        os: [ubuntu-22.04, windows-latest]
         python-version: [3.7, 3.8, '3.10', 3.11]
       fail-fast: false
 
@@ -128,10 +128,10 @@ jobs:
           name: ui_bundle
           path: testplan/web_ui/testing/build
       - name: Set up Zookeeper for tests
-        if: ${{ matrix.os == 'ubuntu-latest' }}
+        if: ${{ matrix.os == 'ubuntu-22.04' }}
         run: sudo apt-get -y install zookeeper zookeeper-bin zookeeperd
       - name: Set up Kafka for tests
-        if: ${{ matrix.os == 'ubuntu-latest' }}
+        if: ${{ matrix.os == 'ubuntu-22.04' }}
         run: |
           wget https://archive.apache.org/dist/kafka/3.5.2/kafka_2.13-3.5.2.tgz -O kafka.tgz
           sudo mkdir /opt/kafka

doc/newsfragments/2348_changed.kill_proc.rst

@@ -0,0 +1 @@
+Checks if a process exists by reading the `/proc/<pid>/stat`.

testplan/common/utils/process.py

@@ -5,6 +5,7 @@
 import subprocess
 import threading
 import time
+import re
 import warnings
 from enum import Enum, auto
 from signal import Signals
@@ -13,7 +14,12 @@
 import psutil
 
 from testplan.common.utils.logger import TESTPLAN_LOGGER
-from testplan.common.utils.timing import exponential_interval, get_sleeper
+from testplan.common.utils.timing import (
+    exponential_interval,
+    get_sleeper,
+    wait,
+    TimeoutException,
+)
 
 
 def _log_proc(msg: Any, warn=False, output: IO = None):
@@ -26,6 +32,31 @@ def _log_proc(msg: Any, warn=False, output: IO = None):
         warnings.warn(msg)
 
 
+def process_is_alive(proc_id: int) -> bool:
+    stat_file_path = f"/proc/{proc_id}/stat"
+    try:
+        with open(stat_file_path, "r") as stat_file:
+            stat_content = stat_file.read()
+        if re.match(r"\d+\s\(.*\)\s(\S+)\s.+", stat_content).group(1) != "Z":
+            return False
+        return True
+    except:
+        return True
+
+
+def wait_process_clean(proc_id: int, timeout: int = 5, output: IO = None):
+    if platform.system() != "Linux":
+        return
+    try:
+        wait(lambda: process_is_alive(proc_id), timeout=timeout)
+    except TimeoutException:
+        _log_proc(
+            msg=f"Process {proc_id} is not cleaned up",
+            warn=True,
+            output=output,
+        )
+
+
 def kill_process(
     proc: subprocess.Popen,
     timeout: int = 5,
@@ -54,6 +85,7 @@ def kill_process(
 
     retcode = proc.poll()
     if retcode is not None:
+        wait_process_clean(proc.pid, timeout=timeout)
         return retcode
 
     child_procs = psutil.Process(proc.pid).children(recursive=True)
@@ -92,7 +124,7 @@ def kill_process(
                 msg="While terminating child process - {}".format(exc),
                 warn=True,
             )
-
+    wait_process_clean(proc.pid, timeout=timeout)
     return proc.returncode