Throwaway dbg commit

NVIDIA · Oct 16, 2024 · d6b5f85 · d6b5f85
1 parent 38c2ccb
commit d6b5f85
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 7 deletions.
diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml
@@ -75,12 +75,12 @@ jobs:
           pip install pytest lightning
           PY_VER_NODOT=$(python -c"import sysconfig; print(sysconfig.get_config_var('py_version_nodot'))")
           pip install ./dist/nvidia_resiliency_ext-*-cp${PY_VER_NODOT}-*.whl
-      - name: Run Fault Tolerance unit tests
-        run: |
-          pytest  -s -vvv -m "not gpu" ./tests/fault_tolerance/unit/ 
+      #- name: Run Fault Tolerance unit tests
+      #  run: |
+      #    pytest  -s -vvv -m "not gpu" ./tests/fault_tolerance/unit/ 
       - name: Run Straggler unit tests
         run: |
-          pytest  -s -vvv -m "not gpu" ./tests/straggler/unit/ 
-      - name: Run PTL callbacks unit tests
-        run: |
-          pytest  -s -vvv -m "not gpu" ./tests/ptl_resiliency/unit/
+          pytest  -s -vvv -m "not gpu" ./tests/straggler/unit/test_data_shared.py
+      #- name: Run PTL callbacks unit tests
+      #  run: |
+      #    pytest  -s -vvv -m "not gpu" ./tests/ptl_resiliency/unit/
diff --git a/tests/straggler/unit/_utils.py b/tests/straggler/unit/_utils.py
@@ -111,10 +111,16 @@ def distributed_worker(
 
     ready_flag.set()
 
+    print(f"RANK {rank} STARTS WORKER FN", flush=True, file=sys.stderr)
+
     worker_fn(**kwargs)
 
+    print(f"RANK {rank} BEFORE destroy_process_group", flush=True, file=sys.stderr)
+
     torch.distributed.destroy_process_group()
 
+    print(f"RANK {rank} AFTER destroy_process_group", flush=True, file=sys.stderr)
+
     sys.exit(0)