chore: Add more verbosity

.github/workflows/tilt.yml

@@ -33,9 +33,18 @@ jobs:
           MONITOR_PID=$!
           echo $MONITOR_PID > monitor.pid
           
+          # Add container status monitoring
+          (while true; do 
+            echo "=== Container Status at $(date '+%Y-%m-%d %H:%M:%S.%N') ===" >> ./logs/container_status.log
+            docker ps -a --format "{{.Names}}\t{{.Status}}\t{{.State}}" >> ./logs/container_status.log
+            sleep 0.1
+          done) &
+          STATUS_PID=$!
+          echo $STATUS_PID > status.pid
+          
           # Wait for monitoring to start and verify it's running
           sleep 5
-          if ! kill -0 $MONITOR_PID 2>/dev/null; then
+          if ! kill -0 $MONITOR_PID 2>/dev/null || ! kill -0 $STATUS_PID 2>/dev/null; then
             echo "Monitoring failed to start"
             exit 1
           fi
@@ -53,22 +62,33 @@ jobs:
       env:
         TF_VAR_sa_creds: ${{ secrets.GOOGLE_SA_BASE64 }}
         # Add these two steps:
+    - name: Collect failure information
+      if: failure()
+      run: |
+        echo "=== Docker Container Status ===" > ./logs/failure_info.log
+        docker ps -a >> ./logs/failure_info.log
+        echo "\n=== Failed Container Logs ===" >> ./logs/failure_info.log
+        for container in $(docker ps -a --filter "status=exited" --format '{{.Names}}'); do
+          echo "\n=== $container logs ===" >> ./logs/failure_info.log
+          docker logs $container &>> ./logs/failure_info.log
+        done
+    # Then update the stop monitoring step to kill both processes:
     - name: Stop monitoring
       if: always()
       run: |
         if [ -f monitor.pid ]; then
           kill $(cat monitor.pid) || echo "Monitoring process already stopped."
           rm -f monitor.pid
         fi
+        if [ -f status.pid ]; then
+          kill $(cat status.pid) || echo "Status monitoring process already stopped."
+          rm -f status.pid
+        fi
         sleep 5  # Allow log flush
-    - name: Debug log file
-      run: |
-        ls -la ./logs
-        cat ./logs/container_memory.log || echo "No logs found."
 
     - name: Upload memory logs
       if: always()
       uses: actions/upload-artifact@v4
       with:
-        name: memory-logs
-        path: ./logs/container_memory.log
+        name: debug-logs
+        path: ./logs/*.log

dev/bin/tilt-ci.sh

@@ -1,17 +1,39 @@
 #!/bin/bash
+set -eo pipefail  # Exit on error, pipe failure
 
 REPO_ROOT=$(git rev-parse --show-toplevel)
+LOGS_DIR="${REPO_ROOT}/logs"
+mkdir -p "${LOGS_DIR}"
 
+# Source CI environment if exists
 [ -f tmp.env.ci ] && source tmp.env.ci || true
 
 cd "${REPO_ROOT}"
-tilt ci --file dev/Tiltfile | tee tilt.log | grep cypress
+
+# Run Tilt with full logging
+echo "Starting Tilt CI at $(date)" | tee -a "${LOGS_DIR}/tilt-full.log"
+tilt ci --file dev/Tiltfile 2>&1 | tee -a "${LOGS_DIR}/tilt-full.log" | tee >(grep cypress > "${LOGS_DIR}/cypress.log") | grep cypress
+
 status=${PIPESTATUS[0]}
 
-if [[ $status -eq 0 ]]; then
-  echo "Tilt CI passed"
+# Collect additional information on failure
+if [[ $status -ne 0 ]]; then
+    echo "Tilt CI failed with status $status at $(date)" | tee -a "${LOGS_DIR}/tilt-full.log"
+    echo "=== Tilt Logs ===" | tee -a "${LOGS_DIR}/failure.log"
+    tail -n 100 "${LOGS_DIR}/tilt-full.log" >> "${LOGS_DIR}/failure.log"
+
+    echo "=== Container Status ===" | tee -a "${LOGS_DIR}/failure.log"
+    docker ps -a >> "${LOGS_DIR}/failure.log"
+
+    echo "=== Failed Container Logs ===" | tee -a "${LOGS_DIR}/failure.log"
+    docker ps -a --filter "status=exited" --format '{{.Names}}' | while read container; do
+        echo "=== $container ===" >> "${LOGS_DIR}/failure.log"
+        docker logs $container &>> "${LOGS_DIR}/failure.log"
+    done
+
+    cat "${LOGS_DIR}/failure.log"
 else
-  cat tilt.log
+    echo "Tilt CI passed at $(date)" | tee -a "${LOGS_DIR}/tilt-full.log"
 fi
 
-exit "$status"
+exit "$status"