diff --git a/.github/workflows/tilt.yml b/.github/workflows/tilt.yml index c339b6b17..dc2f77340 100644 --- a/.github/workflows/tilt.yml +++ b/.github/workflows/tilt.yml @@ -33,9 +33,18 @@ jobs: MONITOR_PID=$! echo $MONITOR_PID > monitor.pid + # Add container status monitoring + (while true; do + echo "=== Container Status at $(date '+%Y-%m-%d %H:%M:%S.%N') ===" >> ./logs/container_status.log + docker ps -a --format "{{.Names}}\t{{.Status}}\t{{.State}}" >> ./logs/container_status.log + sleep 0.1 + done) & + STATUS_PID=$! + echo $STATUS_PID > status.pid + # Wait for monitoring to start and verify it's running sleep 5 - if ! kill -0 $MONITOR_PID 2>/dev/null; then + if ! kill -0 $MONITOR_PID 2>/dev/null || ! kill -0 $STATUS_PID 2>/dev/null; then echo "Monitoring failed to start" exit 1 fi @@ -53,6 +62,17 @@ jobs: env: TF_VAR_sa_creds: ${{ secrets.GOOGLE_SA_BASE64 }} # Add these two steps: + - name: Collect failure information + if: failure() + run: | + echo "=== Docker Container Status ===" > ./logs/failure_info.log + docker ps -a >> ./logs/failure_info.log + echo "\n=== Failed Container Logs ===" >> ./logs/failure_info.log + for container in $(docker ps -a --filter "status=exited" --format '{{.Names}}'); do + echo "\n=== $container logs ===" >> ./logs/failure_info.log + docker logs $container &>> ./logs/failure_info.log + done + # Then update the stop monitoring step to kill both processes: - name: Stop monitoring if: always() run: | @@ -60,15 +80,15 @@ jobs: kill $(cat monitor.pid) || echo "Monitoring process already stopped." rm -f monitor.pid fi + if [ -f status.pid ]; then + kill $(cat status.pid) || echo "Status monitoring process already stopped." + rm -f status.pid + fi sleep 5 # Allow log flush - - name: Debug log file - run: | - ls -la ./logs - cat ./logs/container_memory.log || echo "No logs found." - name: Upload memory logs if: always() uses: actions/upload-artifact@v4 with: - name: memory-logs - path: ./logs/container_memory.log + name: debug-logs + path: ./logs/*.log diff --git a/dev/bin/tilt-ci.sh b/dev/bin/tilt-ci.sh index 896ef7220..86c2d2d9e 100755 --- a/dev/bin/tilt-ci.sh +++ b/dev/bin/tilt-ci.sh @@ -1,17 +1,39 @@ #!/bin/bash +set -eo pipefail # Exit on error, pipe failure REPO_ROOT=$(git rev-parse --show-toplevel) +LOGS_DIR="${REPO_ROOT}/logs" +mkdir -p "${LOGS_DIR}" +# Source CI environment if exists [ -f tmp.env.ci ] && source tmp.env.ci || true cd "${REPO_ROOT}" -tilt ci --file dev/Tiltfile | tee tilt.log | grep cypress + +# Run Tilt with full logging +echo "Starting Tilt CI at $(date)" | tee -a "${LOGS_DIR}/tilt-full.log" +tilt ci --file dev/Tiltfile 2>&1 | tee -a "${LOGS_DIR}/tilt-full.log" | tee >(grep cypress > "${LOGS_DIR}/cypress.log") | grep cypress + status=${PIPESTATUS[0]} -if [[ $status -eq 0 ]]; then - echo "Tilt CI passed" +# Collect additional information on failure +if [[ $status -ne 0 ]]; then + echo "Tilt CI failed with status $status at $(date)" | tee -a "${LOGS_DIR}/tilt-full.log" + echo "=== Tilt Logs ===" | tee -a "${LOGS_DIR}/failure.log" + tail -n 100 "${LOGS_DIR}/tilt-full.log" >> "${LOGS_DIR}/failure.log" + + echo "=== Container Status ===" | tee -a "${LOGS_DIR}/failure.log" + docker ps -a >> "${LOGS_DIR}/failure.log" + + echo "=== Failed Container Logs ===" | tee -a "${LOGS_DIR}/failure.log" + docker ps -a --filter "status=exited" --format '{{.Names}}' | while read container; do + echo "=== $container ===" >> "${LOGS_DIR}/failure.log" + docker logs $container &>> "${LOGS_DIR}/failure.log" + done + + cat "${LOGS_DIR}/failure.log" else - cat tilt.log + echo "Tilt CI passed at $(date)" | tee -a "${LOGS_DIR}/tilt-full.log" fi -exit "$status" +exit "$status" \ No newline at end of file