switch troubleshoot to Inspect where appropriate. (#386)

codebundles/k8s-deployment-healthcheck/deployment_logs.sh

@@ -305,10 +305,10 @@ if [[ -n "$INTERESTING_RESOURCES" ]]; then
         case "$type" in
         pod)
             if [[ "$status" != "Running" ]]; then
-                recommendations+=("Troubleshoot failed pods in namespace \`${NAMESPACE}\`")
+                recommendations+=("Inspect failed pods in namespace \`${NAMESPACE}\`")
             fi
             if ((restarts > 0)); then
-                recommendations+=("Troubleshoot container restarts in namespace \`${NAMESPACE}\`")
+                recommendations+=("Inspect container restarts in namespace \`${NAMESPACE}\`")
             fi
             ;;
         deployment|deployment.apps)

codebundles/k8s-ingress-healthcheck/runbook.robot

@@ -20,7 +20,7 @@ Fetch Ingress Object Health in Namespace `${NAMESPACE}`
     [Documentation]    Fetches all ingress objects in the namespace and outputs the name, health status, services, and endpoints. 
     [Tags]    service    ingress    endpoint   health    ${NAMESPACE}
     ${ingress_object_summary}=    RW.CLI.Run Cli
-    ...    cmd=NAMESPACE="${NAMESPACE}";CONTEXT="${CONTEXT}";ingresses=$(${KUBERNETES_DISTRIBUTION_BINARY} get ingress -n "$NAMESPACE" --context "$CONTEXT" -o jsonpath='{range .items[*]}{.metadata.name}{"\\n"}{end}');for ingress in $ingresses;do echo "Ingress: $ingress";health_status="NA";backend_services=$(${KUBERNETES_DISTRIBUTION_BINARY} get ingress "$ingress" -n "$NAMESPACE" --context "$CONTEXT" -o jsonpath='{range .spec.rules[*].http.paths[*]}{.backend.service.name}{"|"}{.backend.service.port.number}{"\\n"}{end}');while IFS='|' read -r service port;do echo "Backend Service: $service, Port: $port";target_ports=$(${KUBERNETES_DISTRIBUTION_BINARY} get service "$service" -n "$NAMESPACE" --context "$CONTEXT" -o jsonpath="{.spec.ports[?(@.port==$port)].targetPort}");service_exists=$(${KUBERNETES_DISTRIBUTION_BINARY} get service "$service" -n "$NAMESPACE" --context "$CONTEXT" -ojsonpath='{.metadata.name}');if [ -z "$service_exists" ];then health_status="Unhealthy";echo "Error: Service $service does not exist";echo "Next Step: Check namespace $NAMESPACE for service name $service.";continue;else selectors=$(${KUBERNETES_DISTRIBUTION_BINARY} get svc "$service" -n "$NAMESPACE" --context "$CONTEXT" -o jsonpath='{.spec.selector}');label_selector="";for key in $(echo $selectors | jq -r 'keys[]');do value=$(echo $selectors | jq -r --arg key "$key" '.[$key]');label_selector="\${label_selector}\${key}=\${value},";done;label_selector=\${label_selector::-1};found_owner=0;for kind in deployment statefulset daemonset;do matching_owners=$(${KUBERNETES_DISTRIBUTION_BINARY} get $kind -n "$NAMESPACE" --context "$CONTEXT" -l "$label_selector" -o jsonpath='{.items[*].metadata.name}');if [ -n "$matching_owners" ];then for owner in $matching_owners;do echo "Owner Kind: $kind";echo "Owner Name: $owner";found_owner=1;done;fi;done;if [ "$found_owner" == 0 ];then echo "Error: No matching deployment, statefulset, or daemonset found to match label selector \\`"$label_selector"\\`";echo "Next Steps:\\n- Check namespace \\`"$NAMESPACE"\\` for deployment, statefulset, or daemonset with labels that match \\`"$label_selector"\\`";fi;port_found="No";associated_pods=$(${KUBERNETES_DISTRIBUTION_BINARY} get pods -n "$NAMESPACE" --context "$CONTEXT" -l "$label_selector" -o jsonpath='{.items[*].metadata.name}');for pod in $associated_pods;do container_ports=$(${KUBERNETES_DISTRIBUTION_BINARY} get pod "$pod" -n "$NAMESPACE" --context "$CONTEXT" -o jsonpath='{.spec.containers[*].ports[*].containerPort}');for target_port in $target_ports;do if echo "$container_ports" | grep -wq "$target_port";then port_found="Yes";break;fi;done;done;if [ "$port_found" = "No" ];then health_status="Unhealthy";echo "Warning: targetPort $target_ports of service $service is not found as a containerPort in associated pods";else health_status="Healthy";fi;endpoint_pods=$(${KUBERNETES_DISTRIBUTION_BINARY} get endpoints "$service" -n "$NAMESPACE" --context "$CONTEXT" -ojsonpath='{range .subsets[*].addresses[*]}- Pod Name: {.targetRef.name}\\n Pod IP: {.ip}\\n{end}');if [ -z "$endpoint_pods" ];then health_status="Unhealthy";echo "Error: Endpoint for service \\`"$service"\\` does not have any running pods"; echo "Next Steps:\\n- Troubleshoot Container Restarts in Namespace \\`"$NAMESPACE"\\` \\n- Troubleshoot Pending Pods In Namespace \\`"$NAMESPACE"\\`\\n- Troubleshoot Failed Pods In Namespace \\`"$NAMESPACE"\\`";else echo "Endpoint Pod:";echo "$endpoint_pods";health_status="Healthy";fi;fi;done<<<"$backend_services";echo "Health Status: $health_status";echo "------------";done
+    ...    cmd=NAMESPACE="${NAMESPACE}";CONTEXT="${CONTEXT}";ingresses=$(${KUBERNETES_DISTRIBUTION_BINARY} get ingress -n "$NAMESPACE" --context "$CONTEXT" -o jsonpath='{range .items[*]}{.metadata.name}{"\\n"}{end}');for ingress in $ingresses;do echo "Ingress: $ingress";health_status="NA";backend_services=$(${KUBERNETES_DISTRIBUTION_BINARY} get ingress "$ingress" -n "$NAMESPACE" --context "$CONTEXT" -o jsonpath='{range .spec.rules[*].http.paths[*]}{.backend.service.name}{"|"}{.backend.service.port.number}{"\\n"}{end}');while IFS='|' read -r service port;do echo "Backend Service: $service, Port: $port";target_ports=$(${KUBERNETES_DISTRIBUTION_BINARY} get service "$service" -n "$NAMESPACE" --context "$CONTEXT" -o jsonpath="{.spec.ports[?(@.port==$port)].targetPort}");service_exists=$(${KUBERNETES_DISTRIBUTION_BINARY} get service "$service" -n "$NAMESPACE" --context "$CONTEXT" -ojsonpath='{.metadata.name}');if [ -z "$service_exists" ];then health_status="Unhealthy";echo "Error: Service $service does not exist";echo "Next Step: Check namespace $NAMESPACE for service name $service.";continue;else selectors=$(${KUBERNETES_DISTRIBUTION_BINARY} get svc "$service" -n "$NAMESPACE" --context "$CONTEXT" -o jsonpath='{.spec.selector}');label_selector="";for key in $(echo $selectors | jq -r 'keys[]');do value=$(echo $selectors | jq -r --arg key "$key" '.[$key]');label_selector="\${label_selector}\${key}=\${value},";done;label_selector=\${label_selector::-1};found_owner=0;for kind in deployment statefulset daemonset;do matching_owners=$(${KUBERNETES_DISTRIBUTION_BINARY} get $kind -n "$NAMESPACE" --context "$CONTEXT" -l "$label_selector" -o jsonpath='{.items[*].metadata.name}');if [ -n "$matching_owners" ];then for owner in $matching_owners;do echo "Owner Kind: $kind";echo "Owner Name: $owner";found_owner=1;done;fi;done;if [ "$found_owner" == 0 ];then echo "Error: No matching deployment, statefulset, or daemonset found to match label selector \\`"$label_selector"\\`";echo "Next Steps:\\n- Check namespace \\`"$NAMESPACE"\\` for deployment, statefulset, or daemonset with labels that match \\`"$label_selector"\\`";fi;port_found="No";associated_pods=$(${KUBERNETES_DISTRIBUTION_BINARY} get pods -n "$NAMESPACE" --context "$CONTEXT" -l "$label_selector" -o jsonpath='{.items[*].metadata.name}');for pod in $associated_pods;do container_ports=$(${KUBERNETES_DISTRIBUTION_BINARY} get pod "$pod" -n "$NAMESPACE" --context "$CONTEXT" -o jsonpath='{.spec.containers[*].ports[*].containerPort}');for target_port in $target_ports;do if echo "$container_ports" | grep -wq "$target_port";then port_found="Yes";break;fi;done;done;if [ "$port_found" = "No" ];then health_status="Unhealthy";echo "Warning: targetPort $target_ports of service $service is not found as a containerPort in associated pods";else health_status="Healthy";fi;endpoint_pods=$(${KUBERNETES_DISTRIBUTION_BINARY} get endpoints "$service" -n "$NAMESPACE" --context "$CONTEXT" -ojsonpath='{range .subsets[*].addresses[*]}- Pod Name: {.targetRef.name}\\n Pod IP: {.ip}\\n{end}');if [ -z "$endpoint_pods" ];then health_status="Unhealthy";echo "Error: Endpoint for service \\`"$service"\\` does not have any running pods"; echo "Next Steps:\\n- Inspect Container Restarts in Namespace \\`"$NAMESPACE"\\` \\n- Inspect Pending Pods In Namespace \\`"$NAMESPACE"\\`\\n- Inspect Failed Pods In Namespace \\`"$NAMESPACE"\\`";else echo "Endpoint Pod:";echo "$endpoint_pods";health_status="Healthy";fi;fi;done<<<"$backend_services";echo "Health Status: $health_status";echo "------------";done
     ...    env=${env}
     ...    secret_file__kubeconfig=${kubeconfig}
     ...    show_in_rwl_cheatsheet=true

libraries/RW/NextSteps/Kubernetes/mapping.yaml

@@ -17,13 +17,12 @@
 # BEGIN NamespaceTroubleshooting
 NamespaceTroubleshooting: &nst
   - Trace And Troubleshoot Namespace Warning Events And Errors
-  - Troubleshoot Container Restarts In Namespace
-  - Troubleshoot Pending Pods In Namespace
-  - Troubleshoot Failed Pods In Namespace
-  - Troubleshoot Workload Status Conditions In Namespace
+  - Inspect Container Restarts In Namespace
+  - Inspect Pending Pods In Namespace
+  - Inspect Failed Pods In Namespace
+  - Inspect Workload Status Conditions In Namespace
   - Get Listing Of Resources In Namespace
   - Check For Namespace Event Anomalies
-  - Troubleshoot Namespace Services And Application Workloads
   - Check Missing or Risky PodDisruptionBudget Policies
   - Service:$service_name
   - Ingress:$ingress_name