diff --git a/README.md b/README.md index fc1c2f4f..b57a5416 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ Run the codebundle | [Azure Monitor Activity Log SLI](https://github.com/runwhen-contrib/rw-cli-codecollection/blob/main/codebundles/azure-monitor-event-triage/sli.robot) | `Kubernetes`, `AKS`, `Azure` | `Run Azure Monitor Activity Log Triage` | Measures the count of error activity log entries as a SLI metric for the Azure tenancy. [Docs](https://docs.runwhen.com/public/v/cli-codecollection/azure-monitor-event-triage) | | [Azure Monitor Event Triage](https://github.com/runwhen-contrib/rw-cli-codecollection/blob/main/codebundles/azure-monitor-event-triage/runbook.robot) | `Kubernetes`, `AKS`, `Azure` | `Run Azure Monitor Activity Log Triage` | Triages issues related to a Azure Loadbalancers, Kubernetes ingress objects and services. [Docs](https://docs.runwhen.com/public/v/cli-codecollection/azure-monitor-event-triage) | | [GCP Gcloud Log Inspection](https://github.com/runwhen-contrib/rw-cli-codecollection/blob/main/codebundles/gcloud-log-inspection/runbook.robot) | `GCP`, `Gcloud`, `Google Monitoring` | `Inspect GCP Logs For Common Errors` | Fetches logs from a GCP using a configurable query and raises an issue with details on the most common issues. [Docs](https://docs.runwhen.com/public/v/cli-codecollection/gcloud-log-inspection) | -| [GCP Node Prempt List](https://github.com/runwhen-contrib/rw-cli-codecollection/blob/main/codebundles/gcloud-node-preempt/sli.robot) | `GCP`, `GKE` | `Count the number of nodes in active prempt operation` | Check if any GCP nodes have an active preempt operation. [Docs](https://docs.runwhen.com/public/v/cli-codecollection/gcloud-node-preempt) | +| [GCP Node Prempt List](https://github.com/runwhen-contrib/rw-cli-codecollection/blob/main/codebundles/gcloud-node-preempt/runbook.robot) | `GCP`, `GKE` | `List all nodes in an active prempt operation` | List all GCP nodes that have an active preempt operation. [Docs](https://docs.runwhen.com/public/v/cli-codecollection/gcloud-node-preempt) | | [GKE Kong Ingress Host Triage](https://github.com/runwhen-contrib/rw-cli-codecollection/blob/main/codebundles/curl-gmp-kong-ingress-inspection/runbook.robot) | `GCP`, `GMP`, `Ingress`, `Kong`, `Metrics` | `Check If Kong Ingress HTTP Error Rate Violates HTTP Error Threshold`, `Check If Kong Ingress HTTP Request Latency Violates Threshold`, `Check If Kong Ingress Controller Reports Upstream Errors` | Collects Kong ingress host metrics from GMP on GCP and inspects the results for ingress with a HTTP error code rate greater than zero over a configurable duration and raises issues based on the number of ingress with error codes. [Docs](https://docs.runwhen.com/public/v/cli-codecollection/curl-gmp-kong-ingress-inspection) | | [GKE Nginx Ingress Host Triage](https://github.com/runwhen-contrib/rw-cli-codecollection/blob/main/codebundles/curl-gmp-nginx-ingress-inspection/runbook.robot) | `GCP`, `GMP`, `Ingress`, `Nginx`, `Metrics` | `Fetch Nginx Ingress HTTP Errors From GMP And Perform Inspection On Results`, `Find Ingress Owner and Service Health` | Collects Nginx ingress host controller metrics from GMP on GCP and inspects the results for ingress with a HTTP error code rate greater than zero over a configurable duration and raises issues based on the number of ingress with error codes. [Docs](https://docs.runwhen.com/public/v/cli-codecollection/curl-gmp-nginx-ingress-inspection) | | [Kubeprometheus Operator Troubleshoot](https://github.com/runwhen-contrib/rw-cli-codecollection/blob/main/codebundles/k8s-prometheus-healthcheck/runbook.robot) | `Kubernetes`, `AKS`, `EKS`, `GKE`, `OpenShift`, `Prometheus` | `Check Prometheus Service Monitors`, `Check For Successful Rule Setup`, `Verify Prometheus RBAC Can Access ServiceMonitors`, `Identify Endpoint Scraping Errors`, `Check Prometheus API Healthy` | This taskset investigates the logs, state and health of Kubernetes Prometheus operator. [Docs](https://docs.runwhen.com/public/v/cli-codecollection/k8s-prometheus-healthcheck) | @@ -62,7 +62,7 @@ Run the codebundle | [Kubernetes ArgoCD Application Health & Troubleshoot](https://github.com/runwhen-contrib/rw-cli-codecollection/blob/main/codebundles/k8s-argocd-application-health/runbook.robot) | `Kubernetes`, `AKS`, `EKS`, `GKE`, `OpenShift`, `ArgoCD` | `Fetch ArgoCD Application Sync Status & Health`, `Fetch ArgoCD Application Last Sync Operation Details`, `Fetch Unhealthy ArgoCD Application Resources`, `Scan For Errors in Pod Logs Related to ArgoCD Application Deployments`, `Fully Describe ArgoCD Application` | This taskset collects information and runs general troubleshooting checks against argocd application objects within a namespace. [Docs](https://docs.runwhen.com/public/v/cli-codecollection/k8s-argocd-application-health) | | [Kubernetes ArgoCD HelmRelease TaskSet](https://github.com/runwhen-contrib/rw-cli-codecollection/blob/main/codebundles/k8s-argocd-helm-health/runbook.robot) | `Kubernetes`, `AKS`, `EKS`, `GKE`, `OpenShift`, `ArgoCD` | `Fetch all available ArgoCD Helm releases`, `Fetch Installed ArgoCD Helm release versions` | This codebundle runs a series of tasks to identify potential helm release issues related to ArgoCD managed Helm objects. [Docs](https://docs.runwhen.com/public/v/cli-codecollection/k8s-argocd-helm-health) | | [Kubernetes Artifactory Triage](https://github.com/runwhen-contrib/rw-cli-codecollection/blob/main/codebundles/k8s-artifactory-health/runbook.robot) | `Kubernetes`, `AKS`, `EKS`, `GKE`, `OpenShift`, `Artifactory` | `Check Artifactory Liveness and Readiness Endpoints` | Performs a triage on the Open Source version of Artifactory in a Kubernetes cluster. [Docs](https://docs.runwhen.com/public/v/cli-codecollection/k8s-artifactory-health) | -| [Kubernetes CertManager Healthcheck](https://github.com/runwhen-contrib/rw-cli-codecollection/blob/main/codebundles/k8s-certmanager-healthcheck/sli.robot) | `Kubernetes`, `AKS`, `EKS`, `GKE`, `OpenShift` | `Get Health Score of CertManager Workloads` | Check the health of pods deployed by cert-manager. [Docs](https://docs.runwhen.com/public/v/cli-codecollection/k8s-certmanager-healthcheck) | +| [Kubernetes CertManager Healthcheck](https://github.com/runwhen-contrib/rw-cli-codecollection/blob/main/codebundles/k8s-certmanager-healthcheck/runbook.robot) | `Kubernetes`, `AKS`, `EKS`, `GKE`, `OpenShift`, `CertManager` | `Get Namespace Certificate Summary`, `Find Failed Certificate Requests and Identify Issues` | This taskset checks that your cert manager certificates are renewing as expected, raising issues when they are past due in the configured namespace [Docs](https://docs.runwhen.com/public/v/cli-codecollection/k8s-certmanager-healthcheck) | | [Kubernetes Daemonset Triage](https://github.com/runwhen-contrib/rw-cli-codecollection/blob/main/codebundles/k8s-daemonset-healthcheck/runbook.robot) | `Kubernetes`, `AKS`, `EKS`, `GKE`, `OpenShift` | `Get DaemonSet Log Details For Report`, `Get Related Daemonset Events`, `Check Daemonset Replicas` | Triages issues related to a Daemonset and its available replicas. [Docs](https://docs.runwhen.com/public/v/cli-codecollection/k8s-daemonset-healthcheck) | | [Kubernetes Deployment Triage](https://github.com/runwhen-contrib/rw-cli-codecollection/blob/main/codebundles/k8s-deployment-healthcheck/runbook.robot) | `Kubernetes`, `AKS`, `EKS`, `GKE`, `OpenShift` | `Check Deployment Log For Issues`, `Troubleshoot Deployment Warning Events`, `Get Deployment Workload Details For Report`, `Troubleshoot Deployment Replicas`, `Check For Deployment Event Anomalies` | Triages issues related to a deployment and its replicas. [Docs](https://docs.runwhen.com/public/v/cli-codecollection/k8s-deployment-healthcheck) | | [Kubernetes Flux Choas Testing](https://github.com/runwhen-contrib/rw-cli-codecollection/blob/main/codebundles/k8s-chaos-flux/runbook.robot) | `Kubernetes`, `AKS`, `EKS`, `GKE`, `OpenShift` | `Suspend the Flux Resource Reconciliation`, `Find Random FluxCD Workload as Chaos Target`, `Execute Chaos Command`, `Execute Additional Chaos Command`, `Resume Flux Resource Reconciliation` | This taskset is used to suspend a flux resource for the purposes of executing chaos tasks. [Docs](https://docs.runwhen.com/public/v/cli-codecollection/k8s-chaos-flux) | @@ -86,7 +86,7 @@ Run the codebundle | [Kubernetes Vault Triage](https://github.com/runwhen-contrib/rw-cli-codecollection/blob/main/codebundles/k8s-vault-healthcheck/runbook.robot) | `AKS`, `EKS`, `GKE`, `Kubernetes`, `Vault` | `Fetch Vault CSI Driver Logs`, `Get Vault CSI Driver Warning Events`, `Check Vault CSI Driver Replicas`, `Fetch Vault Logs`, `Get Related Vault Events`, `Fetch Vault StatefulSet Manifest Details`, `Fetch Vault DaemonSet Manifest Details`, `Verify Vault Availability`, `Check Vault StatefulSet Replicas` | A suite of tasks that can be used to triage potential issues in your vault namespace. [Docs](https://docs.runwhen.com/public/v/cli-codecollection/k8s-vault-healthcheck) | | [Terraform Cloud Workspace Lock Check](https://github.com/runwhen-contrib/rw-cli-codecollection/blob/main/codebundles/terraform-cloud-workspace-lock-check/runbook.robot) | `Terraform Cloud` | `Checking whether the Terraform Cloud Workspace is in a locked state` | Check whether the Terraform Cloud Workspace is in a locked state. [Docs](https://docs.runwhen.com/public/v/cli-codecollection/terraform-cloud-workspace-lock-check) | | [Test Issues](https://github.com/runwhen-contrib/rw-cli-codecollection/blob/main/codebundles/test-issue/runbook.robot) | `Test` | `Raise Full Issue` | A codebundle for testing the issues feature. Purely for testing flow. [Docs](https://docs.runwhen.com/public/v/cli-codecollection/test-issue) | -| [cURL HTTP OK](https://github.com/runwhen-contrib/rw-cli-codecollection/blob/main/codebundles/curl-http-ok/sli.robot) | `Linux macOS Windows HTTP` | `Checking HTTP URL Is Available And Timely` | This taskset uses curl to validate the response code of the endpoint. Returns ascore of 1 if healthy, an 0 if unhealthy. [Docs](https://docs.runwhen.com/public/v/cli-codecollection/curl-http-ok) | +| [cURL HTTP OK](https://github.com/runwhen-contrib/rw-cli-codecollection/blob/main/codebundles/curl-http-ok/runbook.robot) | `Linux macOS Windows HTTP` | `Checking HTTP URL Is Available And Timely` | This taskset uses curl to validate the response code of the endpoint and provides the total time of the request. [Docs](https://docs.runwhen.com/public/v/cli-codecollection/curl-http-ok) | | [cli-test-taskset](https://github.com/runwhen-contrib/rw-cli-codecollection/blob/main/codebundles/cli-test/runbook.robot) | `cli` | `Run CLI and Parse Output For Issues`, `Exec Test`, `Local Process Test` | This taskset smoketests the CLI codebundle setup and run process [Docs](https://docs.runwhen.com/public/v/cli-codecollection/cli-test) | | [cmd-test-taskset](https://github.com/runwhen-contrib/rw-cli-codecollection/blob/main/codebundles/cmd-test/runbook.robot) | `cmd` | `Run CLI Command`, `Run Bash File`, `Log Suggestion` | This taskset smoketests the CLI codebundle setup and run process by running a bare command [Docs](https://docs.runwhen.com/public/v/cli-codecollection/cmd-test) | diff --git a/SUMMARY.md b/SUMMARY.md index f84ae077..3294edf8 100644 --- a/SUMMARY.md +++ b/SUMMARY.md @@ -1,41 +1,41 @@ # Summary ## Codebundles +* [k8s-image-check](codebundles/k8s-image-check/README.md) +* [k8s-jenkins-healthcheck](codebundles/k8s-jenkins-healthcheck/README.md) +* [azure-loadbalancer-triage](codebundles/azure-loadbalancer-triage/README.md) +* [k8s-fluxcd-kustomization-health](codebundles/k8s-fluxcd-kustomization-health/README.md) * [k8s-deployment-healthcheck](codebundles/k8s-deployment-healthcheck/README.md) -* [azure-monitor-event-triage](codebundles/azure-monitor-event-triage/README.md) -* [k8s-fluxcd-helm-health](codebundles/k8s-fluxcd-helm-health/README.md) -* [k8s-argocd-helm-health](codebundles/k8s-argocd-helm-health/README.md) -* [k8s-certmanager-healthcheck](codebundles/k8s-certmanager-healthcheck/README.md) +* [k8s-restart-resource](codebundles/k8s-restart-resource/README.md) +* [aws-cloudwatch-overused-ec2](codebundles/aws-cloudwatch-overused-ec2/README.md) * [k8s-prometheus-healthcheck](codebundles/k8s-prometheus-healthcheck/README.md) +* [curl-gmp-kong-ingress-inspection](codebundles/curl-gmp-kong-ingress-inspection/README.md) * [gcloud-log-inspection](codebundles/gcloud-log-inspection/README.md) -* [k8s-serviceaccount-check](codebundles/k8s-serviceaccount-check/README.md) -* [k8s-chaos-flux](codebundles/k8s-chaos-flux/README.md) -* [k8s-loki-healthcheck](codebundles/k8s-loki-healthcheck/README.md) +* [azure-monitor-event-triage](codebundles/azure-monitor-event-triage/README.md) +* [k8s-vault-healthcheck](codebundles/k8s-vault-healthcheck/README.md) +* [k8s-certmanager-healthcheck](codebundles/k8s-certmanager-healthcheck/README.md) +* [k8s-ingress-gce-healthcheck](codebundles/k8s-ingress-gce-healthcheck/README.md) +* [k8s-statefulset-healthcheck](codebundles/k8s-statefulset-healthcheck/README.md) +* [k8s-pvc-healthcheck](codebundles/k8s-pvc-healthcheck/README.md) * [curl-http-ok](codebundles/curl-http-ok/README.md) +* [k8s-serviceaccount-check](codebundles/k8s-serviceaccount-check/README.md) * [aws-eks-node-reboot](codebundles/aws-eks-node-reboot/README.md) -* [k8s-argocd-application-health](codebundles/k8s-argocd-application-health/README.md) +* [k8s-fluxcd-helm-health](codebundles/k8s-fluxcd-helm-health/README.md) * [curl-gmp-nginx-ingress-inspection](codebundles/curl-gmp-nginx-ingress-inspection/README.md) -* [k8s-daemonset-healthcheck](codebundles/k8s-daemonset-healthcheck/README.md) -* [k8s-app-troubleshoot](codebundles/k8s-app-troubleshoot/README.md) -* [curl-gmp-kong-ingress-inspection](codebundles/curl-gmp-kong-ingress-inspection/README.md) * [k8s-artifactory-health](codebundles/k8s-artifactory-health/README.md) -* [azure-loadbalancer-triage](codebundles/azure-loadbalancer-triage/README.md) -* [k8s-redis-healthcheck](codebundles/k8s-redis-healthcheck/README.md) -* [k8s-restart-resource](codebundles/k8s-restart-resource/README.md) -* [k8s-fluxcd-kustomization-health](codebundles/k8s-fluxcd-kustomization-health/README.md) -* [k8s-image-check](codebundles/k8s-image-check/README.md) -* [aws-cloudwatch-overused-ec2](codebundles/aws-cloudwatch-overused-ec2/README.md) -* [k8s-statefulset-healthcheck](codebundles/k8s-statefulset-healthcheck/README.md) -* [k8s-ingress-gce-healthcheck](codebundles/k8s-ingress-gce-healthcheck/README.md) -* [k8s-vault-healthcheck](codebundles/k8s-vault-healthcheck/README.md) -* [k8s-jenkins-healthcheck](codebundles/k8s-jenkins-healthcheck/README.md) -* [k8s-pvc-healthcheck](codebundles/k8s-pvc-healthcheck/README.md) * [k8s-ingress-healthcheck](codebundles/k8s-ingress-healthcheck/README.md) +* [k8s-app-troubleshoot](codebundles/k8s-app-troubleshoot/README.md) +* [k8s-daemonset-healthcheck](codebundles/k8s-daemonset-healthcheck/README.md) * [gcloud-node-preempt](codebundles/gcloud-node-preempt/README.md) * [k8s-namespace-healthcheck](codebundles/k8s-namespace-healthcheck/README.md) +* [k8s-argocd-application-health](codebundles/k8s-argocd-application-health/README.md) +* [k8s-loki-healthcheck](codebundles/k8s-loki-healthcheck/README.md) +* [k8s-chaos-flux](codebundles/k8s-chaos-flux/README.md) +* [k8s-argocd-helm-health](codebundles/k8s-argocd-helm-health/README.md) +* [k8s-redis-healthcheck](codebundles/k8s-redis-healthcheck/README.md) ## Keywords * [CLI](libraries/.docs/CLI.md) -* [stdout_parser](libraries/.docs/stdout_parser.md) * [cli_utils](libraries/.docs/cli_utils.md) +* [stdout_parser](libraries/.docs/stdout_parser.md) * [json_parser](libraries/.docs/json_parser.md) diff --git a/codebundles/k8s-ingress-gce-healthcheck/meta.yaml b/codebundles/k8s-ingress-gce-healthcheck/meta.yaml index 6befef72..1674e553 100644 --- a/codebundles/k8s-ingress-gce-healthcheck/meta.yaml +++ b/codebundles/k8s-ingress-gce-healthcheck/meta.yaml @@ -70,46 +70,46 @@ commands: && gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS && gcloud logging read ''severity="ERROR" AND resource.type="gce_network" AND protoPayload.resourceName=~"''\${backend}''"'' --freshness=1d --limit=50 --project - "$GCP_PROJECT_ID" --format=json | jq ''[.[] | {timestamp: .timestamp, ip: .protoPayload.request.networkEndpoints[].ipAddress, - message: .protoPayload.response.error.message}] | group_by(.message) | map(max_by(.timestamp)) - | .[] | (.timestamp + " | IP: " + .ip + " | Error: " + .message)''; done' + "$GCP_PROJECT_ID" --format=json | jq ''if length > 0 then [ .[] | select(.protoPayload.response.error.message? + or .protoPayload.status.message?) | { timestamp: .timestamp, ip: (if .protoPayload.request.networkEndpoints? + then .protoPayload.request.networkEndpoints[].ipAddress else null end), message: + (.protoPayload.response.error.message? // .protoPayload.status.message?) } ] | + group_by(.message) | map(max_by(.timestamp)) | .[] | (.timestamp + " | IP: " + + (.ip // "N/A") + " | Error: " + .message) else "No results found" end''; done' doc_links: ' - - [Kubernetes Documentation - Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/){:target="_blank"} - - - [jq - Command Line JSON Processor](https://stedolan.github.io/jq/){:target="_blank"} - - - [Google Cloud gcloud logging read Command Reference](https://cloud.google.com/sdk/gcloud/reference/logging/read){:target="_blank"} - - - [Google JSON Key file information](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#sa-key-json-file){:target="_blank"}' - explanation: "\n\nThis command is for troubleshooting Ingress resources within Kubernetes.\ - \ It uses the GCP Project ID, Ingress NAME, NAMESPACE and CONTEXT provided to\ - \ output error messages related to backends that aren't HEALTHY. It first makes\ - \ sure that the service account for GCloud can be used with 'gcloud auth activate-service-acccount'\ - \ and then it reads the errors using GCloud logging with 'gcloud logging read'.\ - \ The command is filtered to severity ERROR, about a network of type \u2018gce_network\u2019\ - , associated with backend provided by the Ingress resource, and is limited to\ - \ entries of the last day. Finally, 'jq' is used to output the timestamp, IP address,\ - \ and error message related to each backend specified in the Ingress resources." - multi_line_details: " \n\n# Get name of ingress resource and namespace it's in,\ - \ along with context and GCP project ID \nINGRESS_NAME=${INGRESS}\nNAMESPACE=${NAMESPACE}\n\ - CONTEXT=${CONTEXT}\nGCP_PROJECT_ID=${GCP_PROJECT_ID}\n\n# Use kubectl to get a\ - \ json file that contains annotations about the ingress (metadata about the resources)\ - \ \nJSON=$(kubectl get ingress $INGRESS_NAME -n $NAMESPACE --context $CONTEXT\ - \ -o=json)\n\n# Extract backends from JSON using jq to parse it\nBACKENDS=$(echo\ - \ $JSON | jq -r '.metadata.annotations[\"ingress.kubernetes.io/backends\"] | fromjson\ - \ | to_entries[] | select(.value != \"HEALTHY\") | .key') \n\n# Iterate over each\ - \ backend\nfor backend in $BACKENDS; do \n # Print out the backend name for\ - \ reference\n echo \"Backend: \\${backend}\"\n \n # Authenticate with\ - \ gcloud\n gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS\n\ - \n # Fetch error log entries from logging API for the past day\n ERROR_LOGS=$(gcloud\ + - [Kubernetes Ingress - Managing Traffic to Your Services](https://kubernetes.io/docs/concepts/services-networking/ingress/){:target="_blank"} + + - [Google Cloud Monitor Reading Logs for GCE_NETWORK Errors](https://cloud.google.com/error-reporting/docs/setup/index#logging){:target="_blank"}' + explanation: ' + + + This command is used to debug an Ingress in Kubernetes. It retrieves errors from + GCP''s log streaming and prints out the timestamp, IP address, and error message + associated with it. The command first retrieves the backend values of the Ingress + indicated by the INGRESS_NAME variable and its corresponding NAMESPACE and CONTEXT. + It then uses GCP authentication tokens (GOOGLE_APPLICATION_CREDENTIALS) to access + the GCP logging system. A query is run for each backend which fetches the resourceName + that matches and limits to errors returned in the last 24 hours. Finally, the + information is formatted into a readable output containing the details of the + errors as required.' + multi_line_details: "\n\n# Variable declarations \nINGRESS_NAME=${INGRESS};\nNAMESPACE=${NAMESPACE};\n\ + CONTEXT=${CONTEXT};\nGCP_PROJECT_ID=${GCP_PROJECT_ID};\n\n# Get information from\ + \ resource\nfor backend in $(kubectl get ingress $INGRESS_NAME -n $NAMESPACE --context\ + \ $CONTEXT -o=json | jq -r '.metadata.annotations[\"ingress.kubernetes.io/backends\"\ + ] | fromjson | to_entries[] | select(.value != \"HEALTHY\") | .key'); do\n #\ + \ Print resource name\n echo \"Backend: \\${backend}\"\n\n # Authenticate with\ + \ Google Application credentials\n gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS\n\ + \n # Use the Google Cloud Logging command to read errors from resource\n gcloud\ \ logging read 'severity=\"ERROR\" AND resource.type=\"gce_network\" AND protoPayload.resourceName=~\"\ - '$backend'\"' --freshness=1d --limit=50 --project \"$GCP_PROJECT_ID\" --format=json)\n\ - \n # Parse each log entry in logs returned by logging API, and print out relevant\ - \ information\n LOGS=$(echo $ERROR_LOGS | jq '[.[] | {timestamp: .timestamp,\ - \ ip: .protoPayload.request.networkEndpoints[].ipAddress, message: .protoPayload.response.error.message}]\ + '\\${backend}'\"' --freshness=1d --limit=50 --project \"$GCP_PROJECT_ID\" --format=json\ + \ | jq 'if length > 0 then [ .[] | select(.protoPayload.response.error.message?\ + \ or.protoPayload.status.message?) | { timestamp: .timestamp, ip: (if .protoPayload.request.networkEndpoints?\ + \ then .protoPayload.request.networkEndpoints[].ipAddress else null end), message:\ + \ (.protoPayload.response.error.message? // .protoPayload.status.message?) } ]\ \ | group_by(.message) | map(max_by(.timestamp)) | .[] | (.timestamp + \" | IP:\ - \ \" + .ip + \" | Error: \" + .message)') \ndone" + \ \" + (.ip // \"N/A\") + \" | Error: \" + .message) else \"No results found\"\ + \ end'; done" name: fetch_network_error_logs_from_gcp_operations_manager_for_ingress_backends - command: INGRESS=${INGRESS}; NAMESPACE=${NAMESPACE}; CONTEXT=${CONTEXT}; FORWARDING_RULE=$(kubectl get ingress $INGRESS -n $NAMESPACE --context $CONTEXT -o=jsonpath='{.metadata.annotations.ingress\.kubernetes\.io/forwarding-rule}')