Skip to content

Commit

Permalink
update gce jq filter (#236)
Browse files Browse the repository at this point in the history
* escape backticks

* update gce jq filter
  • Loading branch information
stewartshea authored Nov 2, 2023
1 parent beefedd commit 03bf417
Show file tree
Hide file tree
Showing 2 changed files with 8 additions and 8 deletions.
14 changes: 7 additions & 7 deletions codebundles/k8s-deployment-healthcheck/deployment_logs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ for match in "${FUZZY_ENV_VAR_RESOURCE_MATCHES[@]}"; do
resource=${parts[1]}
env_key=${parts[2]}
env_value=${parts[3]}
echo "Found string `$string` in resource `$resource`. Check manifest and environment variable `$env_key` for accuracy. "
echo "Found string \`$string\` in resource \`$resource\`. Check manifest and environment variable \`$env_key\` for accuracy. "
done

# Fetch namespace events for searching through
Expand Down Expand Up @@ -257,7 +257,7 @@ if [[ ${#FUZZY_ENV_VAR_RESOURCE_MATCHES[@]} -ne 0 ]]; then
env_value=${parts[3]}

if [[ -z ${seen_resources[$resource]} ]]; then
recommendations+=("Review manifest for `$resource` in namespace: `${NAMESPACE}`. Matched error log string `$string` in environment variable `$env_key`. ")
recommendations+=("Review manifest for \`$resource\` in namespace: \`${NAMESPACE}\`. Matched error log string \`$string\` in environment variable \`$env_key\`. ")
seen_resources[$resource]=1
fi
done
Expand All @@ -279,20 +279,20 @@ if [[ -n "$INTERESTING_RESOURCES" ]]; then
case "$type" in
pod)
if [[ "$status" != "Running" ]]; then
recommendations+=("Troubleshoot *failed pods* in *namespace* `${NAMESPACE}` ")
recommendations+=("Troubleshoot *failed pods* in *namespace* \`${NAMESPACE}\` ")
fi
if ((restarts > 0)); then
recommendations+=("Troubleshoot *container restarts* in *namespace* `${NAMESPACE}` ")
recommendations+=("Troubleshoot *container restarts* in *namespace* \`${NAMESPACE}\` ")
fi
;;
deployment|deployment.apps)
recommendations+=("Check *deployment* health `$name` in *namespace* `${NAMESPACE}` ")
recommendations+=("Check *deployment* health \`$name\` in *namespace* \`${NAMESPACE}\` ")
;;
service)
recommendations+=("Check *service* health `$name` in *namespace* `${NAMESPACE}` ")
recommendations+=("Check *service* health \`$name\` in *namespace* \`${NAMESPACE}\` ")
;;
statefulset|statefulset.apps)
recommendations+=("Check *statefulset* health `$name` in *namespace* `${NAMESPACE}` ")
recommendations+=("Check *statefulset* health \`$name\` in *namespace* \`${NAMESPACE}\` ")
;;
esac
done <<< "$INTERESTING_RESOURCES"
Expand Down
2 changes: 1 addition & 1 deletion codebundles/k8s-ingress-gce-healthcheck/runbook.robot
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ Fetch Network Error Logs from GCP Operations Manager for Ingress Backends
[Documentation] Fetch logs from the last 1d that are specific to the HTTP Load Balancer within the last 60 minutes
[Tags] service ingress endpoint health
${network_error_logs}= RW.CLI.Run Cli
... cmd=INGRESS_NAME=${INGRESS}; NAMESPACE=${NAMESPACE}; CONTEXT=${CONTEXT}; GCP_PROJECT_ID=${GCP_PROJECT_ID};for backend in $(${KUBERNETES_DISTRIBUTION_BINARY} get ingress $INGRESS_NAME -n $NAMESPACE --context $CONTEXT -o=json | jq -r '.metadata.annotations["ingress.kubernetes.io/backends"] | fromjson | to_entries[] | select(.value != "HEALTHY") | .key'); do echo "Backend: \${backend}" && gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS && gcloud logging read 'severity="ERROR" AND resource.type="gce_network" AND protoPayload.resourceName=~"'\${backend}'"' --freshness=1d --limit=50 --project "$GCP_PROJECT_ID" --format=json | jq '[.[] | {timestamp: .timestamp, ip: .protoPayload.request.networkEndpoints[].ipAddress, message: .protoPayload.response.error.message}] | group_by(.message) | map(max_by(.timestamp)) | .[] | (.timestamp + " | IP: " + .ip + " | Error: " + .message)'; done
... cmd=INGRESS_NAME=${INGRESS}; NAMESPACE=${NAMESPACE}; CONTEXT=${CONTEXT}; GCP_PROJECT_ID=${GCP_PROJECT_ID};for backend in $(${KUBERNETES_DISTRIBUTION_BINARY} get ingress $INGRESS_NAME -n $NAMESPACE --context $CONTEXT -o=json | jq -r '.metadata.annotations["ingress.kubernetes.io/backends"] | fromjson | to_entries[] | select(.value != "HEALTHY") | .key'); do echo "Backend: \${backend}" && gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS && gcloud logging read 'severity="ERROR" AND resource.type="gce_network" AND protoPayload.resourceName=~"'\${backend}'"' --freshness=1d --limit=50 --project "$GCP_PROJECT_ID" --format=json | jq 'if length > 0 then [ .[] | select(.protoPayload.response.error.message? or .protoPayload.status.message?) | { timestamp: .timestamp, ip: (if .protoPayload.request.networkEndpoints? then .protoPayload.request.networkEndpoints[].ipAddress else null end), message: (.protoPayload.response.error.message? // .protoPayload.status.message?) } ] | group_by(.message) | map(max_by(.timestamp)) | .[] | (.timestamp + " | IP: " + (.ip // "N/A") + " | Error: " + .message) else "No results found" end'; done
... env=${env}
... secret_file__kubeconfig=${kubeconfig}
... secret_file__gcp_credentials_json=${gcp_credentials_json}
Expand Down

0 comments on commit 03bf417

Please sign in to comment.