Notify on pod OOMKilled statuses (#369)

cmd/daemon/kubernetes/pods.go

@@ -131,6 +131,32 @@ func (p *PodInformer) handle(e interface{}) {
 			log.Errorf("Failed to send create container config error: %v", err)
 		}
 	}
+
+	if isPodOOMKilled(pod) {
+		log.Infof("Pod: %s was OOMKilled", pod.Name)
+
+		var errorContainers []http.ContainerError
+		for _, cst := range pod.Status.ContainerStatuses {
+			if isContainerOOMKilled(cst) {
+				errorContainers = append(errorContainers, http.ContainerError{
+					Name:         cst.Name,
+					ErrorMessage: cst.State.Terminated.Message,
+					Type:         "OOMKilled",
+				})
+			}
+		}
+
+		err := p.exporter.SendPodErrorEvent(ctx, http.PodErrorEvent{
+			PodName:     pod.Name,
+			Namespace:   pod.Namespace,
+			Errors:      errorContainers,
+			ArtifactID:  pod.Annotations[artifactIDAnnotationKey],
+			AuthorEmail: pod.Annotations[authorAnnotationKey],
+		})
+		if err != nil {
+			log.Errorf("Failed to send create container config error: %v", err)
+		}
+	}
 }
 
 // Avoid reporting on pods that has been marked for termination
@@ -165,6 +191,25 @@ func isPodInCreateContainerConfigError(pod *corev1.Pod) bool {
 	return false
 }
 
+func isPodOOMKilled(pod *corev1.Pod) bool {
+	if pod.Status.Phase != corev1.PodFailed {
+		return false
+	}
+
+	for _, cst := range pod.Status.ContainerStatuses {
+		return isContainerOOMKilled(cst)
+	}
+	return false
+}
+
+func isContainerOOMKilled(containerStatus corev1.ContainerStatus) bool {
+	if containerStatus.State.Terminated != nil && containerStatus.State.Terminated.Reason == "OOMKilled" {
+		return true
+	}
+
+	return false
+}
+
 // isPodControlledByJob - check the PodStatus and indicates whether the Pod is controlled by a Job
 func isPodControlledByJob(pod *corev1.Pod) bool {
 	if pod.OwnerReferences == nil {

cmd/daemon/kubernetes/pods_test.go

@@ -170,3 +170,57 @@ func TestIsPodInCreateContainerConfigError(t *testing.T) {
 		})
 	}
 }
+
+func TestIsPodOOMKilled(t *testing.T) {
+	testCases := []struct {
+		desc     string
+		pod      *corev1.Pod
+		expected bool
+	}{
+		{
+			desc:     "no container status",
+			pod:      &corev1.Pod{},
+			expected: false,
+		},
+		{
+			desc: "container status is not OOMKilled",
+			pod: &corev1.Pod{
+				Status: corev1.PodStatus{
+					ContainerStatuses: []corev1.ContainerStatus{
+						{
+							State: corev1.ContainerState{
+								Terminated: &corev1.ContainerStateTerminated{},
+							},
+						},
+					},
+				},
+			},
+			expected: false,
+		},
+		{
+			desc: "container status is OOMKilled",
+			pod: &corev1.Pod{
+				Status: corev1.PodStatus{
+					Phase: corev1.PodFailed,
+					ContainerStatuses: []corev1.ContainerStatus{
+						{
+							State: corev1.ContainerState{
+								Terminated: &corev1.ContainerStateTerminated{
+									Reason: "OOMKilled",
+								},
+							},
+						},
+					},
+				},
+			},
+			expected: true,
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			actual := isPodOOMKilled(tc.pod)
+
+			assert.Equal(t, tc.expected, actual, "output not as expected")
+		})
+	}
+}