Make it so Succeeded pods are not deleted immediately (#3878)

Once a pod has succeeded, we will delete it is older than MinimumPodAge

The issue with this is that we start the "timer" when the last container started, so any pod that runs for longer than MinimumPodAge will be deleted immediately on completing

Now we will start the "timer" based on the time it became terminated, so it will stick for a minimum of MinimumPodAge duration after it finishes

Signed-off-by: JamesMurkin <jamesmurkin@hotmail.com>
Co-authored-by: Mustafa Ilyas <MustafaI@users.noreply.github.com>

internal/executor/service/resource_cleanup.go

@@ -175,13 +175,12 @@ func (r *ResourceCleanupService) canPodBeRemoved(pod *v1.Pod) bool {
 		return false
 	}
 
-	lastContainerStart := util.FindLastContainerStartTime(pod)
-	if lastContainerStart.Add(r.kubernetesConfiguration.MinimumPodAge).After(time.Now()) {
+	lastChange, err := util.LastStatusChange(pod)
+	if err == nil && lastChange.Add(r.kubernetesConfiguration.MinimumPodAge).After(time.Now()) {
 		return false
 	}
 
 	if pod.Status.Phase == v1.PodFailed {
-		lastChange, err := util.LastStatusChange(pod)
 		if err == nil && lastChange.Add(r.kubernetesConfiguration.FailedPodExpiry).After(time.Now()) {
 			return false
 		}