added keepLatestPodsOnly to the tf spec

fixes #94

- added `keepLatestPodsOnly` to the terraform spec which will only keep the current tf k8s-resource generation's pods.
- refactor to remove ambiguity of "labels"

**example**

```yaml
kind: Terraform
spec:
  ...
  keepLatestPodsOnly: true
```

deploy/crds/tf.isaaguilar.com_terraforms_crd.yaml

@@ -260,6 +260,11 @@ spec:
                 description: KeepCompletedPods when true will keep completed pods.
                   Default is false and completed pods are removed.
                 type: boolean
+              keepLatestPodsOnly:
+                description: KeepLatestPodsOnly when true will keep only the pods
+                  that match the current generation of the terraform k8s-resource.
+                  This overrides the behavior of `keepCompletedPods`.
+                type: boolean
               outputsSecret:
                 description: OutputsSecret will create a secret with the outputs from
                   the module. All outputs from the module will be written to the secret

pkg/apis/tf/v1alpha1/terraform_types.go

@@ -33,6 +33,11 @@ type TerraformSpec struct {
 	// Important: Run "operator-sdk generate k8s" to regenerate code after modifying this file
 	// Add custom validation using kubebuilder tags: https://book-v1.book.kubebuilder.io/beyond_basics/generating_crd.html
 
+	// KeepLatestPodsOnly when true will keep only the pods that match the
+	// current generation of the terraform k8s-resource. This overrides the
+	// behavior of `keepCompletedPods`.
+	KeepLatestPodsOnly bool `json:"keepLatestPodsOnly,omitempty"`
+
 	// KeepCompletedPods when true will keep completed pods. Default is false
 	// and completed pods are removed.
 	KeepCompletedPods bool `json:"keepCompletedPods,omitempty"`

pkg/controllers/terraform_controller.go

@@ -24,6 +24,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/uuid"
@@ -399,6 +400,9 @@ func (r *ReconcileTerraform) Reconcile(ctx context.Context, request reconcile.Re
 		if err != nil {
 			reqLogger.V(1).Info(err.Error())
 		}
+		if tf.Spec.KeepLatestPodsOnly {
+			go r.backgroundReapOldGenerationPods(tf)
+		}
 		return reconcile.Result{}, nil
 	}
 
@@ -481,11 +485,11 @@ func (r *ReconcileTerraform) Reconcile(ctx context.Context, request reconcile.Re
 	f := fields.Set{
 		"metadata.generateName": fmt.Sprintf("%s-%s-", tf.Status.PodNamePrefix+"-v"+fmt.Sprint(generation), podType),
 	}
-	labels := map[string]string{
+	labelSelector := map[string]string{
 		"terraforms.tf.isaaguilar.com/generation": fmt.Sprintf("%d", generation),
 	}
 	matchingFields := client.MatchingFields(f)
-	matchingLabels := client.MatchingLabels(labels)
+	matchingLabels := client.MatchingLabels(labelSelector)
 	pods := &corev1.PodList{}
 	err = r.Client.List(ctx, pods, inNamespace, matchingFields, matchingLabels)
 	if err != nil {
@@ -570,7 +574,7 @@ func (r *ReconcileTerraform) Reconcile(ctx context.Context, request reconcile.Re
 			reqLogger.V(1).Info(err.Error())
 			return reconcile.Result{}, err
 		}
-		if !tf.Spec.KeepCompletedPods {
+		if !tf.Spec.KeepCompletedPods && !tf.Spec.KeepLatestPodsOnly {
 			err := r.Client.Delete(ctx, &pods.Items[0])
 			if err != nil {
 				reqLogger.V(1).Info(err.Error())
@@ -807,6 +811,33 @@ func checkSetNewStage(tf *tfv1alpha1.Terraform) bool {
 	return isNewStage
 }
 
+func (r ReconcileTerraform) backgroundReapOldGenerationPods(tf *tfv1alpha1.Terraform) {
+	logger := r.Log.WithName("Reaper").WithValues("Terraform", fmt.Sprintf("%s/%s", tf.Namespace, tf.Name))
+	podList := corev1.PodList{}
+	labelSelector, err := labels.Parse(fmt.Sprintf("terraforms.tf.isaaguilar.com/generation!=%d,terraforms.tf.isaaguilar.com/resourceName=%s", tf.Generation, tf.Name))
+	if err != nil {
+		logger.Error(err, "Could not parse labels")
+	}
+	fieldSelector, err := fields.ParseSelector("status.phase!=Running")
+	if err != nil {
+		logger.Error(err, "Could not parse fields")
+	}
+	err = r.Client.List(context.TODO(), &podList)
+	if err != nil {
+		logger.Error(err, "Could not list pods to reap")
+	}
+	err = r.Client.DeleteAllOf(context.TODO(), &corev1.Pod{}, &client.DeleteAllOfOptions{
+		ListOptions: client.ListOptions{
+			LabelSelector: labelSelector,
+			Namespace:     tf.Namespace,
+			FieldSelector: fieldSelector,
+		},
+	})
+	if err != nil {
+		logger.Error(err, "Could not reap old generation pods")
+	}
+}
+
 // updateFinalizer sets and unsets the finalizer on the tf resource. When
 // IgnoreDelete is true, the finalizer is removed. When IgnoreDelete is false,
 // the finalizer is added.
@@ -1935,15 +1966,15 @@ func (r RunOptions) generatePod(podType, preScriptPodType tfv1alpha1.PodType, is
 		}
 	}
 
-	labels := r.runnerLabels
-	labels["terraforms.tf.isaaguilar.com/generation"] = fmt.Sprintf("%d", generation)
-	labels["terraforms.tf.isaaguilar.com/resourceName"] = r.tfName
-	labels["terraforms.tf.isaaguilar.com/podPrefix"] = r.name
-	labels["terraforms.tf.isaaguilar.com/terraformVersion"] = r.terraformVersion
-	labels["app.kubernetes.io/name"] = "terraform-operator"
-	labels["app.kubernetes.io/component"] = "terraform-operator-runner"
-	labels["app.kubernetes.io/instance"] = string(podType)
-	labels["app.kubernetes.io/created-by"] = "controller"
+	runnerLabels := r.runnerLabels
+	runnerLabels["terraforms.tf.isaaguilar.com/generation"] = fmt.Sprintf("%d", generation)
+	runnerLabels["terraforms.tf.isaaguilar.com/resourceName"] = r.tfName
+	runnerLabels["terraforms.tf.isaaguilar.com/podPrefix"] = r.name
+	runnerLabels["terraforms.tf.isaaguilar.com/terraformVersion"] = r.terraformVersion
+	runnerLabels["app.kubernetes.io/name"] = "terraform-operator"
+	runnerLabels["app.kubernetes.io/component"] = "terraform-operator-runner"
+	runnerLabels["app.kubernetes.io/instance"] = string(podType)
+	runnerLabels["app.kubernetes.io/created-by"] = "controller"
 
 	initContainers := []corev1.Container{}
 	containers := []corev1.Container{}
@@ -2090,7 +2121,7 @@ func (r RunOptions) generatePod(podType, preScriptPodType tfv1alpha1.PodType, is
 		ObjectMeta: metav1.ObjectMeta{
 			GenerateName: generateName,
 			Namespace:    r.namespace,
-			Labels:       labels,
+			Labels:       runnerLabels,
 			Annotations:  annotations,
 		},
 		Spec: corev1.PodSpec{
@@ -2429,11 +2460,11 @@ func (r *ReconcileTerraform) exportRepo(ctx context.Context, tf *tfv1alpha1.Terr
 	f := fields.Set{
 		"metadata.generateName": fmt.Sprintf("%s-%s-", tf.Status.PodNamePrefix+"-v"+fmt.Sprint(generation), exportPodType),
 	}
-	labels := map[string]string{
+	labelSelector := map[string]string{
 		"terraforms.tf.isaaguilar.com/generation": fmt.Sprintf("%d", generation),
 	}
 	matchingFields := client.MatchingFields(f)
-	matchingLabels := client.MatchingLabels(labels)
+	matchingLabels := client.MatchingLabels(labelSelector)
 	pods := &corev1.PodList{}
 	err = r.Client.List(ctx, pods, inNamespace, matchingFields, matchingLabels)
 	if err != nil {
@@ -2500,7 +2531,7 @@ func (r *ReconcileTerraform) exportRepo(ctx context.Context, tf *tfv1alpha1.Terr
 
 		if pods.Items[0].Status.Phase == corev1.PodSucceeded {
 			exportedStatus = tfv1alpha1.ExportedTrue
-			if !tf.Spec.KeepCompletedPods {
+			if !tf.Spec.KeepCompletedPods && !tf.Spec.KeepLatestPodsOnly {
 				err := r.Client.Delete(ctx, &pods.Items[0])
 				if err != nil {
 					reqLogger.V(1).Info(err.Error())