From 9fb5355ef0a70af098572898b8a9ff80bc0e68d9 Mon Sep 17 00:00:00 2001
From: Josphat Mutai <josphatkmutai@gmail.com>
Date: Thu, 14 Nov 2024 05:04:19 +0300
Subject: [PATCH 01/16] rook ceph teardown first draft

---
 mojaloop/iac/roles/cc_k8s/tasks/setup_facts.yaml |  1 +
 mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml    | 12 ++++++++++++
 2 files changed, 13 insertions(+)

diff --git a/mojaloop/iac/roles/cc_k8s/tasks/setup_facts.yaml b/mojaloop/iac/roles/cc_k8s/tasks/setup_facts.yaml
index 4481e7e4..c63974c4 100644
--- a/mojaloop/iac/roles/cc_k8s/tasks/setup_facts.yaml
+++ b/mojaloop/iac/roles/cc_k8s/tasks/setup_facts.yaml
@@ -27,6 +27,7 @@
     fact_kubernetes_oidc_groups_claim: "{{ fact_argo_merged_config.apps['security'].sub_apps['zitadel'].zitadel_grant_prefix }}"
     fact_netbird_management_url: "https://netbird.{{ fact_argo_merged_config.apps['dns_utils'].sub_apps['cr_config'].dns_public_subdomain }}:443"
     fact_environment_list: "{{ fact_argo_merged_config.apps['deploy_env'].sub_apps['config'].environment_list.split(',') }}"
+    fact_rook_ceph_namespace: "{{ fact_argo_merged_config.apps['utils'].sub_apps['rook_ceph'].namespace }}"
 
 - name: set gitlab domain to public
   set_fact:
diff --git a/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml b/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
index 527914e4..6b1ccd8d 100644
--- a/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
+++ b/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
@@ -1,3 +1,15 @@
+- name: Tear down ceph cluster
+  shell: |
+    export KUBECONFIG={{ kubeconfig_location }}/kubeconfig
+    kubectl config set-context --current --namespace {{ fact_argocd_helm_namespace }}
+    argocd app set --source-position 1  rook-ceph --sync-policy none
+    argocd app terminate-op root-deployer --grpc-web
+    argocd app terminate-op rook-ceph --grpc-web
+    kubectl -n {{ fact_rook_ceph_namespace }} scale deployment rook-ceph-operator --replicas=0
+    for CRD in $(kubectl get crd -n {{ fact_rook_ceph_namespace }} | awk '/ceph.rook.io/ {print $1}'); do kubectl get -n {{ fact_rook_ceph_namespace }} "$CRD" -o name | xargs -I {} kubectl patch -n {{ fact_rook_ceph_namespace }} {} --type merge -p '{"metadata":{"finalizers": []}}';done
+    kubectl -n {{ fact_rook_ceph_namespace }} delete cephobjectstore ceph-objectstore cephfilesystem ceph-filesystem cephblockpool ceph-blockpool
+    kubectl -n {{ fact_rook_ceph_namespace }} delete cephcluster
+
 - name: Delete resources before infra teardown
   shell: |
     export KUBECONFIG={{ kubeconfig_location }}/kubeconfig

From a0c9b2ac2334767875ac8f1af917a9ff8c8cd34b Mon Sep 17 00:00:00 2001
From: Josphat Mutai <josphatkmutai@gmail.com>
Date: Thu, 14 Nov 2024 05:22:17 +0300
Subject: [PATCH 02/16] include ceph crds removal

---
 mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml b/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
index 6b1ccd8d..2f3953f8 100644
--- a/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
+++ b/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
@@ -9,6 +9,9 @@
     for CRD in $(kubectl get crd -n {{ fact_rook_ceph_namespace }} | awk '/ceph.rook.io/ {print $1}'); do kubectl get -n {{ fact_rook_ceph_namespace }} "$CRD" -o name | xargs -I {} kubectl patch -n {{ fact_rook_ceph_namespace }} {} --type merge -p '{"metadata":{"finalizers": []}}';done
     kubectl -n {{ fact_rook_ceph_namespace }} delete cephobjectstore ceph-objectstore cephfilesystem ceph-filesystem cephblockpool ceph-blockpool
     kubectl -n {{ fact_rook_ceph_namespace }} delete cephcluster
+    kubectl get crd | grep rook.io | awk '{print $1}' | xargs kubectl delete crd
+    kubectl get crd | grep objectbucket.io | awk '{print $1}' | xargs kubectl delete crd
+  ignore_errors: yes
 
 - name: Delete resources before infra teardown
   shell: |

From 06d2d4a5e9d40400e3f9ea05ca6e1e9c0e7a4cfd Mon Sep 17 00:00:00 2001
From: Josphat Mutai <josphatkmutai@gmail.com>
Date: Thu, 14 Nov 2024 05:32:40 +0300
Subject: [PATCH 03/16] ignore resources not found

---
 mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml b/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
index 2f3953f8..0710aaf1 100644
--- a/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
+++ b/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
@@ -6,12 +6,11 @@
     argocd app terminate-op root-deployer --grpc-web
     argocd app terminate-op rook-ceph --grpc-web
     kubectl -n {{ fact_rook_ceph_namespace }} scale deployment rook-ceph-operator --replicas=0
-    for CRD in $(kubectl get crd -n {{ fact_rook_ceph_namespace }} | awk '/ceph.rook.io/ {print $1}'); do kubectl get -n {{ fact_rook_ceph_namespace }} "$CRD" -o name | xargs -I {} kubectl patch -n {{ fact_rook_ceph_namespace }} {} --type merge -p '{"metadata":{"finalizers": []}}';done
-    kubectl -n {{ fact_rook_ceph_namespace }} delete cephobjectstore ceph-objectstore cephfilesystem ceph-filesystem cephblockpool ceph-blockpool
-    kubectl -n {{ fact_rook_ceph_namespace }} delete cephcluster
-    kubectl get crd | grep rook.io | awk '{print $1}' | xargs kubectl delete crd
-    kubectl get crd | grep objectbucket.io | awk '{print $1}' | xargs kubectl delete crd
-  ignore_errors: yes
+    for CRD in $(kubectl get crd -n {{ fact_rook_ceph_namespace }} | awk '/ceph.rook.io/ {print $1}'); do kubectl get -n {{ fact_rook_ceph_namespace }} "$CRD" -o name | xargs -I {} kubectl patch -n {{ fact_rook_ceph_namespace }} {} --type merge -p '{"metadata":{"finalizers": []}}' --ignore-not-found=true;done
+    kubectl -n {{ fact_rook_ceph_namespace }} delete cephobjectstore ceph-objectstore cephfilesystem ceph-filesystem cephblockpool ceph-blockpool --ignore-not-found=true
+    kubectl -n {{ fact_rook_ceph_namespace }} delete cephcluster --ignore-not-found=true
+    kubectl get crd | grep rook.io | awk '{print $1}' | xargs kubectl delete crd --ignore-not-found=true
+    kubectl get crd | grep objectbucket.io | awk '{print $1}' | xargs kubectl delete crd --ignore-not-found=true
 
 - name: Delete resources before infra teardown
   shell: |

From 72517865e525ff971b74c5c5a6ea507d50f848c4 Mon Sep 17 00:00:00 2001
From: Josphat Mutai <josphatkmutai@gmail.com>
Date: Thu, 14 Nov 2024 13:28:44 +0300
Subject: [PATCH 04/16] ceph destroy updates

---
 mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml b/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
index 0710aaf1..b7f4818d 100644
--- a/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
+++ b/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
@@ -1,16 +1,16 @@
 - name: Tear down ceph cluster
   shell: |
     export KUBECONFIG={{ kubeconfig_location }}/kubeconfig
-    kubectl config set-context --current --namespace {{ fact_argocd_helm_namespace }}
-    argocd app set --source-position 1  rook-ceph --sync-policy none
-    argocd app terminate-op root-deployer --grpc-web
-    argocd app terminate-op rook-ceph --grpc-web
+    kubectl patch application -n {{ fact_argo_merged_config.namespace }} root-deployer --type json --patch='[ { "op": "remove", "path": "/spec/syncPolicy/automated" } ]' || true
+    kubectl patch application -n {{ fact_argo_merged_config.namespace }} rook-ceph --type json --patch='[ { "op": "remove", "path": "/spec/syncPolicy/automated" } ]' || true
+    kubectl config set-context --current --namespace {{ fact_argo_merged_config.namespace }}
     kubectl -n {{ fact_rook_ceph_namespace }} scale deployment rook-ceph-operator --replicas=0
-    for CRD in $(kubectl get crd -n {{ fact_rook_ceph_namespace }} | awk '/ceph.rook.io/ {print $1}'); do kubectl get -n {{ fact_rook_ceph_namespace }} "$CRD" -o name | xargs -I {} kubectl patch -n {{ fact_rook_ceph_namespace }} {} --type merge -p '{"metadata":{"finalizers": []}}' --ignore-not-found=true;done
-    kubectl -n {{ fact_rook_ceph_namespace }} delete cephobjectstore ceph-objectstore cephfilesystem ceph-filesystem cephblockpool ceph-blockpool --ignore-not-found=true
-    kubectl -n {{ fact_rook_ceph_namespace }} delete cephcluster --ignore-not-found=true
+    for CRD in $(kubectl get crd -n {{ fact_rook_ceph_namespace }} | awk '/ceph.rook.io/ {print $1}'); do kubectl get -n {{ fact_rook_ceph_namespace }} "$CRD" -o name | xargs -I {} kubectl patch -n {{ fact_rook_ceph_namespace }} {} --type merge -p '{"metadata":{"finalizers": []}}';done
+    # kubectl -n {{ fact_rook_ceph_namespace }} delete cephobjectstore ceph-objectstore cephfilesystem ceph-filesystem cephblockpool ceph-blockpool --ignore-not-found=true
+    # kubectl -n {{ fact_rook_ceph_namespace }} delete cephcluster --ignore-not-found=true
     kubectl get crd | grep rook.io | awk '{print $1}' | xargs kubectl delete crd --ignore-not-found=true
     kubectl get crd | grep objectbucket.io | awk '{print $1}' | xargs kubectl delete crd --ignore-not-found=true
+  ignore_errors: true
 
 - name: Delete resources before infra teardown
   shell: |

From 7063c343a57bd74541987a2c434d4b44be166888 Mon Sep 17 00:00:00 2001
From: David Fry <david@infitx.com>
Date: Thu, 14 Nov 2024 09:21:45 +0000
Subject: [PATCH 05/16] Upgrade/netbird with relay (#99)

* upgrade to latest nb with relay

* rm netbird prefix

* fix bastion sort on destroy
---
 mojaloop/iac/playbooks/ccmicrok8s_cluster_destroy.yaml | 2 +-
 mojaloop/iac/roles/cc_k8s/defaults/main.yaml           | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/mojaloop/iac/playbooks/ccmicrok8s_cluster_destroy.yaml b/mojaloop/iac/playbooks/ccmicrok8s_cluster_destroy.yaml
index 74bdbb9d..48950870 100644
--- a/mojaloop/iac/playbooks/ccmicrok8s_cluster_destroy.yaml
+++ b/mojaloop/iac/playbooks/ccmicrok8s_cluster_destroy.yaml
@@ -1,4 +1,4 @@
-- hosts: bastion
+- hosts: "{{ groups['bastion'] | sort | first }}"
   become: true
   roles:
     - role: geerlingguy.k8s.helm
diff --git a/mojaloop/iac/roles/cc_k8s/defaults/main.yaml b/mojaloop/iac/roles/cc_k8s/defaults/main.yaml
index 673a2c03..62e9a755 100644
--- a/mojaloop/iac/roles/cc_k8s/defaults/main.yaml
+++ b/mojaloop/iac/roles/cc_k8s/defaults/main.yaml
@@ -327,6 +327,8 @@ argocd_default:
           netbird_tf_provider_version: "99.99.99"
           datastore_encryption_key_secret: "db-enc-key-secret"
           datastore_encryption_key_secret_key: "key"
+          relay_secret: "relay-secret"
+          relay_secret_key: "key"
     nexus:
       argocd_app_name: "nexus"
       app_namespace: "argocd"

From e2e035784774858c76a084691bc0abd4d4b94fbe Mon Sep 17 00:00:00 2001
From: Josphat Mutai <josphatkmutai@gmail.com>
Date: Tue, 19 Nov 2024 11:54:35 +0300
Subject: [PATCH 06/16] Update rook teardown task

---
 mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml b/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
index b7f4818d..eaa0faca 100644
--- a/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
+++ b/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
@@ -1,15 +1,17 @@
 - name: Tear down ceph cluster
-  shell: |
+  ansible.builtin.shell: |
     export KUBECONFIG={{ kubeconfig_location }}/kubeconfig
     kubectl patch application -n {{ fact_argo_merged_config.namespace }} root-deployer --type json --patch='[ { "op": "remove", "path": "/spec/syncPolicy/automated" } ]' || true
     kubectl patch application -n {{ fact_argo_merged_config.namespace }} rook-ceph --type json --patch='[ { "op": "remove", "path": "/spec/syncPolicy/automated" } ]' || true
-    kubectl config set-context --current --namespace {{ fact_argo_merged_config.namespace }}
+    kubectl config set-context --current --namespace {{ fact_rook_ceph_namespace }}
     kubectl -n {{ fact_rook_ceph_namespace }} scale deployment rook-ceph-operator --replicas=0
     for CRD in $(kubectl get crd -n {{ fact_rook_ceph_namespace }} | awk '/ceph.rook.io/ {print $1}'); do kubectl get -n {{ fact_rook_ceph_namespace }} "$CRD" -o name | xargs -I {} kubectl patch -n {{ fact_rook_ceph_namespace }} {} --type merge -p '{"metadata":{"finalizers": []}}';done
-    # kubectl -n {{ fact_rook_ceph_namespace }} delete cephobjectstore ceph-objectstore cephfilesystem ceph-filesystem cephblockpool ceph-blockpool --ignore-not-found=true
-    # kubectl -n {{ fact_rook_ceph_namespace }} delete cephcluster --ignore-not-found=true
-    kubectl get crd | grep rook.io | awk '{print $1}' | xargs kubectl delete crd --ignore-not-found=true
-    kubectl get crd | grep objectbucket.io | awk '{print $1}' | xargs kubectl delete crd --ignore-not-found=true
+    kubectl delete cephobjectstore ceph-objectstore --ignore-not-found=true -n {{ fact_rook_ceph_namespace }}
+    kubectl delete cephfilesystem ceph-filesystem --ignore-not-found=true -n {{ fact_rook_ceph_namespace }}
+    kubectl delete cephblockpool ceph-blockpool --ignore-not-found=true -n {{ fact_rook_ceph_namespace }}
+    kubectl delete cephcluster rook-ceph --ignore-not-found=true -n {{ fact_rook_ceph_namespace }}
+  args:
+    executable: /bin/bash
   ignore_errors: true
 
 - name: Delete resources before infra teardown

From f9ae99788a42858cda0f5022b11d2448dd048b84 Mon Sep 17 00:00:00 2001
From: Josphat Mutai <josphatkmutai@gmail.com>
Date: Wed, 20 Nov 2024 02:03:02 +0300
Subject: [PATCH 07/16] teradown updates

---
 mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml | 28 ++++++++++---------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml b/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
index eaa0faca..f2b17367 100644
--- a/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
+++ b/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
@@ -1,7 +1,20 @@
+
+- name: Delete resources before infra teardown
+  shell: |
+    export KUBECONFIG={{ kubeconfig_location }}/kubeconfig
+    kubectl patch application -n {{ fact_argo_merged_config.namespace }} root-deployer --type json --patch='[ { "op": "remove", "path": "/spec/syncPolicy/automated" } ]' || true
+    kubectl patch application -n {{ fact_argo_merged_config.namespace }} vault-post-config --type json --patch='[ { "op": "remove", "path": "/spec/syncPolicy/automated" } ]' || true
+    kubectl patch application -n {{ fact_argo_merged_config.namespace }} gitlab-pre --type json --patch='[ { "op": "remove", "path": "/spec/syncPolicy/automated" } ]' || true
+    kubectl delete workspaces.tf.upbound.io vault-{{ cluster_cloud_provider }}-post-config
+    kubectl delete workspaces.tf.upbound.io vault-post-config
+    kubectl delete dbinstanceclaim -n {{ fact_argo_merged_config.apps['gitlab'].sub_apps['gitlab'].namespace }} --ignore-not-found=true --all
+  args:
+    executable: /bin/bash
+  ignore_errors: true
+
 - name: Tear down ceph cluster
   ansible.builtin.shell: |
     export KUBECONFIG={{ kubeconfig_location }}/kubeconfig
-    kubectl patch application -n {{ fact_argo_merged_config.namespace }} root-deployer --type json --patch='[ { "op": "remove", "path": "/spec/syncPolicy/automated" } ]' || true
     kubectl patch application -n {{ fact_argo_merged_config.namespace }} rook-ceph --type json --patch='[ { "op": "remove", "path": "/spec/syncPolicy/automated" } ]' || true
     kubectl config set-context --current --namespace {{ fact_rook_ceph_namespace }}
     kubectl -n {{ fact_rook_ceph_namespace }} scale deployment rook-ceph-operator --replicas=0
@@ -13,18 +26,7 @@
   args:
     executable: /bin/bash
   ignore_errors: true
-
-- name: Delete resources before infra teardown
-  shell: |
-    export KUBECONFIG={{ kubeconfig_location }}/kubeconfig
-    kubectl patch application -n {{ fact_argo_merged_config.namespace }} root-deployer --type json --patch='[ { "op": "remove", "path": "/spec/syncPolicy/automated" } ]' || true
-    kubectl patch application -n {{ fact_argo_merged_config.namespace }} vault-post-config --type json --patch='[ { "op": "remove", "path": "/spec/syncPolicy/automated" } ]' || true
-    kubectl patch application -n {{ fact_argo_merged_config.namespace }} gitlab-pre --type json --patch='[ { "op": "remove", "path": "/spec/syncPolicy/automated" } ]' || true
-    kubectl delete workspaces.tf.upbound.io vault-{{ cluster_cloud_provider }}-post-config
-    kubectl delete workspaces.tf.upbound.io vault-post-config
-    kubectl delete dbinstanceclaim -n {{ fact_argo_merged_config.apps['gitlab'].sub_apps['gitlab'].namespace }} --ignore-not-found=true --all
-  args:
-    executable: /bin/bash
+  
 # - name: Wait for vault to be gone
 #   shell: |
 #     export KUBECONFIG={{ kubeconfig_location }}/kubeconfig

From 1602ffb474f2e987cc5f7d591c8983bacd3fb9a0 Mon Sep 17 00:00:00 2001
From: Josphat Mutai <josphatkmutai@gmail.com>
Date: Tue, 26 Nov 2024 22:27:25 +0300
Subject: [PATCH 08/16] update default values for rook ceph

---
 mojaloop/iac/roles/cc_k8s/defaults/main.yaml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/mojaloop/iac/roles/cc_k8s/defaults/main.yaml b/mojaloop/iac/roles/cc_k8s/defaults/main.yaml
index 62e9a755..7d95177f 100644
--- a/mojaloop/iac/roles/cc_k8s/defaults/main.yaml
+++ b/mojaloop/iac/roles/cc_k8s/defaults/main.yaml
@@ -41,8 +41,14 @@ argocd_default:
         rook_ceph:
           app_name: "rook-ceph"
           namespace: "rook-ceph"
-          helm_version: "1.14.5"
+          helm_version: "1.15.5"
           rook_csi_kubelet_dir_path: "/var/lib/kubelet"
+          mon_volumes_storage_class: "gp3"
+          mon_volume_size: "10Gi"
+          osd_volumes_storage_class: "gp3"
+          osd_count: "'3'"
+          volume_size_per_osd: "500Gi"
+          object_store_region: "us-east-1"
           sync_wave: "'-10'"
           sub_app_enabled: "true"
         reflector:

From c4f631439a73eaac8569d1765b8d45951e6c2cc1 Mon Sep 17 00:00:00 2001
From: Josphat Mutai <josphatkmutai@gmail.com>
Date: Wed, 27 Nov 2024 16:29:44 +0300
Subject: [PATCH 09/16] test pause in ebs volumes deletion

---
 mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml b/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
index f2b17367..ac51b0d0 100644
--- a/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
+++ b/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
@@ -26,7 +26,10 @@
   args:
     executable: /bin/bash
   ignore_errors: true
-  
+
+- name: Pause for 2 minutes to delete pvc volumes
+  ansible.builtin.pause:
+    seconds: 120
 # - name: Wait for vault to be gone
 #   shell: |
 #     export KUBECONFIG={{ kubeconfig_location }}/kubeconfig

From 79fba46e3509c91658df57ca4bebe9539440c1b6 Mon Sep 17 00:00:00 2001
From: Josphat Mutai <josphatkmutai@gmail.com>
Date: Wed, 27 Nov 2024 18:27:51 +0300
Subject: [PATCH 10/16] update teardown task

---
 mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml b/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
index ac51b0d0..a9eee1a7 100644
--- a/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
+++ b/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
@@ -23,13 +23,14 @@
     kubectl delete cephfilesystem ceph-filesystem --ignore-not-found=true -n {{ fact_rook_ceph_namespace }}
     kubectl delete cephblockpool ceph-blockpool --ignore-not-found=true -n {{ fact_rook_ceph_namespace }}
     kubectl delete cephcluster rook-ceph --ignore-not-found=true -n {{ fact_rook_ceph_namespace }}
+    kubectl get crd | grep rook.io | awk '{print $1}' | xargs kubectl delete crd --ignore-not-found=true
   args:
     executable: /bin/bash
   ignore_errors: true
 
-- name: Pause for 2 minutes to delete pvc volumes
+- name: Pause to finish resources deletion
   ansible.builtin.pause:
-    seconds: 120
+    seconds: 300
 # - name: Wait for vault to be gone
 #   shell: |
 #     export KUBECONFIG={{ kubeconfig_location }}/kubeconfig

From e75291d64fb6fdd30a8b4541bd93f513dab638b2 Mon Sep 17 00:00:00 2001
From: Josphat Mutai <josphatkmutai@gmail.com>
Date: Thu, 28 Nov 2024 10:06:31 +0300
Subject: [PATCH 11/16] try sleep

---
 mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml b/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
index a9eee1a7..feefb660 100644
--- a/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
+++ b/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
@@ -24,13 +24,14 @@
     kubectl delete cephblockpool ceph-blockpool --ignore-not-found=true -n {{ fact_rook_ceph_namespace }}
     kubectl delete cephcluster rook-ceph --ignore-not-found=true -n {{ fact_rook_ceph_namespace }}
     kubectl get crd | grep rook.io | awk '{print $1}' | xargs kubectl delete crd --ignore-not-found=true
+    sleep 120  # Wait for 120 seconds
   args:
     executable: /bin/bash
   ignore_errors: true
 
-- name: Pause to finish resources deletion
-  ansible.builtin.pause:
-    seconds: 300
+# - name: Pause to finish resources deletion
+#   ansible.builtin.pause:
+#     seconds: 300
 # - name: Wait for vault to be gone
 #   shell: |
 #     export KUBECONFIG={{ kubeconfig_location }}/kubeconfig

From 51cbb324df1b7099da11000fb161573e07f38675 Mon Sep 17 00:00:00 2001
From: Josphat Mutai <josphatkmutai@gmail.com>
Date: Thu, 28 Nov 2024 13:11:53 +0300
Subject: [PATCH 12/16] sleep to for x time

---
 mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml b/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
index feefb660..b559fedc 100644
--- a/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
+++ b/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
@@ -8,6 +8,7 @@
     kubectl delete workspaces.tf.upbound.io vault-{{ cluster_cloud_provider }}-post-config
     kubectl delete workspaces.tf.upbound.io vault-post-config
     kubectl delete dbinstanceclaim -n {{ fact_argo_merged_config.apps['gitlab'].sub_apps['gitlab'].namespace }} --ignore-not-found=true --all
+    sleep 300  # Wait for 300 seconds
   args:
     executable: /bin/bash
   ignore_errors: true
@@ -29,6 +30,9 @@
     executable: /bin/bash
   ignore_errors: true
 
+# - name: Sleep for 300 seconds and continue with play
+#   ansible.builtin.wait_for:
+#     timeout: 300
 # - name: Pause to finish resources deletion
 #   ansible.builtin.pause:
 #     seconds: 300

From 5a20c472eaedc60f2700754e2a106df1532902d2 Mon Sep 17 00:00:00 2001
From: Josphat Mutai <josphatkmutai@gmail.com>
Date: Thu, 28 Nov 2024 15:37:31 +0300
Subject: [PATCH 13/16] sleep for 2 mins max

---
 mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml b/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
index b559fedc..e36d9093 100644
--- a/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
+++ b/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
@@ -8,7 +8,6 @@
     kubectl delete workspaces.tf.upbound.io vault-{{ cluster_cloud_provider }}-post-config
     kubectl delete workspaces.tf.upbound.io vault-post-config
     kubectl delete dbinstanceclaim -n {{ fact_argo_merged_config.apps['gitlab'].sub_apps['gitlab'].namespace }} --ignore-not-found=true --all
-    sleep 300  # Wait for 300 seconds
   args:
     executable: /bin/bash
   ignore_errors: true
@@ -24,7 +23,6 @@
     kubectl delete cephfilesystem ceph-filesystem --ignore-not-found=true -n {{ fact_rook_ceph_namespace }}
     kubectl delete cephblockpool ceph-blockpool --ignore-not-found=true -n {{ fact_rook_ceph_namespace }}
     kubectl delete cephcluster rook-ceph --ignore-not-found=true -n {{ fact_rook_ceph_namespace }}
-    kubectl get crd | grep rook.io | awk '{print $1}' | xargs kubectl delete crd --ignore-not-found=true
     sleep 120  # Wait for 120 seconds
   args:
     executable: /bin/bash

From dbcee61cee369f757a9381a2af28284c1bbc2d9c Mon Sep 17 00:00:00 2001
From: Josphat Mutai <josphatkmutai@gmail.com>
Date: Sat, 30 Nov 2024 17:34:29 +0300
Subject: [PATCH 14/16] set some defaults for rook ceph

---
 mojaloop/iac/roles/cc_k8s/defaults/main.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/mojaloop/iac/roles/cc_k8s/defaults/main.yaml b/mojaloop/iac/roles/cc_k8s/defaults/main.yaml
index fad4a716..fa7831b7 100644
--- a/mojaloop/iac/roles/cc_k8s/defaults/main.yaml
+++ b/mojaloop/iac/roles/cc_k8s/defaults/main.yaml
@@ -45,13 +45,15 @@ argocd_default:
           app_name: "rook-ceph"
           namespace: "rook-ceph"
           helm_version: "1.15.5"
+          image_version: "v18.2.4"
           rook_csi_kubelet_dir_path: "/var/lib/kubelet"
           mon_volumes_storage_class: "gp3"
-          mon_volume_size: "10Gi"
+          mon_volumes_size: "10Gi"
           osd_volumes_storage_class: "gp3"
           osd_count: "'3'"
           volume_size_per_osd: "500Gi"
           object_store_region: "us-east-1"
+          volumes_provider: "host"
           sync_wave: "'-10'"
           sub_app_enabled: "true"
         reflector:

From 0e3fa30f960cd93d6126e96d7e00812b397f18f9 Mon Sep 17 00:00:00 2001
From: Josphat Mutai <josphatkmutai@gmail.com>
Date: Mon, 2 Dec 2024 15:28:29 +0300
Subject: [PATCH 15/16] remove ceph object default region var

---
 mojaloop/iac/roles/cc_k8s/defaults/main.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/mojaloop/iac/roles/cc_k8s/defaults/main.yaml b/mojaloop/iac/roles/cc_k8s/defaults/main.yaml
index fa7831b7..15d6dfa6 100644
--- a/mojaloop/iac/roles/cc_k8s/defaults/main.yaml
+++ b/mojaloop/iac/roles/cc_k8s/defaults/main.yaml
@@ -52,8 +52,7 @@ argocd_default:
           osd_volumes_storage_class: "gp3"
           osd_count: "'3'"
           volume_size_per_osd: "500Gi"
-          object_store_region: "us-east-1"
-          volumes_provider: "host"
+          volumes_provider: "pvc"
           sync_wave: "'-10'"
           sub_app_enabled: "true"
         reflector:

From de31719dadc0e0ea049283d4fb37f72c9568c296 Mon Sep 17 00:00:00 2001
From: Josphat Mutai <josphatkmutai@gmail.com>
Date: Mon, 2 Dec 2024 15:30:40 +0300
Subject: [PATCH 16/16] sleep 300 seconds to wait resource deletion

---
 mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml b/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
index eb1bf899..6a395671 100644
--- a/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
+++ b/mojaloop/iac/roles/cc_k8s/tasks/teardown.yaml
@@ -25,7 +25,7 @@
     kubectl delete cephfilesystem ceph-filesystem --ignore-not-found=true -n {{ fact_rook_ceph_namespace }}
     kubectl delete cephblockpool ceph-blockpool --ignore-not-found=true -n {{ fact_rook_ceph_namespace }}
     kubectl delete cephcluster rook-ceph --ignore-not-found=true -n {{ fact_rook_ceph_namespace }}
-    sleep 120  # Wait for 120 seconds
+    sleep 300  # Wait for 300 seconds
   args:
     executable: /bin/bash
   ignore_errors: true