feat: (IAC-1108) Multi-tenancy process change - apply podtemplates be…

…fore running onboard (#481)

docs/user/Multi-Tenancy.md

@@ -106,12 +106,19 @@ Step 3. Onboard tenants. Run the following command:
     -e JUMP_SVR_PRIVATE_KEY=$HOME/.ssh/id_rsa \
     playbooks/playbook.yaml --tags "multi-tenancy,onboard"
   ```
-**Note:** As part of setup in the above `Onboard tenants` step, for every onboarded tenant, 
+**Notes:** 
+- As part of the setup in the above `Onboard tenants` step, for every onboarded tenant, 
 
-- A CAS server directory containing the configuration artifacts is created under the `/site-config` folder. 
-For example,if you have tenant with the ID `acme`, then a CAS server directory named `cas-acme-default` will be created.
+  >- A CAS server directory containing the configuration artifacts is created under the `/site-config` folder. 
+For example, if you have tenant with the ID `acme`, then a CAS server directory named `cas-acme-default` will be created.
+  >
+  >- Starting with SAS Viya Platform cadence 2023.03, each tenant will require their own copy of certain Kubernetes resources. Hence a new directory for each tenant containing all the `sas-programming-environment` files will be created under `$deploy/site-config/multi-tenant/`. For example, if you have a tenant with the ID `acme`, then a directory named `$deploy/site-config/multi-tenant/acme` will be created.
+  >
+  >- The base `kustomization.yaml` file will be updated to add a reference to the tenant directories in the resources block. And the changes will be applied to create the tenant-specific resources before running the `sas-tenant-onboard` job. A short delay is introduced here to allow the apply commands to finish.
 
-- Starting with SAS Viya Platform cadence 2023.03, each tenant will require their own copy of certain Kubernetes resources. Hence a new directory for each tenant containing all the `sas-programming-environment` files will be created under `$deploy/site-config/multi-tenant/`. For example, if you have a tenant with the ID `acme`, then a directory named `$deploy/site-config/multi-tenant/acme` will be created.
+- Starting with SAS Viya Platform cadence 2023.07, the `sas-tenant-onboard-job` continues to run until the conclusion of the rolling restart of all SAS Viya services. The `Onboard tenants` command run above does not wait for the conclusion of the rolling restart of all SAS Viya services. The action concludes after the `sas-tenant-onboard-job` has reached `Running` state. User should continue to monitor the status of `sas-tenant-onboard-job` manually.
+
+- It is recommended that User proceeds with the `cas-onboard` command below if it was not applied together with the `multi-tenancy,onboard` action.
 
 Step 4. Add or update CAS customizations for tenants as needed and then run following command to onboard the CAS servers:
 
@@ -128,6 +135,7 @@ Step 4. Add or update CAS customizations for tenants as needed and then run foll
 **Note:** 
 - If there are no additional CAS customizations required for tenants then run 'onboard' and 'cas-onboard' tags together in Step 3 and skip Step 4.
 - The tenant CAS servers might take several mins to stabilize after the cas-onboard command above has completed successfully.
+- The successful conclusion of the `sas-tenant-onboard-job` is a clear indication that administrators can sign on to the new tenant, or run another instance of the sas-tenant-job.
 
 ## Log In and Validate an Onboarded Tenant
 After the onboard and cas-onboard steps are complete see the steps [here](https://go.documentation.sas.com/doc/en/itopscdc/default/caltenants/p0emzq13c0zbhxn1hktsdlmig934.htm#n05u0e3vmr5lcqn1l5xa2rhkdu6x) to login and validate an onboarded tenant.

playbooks/playbook.yaml

@@ -67,7 +67,6 @@
         name: vdm
       tags:
         - viya
-        - multi-tenancy
     - name: Monitoring role - namespace
       include_role:
         name: monitoring

roles/multi-tenancy/defaults/main.yml

@@ -11,6 +11,16 @@ V4_CFG_CR_HOST: '{{ V4_CFG_CR_URL | regex_replace("^https?:\/\/(.*)\/?", "\1") }
 V4_CFG_CR_USER: null
 V4_CFG_CR_PASSWORD: null
 
+# Deployment Operator
+V4_DEPLOYMENT_OPERATOR_ENABLED: true
+V4_DEPLOYMENT_OPERATOR_SCOPE: cluster
+V4_DEPLOYMENT_OPERATOR_NAMESPACE: sasoperator
+V4_DEPLOYMENT_OPERATOR_CRB: sasoperator
+
+## Below the line deployment -- internal use only
+## Setting true enables using custom du for below the line testing
+V4_CFG_BELOW_THE_LINE: false
+
 # Multi-tenant is enabled within SAS Viya deployment
 V4MT_ENABLE: false
 

roles/multi-tenancy/tasks/main.yaml

@@ -15,6 +15,24 @@
     - cas-onboard
     - offboard
 
+# Deploy the Software for tenant pod-templates
+- name: Include Deployment assets - onboard
+  include_tasks: ../../vdm/tasks/assets.yaml
+  tags:
+    - onboard
+
+- name: Include SASDeployment Custom Resource - onboard
+  include_tasks: ../../vdm/tasks/sasdeployment_custom_resource.yaml
+  tags:
+    - onboard
+
+- name: Include Deploy - onboard
+  include_tasks: ../../vdm/tasks/deploy.yaml
+  when:
+    - DEPLOY
+  tags:
+    - onboard
+
 - name: Multi-tenant role - onboard offboard
   include_tasks: multi-tenant-onboard-offboard.yaml
   tags:
@@ -26,3 +44,24 @@
   tags:
     - cas-onboard
     - offboard
+
+# Deploy the Software for cas-onboard
+- name: Include Deployment assets - cas onboard
+  include_tasks: ../../vdm/tasks/assets.yaml
+  tags:
+    - cas-onboard
+    - offboard
+
+- name: Include SASDeployment Custom Resource - cas onboard
+  include_tasks: ../../vdm/tasks/sasdeployment_custom_resource.yaml
+  tags:
+    - cas-onboard
+    - offboard
+
+- name: Include Deploy - cas onboard
+  include_tasks: ../../vdm/tasks/deploy.yaml
+  when:
+    - DEPLOY
+  tags:
+    - cas-onboard
+    - offboard

roles/multi-tenancy/tasks/multi-tenant-onboard-offboard.yaml

@@ -2,6 +2,17 @@
 # SPDX-License-Identifier: Apache-2.0
 
 ---
+- name: Check tenant pod-templates are created
+  ansible.builtin.shell: |
+    kubectl --kubeconfig {{ KUBECONFIG }} get podtemplate -n {{ NAMESPACE }} | egrep '{{ V4MT_TENANT_IDS | replace(",", "|") | replace(" ", "") }}'
+  register: podtemplate_status
+  until: podtemplate_status.stdout | length > 0
+  failed_when: podtemplate_status.stderr | length > 0
+  retries: 5
+  delay: 90
+  tags:
+    - onboard
+
 # Apply the service account role
 - name: Apply service account role
   ansible.builtin.shell: |
@@ -182,7 +193,7 @@
   ansible.builtin.shell: |
     kubectl --kubeconfig {{ KUBECONFIG }} get pods -n {{ NAMESPACE }} --sort-by=.metadata.creationTimestamp | tac | grep sas-tenant-onboard | awk '{print $3}'
   register: pod_status
-  until: pod_status.stdout_lines[0] == "Completed" or pod_status.stdout_lines[0] in pod_fail_list
+  until: pod_status.stdout_lines[0] == "Running" or pod_status.stdout_lines[0] in pod_fail_list
   retries: "{{ V4MT_ONBOARD_RETRY | int }}"
   delay: "{{ V4MT_ONBOARD_DELAY | int }}"
   tags:
@@ -207,3 +218,16 @@
   tags:
     - onboard
     - offboard
+
+- name: Sleep for 120 seconds
+  debug:
+    msg:
+      - "Sleeping for 120 seconds before continuing with cas-onboard steps"
+  tags:
+    - onboard
+
+- name: Sleep for 120 seconds and continue with cas-onboard steps
+  ansible.builtin.wait_for:
+    timeout: 120
+  tags:
+    - onboard

roles/multi-tenancy/tasks/multi-tenant-setup.yaml

@@ -188,6 +188,7 @@
     - V4_CFG_CADENCE_VERSION is version('2023.03', ">=") or V4_CFG_CADENCE_NAME|lower == "fast"
   tags:
     - onboard
+    - offboard
 
 - name: Add env variable for pod templates in sas-tenant-job
   lineinfile:
@@ -212,3 +213,4 @@
     - V4_CFG_CADENCE_VERSION is version('2023.03', ">=") or V4_CFG_CADENCE_NAME|lower == "fast"
   tags:
     - onboard
+    - offboard

roles/multi-tenancy/tasks/onboard-offboard-cas-servers.yaml

@@ -35,43 +35,6 @@
   tags:
     - offboard
 
-## Add tenant specific pod templates
-- name: Check if tenant resources are present
-  lineinfile:
-    path: "{{ DEPLOY_DIR }}/kustomization.yaml"
-    line: "- site-config/multi-tenant/{{ item | trim }}"
-    state: present
-  check_mode: true
-  register: result
-  with_items: "{{ V4MT_TENANT_IDS.split(',') }}"
-  when: V4_CFG_CADENCE_VERSION is version('2023.03', ">=") or V4_CFG_CADENCE_NAME|lower == "fast"
-  tags:
-    - cas-onboard
-
-- name: Add tenant directories to resources
-  lineinfile:
-    path: "{{ DEPLOY_DIR }}/kustomization.yaml"
-    insertafter: "resources:"
-    line: "- site-config/multi-tenant/{{ item | trim }}"
-    state: present
-  with_items: "{{ V4MT_TENANT_IDS.split(',') }}"
-  when:
-    - result.changed
-    - V4_CFG_CADENCE_VERSION is version('2023.03', ">=") or V4_CFG_CADENCE_NAME|lower == "fast"
-  tags:
-    - cas-onboard
-
-# On offboard remove all the tenant pod template resources
-- name: Remove all tenant resources
-  lineinfile:
-    path: "{{ DEPLOY_DIR }}/kustomization.yaml"
-    regexp: .*site-config/multi-tenant/{{ item | trim }}.*$
-    state: absent
-  with_items: "{{ V4MT_TENANT_IDS.split(',') }}"
-  when: V4_CFG_CADENCE_VERSION is version('2023.03', ">=") or V4_CFG_CADENCE_NAME|lower == "fast"
-  tags:
-    - offboard
-
 # Offboard CAS servers
 - name: Kubectl delete cas servers for tenants
   ansible.builtin.shell: |

roles/multi-tenancy/tasks/tenant-pod-templates.yaml

@@ -31,3 +31,43 @@
       with_items: "{{ result.files }}"
       loop_control:
         loop_var: outer_item
+
+## Add tenant specific pod templates
+- name: Check if tenant resources are present
+  lineinfile:
+    path: "{{ DEPLOY_DIR }}/kustomization.yaml"
+    line: "- site-config/multi-tenant/{{ tenant | trim }}"
+    state: present
+  check_mode: true
+  register: result
+  when:
+    - V4_CFG_CADENCE_VERSION is version('2023.03', ">=") or V4_CFG_CADENCE_NAME|lower == "fast"
+    - V4MT_TENANT_IDS is search(tenant)
+  tags:
+    - onboard
+
+- name: Add tenant directories to resources
+  lineinfile:
+    path: "{{ DEPLOY_DIR }}/kustomization.yaml"
+    insertafter: "resources:"
+    line: "- site-config/multi-tenant/{{ tenant | trim }}"
+    state: present
+  when:
+    - result.changed
+    - V4_CFG_CADENCE_VERSION is version('2023.03', ">=") or V4_CFG_CADENCE_NAME|lower == "fast"
+    - V4MT_TENANT_IDS is search(tenant)
+  tags:
+    - onboard
+
+# On offboard remove all the tenant pod template resources
+- name: Remove all tenant resources
+  lineinfile:
+    path: "{{ DEPLOY_DIR }}/kustomization.yaml"
+    regexp: '.*site-config/multi-tenant/{{ tenant | trim }}.*$'
+    state: absent
+  # with_items: "{{ V4MT_TENANT_IDS.split(',') }}"
+  when:
+    - V4_CFG_CADENCE_VERSION is version('2023.03', ">=") or V4_CFG_CADENCE_NAME|lower == "fast"
+    - V4MT_TENANT_IDS is search(tenant)
+  tags:
+    - offboard

roles/vdm/tasks/assets.yaml

@@ -21,6 +21,7 @@
     - install
     - uninstall
     - update
+    - onboard
     - cas-onboard
     - offboard
 

roles/vdm/tasks/deploy.yaml

@@ -13,6 +13,7 @@
   tags:
     - install
     - update
+    - onboard
     - cas-onboard
 
 - name: Deploy - Apply SAS Viya deployment
@@ -21,6 +22,7 @@
   tags:
     - install
     - update
+    - onboard
     - cas-onboard
   block:
     - name: Deploy - Gather all resource files

roles/vdm/tasks/main.yaml

@@ -41,7 +41,6 @@
     - install
     - uninstall
     - update
-    - multi-tenancy
 
 - name: Base overlays
   overlay_facts:
@@ -235,7 +234,6 @@
     - install
     - uninstall
     - update
-    - multi-tenancy
 
 - name: Include Deploy
   include_tasks: deploy.yaml
@@ -245,7 +243,6 @@
     - install
     - uninstall
     - update
-    - multi-tenancy
 
 - name: Include Deployment Operator - Uninstall
   include_tasks: deployment_operator.yaml

roles/vdm/tasks/sasdeployment_custom_resource.yaml

@@ -14,6 +14,7 @@
     - install
     - uninstall
     - update
+    - onboard
     - cas-onboard
     - offboard
 
@@ -22,6 +23,7 @@
     - install
     - uninstall
     - update
+    - onboard
     - cas-onboard
     - offboard
   block:
@@ -42,6 +44,7 @@
     - install
     - uninstall
     - update
+    - onboard
     - cas-onboard
     - offboard
   block:
@@ -77,6 +80,7 @@
   tags:
     - install
     - update
+    - onboard
     - cas-onboard
     - offboard
   block:
@@ -108,6 +112,7 @@
   tags:
     - install
     - update
+    - onboard
     - cas-onboard
     - offboard
   block:
@@ -140,6 +145,7 @@
   tags:
     - install
     - update
+    - onboard
     - cas-onboard
     - offboard
   block: