Solve issue Object diff for roles removing most roles (#562)

* added singulars to be treated as well

* new attributes for roles

* new attributes for roles

* added changelog fragment

* fix on map_item function

* removed extra empty line

* fixes on object_diff inputs

* removed debug information. added ORGANIZATIONLESS to credentials and users without an organization

* fix lintering issues

* fix lintering issues

* fix lintering issues

* tests fixes. multiple list from #647 fixed. Test ping URL fixed

changelogs/fragments/object_diff.yml

@@ -0,0 +1,5 @@
+---
+bugfixes:
+  - Added more attributes to be expanded and used by the comparison
+  - Fixed lintering issues
+...

plugins/lookup/controller_object_diff.py

@@ -89,11 +89,11 @@
   returned: on successful differential
 """
 
-from ansible.plugins.lookup import LookupBase
+import copy
 from ansible.errors import AnsibleError, AnsibleLookupError
 from ansible.module_utils._text import to_native
+from ansible.plugins.lookup import LookupBase
 from ansible.utils.display import Display
-import copy
 
 
 class LookupModule(LookupBase):
@@ -114,6 +114,13 @@ def create_present_list(self, compare_list):
 
         return compare_list
 
+    def map_item(self, item, new_attribute_name, attribute_value, dupitems):
+        new_item = copy.deepcopy(item)
+        new_item.update({new_attribute_name: attribute_value})
+        for dupitem in [dupitem for dupitem in dupitems if dupitem in new_item]:
+            new_item.pop(dupitem)
+        return new_item
+
     def run(self, terms, variables=None, **kwargs):
         self.set_options(direct=kwargs)
 
@@ -179,7 +186,12 @@ def run(self, terms, variables=None, **kwargs):
             api_list_reduced = copy.deepcopy(api_list)
         elif api_list[0]["type"] == "instance_group":
             compare_list_reduced = [{key: item[key] for key in keys_to_keep} for item in compare_list]
-            api_list_reduced = [{key: item[key] for key in api_keys_to_keep} for item in api_list if item["summary_fields"]["user_capabilities"]["delete"]]
+            api_list_reduced = [
+                {key: item[key] for key in api_keys_to_keep}
+                for item in api_list
+                if (item["summary_fields"] and item["summary_fields"]["user_capabilities"]["delete"])
+            ]
+
         else:
             compare_list_reduced = [{key: item[key] for key in keys_to_keep} for item in compare_list]
             api_list_reduced = [{key: item[key] for key in api_keys_to_keep} for item in api_list]
@@ -196,7 +208,7 @@ def run(self, terms, variables=None, **kwargs):
                 item.pop("summary_fields")
         elif api_list[0]["type"] == "credential":
             for item in api_list_reduced:
-                item.update({"organization": item["summary_fields"]["organization"]["name"]})
+                item.update({"organization": item["summary_fields"]["organization"]["name"] if item["summary_fields"]["organization"] else ""})
                 item.update({"credential_type": item["summary_fields"]["credential_type"]["name"]})
                 item.pop("summary_fields")
         elif api_list[0]["type"] == "workflow_job_template_node":
@@ -225,43 +237,57 @@ def run(self, terms, variables=None, **kwargs):
             list_to_extend = []
             list_to_remove = []
             for item in compare_list_reduced:
-                target_teams_expanded = False
-                job_templates_expanded = False
-                workflows_expanded = False
+                expanded = False
+                dupitems = [
+                    "target_team",
+                    "target_teams",
+                    "job_template",
+                    "job_templates",
+                    "workflow",
+                    "workflows",
+                    "inventory",
+                    "inventories",
+                    "project",
+                    "projects",
+                    "credential",
+                    "credentials",
+                ]
+                if "target_team" in item:
+                    list_to_extend.append(self.map_item(item, "team", item["target_team"], dupitems))
+                    expanded = True
                 if "target_teams" in item:
                     for team in item["target_teams"]:
-                        new_item = copy.deepcopy(item)
-                        new_item.update({"team": team})
-                        new_item.pop("target_teams")
-                        if "job_templates" in new_item:
-                            new_item.pop("job_templates")
-                        if "workflows" in new_item:
-                            new_item.pop("workflows")
-                        list_to_extend.append(new_item)
-                    target_teams_expanded = True
+                        list_to_extend.append(self.map_item(item, "team", team, dupitems))
+                    expanded = True
+                if "job_template" in item:
+                    list_to_extend.append(self.map_item(item, "job_template", item["job_template"], dupitems))
+                    expanded = True
                 if "job_templates" in item:
                     for job_template in item["job_templates"]:
-                        new_item = copy.deepcopy(item)
-                        new_item.update({"job_template": job_template})
-                        new_item.pop("job_templates")
-                        if "target_teams" in new_item:
-                            new_item.pop("target_teams")
-                        if "workflows" in new_item:
-                            new_item.pop("workflows")
-                        list_to_extend.append(new_item)
-                    job_templates_expanded = True
+                        list_to_extend.append(self.map_item(item, "job_template", job_template, dupitems))
+                    expanded = True
+                if "workflow" in item:
+                    list_to_extend.append(self.map_item(item, "workflow_job_template", item["workflow"], dupitems))
+                    expanded = True
                 if "workflows" in item:
                     for workflow in item["workflows"]:
-                        new_item = copy.deepcopy(item)
-                        new_item.update({"workflow_job_template": workflow})
-                        new_item.pop("workflows")
-                        if "target_teams" in new_item:
-                            new_item.pop("target_teams")
-                        if "job_templates" in new_item:
-                            new_item.pop("job_templates")
-                        list_to_extend.append(new_item)
-                    workflows_expanded = True
-                if target_teams_expanded or job_templates_expanded or workflows_expanded:
+                        list_to_extend.append(self.map_item(item, "workflow_job_template", workflow, dupitems))
+                    expanded = True
+                if "inventory" in item:
+                    list_to_extend.append(self.map_item(item, "inventory", item["inventory"], dupitems))
+                    expanded = True
+                if "inventories" in item:
+                    for inventory in item["inventories"]:
+                        list_to_extend.append(self.map_item(item, "inventory", inventory, dupitems))
+                    expanded = True
+                if "project" in item:
+                    list_to_extend.append(self.map_item(item, "project", item["project"], dupitems))
+                    expanded = True
+                if "projects" in item:
+                    for project in item["projects"]:
+                        list_to_extend.append(self.map_item(item, "project", project, dupitems))
+                    expanded = True
+                if expanded:
                     list_to_remove.append(item)
             for item in list_to_remove:
                 compare_list_reduced.remove(item)
@@ -295,7 +321,8 @@ def run(self, terms, variables=None, **kwargs):
                 item.update({"state": "absent"})
         # Combine Lists
         if self.get_option("with_present"):
-            compare_list = self.create_present_list(compare_list)
+            for item in compare_list_reduced:
+                item.update({"state": "present"})
             compare_list.extend(difference)
             # Return Compare list with difference attached
             difference = compare_list
@@ -308,4 +335,4 @@ def run(self, terms, variables=None, **kwargs):
             for item in difference_to_remove:
                 difference.remove(item)
 
-        return difference
+        return [difference]

roles/applications/tasks/main.yml

@@ -5,11 +5,11 @@
     name:                           "{{ __application_item.name | mandatory }}"
     new_name:                       "{{ __application_item.new_name | default(omit, true) }}"
     organization:                   "{{ __application_item.organization | mandatory }}"
-    description:                    "{{ __application_item.description | default(( '' if controller_configuration_applications_enforce_defaults else omit), true) }}"
+    description:                    "{{ __application_item.description | default(('' if controller_configuration_applications_enforce_defaults else omit), true) }}"
     authorization_grant_type:       "{{ __application_item.authorization_grant_type | default('password') }}"
     client_type:                    "{{ __application_item.client_type | default('public') }}"
     redirect_uris:                  "{{ __application_item.redirect_uris | default([]) }}"
-    skip_authorization:             "{{ __application_item.skip_authorization | default(( false if controller_configuration_applications_enforce_defaults else omit), true) }}"
+    skip_authorization:             "{{ __application_item.skip_authorization | default((false if controller_configuration_applications_enforce_defaults else omit), true) }}"
     state:                          "{{ __application_item.state | default(controller_state | default('present')) }}"
 
     # Role specific options

roles/credential_input_sources/tasks/main.yml

@@ -4,8 +4,8 @@
     target_credential:              "{{ __cred_input_src_item.target_credential | mandatory }}"
     input_field_name:               "{{ __cred_input_src_item.input_field_name | mandatory }}"
     source_credential:              "{{ __cred_input_src_item.source_credential | default(omit, true) }}"
-    description:                    "{{ __cred_input_src_item.description | default(( '' if controller_configuration_credential_input_sources_enforce_defaults else omit), true) }}"
-    metadata:                       "{{ __cred_input_src_item.metadata | default(( {} if controller_configuration_credential_input_sources_enforce_defaults else omit), true) }}"
+    description:                    "{{ __cred_input_src_item.description | default(('' if controller_configuration_credential_input_sources_enforce_defaults else omit), true) }}"
+    metadata:                       "{{ __cred_input_src_item.metadata | default(({} if controller_configuration_credential_input_sources_enforce_defaults else omit), true) }}"
     state:                          "{{ __cred_input_src_item.state | default(controller_state | default('present')) }}"
 
     # Role specific options

roles/credential_types/tasks/main.yml

@@ -3,9 +3,9 @@
   credential_type:
     name:                           "{{ __controller_credential_type_item.name | mandatory }}"
     new_name:                       "{{ __controller_credential_type_item.new_name | default(omit, true) }}"
-    description:                    "{{ __controller_credential_type_item.description | default(( '' if controller_configuration_credential_types_enforce_defaults else omit), true) }}"
-    injectors:                      "{{ __controller_credential_type_item.injectors | default(( {} if controller_configuration_credential_types_enforce_defaults else omit), true) | regex_replace('[ ]{2,}', '') }}"
-    inputs:                         "{{ __controller_credential_type_item.inputs | default(( {} if controller_configuration_credential_types_enforce_defaults else omit), true) }}"
+    description:                    "{{ __controller_credential_type_item.description | default(('' if controller_configuration_credential_types_enforce_defaults else omit), true) }}"
+    injectors:                      "{{ __controller_credential_type_item.injectors | default(({} if controller_configuration_credential_types_enforce_defaults else omit), true) | regex_replace('[ ]{2,}', '') }}"
+    inputs:                         "{{ __controller_credential_type_item.inputs | default(({} if controller_configuration_credential_types_enforce_defaults else omit), true) }}"
     kind:                           "{{ __controller_credential_type_item.kind | default('cloud') }}"
     state:                          "{{ __controller_credential_type_item.state | default(controller_state | default('present')) }}"
 

roles/credentials/tasks/main.yml

@@ -4,13 +4,13 @@
     name:                           "{{ __controller_credentials_item.name | mandatory }}"
     new_name:                       "{{ __controller_credentials_item.new_name | default(omit, true) }}"
     copy_from:                      "{{ __controller_credentials_item.copy_from | default(omit, true) }}"
-    description:                    "{{ __controller_credentials_item.description | default(( '' if controller_configuration_credentials_enforce_defaults else omit), true) }}"
-    organization:                   "{{ __controller_credentials_item.organization.name | default(__controller_credentials_item.organization | default(( '' if controller_configuration_credentials_enforce_defaults else omit), true)) }}"
+    description:                    "{{ __controller_credentials_item.description | default(('' if controller_configuration_credentials_enforce_defaults else omit), true) }}"
+    organization:                   "{{ __controller_credentials_item.organization.name | default(__controller_credentials_item.organization | default(('' if controller_configuration_credentials_enforce_defaults else omit), true)) }}"
     credential_type:                "{{ __controller_credentials_item.credential_type.name | default(__controller_credentials_item.credential_type | mandatory ) }}"
-    inputs:                         "{{ __controller_credentials_item.inputs | default(( {} if controller_configuration_credentials_enforce_defaults else omit), true) }}"
-    user:                           "{{ __controller_credentials_item.user.username | default(__controller_credentials_item.user | default(( '' if controller_configuration_credentials_enforce_defaults else omit), true)) }}"
-    team:                           "{{ __controller_credentials_item.team.name | default(__controller_credentials_item.team | default(( '' if controller_configuration_credentials_enforce_defaults else omit), true)) }}"
-    update_secrets:                 "{{ __controller_credentials_item.update_secrets | default( true if controller_configuration_credentials_enforce_defaults else omit) }}"
+    inputs:                         "{{ __controller_credentials_item.inputs | default(({} if controller_configuration_credentials_enforce_defaults else omit), true) }}"
+    user:                           "{{ __controller_credentials_item.user.username | default(__controller_credentials_item.user | default(('' if controller_configuration_credentials_enforce_defaults else omit), true)) }}"
+    team:                           "{{ __controller_credentials_item.team.name | default(__controller_credentials_item.team | default(('' if controller_configuration_credentials_enforce_defaults else omit), true)) }}"
+    update_secrets:                 "{{ __controller_credentials_item.update_secrets | default(true if controller_configuration_credentials_enforce_defaults else omit) }}"
     state:                          "{{ __controller_credentials_item.state | default(controller_state | default('present')) }}"
 
     # Role specific options

roles/filetree_create/tasks/all.yml

@@ -25,7 +25,7 @@
         msg: "The organization {{ organization_filter }} has the ID {{ organization_id }}"
 
 - name: Include tasks (block)
-  when: "['all', 'labels', 'applications', 'instance_groups', 'settings', 'inventory', 'credentials', 'credential_types', 'notification_templates', 'users', 'teams', 'organizations', 'projects', 'execution_environments', 'job_templates', 'workflow_job_templates', 'workflow_job_template_nodes', 'schedules'] | intersect(input_tag) | length > 0"
+  when: "['all', 'labels', 'applications', 'instance_groups', 'settings', 'inventory', 'credentials', 'credential_types', 'notification_templates', 'users', 'teams', 'roles', 'organizations', 'projects', 'execution_environments', 'job_templates', 'workflow_job_templates', 'workflow_job_template_nodes', 'schedules'] | intersect(input_tag) | length > 0"
   block:
     - name: "Export Inventories and related Groups and Hosts"
       ansible.builtin.include_tasks: "inventory.yml"
@@ -41,10 +41,10 @@
       when: "'notification_templates' in input_tag or 'all' in input_tag"
     - name: "Export Users"
       ansible.builtin.include_tasks: "users.yml"
-      when: "'users' in input_tag or 'all' in input_tag"
+      when: "'users' in input_tag or 'roles' in input_tag or 'all' in input_tag"
     - name: "Export Teams"
       ansible.builtin.include_tasks: "teams.yml"
-      when: "'teams' in input_tag or 'all' in input_tag"
+      when: "'teams' in input_tag or 'roles' in input_tag or 'all' in input_tag"
     - name: "Export Organizations"
       ansible.builtin.include_tasks: "organizations.yml"
       when: "'organizations' in input_tag or 'all' in input_tag"

roles/filetree_create/tasks/users.yml

@@ -9,7 +9,7 @@
 
 - name: "Add the users the Organizations information"  # noqa jinja[spacing]
   ansible.builtin.set_fact:
-    current_users: "{{ (current_users | default([])) + [user_lookvar_item | combine({'organizations': user_lookvar_item_organizations})] }}"
+    current_users: "{{ (current_users | default([])) + [user_lookvar_item | combine({'organizations': user_lookvar_item_organizations if (user_lookvar_item_organizations | length > 1) else ['ORGANIZATIONLESS']})] }}"
   vars:
     user_lookvar_item_organizations: "{{ query(controller_api_plugin, user_lookvar_item.related.organizations,
                                                host=controller_hostname, oauth_token=controller_oauthtoken, verify_ssl=controller_validate_certs,

roles/filetree_create/templates/current_credentials.j2

@@ -5,6 +5,8 @@ controller_credentials:
     credential_type: "{{ current_credentials_asset_value.summary_fields.credential_type.name }}"
 {% if current_credentials_asset_value.organization is defined and current_credentials_asset_value.organization is not none %}
     organization: "{{ current_credentials_asset_value.summary_fields.organization.name }}"
+{% else %}
+    organization: "ORGANIZATIONLESS"
 {% endif %}
     inputs:
 {{ current_credentials_asset_value.inputs | to_nice_yaml(indent=2) | indent(width=6, first=True) | replace("$encrypted$", "\'\'") }}

roles/object_diff/tasks/applications.yml

@@ -9,17 +9,17 @@
 - name: "Get the API list of all Applications in Organization {{ orgs }}"
   ansible.builtin.set_fact:
     __controller_api_applications: "{{ query(controller_api_plugin, 'applications',
-                                          query_params={'organization': __controller_organization_id.id},
-                                          host=controller_hostname, oauth_token=controller_oauthtoken, verify_ssl=controller_validate_certs,
-                                          return_all=true, max_objects=query_controller_api_max_objects)
-                                }}"
+                                             query_params={'organization': __controller_organization_id.id},
+                                             host=controller_hostname, oauth_token=controller_oauthtoken, verify_ssl=controller_validate_certs,
+                                             return_all=true, max_objects=query_controller_api_max_objects)
+                                    }}"
 
 - name: "Find the difference of Application between what is on the Controller versus CasC on SCM"
   ansible.builtin.set_fact:
     __applications_difference: "{{ query(controller_role_plugin,
-                                      api_list=__controller_api_applications, compare_list=controller_applications,
-                                      with_present=include_present_state, set_absent=true)
-                            }}"
+                                         api_list=__controller_api_applications, compare_list=controller_applications,
+                                         with_present=include_present_state, set_absent=true) | flatten
+                                }}"
 
 - name: "Set application's list to be configured"
   ansible.builtin.set_fact: