Merge pull request #7 from zerwes/improve-task-detection

Improve task detection

.github/workflows/test.yml

@@ -0,0 +1,24 @@
+# vim: tabstop=2 expandtab shiftwidth=2 softtabstop=2 smartindent nu
+---
+name: test
+on:
+  pull_request:
+  push:
+  schedule:
+    - cron: "20 6 * * 5"
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - name: install pyyaml
+        run: |
+          python -m pip install --upgrade pip
+          pip install pyyaml
+      - name: run test
+        run: |
+          ./test.sh

.gitignore

@@ -2,3 +2,5 @@
 *.pyc
 *.bak
 test*.yml
+xyz/
+.collections/

README.md

@@ -1,4 +1,5 @@
 [![pylint](https://github.com/zerwes/ansible-fqcn-converter/actions/workflows/pylint.yml/badge.svg)](https://github.com/zerwes/ansible-fqcn-converter/actions/workflows/pylint.yml)
+[![test](https://github.com/zerwes/ansible-fqcn-converter/actions/workflows/test.yml/badge.svg)](https://github.com/zerwes/ansible-fqcn-converter/actions/workflows/test.yml)
 
 # Ansible FQCN converter
 Update ansible tasks, playbooks, handlers etc. to use fully qualified module names (even for ansible buildins) by searching for all known modules that are not in fqcn notation and replacing them with the fqcn name.

example.yml

@@ -0,0 +1,92 @@
+---
+- name: ansible fqcn test playbook
+  hosts: localhost
+  gather_facts: true
+  tasks:
+    - name: test task 1
+      command: cat dog
+
+    - name: test task 2
+      command:
+        cmd: cat dog
+
+    - name: test task 1 fqcn
+      ansible.builtin.command: cat dog
+
+    - name: test task 2 fqcn
+      ansible.builtin.command:
+        cmd: cat dog
+
+- name: roles
+  collections:
+    - 'debops.debops'
+    - 'debops.roles01'
+    - 'debops.roles02'
+    - 'debops.roles03'
+  roles:
+    - role: bootstrap
+      tags:
+        - bootstrap
+    - role: common
+      tags:
+        - common
+
+---
+# just tasks list
+
+# named task
+- name: test apt unnamed
+  apt:
+    name: ipsum
+
+# named task w/ group as pitfall
+- name: test copy with group as pitfall
+  copy:
+    content: aaa
+    dest: xyz
+    group: lorem
+
+# galaxy module
+- name: general command os6_config
+  os6_config:
+    commands: "{{ dellos_general_command | default([]) }}"
+  when: ansible_network_os == "dellemc.os6.os6"
+
+# unnamed task
+- file:
+    path: /etc/apt/sources.list.d/pve-enterprise.list
+    state: absent
+  when: "'ox' in group_names"
+
+# unnamed fqcn
+- ansible.builtin.copy:
+    src: a
+    dest: b
+
+# named fqcn
+- name: lorem ipsum
+  ansible.builtin.copy:
+    src: lorem
+    dest: ipsum
+
+# block
+- name: "configure ..."
+  block:
+    - name: "read ..."
+      command: cat dog
+      become: true
+      check_mode: false
+      changed_when: false
+      register: _local_json
+
+    - name: read json
+      set_fact:
+        current_of_local_json: "{{ _local_json.stdout | from_json }}"
+
+    - name: "write {{ of_default_config }}"
+      ansible.builtin.copy:
+        dest: "{{ of_default_config }}"
+        content: "{{ current_of_local_json | combine(of_local_json, recursive=true) | to_nice_json(indent=2, sort_keys=false) }}\n"
+      notify: restart-xyz
+  when: of_local_json
+

exampleconverted.yml

@@ -0,0 +1,95 @@
+---
+- name: ansible fqcn test playbook
+  hosts: localhost
+  gather_facts: true
+  tasks:
+    - name: test task 1
+      # possible ambiguous replacement: command : ansible.builtin.command | community.ciscosmb.command | community.routeros.command
+      ansible.builtin.command: cat dog
+
+    - name: test task 2
+      # possible ambiguous replacement: command : ansible.builtin.command | community.ciscosmb.command | community.routeros.command
+      ansible.builtin.command:
+        cmd: cat dog
+
+    - name: test task 1 fqcn
+      ansible.builtin.command: cat dog
+
+    - name: test task 2 fqcn
+      ansible.builtin.command:
+        cmd: cat dog
+
+- name: roles
+  collections:
+    - 'debops.debops'
+    - 'debops.roles01'
+    - 'debops.roles02'
+    - 'debops.roles03'
+  roles:
+    - role: bootstrap
+      tags:
+        - bootstrap
+    - role: common
+      tags:
+        - common
+
+---
+# just tasks list
+
+# named task
+- name: test apt unnamed
+  ansible.builtin.apt:
+    name: ipsum
+
+# named task w/ group as pitfall
+- name: test copy with group as pitfall
+  ansible.builtin.copy:
+    content: aaa
+    dest: xyz
+    group: lorem
+
+# galaxy module
+- name: general command os6_config
+  dellemc.os6.os6_config:
+    commands: "{{ dellos_general_command | default([]) }}"
+  when: ansible_network_os == "dellemc.os6.os6"
+
+# unnamed task
+- ansible.builtin.file:
+    path: /etc/apt/sources.list.d/pve-enterprise.list
+    state: absent
+  when: "'ox' in group_names"
+
+# unnamed fqcn
+- ansible.builtin.copy:
+    src: a
+    dest: b
+
+# named fqcn
+- name: lorem ipsum
+  ansible.builtin.copy:
+    src: lorem
+    dest: ipsum
+
+# block
+- name: "configure ..."
+  block:
+    - name: "read ..."
+      # possible ambiguous replacement: command : ansible.builtin.command | community.ciscosmb.command | community.routeros.command
+      ansible.builtin.command: cat dog
+      become: true
+      check_mode: false
+      changed_when: false
+      register: _local_json
+
+    - name: read json
+      ansible.builtin.set_fact:
+        current_of_local_json: "{{ _local_json.stdout | from_json }}"
+
+    - name: "write {{ of_default_config }}"
+      ansible.builtin.copy:
+        dest: "{{ of_default_config }}"
+        content: "{{ current_of_local_json | combine(of_local_json, recursive=true) | to_nice_json(indent=2, sort_keys=false) }}\n"
+      notify: restart-xyz
+  when: of_local_json
+

fqcn-fixer.py

@@ -35,6 +35,18 @@ def isexcluded(path, _exclude_paths):
         fnmatch.fnmatch(ppath, ep)
         for ep in _exclude_paths
         )
+
+def debugmsg(msg):
+    """debug msg"""
+    print(msg, file=sys.stderr, flush=True)
+
+def checkignoreregex(checkline):
+    """check if we should ignore replacement"""
+    for exre in _general_exclude_regex:
+        if exre.match(checkline):
+            return True
+    return False
+
 class Dumper(yaml.Dumper): # pylint: disable=too-many-ancestors
     """https://github.com/yaml/pyyaml/issues/234"""
     def increase_indent(self, flow=False, *dargs): # pylint: disable=keyword-arg-before-vararg
@@ -59,6 +71,12 @@ def increase_indent(self, flow=False, *dargs): # pylint: disable=keyword-arg-bef
     "*/meta/*",
     ]
 
+# case insensitive list of regex to exclude / skip replacements
+_general_exclude_regex = [
+    re.compile(r'\s*gather_facts:\s*(no|yes|true|false)', re.IGNORECASE),
+    re.compile(r'\s*-\srole:\s*\w+'),
+]
+
 required_fqcnconverter_file_version = '0.0.5'
 
 argparser = argparse.ArgumentParser(description=__doc__)
@@ -137,6 +155,13 @@ def increase_indent(self, flow=False, *dargs): # pylint: disable=keyword-arg-bef
     default=False,
     help="update the fqcn-map-file"
     )
+argparser.add_argument(
+    '-D', '--debug',
+    dest='debug',
+    action='store_true',
+    default=False,
+    help="debug output"
+    )
 
 args = argparser.parse_args()
 
@@ -247,6 +272,10 @@ def increase_indent(self, flow=False, *dargs): # pylint: disable=keyword-arg-bef
 
 # prepare regex
 _fqcnregex = re.compile(r'^(?P<white>\s*-?\s+)(?P<module>%s):' % '|'.join(fqcndict.keys()))
+_taskstartregex = re.compile(
+    r'^(?P<white>\s*-\s+)(?P<nm>%s):' %
+        '|'.join(['name'] + list(fqcndict.keys()))
+    )
 
 # do it
 for f in parsefiles:
@@ -257,40 +286,84 @@ def increase_indent(self, flow=False, *dargs): # pylint: disable=keyword-arg-bef
             backup=args.backupextension) as fi:
         originallines = []
         changedlines = []
-        startingwhitespaces = False
+        startingwhitespaces = r'\s*-?\s+'
+        startingwhitespacesaftertask = 0
+        startingwhitespaces4comments = 0
+        in_task = False
+        in_task_done = False
         fqcnregex = _fqcnregex
         for line in fi:
+            if args.debug:
+                debugmsg(
+                    'STARTLINE : line: %s in_task: %s in_task_done: %s\n' %
+                    (line, in_task, in_task_done,)
+                    )
             if args.printdiff:
                 originallines.append(line)
             nline = line
-            fqcnmatch = fqcnregex.match(line)
-            if fqcnmatch:
-                if not startingwhitespaces:
-                    startingwhitespaces = fqcnmatch.group('white')
-                    fqcnregex = re.compile('^%s(?P<module>%s):' %
-                        (startingwhitespaces, '|'.join(fqcndict.keys()))
+            taskmatch = _taskstartregex.match(line)
+            if taskmatch:
+                in_task_done = False
+                in_task = False
+            if in_task_done and not taskmatch:
+                if args.debug:
+                    debugmsg('SKIPLINE! %s\n' % (in_task_done and not in_task))
+                print('.', file=sys.stderr, end='', flush=True)
+            else:
+                if not in_task:
+                    if args.debug:
+                        debugmsg('TASKMATCH : line: %s taskmatch: %s\n' % (line, taskmatch,))
+                    if taskmatch:
+                        in_task = True
+                        in_task_done = False
+                        fqcnregex = _fqcnregex
+                        startingwhitespaces = r'\s*-?\s+'
+                        startingwhitespacesaftertask = len(taskmatch.group('white'))
+                        if args.debug:
+                            debugmsg('line: %s taskmatch: %s' % (line, taskmatch,))
+                            debugmsg(
+                                'startingwhitespaces "%s" startingwhitespacesaftertask "%s"' %
+                                (startingwhitespaces, startingwhitespacesaftertask,)
+                            )
+                fqcnmatch = fqcnregex.match(line)
+                if args.debug:
+                    debugmsg('FQCNMATCH : line: %s fqcnmatch: %s\n' % (line, fqcnmatch,))
+                    debugmsg('fqcnregex: %s' % fqcnregex)
+                if fqcnmatch and not checkignoreregex(line):
+                    in_task_done = True
+                    in_task = False
+                    fqcnmodule = fqcnmatch.group('module')
+                    nline = re.sub(
+                        '^(%s)%s:' % (startingwhitespaces, fqcnmodule),
+                        '\\1%s:' % fqcndict[fqcnmodule][0],
+                        line
                         )
-                fqcnmodule = fqcnmatch.group('module')
-                nline = re.sub(
-                    '^(%s)%s:' % (startingwhitespaces, fqcnmodule),
-                    '\\1%s:' % fqcndict[fqcnmodule][0],
-                    line
-                    )
-                if fqcnmodule == fqcndict[fqcnmodule][0]:
-                    print('.', file=sys.stderr, end='', flush=True)
+                    if fqcnmodule == fqcndict[fqcnmodule][0]:
+                        print('.', file=sys.stderr, end='', flush=True)
+                    else:
+                        print('*', file=sys.stderr, end='', flush=True)
+                        if len(fqcndict[fqcnmodule]) > 1:
+                            wtxt = ('possible ambiguous replacement: %s : %s' %
+                                   (fqcnmodule, ' | '.join(fqcndict[fqcnmodule])))
+                            warnings.append(wtxt)
+                            if args.writewarnings:
+                                if args.writefiles:
+                                    print('%s# %s' % (' '*startingwhitespaces4comments, wtxt))
+                                if args.printdiff:
+                                    changedlines.append(
+                                        '%s# %s\n' % (' '*startingwhitespaces4comments, wtxt)
+                                        )
                 else:
-                    print('*', file=sys.stderr, end='', flush=True)
-                    if len(fqcndict[fqcnmodule]) > 1:
-                        wtxt = ('possible ambiguous replacement: %s : %s' %
-                               (fqcnmodule, ' | '.join(fqcndict[fqcnmodule])))
-                        warnings.append(wtxt)
-                        if args.writewarnings:
-                            if args.writefiles:
-                                print('# %s\n' % wtxt)
-                            if args.printdiff:
-                                changedlines.append('# %s\n' % wtxt)
-            else:
-                print('.', file=sys.stderr, end='', flush=True)
+                    print('.', file=sys.stderr, end='', flush=True)
+                    if startingwhitespacesaftertask > 0:
+                        startingwhitespaces = ' ' * startingwhitespacesaftertask
+                        startingwhitespaces4comments = startingwhitespacesaftertask
+                        startingwhitespacesaftertask = 0
+                        fqcnregex = re.compile('^%s(?P<module>%s):' %
+                            (startingwhitespaces, '|'.join(fqcndict.keys()))
+                            )
+                        if args.debug:
+                            debugmsg('set STARTINGWHITESPACES to "%s"' % startingwhitespaces)
 
             if args.writefiles:
                 print(nline, end='')