[CICD] Add CLI Extensions Command Test Coverage (#383)

* add command coverage

* commit

* regex

* command coverage 1.0

* upload html

* modify report

* refactor

* add annotation

* Update test_cmdcov.py

* modify detect command function

* detect from live test file

* update func _get_all_commands

* Update __init__.py

* update sort function

* update sort function

* add level argument

* add cli own

* add EXCLUDE MODULES

* Update component.css

* delete unused code

* delete unused file

* modify regex

* add help info

* add description

* exclude command wait

* detect from recording file

* delete exception already fixed

* add newline

* add license headers

* fix tox

* refactoring

* fix bug

* exclude deprecated commands

* fix tox

* fix tox

* add network exclude commands

* add number sign exclude

* add comment

* add linter rule comcov

* update

* regex

* add log add exec_state

* add rule type command_coverage

* update diff branches

* support exclusion

* linter refactoring

* move constant and regex out

* suppport linter_exclusion.yml in cmdcov

* add license headers

* fix pylint

* fix flake8

* Update test_cmdcov.py

* fix bug for linter rule

* add resource skip commands

* update

1. Change the selected color to pink
2. Mouse hover shows percentage details
3. Modify Coverage Report description

* role commands exclusion

* modify HISTORY.rst and setup.py

* resource skip two commands - hard to test

* bug fixes: search argument

* modify command coverage to command test coverage

* bug fixes: Add three new scenarios

* update CLI own and exclude commands

* move private dns out of cli own

* add log for exclusions

* exclude vm host restart

* config

* Update test_config.py

* Add Extension Command Coverage Report

* Update __init__.py

* Update __init__.py

* Update __init__.py

* update

* update

* Update test_config.py

* Update constant.py

* Update constant.py

HISTORY.rst

@@ -5,8 +5,9 @@ Release History
 
 0.1.49
 ++++++
-* Add Command Coverage Report  (#323)
+* Add Command Coverage Report for CLI modules (#323)
 * Add Linter rule missing_command_coverage and missing_parameter_coverage  (#323)
+* Add Command Coverage Report for CLI extensions (#383)
 
 0.1.48
 ++++++

azdev/operations/cmdcov/__init__.py

@@ -15,14 +15,15 @@
 from azdev.utilities.path import get_cli_repo_path, get_ext_repo_paths
 from .cmdcov import CmdcovManager
 
-
 logger = get_logger(__name__)
+
 try:
     with open(os.path.join(get_cli_repo_path(), 'scripts', 'ci', 'cmdcov.yml'), 'r') as file:
         config = yaml.safe_load(file)
         EXCLUDE_MODULES = config['EXCLUDE_MODULES']
 except CLIError as ex:
-    logger.warning('Failed to load cmdcov.yml: %s', ex)
+    logger.warning('Failed to load cmdcov.yml: %s, please make sure your repo contains the following file '
+                   'https://github.com/Azure/azure-cli/blob/dev/scripts/ci/cmdcov.yml', str(ex))
 
 
 # pylint:disable=too-many-locals, too-many-statements, too-many-branches, duplicate-code
@@ -43,12 +44,14 @@ def run_cmdcov(modules=None, git_source=None, git_target=None, git_repo=None, le
 
     # allow user to run only on CLI or extensions
     cli_only = modules == ['CLI']
-    # ext_only = modules == ['EXT']
-    enable_cli_own = bool(cli_only or modules is None)
-    if cli_only:
+    ext_only = modules == ['EXT']
+    both_cli_ext = False
+    if modules == ['ALL']:
+        both_cli_ext = True
+    if cli_only or ext_only or both_cli_ext:
         modules = None
-    # if cli_only or ext_only:
-    #     modules = None
+
+    enable_cli_own = bool(modules is None)
 
     selected_modules = get_path_table(include_only=modules)
 
@@ -58,19 +61,64 @@ def run_cmdcov(modules=None, git_source=None, git_target=None, git_repo=None, le
     if not any(selected_modules.values()):
         logger.warning('No commands selected to check.')
 
+    # mapping special extension name
+    def _map_extension_module(selected_modules):
+        # azext_applicationinsights -> azext_application_insights
+        # azext_firewall -> azext_azure_firewall
+        # azext_dms -> azext_dms_preview
+        # azext_dnsresolver -> azext_dns_resolver
+        # azext_expressroutecrossconnection -> azext_express_route_cross_connection
+        # azext_imagecopy -> azext_image_copy
+        # azext_loganalytics -> azext_log_analytics
+        # azext_amcs -> azext_monitor_control_service
+        # azext_resourcegraph -> azext_resource_graph
+        # azext_sentinel -> azext_securityinsight
+        # azext_serialconsole -> azext_serial_console
+        # azext_vnettap -> azext_virtual_network_tap
+        # azext_vwan -> azext_virtual_wan
+        # azext_connection_monitor_preview -> azure cli 2.0.81
+        # azext_spring_cloud -> deprecate
+        # azext_interactive -> no command
+        special_extensions_name = {
+            'azext_applicationinsights': 'azext_application_insights',
+            'azext_firewall': 'azext_azure_firewall',
+            'azext_dms': 'azext_dms_preview',
+            'azext_dnsresolver': 'azext_dns_resolver',
+            'azext_expressroutecrossconnection': 'azext_express_route_cross_connection',
+            'azext_imagecopy': 'azext_image_copy',
+            'azext_loganalytics': 'azext_log_analytics',
+            'azext_amcs': 'azext_monitor_control_service',
+            'azext_resourcegraph': 'azext_resource_graph',
+            'azext_sentinel': 'azext_securityinsight',
+            'azext_serialconsole': 'azext_serial_console',
+            'azext_vnettap': 'azext_virtual_network_tap',
+            'azext_vwan': 'azext_virtual_wan',
+        }
+        import copy
+        copied_selected_modules = copy.deepcopy(selected_modules)
+        unsupported_extensions = ['azext_connection_monitor_preview', 'azext_spring_cloud', 'azext_interactive']
+        for k, v in copied_selected_modules['ext'].items():
+            if k in special_extensions_name:
+                selected_modules['ext'][special_extensions_name[k]] = v
+                del selected_modules['ext'][k]
+            if k in unsupported_extensions:
+                del selected_modules['ext'][k]
+
+    _map_extension_module(selected_modules)
+
     if EXCLUDE_MODULES:
         selected_modules['mod'] = {k: v for k, v in selected_modules['mod'].items() if k not in EXCLUDE_MODULES}
         selected_modules['ext'] = {k: v for k, v in selected_modules['ext'].items() if k not in EXCLUDE_MODULES}
 
-    if cli_only:
+    if cli_only and not both_cli_ext:
         selected_mod_names = list(selected_modules['mod'].keys())
         selected_mod_paths = list(selected_modules['mod'].values())
-    # elif ext_only:
-    #     selected_mod_names = list(selected_modules['ext'].keys())
-    #     selected_mod_paths = list(selected_modules['ext'].values())
+    elif ext_only and not both_cli_ext:
+        selected_mod_names = list(selected_modules['ext'].keys())
+        selected_mod_paths = list(selected_modules['ext'].values())
     else:
-        selected_mod_names = list(selected_modules['mod'].keys())
-        selected_mod_paths = list(selected_modules['mod'].values())
+        selected_mod_names = list(selected_modules['mod'].keys()) + list(selected_modules['ext'].keys())
+        selected_mod_paths = list(selected_modules['mod'].values()) + list(selected_modules['ext'].values())
 
     if selected_mod_names:
         display('Modules: {}\n'.format(', '.join(selected_mod_names)))

azdev/operations/cmdcov/cmdcov.py

@@ -13,11 +13,12 @@
 from jinja2 import FileSystemLoader, Environment
 from knack.log import get_logger
 from knack.util import CLIError
+from tqdm import tqdm
 from azdev.operations.regex import get_all_tested_commands_from_regex
 from azdev.utilities.path import get_azdev_repo_path, get_cli_repo_path, find_files
 
-
 logger = get_logger(__name__)
+
 try:
     with open(os.path.join(get_cli_repo_path(), 'scripts', 'ci', 'cmdcov.yml'), 'r') as file:
         config = yaml.safe_load(file)
@@ -39,7 +40,8 @@
         EXCLUDE_COMMANDS = config['EXCLUDE_COMMANDS']
         GLOBAL_EXCLUDE_COMMANDS = config['GLOBAL_EXCLUDE_COMMANDS']
 except CLIError as ex:
-    logger.warning('Failed to load cmdcov.yml: %s', ex)
+    logger.warning('Failed to load cmdcov.yml: %s, please make sure your repo contains the following file '
+                   'https://github.com/Azure/azure-cli/blob/dev/scripts/ci/cmdcov.yml', str(ex))
 
 
 # pylint: disable=too-many-instance-attributes
@@ -62,6 +64,8 @@ def __init__(self, selected_mod_names=None, selected_mod_paths=None, loaded_help
         self.cmdcov_path = os.path.join(get_azdev_repo_path(), 'azdev', 'operations', 'cmdcov')
         self.template_path = os.path.join(self.cmdcov_path, 'template')
         self.exclusions = exclusions
+        self.width = 60
+        self.fillchar = '-'
 
     def run(self):
         self._get_all_commands()
@@ -111,15 +115,18 @@ def _get_all_commands(self):
             elif 'rule_exclusions' in v:
                 if 'missing_command_test_coverage' in v['rule_exclusions']:
                     exclusions_comands.append(c)
-        for _, y in self.loaded_help.items():
+        print("\033[31m" + "Get all commands".center(self.width, self.fillchar) + "\033[0m")
+        time.sleep(0.1)
+        for _, y in tqdm(self.loaded_help.items()):
             if hasattr(y, 'command_source') and y.command_source in self.selected_mod_names:
                 module = y.command_source
-            elif hasattr(y, 'command_source') and hasattr(y.command_source, 'extension_name') and \
-                    y.command_source.extension_name in self.selected_mod_names:
-                module = y.command_source.extension_name
+            elif hasattr(y, 'command_source') and hasattr(y.command_source, 'extension_name'):
+                module = 'azext_' + y.command_source.extension_name.replace('-', '_')
+                if module not in self.selected_mod_names:
+                    module = None
             else:
                 continue
-            if not y.deprecate_info:
+            if (not y.deprecate_info) and module:
                 if y.command.split()[-1] not in GLOBAL_EXCLUDE_COMMANDS and \
                         y.command not in EXCLUDE_COMMANDS.get(module, []) and \
                         y.command not in exclusions_comands:
@@ -138,8 +145,16 @@ def _get_all_tested_commands_from_regex(self):
         get all tested commands from test_*.py
         """
         # pylint: disable=too-many-nested-blocks
-        for idx, path in enumerate(self.selected_mod_paths):
-            test_dir = os.path.join(path, 'tests')
+        print("\033[31m" + "Get tested commands from regex".center(self.width, self.fillchar) + "\033[0m")
+        time.sleep(0.1)
+        for idx, path in enumerate(tqdm(self.selected_mod_paths)):
+            if 'azure-cli-extensions' in path:
+                for dirname in os.listdir(path):
+                    if dirname.startswith('azext'):
+                        test_dir = os.path.join(path, dirname, 'tests')
+                        break
+            else:
+                test_dir = os.path.join(path, 'tests')
             files = find_files(test_dir, '*.py')
             for f in files:
                 with open(os.path.join(test_dir, f), 'r', encoding=ENCODING) as f:
@@ -151,8 +166,16 @@ def _get_all_tested_commands_from_record(self):
         """
         get all tested commands from recording files
         """
-        for idx, path in enumerate(self.selected_mod_paths):
-            test_dir = os.path.join(path, 'tests')
+        print("\033[31m" + "Get tested commands from recording files".center(self.width, self.fillchar) + "\033[0m")
+        time.sleep(0.1)
+        for idx, path in enumerate(tqdm(self.selected_mod_paths)):
+            if 'azure-cli-extensions' in path:
+                for dirname in os.listdir(path):
+                    if dirname.startswith('azext'):
+                        test_dir = os.path.join(path, dirname, 'tests')
+                        break
+            else:
+                test_dir = os.path.join(path, 'tests')
             files = find_files(test_dir, 'test*.yaml')
             for f in files:
                 with open(os.path.join(test_dir, f)) as f:
@@ -297,7 +320,9 @@ def _render_html(self):
             f.write(content)
 
         # render child html
-        for module, coverage in self.command_test_coverage.items():
+        print("\033[31m" + "Render test coverage report".center(self.width, self.fillchar) + "\033[0m")
+        time.sleep(0.1)
+        for module, coverage in tqdm(self.command_test_coverage.items()):
             if coverage:
                 self._render_child_html(module, coverage, self.all_untested_commands[module])