Check installed Ansible collections during 'netlab up' and 'netlab in…

…itial' (#1182)

The 'devices' module contains code that fetches the installed
Ansible collections and allows device-specific quirks code to
check for their presence. The 'process_config_sw_check' is
executed early in 'netlab up' processing unless the '--no-config'
flag is set, and in 'netlab initial' unless it's invoked with
the '--output' option.

netsim/cli/initial.py

@@ -11,6 +11,7 @@
 from . import external_commands
 from . import ansible
 from ..utils import log,status as _status
+from .. import devices
 from box import Box
 
 #
@@ -94,7 +95,8 @@ def run(cli_args: typing.List[str]) -> None:
   else:
     external_commands.LOG_COMMANDS = True
     deploy_text = ', '.join(deploy_parts) or 'complete configuration'
-    if not topology is None:
+    if topology is not None:
+      devices.process_config_sw_check(topology)
       lab_status_change(topology,f'deploying configuration: {deploy_text}')
 
     ansible.playbook('initial-config.ansible',rest)

netsim/cli/up.py

@@ -20,6 +20,7 @@
 from .. import providers
 from ..utils import log,strings,status as _status, read as _read
 from ..data import global_vars
+from ..devices import process_config_sw_check
 
 #
 # Extra arguments for 'netlab up' command
@@ -312,6 +313,8 @@ def run(cli_args: typing.List[str]) -> None:
 
   external_commands.LOG_COMMANDS = True
   provider_probes(topology)
+  if not args.no_config:
+    process_config_sw_check(topology)
 
   p_provider = topology.provider
   p_module = providers.get_provider_module(topology,p_provider)

netsim/devices/__init__.py

@@ -5,6 +5,7 @@
 #
 import typing
 import os
+import json
 
 from box import Box
 
@@ -31,34 +32,103 @@ def load(self, device: str, data: Box) -> typing.Any:
   def device_quirks(self, node: Box, topology: Box) -> None:
     log.fatal(f'{node.device} quirks module does not implement device_quirks method')
 
-"""
-Callback transformation routines
+  def check_config_sw(self, node: Box, topology: Box) -> None:
+    pass
 
-* node_transform: for all nodes, call specified method for every module used by the node
-* link_transform: for all links, call specified method for every module used by any node on the link
+"""
+Get the quirks module for the specified device
 
-Note: mod_load is a global cache of loaded modules
+Note: DEVICE_MODULE is a global cache of loaded modules
 """
 
-mod_load: typing.Dict = {}
+DEVICE_MODULE: typing.Dict = {}
+
+def get_device_module(device: str, topology: Box) -> typing.Optional[typing.Any]:
+  global DEVICE_MODULE
 
-def device_quirk(node: Box, topology: Box, method: str = 'device_quirks') -> None:
-  global mod_load
+  if device in DEVICE_MODULE:
+    return DEVICE_MODULE[device]
+
+  dev_quirk = os.path.dirname(__file__)+"/"+device+".py"
+  if os.path.exists(dev_quirk):
+    DEVICE_MODULE[device] = _Quirks.load(device,topology.defaults.devices.get(device))
+  else:
+    DEVICE_MODULE[device] = None
+
+  return DEVICE_MODULE[device]
+
+"""
+Execute a device quirk callback
+"""
+def exec_device_quirk(node: Box, topology: Box, method: str = 'device_quirks') -> None:
 
   if log.debug_active('quirks'):
     print(f'Processing device quirks: method {method}, node {node.name}/{node.device}')
-  device = node.device
-
-  if not device in mod_load:
-    dev_quirk = os.path.dirname(__file__)+"/"+device+".py"
-    if os.path.exists(dev_quirk):
-      mod_load[device] = _Quirks.load(device,topology.defaults.devices.get(device))
-    else:
-      mod_load[device] = None
 
-  if mod_load[device]:
-    mod_load[device].call(method,node,topology)
+  q_module = get_device_module(node.device,topology)
+  if q_module is not None:
+    q_module.call(method,node,topology)
 
+"""
+Process device quirks at the end of the topology transformation
+"""
 def process_quirks(topology: Box) -> None:
   for n in topology.nodes.values():
-    device_quirk(n,topology)
+    exec_device_quirk(n,topology)
+
+ANSIBLE_COLLECTIONS: dict = {}
+
+def get_ansible_collection(cname: str) -> typing.Optional[dict]:
+  global ANSIBLE_COLLECTIONS
+  from ..cli import external_commands
+
+  if ANSIBLE_COLLECTIONS:
+    return ANSIBLE_COLLECTIONS.get(cname,None)
+
+  result = external_commands.run_command(
+            cmd='ansible-galaxy collection list --format json',
+            check_result=True,return_stdout=True,run_always=True)
+  if not isinstance(result,str):
+    ANSIBLE_COLLECTIONS['_failed'] = True
+    log.error('Cannot run ansible-galaxy to get the list of installed collections',log.MissingDependency)
+    return None
+
+  try:
+    ac_list = json.loads(result)              # ansible-galaxy returns a dictionary of dicts
+    for loc_v in ac_list.values():            # Iterate over returned locations
+      for ack,acv in loc_v.items():           # Iterate over collections
+        if ack not in ANSIBLE_COLLECTIONS:    # ... and keep the first one found
+          ANSIBLE_COLLECTIONS[ack] = acv
+
+  except Exception as ex:
+    log.fatal('Cannot parse the ansible-galaxy JSON printout: {ex}')
+
+  return ANSIBLE_COLLECTIONS.get(cname,None)
+
+COLLECTION_WARNING: dict = {}
+
+def need_ansible_collection(node: Box, cname: str, install: str = '') -> bool:
+  global COLLECTION_WARNING
+
+  cdata = get_ansible_collection(cname)
+  if cdata is not None:
+    return True
+
+  if node.device in COLLECTION_WARNING:
+    return False
+
+  if not install:
+    install = f'ansible-galaxy collection install {cname}'
+  log.error(
+        f'We need {cname} Ansible collection to configure {node.device} devices',
+        category=log.MissingDependency,
+        more_hints = [ f'Use "{install}" to install it' ],
+        module='devices')
+  COLLECTION_WARNING[node.device] = True
+  return False
+
+def process_config_sw_check(topology: Box) -> None:
+  for n in topology.nodes.values():
+    exec_device_quirk(n,topology,method='check_config_sw')
+
+  log.exit_on_error()

netsim/devices/arubacx.py

@@ -7,7 +7,7 @@
 #
 from box import Box
 
-from . import _Quirks
+from . import _Quirks,need_ansible_collection
 from ..augment import devices
 from ..utils import log
 
@@ -27,3 +27,6 @@ def device_quirks(self, node: Box, topology: Box) -> None:
                 return
             node.vrfs[vrf]['ospfidx'] = ospfidx
             ospfidx = ospfidx + 1
+
+  def check_config_sw(self, node: Box, topology: Box) -> None:
+    need_ansible_collection(node,'arubanetworks.aoscx')

netsim/devices/dellos10.py

@@ -3,7 +3,7 @@
 #
 from box import Box
 
-from . import _Quirks
+from . import _Quirks,need_ansible_collection
 from ..utils import log
 from ..augment import devices
 
@@ -28,3 +28,6 @@ def device_quirks(self, node: Box, topology: Box) -> None:
     check_vlan_ospf(node.name,node.get('interfaces',[]),'default')
     for vname,vdata in node.get('vrfs',{}).items():
       check_vlan_ospf(node.name,vdata.get('ospf.interfaces',[]),vname)
+
+  def check_config_sw(self, node: Box, topology: Box) -> None:
+    need_ansible_collection(node,'dellemc.os10')

netsim/devices/srlinux.py

@@ -6,7 +6,7 @@
 #
 from box import Box
 
-from . import _Quirks
+from . import _Quirks,need_ansible_collection
 from ..utils import log
 
 class SRLINUX(_Quirks):
@@ -30,3 +30,6 @@ def device_quirks(self, node: Box, topology: Box) -> None:
                     f'SR Linux on ({node.name}) does not support IS-IS multi-topology required for ipv6.\n',
                     log.IncorrectType,
                     'quirks')
+
+  def check_config_sw(self, node: Box, topology: Box) -> None:
+    need_ansible_collection(node,'nokia.grpc')