Merge pull request #84 from networktocode/develop

v1.0.0 Release
networktocode · Dec 2, 2021 · 2e48376 · 2e48376
2 parents e753e4e + c2d7517
commit 2e48376
Show file tree

Hide file tree

Showing 19 changed files with 402 additions and 17 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,19 @@
 # Changelog
 
+## v1.0.0 - 2021-11
+
+### Added
+
+- #69 Normalise banner demiliter for IOS to ^C & support parsing delimiter ^
+
+### Fixed
+
+- #79 F5 parser fix for irules with multiline single command lines.
+
+### Removed
+
+- #83 remove support for old function 'is_fqdn_valid' as prep for 1.0.0
+
 ## v0.2.5 - 2021-11
 
 ### Added

diff --git a/docs/source/netutils/banner/index.rst b/docs/source/netutils/banner/index.rst
@@ -0,0 +1,6 @@
+*******
+Banner
+*******
+
+.. automodule:: netutils.banner
+    :members:
diff --git a/docs/source/netutils/configs/index.rst b/docs/source/netutils/configs/index.rst
@@ -17,3 +17,7 @@ Edge Cases
 Fortinet Fortios Parser
 -----------------------
 - In order to support html blocks that exist in Fortios configurations, some preprocessing is executed, this is a regex that specifically grabs everything between quotes after the 'set buffer' sub-command. It's explicitly looking for double quote followed by a newline ("\n) to end the captured data.  This support for html data will not support any other html that doesn't follow this convention.
+
+F5 Parser
+-----------------------
+- The "ltm rule" configuration sections are not uniform nor standardized; therefor, these sections are completely removed from the configuration in a preprocessing event.
diff --git a/docs/source/netutils/index.rst b/docs/source/netutils/index.rst
@@ -7,6 +7,7 @@ Netutils Functions
 
    asn/index
    bandwidth/index
+   banner/index
    configs/index
    dns/index
    interface/index

diff --git a/netutils/__init__.py b/netutils/__init__.py
@@ -1,3 +1,3 @@
 """Initialization file for library."""
 
-__version__ = "0.2.5"
+__version__ = "1.0.0"
diff --git a/netutils/banner.py b/netutils/banner.py
@@ -0,0 +1,50 @@
+"""Functions for working with the banner configuration."""
+import re
+from netutils.constants import CARET_C
+
+
+def delimiter_change(config, from_delimiter, to_delimiter):
+    r"""Change the banner delimiter.
+
+    Args:
+        config (str): Configuration line containing banner delimiter.
+        from_delimiter (str): Delimiter to replace in the banner.
+        to_delimiter (str): Delimiter to include in the config.
+
+    Returns:
+        str: Configuration with delimiter replaced.
+
+    Example:
+        >>> from netutils.banner import delimiter_change
+        >>> delimiter_change("banner login ^\n******************\n    TEST BANNER\n******************^", "^", "^C")
+        'banner login ^C\n******************\n    TEST BANNER\n******************^C'
+        >>> delimiter_change("banner login #\n******************\n    TEST BANNER\n******************#", "#", "^C")
+        'banner login ^C\n******************\n    TEST BANNER\n******************^C'
+        >>> delimiter_change("banner login ^CCCCC\n******************\n    TEST BANNER\n******************^C", "^C", "^C")
+        'banner login ^C\n******************\n    TEST BANNER\n******************^C'
+    """
+    config_line = config.replace(from_delimiter, to_delimiter)
+    if to_delimiter == CARET_C:
+        config_line = re.sub(r"\^C+", CARET_C, config_line)
+    return config_line
+
+
+def normalise_delimiter_caret_c(delimiter, config):
+    r"""Normalise delimiter to ^C.
+
+    Args:
+        delimiter (str): Banner delimiter.
+        config (str): Configuration line containing banner delimiter.
+
+    Returns:
+        str: Configuration with delimiter normalised to ^C.
+
+    Example:
+        >>> from netutils.banner import normalise_delimiter_caret_c
+        >>> normalise_delimiter_caret_c("^", "banner login ^\n******************\n    TEST BANNER\n******************^")
+        'banner login ^C\n******************\n    TEST BANNER\n******************^C'
+        >>> normalise_delimiter_caret_c("^C", "banner login ^CCCCC\n******************\n    TEST BANNER\n******************^C")
+        'banner login ^C\n******************\n    TEST BANNER\n******************^C'
+    """
+    config_line = delimiter_change(config, delimiter, CARET_C)
+    return config_line
diff --git a/netutils/config/parser.py b/netutils/config/parser.py
@@ -3,6 +3,7 @@
 
 import re
 from collections import namedtuple
+from netutils.banner import normalise_delimiter_caret_c
 
 ConfigLine = namedtuple("ConfigLine", "config_line,parents")
 
@@ -196,7 +197,7 @@ def _build_banner(self, config_line):
             if not self.is_banner_end(line):
                 banner_config.append(line)
             else:
-                line = line.replace("\x03", "^C")
+                line = normalise_delimiter_caret_c(self.banner_end, line)
                 banner_config.append(line)
                 line = "\n".join(banner_config)
                 if line.endswith("^C"):
@@ -417,7 +418,7 @@ def _build_multiline_config(self, delimiter):
 class CiscoConfigParser(BaseSpaceConfigParser):
     """Cisco Implementation of ConfigParser Class."""
 
-    regex_banner = re.compile(r"^(banner\s+\S+|\s*vacant-message)\s+(?P<banner_delimiter>\^C|\x03)")
+    regex_banner = re.compile(r"^(banner\s+\S+|\s*vacant-message)\s+(?P<banner_delimiter>\^C|.)")
 
     def __init__(self, config):
         """Create ConfigParser Object.
@@ -449,9 +450,10 @@ def _build_banner(self, config_line):
                 return None
         return super(CiscoConfigParser, self)._build_banner(config_line)
 
-    def is_banner_one_line(self, config_line):
+    @staticmethod
+    def is_banner_one_line(config_line):
         """Determine if all banner config is on one line."""
-        _, delimeter, banner = config_line.partition(self.banner_end)
+        _, delimeter, banner = config_line.partition("^C")
         # Based on NXOS configs, the banner delimeter is ignored until another char is used
         banner_config_start = banner.lstrip(delimeter)
         if delimeter not in banner_config_start:
@@ -504,8 +506,7 @@ def _build_banner(self, config_line):
         Raises:
             ValueError: When the parser is unable to identify the End of the Banner.
         """
-        config_line = config_line.replace("\x03", "^C")
-        config_line = re.sub(r"\^C+", "^C", config_line)
+        config_line = normalise_delimiter_caret_c(self.banner_end, config_line)
         return super(IOSConfigParser, self)._build_banner(config_line)
 
     def _update_same_line_children_configs(self):
@@ -597,6 +598,130 @@ class F5ConfigParser(BaseBraceConfigParser):
 
     multiline_delimiters = ['"']
 
+    def __init__(self, config):
+        """Create ConfigParser Object.
+
+        Args:
+            config (str): The config text to parse.
+        """
+        super().__init__(self._clean_config_f5(config))
+
+    def _clean_config_f5(self, config_text):  # pylint: disable=no-self-use
+        """Removes all configuration items with 'ltm rule'.
+
+        iRules are essentially impossible to parse with the lack of uniformity,
+        therefore, this method ensures they are not included in ``self.config``.
+
+        Args:
+            config_text (str): The entire config as a string.
+
+        Returns:
+            str: The sanitized config with all iRules (ltm rule) stanzas removed.
+        """
+        config_split = config_text.split("ltm rule")
+        if len(config_split) > 1:
+            start_config = config_split[0]
+            end_config = config_split[-1]
+            _, ltm, clean_config = end_config.partition("ltm")
+            final_config = start_config + ltm + clean_config
+        else:
+            final_config = config_text
+        return final_config
+
+    def build_config_relationship(self):
+        r"""Parse text tree of config lines and their parents.
+
+        Example:
+            >>> config = '''apm resource webtop-link aShare {
+            ...     application-uri http://funshare.example.com
+            ...     customization-group a_customization_group
+            ... }
+            ... apm sso form-based portal_ext_sso_form_based {
+            ...     form-action /Citrix/Example/ExplicitAuth/LoginAttempt
+            ...     form-field "LoginBtn Log+On
+            ... StateContext "
+            ...     form-password password
+            ...     form-username username
+            ...     passthru true
+            ...     start-uri /Citrix/Example/ExplicitAuth/Login*
+            ...     success-match-type cookie
+            ...     success-match-value CtxsAuthId
+            ... }
+            ... '''
+            >>>
+            >>> config_tree = F5ConfigParser(config)
+            >>> print(config_tree.build_config_relationship())
+            [ConfigLine(config_line='apm resource webtop-link aShare {', parents=()), ConfigLine(config_line='    application-uri http://funshare.example.com', parents=('apm resource webtop-link aShare {',)), ConfigLine(config_line='    customization-group a_customization_group', parents=('apm resource webtop-link aShare {',)), ConfigLine(config_line='}', parents=('apm resource webtop-link aShare {',)), ConfigLine(config_line='apm sso form-based portal_ext_sso_form_based {', parents=()), ConfigLine(config_line='    form-action /Citrix/Example/ExplicitAuth/LoginAttempt', parents=('apm sso form-based portal_ext_sso_form_based {',)), ConfigLine(config_line='    form-field "LoginBtn Log+On\nStateContext "', parents=('apm sso form-based portal_ext_sso_form_based {',)), ConfigLine(config_line='    form-password password', parents=()), ConfigLine(config_line='    form-username username', parents=()), ConfigLine(config_line='    passthru true', parents=()), ConfigLine(config_line='    start-uri /Citrix/Example/ExplicitAuth/Login*', parents=()), ConfigLine(config_line='    success-match-type cookie', parents=()), ConfigLine(config_line='    success-match-value CtxsAuthId', parents=()), ConfigLine(config_line='}', parents=())]
+        """
+        for line in self.generator_config:
+            self.config_lines.append(ConfigLine(line, self._current_parents))
+            line_end = line[-1]
+            if line.endswith("{"):
+                self._current_parents += (line,)
+            elif line.lstrip() == "}":
+                self._current_parents = self._current_parents[:-1]
+            elif any(
+                delimiters in self.multiline_delimiters and line.count(delimiters) == 1
+                for delimiters in self.multiline_delimiters
+            ):
+                for delimiter in self.multiline_delimiters:
+                    if line.count(delimiter) == 1:
+                        self._build_multiline_single_configuration_line(delimiter, line)
+            elif line_end in self.multiline_delimiters and line.count(line_end) == 1:
+                self._current_parents += (line,)
+                self._build_multiline_config(line_end)
+
+        return self.config_lines
+
+    def _build_multiline_single_configuration_line(self, delimiter, prev_line):
+        r"""Concatenate Multiline strings between delimiter when newlines causes string to traverse multiple lines.
+
+        Args:
+            delimiter (str): The text to look for to end multiline config.
+            prev_line (str): The text from the previously analyzed line.
+
+        Returns:
+            ConfigLine: The multiline string text that was added to ``self.config_lines``.
+
+        Example:
+            config = '''apm resource webtop-link aShare {
+                application-uri http://funshare.example.com
+                customization-group a_customization_group
+            }
+            apm sso form-based portal_ext_sso_form_based {
+                form-action /Citrix/Example/ExplicitAuth/LoginAttempt
+            >>>
+            >>> config = '''apm resource webtop-link aShare {
+            ...     application-uri http://funshare.example.com
+            ...     customization-group a_customization_group
+            ... }
+            ... apm sso form-based portal_ext_sso_form_based {
+            ...     form-action /Citrix/Example/ExplicitAuth/LoginAttempt
+            ...     form-field "LoginBtn Log+On
+            ... StateContext "
+            ...     form-password password
+            ...     form-username username
+            ...     passthru true
+            ...     start-uri /Citrix/Example/ExplicitAuth/Login*
+            ...     success-match-type cookie
+            ...     success-match-value CtxsAuthId
+            ... }
+            ... '''
+            >>>
+            >>>
+            >>> config_tree = F5ConfigParser(str(config))
+            >>> print(config_tree.build_config_relationship())
+            [ConfigLine(config_line='apm resource webtop-link aShare {', parents=()), ConfigLine(config_line='    application-uri http://funshare.example.com', parents=('apm resource webtop-link aShare {',)), ConfigLine(config_line='    customization-group a_customization_group', parents=('apm resource webtop-link aShare {',)), ConfigLine(config_line='}', parents=('apm resource webtop-link aShare {',)), ConfigLine(config_line='apm sso form-based portal_ext_sso_form_based {', parents=()), ConfigLine(config_line='    form-action /Citrix/Example/ExplicitAuth/LoginAttempt', parents=('apm sso form-based portal_ext_sso_form_based {',)), ConfigLine(config_line='    form-field "LoginBtn Log+On\nStateContext "', parents=('apm sso form-based portal_ext_sso_form_based {',)), ConfigLine(config_line='    form-password password', parents=()), ConfigLine(config_line='    form-username username', parents=()), ConfigLine(config_line='    passthru true', parents=()), ConfigLine(config_line='    start-uri /Citrix/Example/ExplicitAuth/Login*', parents=()), ConfigLine(config_line='    success-match-type cookie', parents=()), ConfigLine(config_line='    success-match-value CtxsAuthId', parents=()), ConfigLine(config_line='}', parents=())]
+        """
+        multiline_config = [prev_line]
+        for line in self.generator_config:
+            multiline_config.append(line)
+            if line.endswith(delimiter):
+                multiline_entry = ConfigLine("\n".join(multiline_config), self._current_parents)
+                self.config_lines[-1] = multiline_entry
+                self._current_parents = self._current_parents[:-1]
+                return multiline_entry
+
 
 class JunosConfigParser(BaseSpaceConfigParser):
     """Junos config parser."""

diff --git a/netutils/constants.py b/netutils/constants.py
@@ -380,3 +380,8 @@
     340282366920938312347647155603121373184,
     340282366920919120650260773364972912640,
 }
+
+# End of Text characters for banner
+ETX_HEX = "\x03"
+CARET_C = "^C"
+CARET = "^"
diff --git a/netutils/dns.py b/netutils/dns.py
@@ -53,7 +53,3 @@ def is_fqdn_resolvable(hostname):
         return True
     except socket.error:
         return False
-
-
-# Provide until transition to 1.0
-is_fqdn_valid = is_fqdn_resolvable
diff --git a/netutils/utils.py b/netutils/utils.py
@@ -17,7 +17,6 @@
     "find_unordered_cfg_lines": "config.compliance.find_unordered_cfg_lines",
     "section_config": "config.compliance.section_config",
     "fqdn_to_ip": "dns.fqdn_to_ip",
-    "is_fqdn_valid": "dns.is_fqdn_valid",
     "is_fqdn_resolvable": "dns.is_fqdn_resolvable",
     "interface_range_expansion": "interface.interface_range_expansion",
     "interface_range_compress": "interface.interface_range_compress",
@@ -59,6 +58,8 @@
     "longest_prefix_match": "route.longest_prefix_match",
     "vlanlist_to_config": "vlan.vlanlist_to_config",
     "vlanconfig_to_list": "vlan.vlanconfig_to_list",
+    "normalise_delimiter_caret_c": "banner.normalise_delimiter_caret_c",
+    "delimiter_change": "banner.delimiter_change",
 }
 
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "netutils"
-version = "0.2.5"
+version = "1.0.0"
 description = "Common helper functions useful in network automation."
 authors = ["Network to Code, LLC <opensource@networktocode.com>"]
 license = "Apache-2.0"

diff --git a/tests/unit/mock/config/compliance/compliance/cisco_ios/ios_basic_backup.txt b/tests/unit/mock/config/compliance/compliance/cisco_ios/ios_basic_backup.txt
@@ -8,4 +8,10 @@ access-list 1 permit 10.10.15.15
 access-list 1 permit 10.10.20.20
 !
 ntp server 192.168.0.100
-ntp server 192.168.0.101
+ntp server 192.168.0.101
+!
+banner login ^C
+******************
+    TEST BANNER
+******************
+^C
diff --git a/tests/unit/mock/config/compliance/compliance/cisco_ios/ios_basic_feature.py b/tests/unit/mock/config/compliance/compliance/cisco_ios/ios_basic_feature.py
@@ -2,4 +2,5 @@
     {"name": "bgp", "ordered": True, "section": ["router bgp "]},
     {"name": "snmp", "ordered": True, "section": ["snmp-server "]},
     {"name": "ntp", "ordered": False, "section": ["ntp server "]},
+    {"name": "banner", "ordered": True, "section": ["banner "]},
 ]
diff --git a/tests/unit/mock/config/compliance/compliance/cisco_ios/ios_basic_intended.txt b/tests/unit/mock/config/compliance/compliance/cisco_ios/ios_basic_intended.txt
@@ -8,4 +8,10 @@ access-list 1 permit 10.10.15.15
 access-list 1 permit 10.10.20.20
 !
 ntp server 192.168.0.101
-ntp server 192.168.0.100
+ntp server 192.168.0.100
+!
+banner login ^
+******************
+    TEST BANNER
+******************
+^
diff --git a/tests/unit/mock/config/compliance/compliance/cisco_ios/ios_basic_received.json b/tests/unit/mock/config/compliance/compliance/cisco_ios/ios_basic_received.json
@@ -28,5 +28,15 @@
       "missing": "",
       "ordered_compliant": false,
       "unordered_compliant": true
-   }
+   },
+   "banner": {
+      "actual": "banner login ^C\n******************\n    TEST BANNER\n******************^C",
+      "cannot_parse": true,
+      "compliant": true,
+      "extra": "",
+      "intended": "banner login ^C\n******************\n    TEST BANNER\n******************^C",
+      "missing": "",
+      "ordered_compliant": true,
+      "unordered_compliant": true
+   }   
 }