Merge pull request #501 from networktocode/develop

Merge 1.8.0 into main
networktocode · Apr 3, 2024 · f6a1500 · f6a1500
2 parents 89449c7 + 3fb7206
commit f6a1500
Show file tree

Hide file tree

Showing 60 changed files with 2,220 additions and 719 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -18,7 +18,7 @@ jobs:
       - name: "Setup environment"
         uses: "networktocode/gh-action-setup-poetry-environment@v2"
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - name: "Linting: black"
         run: "poetry run invoke black"
   bandit:
@@ -31,7 +31,7 @@ jobs:
       - name: "Setup environment"
         uses: "networktocode/gh-action-setup-poetry-environment@v2"
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - name: "Linting: bandit"
         run: "poetry run invoke bandit"
     needs:
@@ -46,7 +46,7 @@ jobs:
       - name: "Setup environment"
         uses: "networktocode/gh-action-setup-poetry-environment@v2"
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - name: "Type-Hints: mypy"
         run: "poetry run invoke mypy"
     needs:
@@ -61,7 +61,7 @@ jobs:
       - name: "Setup environment"
         uses: "networktocode/gh-action-setup-poetry-environment@v2"
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - name: "Linting: pydocstyle"
         run: "poetry run invoke pydocstyle"
     needs:
@@ -76,7 +76,7 @@ jobs:
       - name: "Setup environment"
         uses: "networktocode/gh-action-setup-poetry-environment@v2"
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - name: "Linting: flake8"
         run: "poetry run invoke flake8"
     needs:
@@ -91,7 +91,7 @@ jobs:
       - name: "Setup environment"
         uses: "networktocode/gh-action-setup-poetry-environment@v2"
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - name: "Linting: yamllint"
         run: "poetry run invoke yamllint"
     needs:
@@ -100,7 +100,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
     runs-on: "ubuntu-20.04"
     env:
       PYTHON_VER: "${{ matrix.python-version }}"
@@ -137,7 +137,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: ["3.11"]
+        python-version: ["3.12"]
     env:
       PYTHON_VER: "${{ matrix.python-version }}"
     steps:
@@ -173,7 +173,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
     runs-on: "ubuntu-20.04"
     env:
       PYTHON_VER: "${{ matrix.python-version }}"
@@ -220,7 +220,7 @@ jobs:
       - name: "Set up Python"
         uses: "actions/setup-python@v2"
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - name: "Install Python Packages"
         run: "pip install poetry"
       - name: "Set env"
@@ -249,7 +249,7 @@ jobs:
       - name: "Set up Python"
         uses: "actions/setup-python@v2"
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - name: "Install Python Packages"
         run: "pip install poetry"
       - name: "Set env"

diff --git a/.github/workflows/proto_num_filepull.yml b/.github/workflows/proto_num_filepull.yml
@@ -0,0 +1,56 @@
+---
+name: "Protocol-Number-Updates"
+
+on:   # yamllint disable-line rule:truthy
+  schedule:
+    - cron: "0 2 1 * *"
+
+jobs:
+  data_gathering:
+    runs-on: "ubuntu-latest"
+    env:
+      BRANCH_NAME: "PROTO_NUM_Updates"
+    steps:
+      # Checkout repo
+      - name: "Check out code"
+        uses: "actions/checkout@v2"
+        with:
+          ref: "develop"
+      # Delete old branch if it exists
+      - name: "Delete existing branch"
+        run: "git branch -D $BRANCH_NAME || true"
+      # Create branch for Flatbot
+      - name: "Create Flatbot branch"
+        run: "git checkout -b $BRANCH_NAME"
+      # Push new branch so Flatbot can make its commit
+      - name: "Push Flatbot branch"
+        run: "git push -f --set-upstream origin $BRANCH_NAME"
+      # Install Black
+      - name: "Install Python Black"
+        run: "pip install black"
+      # This step installs Deno, which is a new Javascript runtime that improves on Node. Can be used for an optional postprocessing step
+      - name: "Setup deno"
+        uses: "denoland/setup-deno@main"
+        with:
+          deno-version: "v1.10.x"
+      # The Flat Action step. We fetch the data in the http_url and save it as downloaded_filename
+      - name: "Fetch data"
+        uses: "githubocto/flat@v3"
+        with:
+          http_url: "https://www.iana.org/assignments/protocol-numbers/protocol-numbers-1.csv"
+          downloaded_filename: "./netutils/data_files/protocol_number_mappings.py"
+          postprocess: "./flat_postprocess/protocol_number_postprocess.ts"
+  pr_creation:
+    runs-on: "ubuntu-latest"
+    needs: "data_gathering"
+    steps:
+      # Checkout repo
+      - name: "Check out code"
+        uses: "actions/checkout@v2"
+        with:
+          ref: "PROTO_NUM_Updates"
+      # Create PR from branch created above into develop
+      - name: "Create a Pull Request"
+        run: "gh pr create -B develop -H PROTO_NUM_Updates --title 'Flatbot PROTOCOL Number File Updates' --body 'Created by Flatbot action'"
+        env:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
diff --git a/docs/admin/release_notes/version_1.8.md b/docs/admin/release_notes/version_1.8.md
@@ -0,0 +1,31 @@
+# v1.8 Release Notes
+
+## Release Overview
+
+- Added support for Python 3.12.
+- Added local support of `distutils.version` now that Python 3.12 deprecates disutils.
+- Added functions `compare_version_loose` and `compare_version_strict` based on `distutils.version`.
+- Added function `paloalto_panos_clean_newlines`.
+- Added various lib_mapper updates.
+- Added HP Comware parser.
+
+## [v1.8.0] 2024-04
+
+### Added
+
+- [#483](https://github.com/networktocode/netutils/pull/483) Added support for Python 3.12.
+- [#483](https://github.com/networktocode/netutils/pull/483) Added local support of `distutils.version` now that Python 3.12 deprecates disutils.
+- [#490](https://github.com/networktocode/netutils/pull/490) Add JunOS and Vyatta to HierConfig mappers.
+- [#416](https://github.com/networktocode/netutils/pull/416) Added `paloalto_panos_clean_newlines` function.
+- [#467](https://github.com/networktocode/netutils/pull/467) Added HP Comware parser.
+
+### Changed
+
+- [#485](https://github.com/networktocode/netutils/pull/485) Changed order of changelog menu.
+- [#494](https://github.com/networktocode/netutils/pull/494) Changed protocol number import to be dynamic update via flatbot.
+- [#495](https://github.com/networktocode/netutils/pull/495) Changed XR mapping, add tests to ensure always using normalized name, various lib_mapper fixes.
+
+### Fixed
+
+- [#496](https://github.com/networktocode/netutils/pull/496) Fixed vyos lib_mapper.
+- [#416](https://github.com/networktocode/netutils/pull/416) Fixed for `\n` characters in parsing bug in palo parser.
diff --git a/docs/dev/attribution.md b/docs/dev/attribution.md
@@ -10,6 +10,7 @@ Influencers
 - [IPCal](https://github.com/ammyblabla/ipcal)
 - [StackOverflow](https://stackoverflow.com/)
 - [Python 3 Docs](https://docs.python.org/3/library/)
+- [Python distutils version](https://github.com/python/cpython/blob/3.11/Lib/distutils/version.py)
 
 In many instances variables and function names were reused, but the code was built from scratch to avoid any potential licensing issues. Functions that were known to be rewritten and their known origin.
 

diff --git a/docs/dev/include_parser_list.md b/docs/dev/include_parser_list.md
@@ -11,6 +11,7 @@
 | citrix_netscaler | netutils.config.parser.NetscalerConfigParser |
 | extreme_netiron | netutils.config.parser.NetironConfigParser |
 | fortinet_fortios | netutils.config.parser.FortinetConfigParser |
+| hp_comware | netutils.config.parser.HPComwareConfigParser |
 | juniper_junos | netutils.config.parser.JunosConfigParser |
 | linux | netutils.config.parser.LINUXConfigParser |
 | mikrotik_routeros | netutils.config.parser.RouterOSConfigParser |

diff --git a/docs/user/include_jinja_list.md b/docs/user/include_jinja_list.md
@@ -18,6 +18,7 @@
 | find_unordered_cfg_lines | netutils.config.compliance.find_unordered_cfg_lines |
 | section_config | netutils.config.compliance.section_config |
 | paloalto_panos_brace_to_set | netutils.config.conversion.paloalto_panos_brace_to_set |
+| paloalto_panos_clean_newlines | netutils.config.conversion.paloalto_panos_clean_newlines |
 | fqdn_to_ip | netutils.dns.fqdn_to_ip |
 | is_fqdn_resolvable | netutils.dns.is_fqdn_resolvable |
 | hash_data | netutils.hash.hash_data |
@@ -59,6 +60,8 @@
 | mac_to_format | netutils.mac.mac_to_format |
 | mac_to_int | netutils.mac.mac_to_int |
 | mac_type | netutils.mac.mac_type |
+| compare_version_loose | netutils.os_version.compare_version_loose |
+| compare_version_strict | netutils.os_version.compare_version_strict |
 | get_upgrade_path | netutils.os_version.get_upgrade_path |
 | compare_cisco_type5 | netutils.password.compare_cisco_type5 |
 | compare_cisco_type7 | netutils.password.compare_cisco_type7 |

diff --git a/docs/user/lib_mapper/aerleon.md b/docs/user/lib_mapper/aerleon.md
@@ -2,20 +2,20 @@
 | ---------- | -- | ------ |
 | arista | → | arista_eos |
 | aruba | → | aruba_aoscx |
-| brocade | → | ruckus_fastiron |
+| brocade | → | brocade_nos |
 | cisco | → | cisco_ios |
 | ciscoasa | → | cisco_asa |
 | cisconx | → | cisco_nxos |
-| ciscoxr | → | cisco_iosxr |
+| ciscoxr | → | cisco_xr |
 | cloudarmor | → | cloudarmor |
 | gce | → | gce |
 | gcp_hf | → | gcp_hf |
 | ipset | → | ipset |
 | iptables | → | iptables |
 | juniper | → | juniper_junos |
-| juniperevo | → | juniper_evo |
+| juniperevo | → | juniper_junos |
 | k8s | → | k8s |
-| msmpc | → | juniper_msmpc |
+| msmpc | → | juniper_junos |
 | nsxt | → | vmware_nsxt |
 | nsxv | → | vmware_nsxv |
 | openconfig | → | openconfig |
@@ -24,7 +24,7 @@
 | pcap | → | pcap |
 | sonic | → | sonic |
 | speedway | → | speedway |
-| srx | → | juniper_srx |
-| srxlo | → | juniper_srx |
+| srx | → | juniper_junos |
+| srxlo | → | juniper_junos |
 | windows | → | windows |
-| windows_advfirewall | → | windows_advfirewall |
+| windows_advfirewall | → | windows |
diff --git a/docs/user/lib_mapper/aerleon_reverse.md b/docs/user/lib_mapper/aerleon_reverse.md
@@ -2,30 +2,25 @@
 | ---------- | -- | ------ |
 | arista_eos | → | arista |
 | aruba_aoscx | → | aruba |
-| brocade_fastiron | → | brocade |
-| brocade_netiron | → | brocade |
+| brocade_nos | → | brocade |
 | cisco_asa | → | ciscoasa |
 | cisco_ios | → | cisco |
-| cisco_iosxr | → | ciscoxr |
 | cisco_nxos | → | cisconx |
+| cisco_xe | → | cisco |
+| cisco_xr | → | ciscoxr |
 | cloudarmor | → | cloudarmor |
 | gce | → | gce |
 | gcp_hf | → | gcp_hf |
 | ipset | → | ipset |
 | iptables | → | iptables |
-| juniper_evo | → | juniperevo |
 | juniper_junos | → | juniper |
-| juniper_msmpc | → | msmpc |
-| juniper_srx | → | srxlo |
 | k8s | → | k8s |
 | openconfig | → | openconfig |
 | packetfilter | → | packetfilter |
 | paloalto_panos | → | paloalto |
 | pcap | → | pcap |
-| ruckus_fastiron | → | brocade |
 | sonic | → | sonic |
 | speedway | → | speedway |
 | vmware_nsxt | → | nsxt |
 | vmware_nsxv | → | nsxv |
-| windows_advfirewall | → | windows_advfirewall |
-| windows_ipsec | → | windows |
+| windows | → | windows |
diff --git a/docs/user/lib_mapper/capirca.md b/docs/user/lib_mapper/capirca.md
@@ -2,20 +2,20 @@
 | ---------- | -- | ------ |
 | arista | → | arista_eos |
 | aruba | → | aruba_aoscx |
-| brocade | → | ruckus_fastiron |
+| brocade | → | brocade_nos |
 | cisco | → | cisco_ios |
 | ciscoasa | → | cisco_asa |
 | cisconx | → | cisco_nxos |
-| ciscoxr | → | cisco_iosxr |
+| ciscoxr | → | cisco_xr |
 | cloudarmor | → | cloudarmor |
 | gce | → | gce |
 | gcp_hf | → | gcp_hf |
 | ipset | → | ipset |
 | iptables | → | iptables |
 | juniper | → | juniper_junos |
-| juniperevo | → | juniper_evo |
+| juniperevo | → | juniper_junos |
 | k8s | → | k8s |
-| msmpc | → | juniper_msmpc |
+| msmpc | → | juniper_junos |
 | nsxt | → | vmware_nsxt |
 | nsxv | → | vmware_nsxv |
 | openconfig | → | openconfig |
@@ -24,7 +24,7 @@
 | pcap | → | pcap |
 | sonic | → | sonic |
 | speedway | → | speedway |
-| srx | → | juniper_srx |
-| srxlo | → | juniper_srx |
+| srx | → | juniper_junos |
+| srxlo | → | juniper_junos |
 | windows | → | windows |
-| windows_advfirewall | → | windows_advfirewall |
+| windows_advfirewall | → | windows |
diff --git a/docs/user/lib_mapper/capirca_reverse.md b/docs/user/lib_mapper/capirca_reverse.md
@@ -2,30 +2,25 @@
 | ---------- | -- | ------ |
 | arista_eos | → | arista |
 | aruba_aoscx | → | aruba |
-| brocade_fastiron | → | brocade |
-| brocade_netiron | → | brocade |
+| brocade_nos | → | brocade |
 | cisco_asa | → | ciscoasa |
 | cisco_ios | → | cisco |
-| cisco_iosxr | → | ciscoxr |
 | cisco_nxos | → | cisconx |
+| cisco_xe | → | cisco |
+| cisco_xr | → | ciscoxr |
 | cloudarmor | → | cloudarmor |
 | gce | → | gce |
 | gcp_hf | → | gcp_hf |
 | ipset | → | ipset |
 | iptables | → | iptables |
-| juniper_evo | → | juniperevo |
 | juniper_junos | → | juniper |
-| juniper_msmpc | → | msmpc |
-| juniper_srx | → | srxlo |
 | k8s | → | k8s |
 | openconfig | → | openconfig |
 | packetfilter | → | packetfilter |
 | paloalto_panos | → | paloalto |
 | pcap | → | pcap |
-| ruckus_fastiron | → | brocade |
 | sonic | → | sonic |
 | speedway | → | speedway |
 | vmware_nsxt | → | nsxt |
 | vmware_nsxv | → | nsxv |
-| windows_advfirewall | → | windows_advfirewall |
-| windows_ipsec | → | windows |
+| windows | → | windows |
diff --git a/docs/user/lib_mapper/forwardnetworks.md b/docs/user/lib_mapper/forwardnetworks.md
@@ -5,10 +5,10 @@
 | ASA | → | cisco_asa |
 | EXTREME_NOS | → | extreme_netiron |
 | F5 | → | bigip_f5 |
-| FORTINET | → | fortinet_fortios |
+| FORTINET | → | fortinet |
 | IOS | → | cisco_ios |
 | IOS_XE | → | cisco_ios |
-| IOS_XR | → | cisco_iosxr |
+| IOS_XR | → | cisco_xr |
 | JUNOS | → | juniper_junos |
 | LINUX | → | linux |
 | LINUX_OVS_OFCTL | → | linux |

diff --git a/docs/user/lib_mapper/forwardnetworks_reverse.md b/docs/user/lib_mapper/forwardnetworks_reverse.md
@@ -5,10 +5,11 @@
 | bigip_f5 | → | F5 |
 | cisco_asa | → | ASA |
 | cisco_ios | → | IOS |
-| cisco_iosxr | → | IOS_XR |
 | cisco_nxos | → | NXOS |
+| cisco_xe | → | IOS |
+| cisco_xr | → | IOS_XR |
 | extreme_netiron | → | EXTREME_NOS |
-| fortinet_fortios | → | FORTINET |
+| fortinet | → | FORTINET |
 | juniper_junos | → | JUNOS |
 | linux | → | LINUX |
 | netscaler | → | NETSCALER |