From c919f02b88a2c1743d7782615e40c0b0aa176d1e Mon Sep 17 00:00:00 2001 From: jhughesbiot Date: Thu, 15 Aug 2024 05:43:19 -0600 Subject: [PATCH] Feature/dhcp disconnect (#626) * Initial work on giving host access to test containers * Change external ip resolving method to use docker0 * Resolve device interface for test containers * Add gRPC methods to control the interface state Implement dhcp disconnect test * Fix host network module option * Misc cleanup * Move ip resolving method to IPControl class * pylint fixes * refactor func to handle case when network interface not exists * set test result "Error" * check device connected * thread for monitoring device connection * Minor changes * check the device connection only before each test * send testrun status using mqtt * remove duplicatied line * refactor setting remaining tests to error * Pylint fixes * pylint * Use mqtt service instead of calling GET /status every 5 seconds. (#644) * Use mqtt service instead of calling GET /status every 5 seconds. * Adds tooltip (#638) Adds tooltip * Fix focus after profile delete - track by name (#640) Fix focus after profile delete - track by name * Update the requests dependency (#643) * Update requests dependency * Update requests dependency * Update dependency in TLS test * Update docker dependency --------- Signed-off-by: J Boddey * remove unused output * encode mqtt message to json * Revert "Expired profile (#619)" (#645) Prevent opening of Expired risk profile * Improve documentation (#639) * Improve docs * Remove paragraph * Text changes * Fix text for the BE error * Change tooltip (#650) * Change tooltip * Allows draft profiles to become expired (#636) * Allow draft profiles to expire * Move status method into risk profile class * Use existing method * Check for expiry in validate method * Remove unused variable * Build UI during package instead of install (#621) * Build UI during package * Fix local build * Install npm * Remove duplicate build message * Fix ESLint * Fix script * Modify scripts * Improve scripts * Fix copy command * Try installing package * Depend on package job * Add sudo * Add sudo * Troubleshoot * Fix workflow * Checkout source for prepare command * Built ui within a container * Mount src files for build instead of static copy in build image * Attempt to fix actions * Remove manual build container cleanup methods * undo failed attempts to fix actions * Fix path * Remove -it flag --------- Signed-off-by: J Boddey Co-authored-by: kurilova Co-authored-by: jhughesbiot * Feature/risk in selected (#654) * Adds risk to selected value * Adds risk to selected value --------- Co-authored-by: J Boddey * Show risk for each question in the Risk profile (#647) * Show risk for each question in the Risk profile * set top position to 0 --------- Co-authored-by: J Boddey * Use mqtt service instead of calling GET /status every 5 seconds. * Use mqtt service instead of calling GET /status every 5 seconds. * Use mqtt service instead of calling GET /status every 5 seconds. * pylint --------- Signed-off-by: J Boddey Co-authored-by: J Boddey Co-authored-by: Aliaksandr Nikitsin Co-authored-by: jhughesbiot * Allow ICMP response to DHCP messages in DHCP snooping test (#608) * Allow ICMP response to DHCP messages * Bug/unit test runtime (#655) * Change base test module startup to allow setup script to run independent of module startup process Update connection_module to allow for unit testing Update unit test run script to use new process * enable all unit tests update google cert * Remove binary fix lines from docker files pylint updates * pylint updates --------- Co-authored-by: jhughesbiot * The risk profile saved with old format is shown improperly while loading based on a new format (#664) * Fill only fields that are present in profile * GAR : The alt text for the expired risk profile should be communicated on Enter key (#662) * Change Expired profile title on Enter; announce Expired profile title on Enter * Update wording of tls cipher results (#671) * Show error message if provided; show default message if no (#680) * Test install on supported operating systems (#675) * Test install on multiple versions * Update step names * Do not give Non-Compliant if error given --------- Signed-off-by: J Boddey Co-authored-by: J Boddey Co-authored-by: Aliaksandr Nikitsin Co-authored-by: Sofia Kurilova --- framework/python/src/common/mqtt.py | 1 + framework/python/src/net_orc/ip_control.py | 9 + .../src/net_orc/network_orchestrator.py | 47 ++-- .../python/src/test_orc/test_orchestrator.py | 80 +++---- modules/network/base/bin/start_module | 202 +++++++++--------- .../python/src/grpc_server/start_server.py | 1 - .../network/host/bin/start_network_service | 23 ++ modules/network/host/conf/module_config.json | 24 +++ modules/network/host/host.Dockerfile | 34 +++ .../python/src/grpc_server/network_service.py | 120 +++++++++++ .../python/src/grpc_server/proto/grpc.proto | 37 ++++ .../python/src/grpc_server/start_server.py | 50 +++++ modules/test/base/base.Dockerfile | 1 + .../base/python/src/grpc/proto/host/client.py | 63 ++++++ modules/test/conn/conf/module_config.json | 6 + .../test/conn/python/src/connection_module.py | 70 ++++++ 16 files changed, 611 insertions(+), 157 deletions(-) create mode 100644 modules/network/host/bin/start_network_service create mode 100644 modules/network/host/conf/module_config.json create mode 100644 modules/network/host/host.Dockerfile create mode 100644 modules/network/host/python/src/grpc_server/network_service.py create mode 100644 modules/network/host/python/src/grpc_server/proto/grpc.proto create mode 100644 modules/network/host/python/src/grpc_server/start_server.py create mode 100644 modules/test/base/python/src/grpc/proto/host/client.py diff --git a/framework/python/src/common/mqtt.py b/framework/python/src/common/mqtt.py index 32cb421eb..c58d24d3f 100644 --- a/framework/python/src/common/mqtt.py +++ b/framework/python/src/common/mqtt.py @@ -33,6 +33,7 @@ class MQTT: def __init__(self) -> None: self._host = WEBSOCKETS_HOST self._client = mqtt_client.Client(mqtt_client.CallbackAPIVersion.VERSION2) + LOGGER.setLevel(logger.logging.INFO) self._client.enable_logger(LOGGER) def _connect(self): diff --git a/framework/python/src/net_orc/ip_control.py b/framework/python/src/net_orc/ip_control.py index 04686f0cd..544b1537b 100644 --- a/framework/python/src/net_orc/ip_control.py +++ b/framework/python/src/net_orc/ip_control.py @@ -17,6 +17,7 @@ from common import logger from common import util import re +import socket LOGGER = logger.get_logger('ip_ctrl') @@ -96,6 +97,14 @@ def get_iface_port_stats(self, iface): else: return None + def get_ip_address(self, iface): + addrs = psutil.net_if_addrs() + if iface in addrs: + for addr in addrs[iface]: + if addr.family == socket.AF_INET: + return addr.address + return None + def get_namespaces(self): result = util.run_command('ip netns list') # Strip ID's from the namespace results diff --git a/framework/python/src/net_orc/network_orchestrator.py b/framework/python/src/net_orc/network_orchestrator.py index a94bca89b..6aee29d95 100644 --- a/framework/python/src/net_orc/network_orchestrator.py +++ b/framework/python/src/net_orc/network_orchestrator.py @@ -138,6 +138,9 @@ def start_network(self): # Get network ready (via Network orchestrator) LOGGER.debug('Network is ready') + def get_ip_address(self, iface): + return self._ip_ctrl.get_ip_address(iface) + def get_listener(self): return self._listener @@ -494,14 +497,15 @@ def _load_network_module(self, module_dir): # Load network service networking configuration if net_module.enable_container: - net_module.net_config.enable_wan = net_module_json['config']['network'][ - 'enable_wan'] - net_module.net_config.ip_index = net_module_json['config']['network'][ - 'ip_index'] - net_module.net_config.host = False if not 'host' in net_module_json[ 'config']['network'] else net_module_json['config']['network']['host'] + if not net_module.net_config.host: + net_module.net_config.enable_wan = net_module_json['config']['network'][ + 'enable_wan'] + net_module.net_config.ip_index = net_module_json['config']['network'][ + 'ip_index'] + net_module.net_config.ipv4_address = self.network_config.ipv4_network[ net_module.net_config.ip_index] net_module.net_config.ipv4_network = self.network_config.ipv4_network @@ -538,26 +542,29 @@ def _get_network_module(self, name): def _start_network_service(self, net_module): LOGGER.debug('Starting network service ' + net_module.display_name) - network = 'host' if net_module.net_config.host else PRIVATE_DOCKER_NET + network = 'host' if net_module.net_config.host else 'bridge' LOGGER.debug(f"""Network: {network}, image name: {net_module.image_name}, container name: {net_module.container_name}""") try: client = docker.from_env() net_module.container = client.containers.run( - net_module.image_name, - auto_remove=True, - cap_add=['NET_ADMIN'], - name=net_module.container_name, - hostname=net_module.container_name, - network_mode='none', - privileged=True, - detach=True, - mounts=net_module.mounts, - environment={ - 'TZ': self.get_session().get_timezone(), - 'HOST_USER': util.get_host_user() - }) + net_module.image_name, + auto_remove=True, + cap_add=['NET_ADMIN'], + name=net_module.container_name, + hostname=net_module.container_name, + # Undetermined version of docker seems to have broken + # DNS configuration (/etc/resolv.conf) Re-add when/if + # this network is utilized and DNS issue is resolved + network=network, + privileged=True, + detach=True, + mounts=net_module.mounts, + environment={ + 'TZ': self.get_session().get_timezone(), + 'HOST_USER': util.get_host_user() + }) except docker.errors.ContainerError as error: LOGGER.error('Container run error') LOGGER.error(error) @@ -793,7 +800,7 @@ def network_adapters_checker(self, mqtt_client: mqtt.MQTT, topic: str): adapters = self._session.detect_network_adapters_change() if adapters: mqtt_client.send_message(topic, adapters) - except Exception: + except Exception: # pylint: disable=W0718 LOGGER.error(traceback.format_exc()) def is_device_connected(self): diff --git a/framework/python/src/test_orc/test_orchestrator.py b/framework/python/src/test_orc/test_orchestrator.py index f5dc0e252..afd748d35 100644 --- a/framework/python/src/test_orc/test_orchestrator.py +++ b/framework/python/src/test_orc/test_orchestrator.py @@ -30,7 +30,7 @@ LOG_NAME = "test_orc" LOGGER = logger.get_logger("test_orc") RUNTIME_DIR = "runtime" -RUNTIME_TEST_DIR = os.path.join(RUNTIME_DIR,"test") +RUNTIME_TEST_DIR = os.path.join(RUNTIME_DIR, "test") TEST_MODULES_DIR = "modules/test" MODULE_CONFIG = "conf/module_config.json" LOG_REGEX = r"^[A-Z][a-z]{2} [0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} test_" @@ -182,9 +182,7 @@ def _write_reports(self, test_report): def _generate_report(self): report = {} - report["testrun"] = { - "version": self.get_session().get_version() - } + report["testrun"] = {"version": self.get_session().get_version()} report["mac_addr"] = self.get_session().get_target_device().mac_addr report["device"] = self.get_session().get_target_device().to_dict() @@ -207,7 +205,10 @@ def _calculate_result(self): for test_result in self._session.get_test_results(): # Check Required tests if (test_result.required_result.lower() == "required" - and test_result.result.lower() != "compliant"): + and test_result.result.lower() not in [ + "compliant", + "error" + ]): result = "Non-Compliant" # Check Required if Applicable tests elif (test_result.required_result.lower() == "required if applicable" @@ -295,25 +296,20 @@ def _timestamp_results(self, device): return completed_results_dir - def zip_results(self, - device, - timestamp, - profile): + def zip_results(self, device, timestamp, profile): try: LOGGER.debug("Archiving test results") - src_path = os.path.join(LOCAL_DEVICE_REPORTS.replace( - "{device_folder}", - device.device_folder), - timestamp) + src_path = os.path.join( + LOCAL_DEVICE_REPORTS.replace("{device_folder}", device.device_folder), + timestamp) # Define temp directory to store files before zipping results_dir = os.path.join(f"/tmp/testrun/{time.time()}") # Define where to save the zip file - zip_location = os.path.join("/tmp/testrun", - timestamp) + zip_location = os.path.join("/tmp/testrun", timestamp) # Delete zip_temp if it already exists if os.path.exists(results_dir): @@ -323,16 +319,13 @@ def zip_results(self, if os.path.exists(zip_location + ".zip"): os.remove(zip_location + ".zip") - shutil.copytree(src_path,results_dir) + shutil.copytree(src_path, results_dir) # Include profile if specified if profile is not None: - LOGGER.debug( - f"Copying profile {profile.name} to results directory") + LOGGER.debug(f"Copying profile {profile.name} to results directory") shutil.copy(profile.get_file_path(), - os.path.join( - results_dir, - "profile.json")) + os.path.join(results_dir, "profile.json")) with open(os.path.join(results_dir, "profile.pdf"), "wb") as f: f.write(profile.to_pdf(device).getvalue()) @@ -349,10 +342,9 @@ def zip_results(self, if os.path.exists(zip_file) else'creation failed'}''') - return zip_file - except Exception as error: # pylint: disable=W0703 + except Exception as error: # pylint: disable=W0703 LOGGER.error("Failed to create zip file") LOGGER.debug(error) return None @@ -426,6 +418,17 @@ def _run_test_module(self, module): device_monitor_capture = os.path.join(device_test_dir, "monitor.pcap") util.run_command(f"chown -R {self._host_user} {device_monitor_capture}") + # Resolve the main docker interface (docker0) for host interaction + # Can't use device or internet iface since these are not in a stable + # state for this type of communication during testing but docker0 has + # to exist and should always be available + external_ip = self._net_orc.get_ip_address("docker0") + LOGGER.debug(f"Using external IP: {external_ip}") + extra_hosts = { + "external.localhost": external_ip + } if external_ip is not None else {} + + #extra_hosts = {"external.localhost":"172.17.0.1"} client = docker.from_env() module.container = client.containers.run( @@ -468,8 +471,10 @@ def _run_test_module(self, module): "IPV4_ADDR": device.ip_addr, "DEVICE_TEST_MODULES": json.dumps(device.test_modules), "IPV4_SUBNET": self._net_orc.network_config.ipv4_network, - "IPV6_SUBNET": self._net_orc.network_config.ipv6_network - }) + "IPV6_SUBNET": self._net_orc.network_config.ipv6_network, + "DEV_IFACE": self._session.get_device_interface() + }, + extra_hosts=extra_hosts) except (docker.errors.APIError, docker.errors.ContainerError) as container_error: LOGGER.error("Test module " + module.name + " has failed to start") @@ -526,19 +531,19 @@ def _run_test_module(self, module): # Convert dict from json into TestCase object test_case = TestCase( - name=test_result["name"], - description=test_result["description"], - expected_behavior=test_result["expected_behavior"], - required_result=test_result["required_result"], - result=test_result["result"]) + name=test_result["name"], + description=test_result["description"], + expected_behavior=test_result["expected_behavior"], + required_result=test_result["required_result"], + result=test_result["result"]) # Any informational test should always report informational if test_case.required_result == "Informational": test_case.result = "Informational" # Add steps to resolve if test is non-compliant - if (test_case.result == "Non-Compliant" and - "recommendations" in test_result): + if (test_case.result == "Non-Compliant" + and "recommendations" in test_result): test_case.recommendations = test_result["recommendations"] else: test_case.recommendations = None @@ -548,7 +553,7 @@ def _run_test_module(self, module): except (FileNotFoundError, PermissionError, json.JSONDecodeError) as results_error: LOGGER.error( - f"Error occurred whilst obtaining results for module {module.name}") + f"Error occurred whilst obtaining results for module {module.name}") LOGGER.error(results_error) # Get the markdown report from the module if generated @@ -666,11 +671,10 @@ def _load_test_module(self, module_dir): for test_case_json in module_json["config"]["tests"]: try: test_case = TestCase( - name=test_case_json["name"], - description=test_case_json["test_description"], - expected_behavior=test_case_json["expected_behavior"], - required_result=test_case_json["required_result"] - ) + name=test_case_json["name"], + description=test_case_json["test_description"], + expected_behavior=test_case_json["expected_behavior"], + required_result=test_case_json["required_result"]) if "recommendations" in test_case_json: test_case.recommendations = test_case_json["recommendations"] diff --git a/modules/network/base/bin/start_module b/modules/network/base/bin/start_module index 8e8cb5e4b..7b1ea8e35 100644 --- a/modules/network/base/bin/start_module +++ b/modules/network/base/bin/start_module @@ -1,99 +1,105 @@ -#!/bin/bash - -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Directory where all binaries will be loaded -BIN_DIR="/testrun/bin" - -# Default interface should be veth0 for all containers -DEFAULT_IFACE=veth0 - -# Create a local user that matches the same as the host -# to be used for correct file ownership for various logs -# HOST_USER mapped in via docker container environemnt variables -useradd $HOST_USER - -# Enable IPv6 for all containers -sysctl net.ipv6.conf.all.disable_ipv6=0 -sysctl -p - -# Read in the config file -CONF_FILE="/testrun/conf/module_config.json" -CONF=`cat $CONF_FILE` - -if [[ -z $CONF ]] -then - echo "No config file present at $CONF_FILE. Exiting startup." - exit 1 -fi - -# Extract the necessary config parameters -MODULE_NAME=$(echo "$CONF" | jq -r '.config.meta.name') -DEFINED_IFACE=$(echo "$CONF" | jq -r '.config.network.interface') -GRPC=$(echo "$CONF" | jq -r '.config.grpc') - -# Validate the module name is present -if [[ -z "$MODULE_NAME" || "$MODULE_NAME" == "null" ]] -then - echo "No module name present in $CONF_FILE. Exiting startup." - exit 1 -fi - -# Select which interace to use -if [[ -z $DEFINED_IFACE || "$DEFINED_IFACE" == "null" ]] -then - echo "No Interface Defined, defaulting to veth0" - INTF=$DEFAULT_IFACE -else - INTF=$DEFINED_IFACE -fi - -# Setup the PYTHONPATH so all imports work as expected -echo "Setting up PYTHONPATH..." -export PYTHONPATH=$($BIN_DIR/setup_python_path) -echo "PYTHONPATH: $PYTHONPATH" - -echo "Configuring binary files..." -$BIN_DIR/setup_binaries $BIN_DIR - -echo "Starting module $MODULE_NAME on local interface $INTF..." - -# Wait for interface to become ready -$BIN_DIR/wait_for_interface $INTF - -# Small pause to let the interface stabalize before starting the capture -#sleep 1 - -# Start network capture -$BIN_DIR/capture $MODULE_NAME $INTF - -# Start the grpc server -if [[ ! -z $GRPC && ! $GRPC == "null" ]] -then - GRPC_PORT=$(echo "$GRPC" | jq -r '.port') - if [[ ! -z $GRPC_PORT && ! $GRPC_PORT == "null" ]] - then - echo "gRPC port resolved from config: $GRPC_PORT" - $BIN_DIR/start_grpc "-p $GRPC_PORT" - else - $BIN_DIR/start_grpc - fi -fi - -# Small pause to let all core services stabalize -sleep 3 - -# Start the networking service +#!/bin/bash + +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Directory where all binaries will be loaded +BIN_DIR="/testrun/bin" + +# Default interface should be veth0 for all containers +DEFAULT_IFACE=veth0 + +# Create a local user that matches the same as the host +# to be used for correct file ownership for various logs +# HOST_USER mapped in via docker container environemnt variables +useradd $HOST_USER + +# Enable IPv6 for all containers +sysctl net.ipv6.conf.all.disable_ipv6=0 +sysctl -p + +# Read in the config file +CONF_FILE="/testrun/conf/module_config.json" +CONF=`cat $CONF_FILE` + +if [[ -z $CONF ]] +then + echo "No config file present at $CONF_FILE. Exiting startup." + exit 1 +fi + +# Extract the necessary config parameters +MODULE_NAME=$(echo "$CONF" | jq -r '.config.meta.name') +DEFINED_IFACE=$(echo "$CONF" | jq -r '.config.network.interface') +HOST=$(echo "$CONF" | jq -r '.config.network.host') +GRPC=$(echo "$CONF" | jq -r '.config.grpc') + +# Validate the module name is present +if [[ -z "$MODULE_NAME" || "$MODULE_NAME" == "null" ]] +then + echo "No module name present in $CONF_FILE. Exiting startup." + exit 1 +fi + +# Select which interace to use +if [[ -z $DEFINED_IFACE || "$DEFINED_IFACE" == "null" ]] +then + echo "No Interface Defined, defaulting to veth0" + INTF=$DEFAULT_IFACE +else + INTF=$DEFINED_IFACE +fi + +# Setup the PYTHONPATH so all imports work as expected +echo "Setting up PYTHONPATH..." +export PYTHONPATH=$($BIN_DIR/setup_python_path) +echo "PYTHONPATH: $PYTHONPATH" + +echo "Configuring binary files..." +$BIN_DIR/setup_binaries $BIN_DIR + +echo "Starting module $MODULE_NAME on local interface $INTF..." + +# Only non-host containers will have a specific +# interface for capturing +if [[ "$HOST" != "true" ]]; then + + # Wait for interface to become ready + $BIN_DIR/wait_for_interface $INTF + + # Small pause to let the interface stabalize before starting the capture + #sleep 1 + + # Start network capture + $BIN_DIR/capture $MODULE_NAME $INTF +fi + +# Start the grpc server +if [[ ! -z $GRPC && ! $GRPC == "null" ]] +then + GRPC_PORT=$(echo "$GRPC" | jq -r '.port') + if [[ ! -z $GRPC_PORT && ! $GRPC_PORT == "null" ]] + then + echo "gRPC port resolved from config: $GRPC_PORT" + $BIN_DIR/start_grpc "-p $GRPC_PORT" + else + $BIN_DIR/start_grpc + fi +fi + +# Small pause to let all core services stabalize +sleep 3 + +# Start the networking service $BIN_DIR/start_network_service $MODULE_NAME $INTF \ No newline at end of file diff --git a/modules/network/base/python/src/grpc_server/start_server.py b/modules/network/base/python/src/grpc_server/start_server.py index d372949e5..9c34ec736 100644 --- a/modules/network/base/python/src/grpc_server/start_server.py +++ b/modules/network/base/python/src/grpc_server/start_server.py @@ -46,6 +46,5 @@ def run(): print('gRPC server starting on port ' + port) serve(port) - if __name__ == '__main__': run() diff --git a/modules/network/host/bin/start_network_service b/modules/network/host/bin/start_network_service new file mode 100644 index 000000000..2d5b6780b --- /dev/null +++ b/modules/network/host/bin/start_network_service @@ -0,0 +1,23 @@ +#!/bin/bash + +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "Starting Host Service..." + + +# Keep host container running until stopped +while true; do + sleep 3 +done \ No newline at end of file diff --git a/modules/network/host/conf/module_config.json b/modules/network/host/conf/module_config.json new file mode 100644 index 000000000..87ec39a35 --- /dev/null +++ b/modules/network/host/conf/module_config.json @@ -0,0 +1,24 @@ +{ + "config": { + "meta": { + "name": "host", + "display_name": "Host", + "description": "Used to access host level networking operations" + }, + "network": { + "host": true + }, + "grpc":{ + "port": 5001 + }, + "docker": { + "depends_on": "base", + "mounts": [ + { + "source": "runtime/network", + "target": "/runtime/network" + } + ] + } + } +} \ No newline at end of file diff --git a/modules/network/host/host.Dockerfile b/modules/network/host/host.Dockerfile new file mode 100644 index 000000000..579d62a2d --- /dev/null +++ b/modules/network/host/host.Dockerfile @@ -0,0 +1,34 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Image name: test-run/host +FROM test-run/base:latest + +ARG MODULE_NAME=host +ARG MODULE_DIR=modules/network/$MODULE_NAME + +#Update and get all additional requirements not contained in the base image +RUN apt-get update --fix-missing + +# Install all necessary packages +RUN apt-get install -y net-tools ethtool + +# Copy over all configuration files +COPY $MODULE_DIR/conf /testrun/conf + +# Copy over all binary files +COPY $MODULE_DIR/bin /testrun/bin + +# Copy over all python files +COPY $MODULE_DIR/python /testrun/python diff --git a/modules/network/host/python/src/grpc_server/network_service.py b/modules/network/host/python/src/grpc_server/network_service.py new file mode 100644 index 000000000..cbb3a1b7a --- /dev/null +++ b/modules/network/host/python/src/grpc_server/network_service.py @@ -0,0 +1,120 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""gRPC Network Service for the Host network module""" +import proto.grpc_pb2_grpc as pb2_grpc +import proto.grpc_pb2 as pb2 + +import traceback +from common import logger +from common import util + +LOG_NAME = 'network_service' +LOGGER = None + + +class NetworkService(pb2_grpc.HostNetworkModule): + """gRPC endpoints for the Host container""" + + def __init__(self): + global LOGGER + LOGGER = logger.get_logger(LOG_NAME, 'host') + + def CheckInterfaceStatus(self, request, context): # pylint: disable=W0613 + try: + status = self.check_interface_status(request.iface_name) + return pb2.CheckInterfaceStatusResponse(code=200, status=status) + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to read iface status: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.CheckInterfaceStatusResponse(code=500, status=False) + + def GetIfaceConnectionStats(self, request, context): # pylint: disable=W0613 + try: + stats = self.get_iface_connection_stats(request.iface_name) + return pb2.GetIfaceStatsResponse(code=200, stats=stats) + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to read connection stats: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.GetIfaceStatsResponse(code=500, stats=False) + + def GetIfacePortStats(self, request, context): # pylint: disable=W0613 + try: + stats = self.get_iface_port_stats(request.iface_name) + return pb2.GetIfaceStatsResponse(code=200, stats=stats) + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to read port stats: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.GetIfaceStatsResponse(code=500, stats=False) + + def SetIfaceDown(self, request, context): # pylint: disable=W0613 + try: + success = self.set_interface_down(request.iface_name) + return pb2.SetIfaceResponse(code=200, success=success) + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to set interface down: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.SetIfaceResponse(code=500, success=False) + + def SetIfaceUp(self, request, context): # pylint: disable=W0613 + try: + success = self.set_interface_up(request.iface_name) + return pb2.SetIfaceResponse(code=200, success=success) + except Exception as e: # pylint: disable=W0718 + fail_message = 'Failed to set interface up: ' + str(e) + LOGGER.error(fail_message) + LOGGER.error(traceback.format_exc()) + return pb2.SetIfaceResponse(code=500, success=False) + + def check_interface_status(self, interface_name): + output = util.run_command(cmd=f'ip link show {interface_name}', output=True) + if 'state DOWN ' in output[0]: + return False + else: + return True + + def get_iface_connection_stats(self, iface): + """Extract information about the physical connection""" + response = util.run_command(f'ethtool {iface}') + if len(response[1]) == 0: + return response[0] + else: + return None + + def get_iface_port_stats(self, iface): + """Extract information about packets connection""" + response = util.run_command(f'ethtool -S {iface}') + if len(response[1]) == 0: + return response[0] + else: + return None + + def set_interface_up(self, interface_name): + """Set the interface to the up state""" + response = util.run_command('ip link set dev ' + interface_name + ' up') + if len(response[1]) == 0: + return response[0] + else: + return None + + def set_interface_down(self, interface_name): + """Set the interface to the up state""" + response = util.run_command('ip link set dev ' + interface_name + ' down') + if len(response[1]) == 0: + return response[0] + else: + return None diff --git a/modules/network/host/python/src/grpc_server/proto/grpc.proto b/modules/network/host/python/src/grpc_server/proto/grpc.proto new file mode 100644 index 000000000..c881b13f7 --- /dev/null +++ b/modules/network/host/python/src/grpc_server/proto/grpc.proto @@ -0,0 +1,37 @@ +syntax = "proto3"; + +service HostNetworkModule { + + rpc CheckInterfaceStatus(CheckInterfaceStatusRequest) returns (CheckInterfaceStatusResponse) {}; + rpc GetIfaceConnectionStats(GetIfaceStatsRequest) returns (GetIfaceStatsResponse) {}; + rpc SetIfaceDown(SetIfaceRequest) returns (SetIfaceResponse) {}; + rpc SetIfaceUp(SetIfaceRequest) returns (SetIfaceResponse) {}; +} + +message CheckInterfaceStatusRequest { + string iface_name = 1; +} + +message CheckInterfaceStatusResponse { + int32 code = 1; + bool status = 2; +} + +message GetIfaceStatsRequest { + string iface_name = 1; +} + +message GetIfaceStatsResponse { + int32 code = 1; + string stats = 2; +} + +message SetIfaceRequest { + string iface_name = 1; +} + +message SetIfaceResponse { + int32 code = 1; + bool success = 2; +} + diff --git a/modules/network/host/python/src/grpc_server/start_server.py b/modules/network/host/python/src/grpc_server/start_server.py new file mode 100644 index 000000000..962277188 --- /dev/null +++ b/modules/network/host/python/src/grpc_server/start_server.py @@ -0,0 +1,50 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Base class for starting the gRPC server for a network module.""" +from concurrent import futures +import grpc +import proto.grpc_pb2_grpc as pb2_grpc +from network_service import NetworkService +import argparse + +DEFAULT_PORT = '5001' + + +def serve(port): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + pb2_grpc.add_HostNetworkModuleServicer_to_server(NetworkService(), server) + server.add_insecure_port('[::]:' + port) + server.start() + server.wait_for_termination() + + +def run(): + parser = argparse.ArgumentParser( + description='GRPC Server for Network Module', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('-p', + '--port', + default=DEFAULT_PORT, + help='Define the default port to run the server on.') + + args = parser.parse_args() + + port = args.port + + print('gRPC server starting on port ' + port) + serve(port) + +if __name__ == '__main__': + run() diff --git a/modules/test/base/base.Dockerfile b/modules/test/base/base.Dockerfile index 4d8c0399a..5a9013cf1 100644 --- a/modules/test/base/base.Dockerfile +++ b/modules/test/base/base.Dockerfile @@ -49,6 +49,7 @@ ARG CONTAINER_PROTO_DIR=testrun/python/src/grpc_server/proto COPY $NET_MODULE_DIR/dhcp-1/$NET_MODULE_PROTO_DIR $CONTAINER_PROTO_DIR/dhcp1/ COPY $NET_MODULE_DIR/dhcp-2/$NET_MODULE_PROTO_DIR $CONTAINER_PROTO_DIR/dhcp2/ +COPY $NET_MODULE_DIR/host/$NET_MODULE_PROTO_DIR $CONTAINER_PROTO_DIR/host/ # Copy the cached version of oui.txt incase the download fails RUN mkdir -p /usr/local/etc diff --git a/modules/test/base/python/src/grpc/proto/host/client.py b/modules/test/base/python/src/grpc/proto/host/client.py new file mode 100644 index 000000000..e08d3376a --- /dev/null +++ b/modules/test/base/python/src/grpc/proto/host/client.py @@ -0,0 +1,63 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +"""gRPC client module for the secondary DHCP Server""" +import grpc +import host.grpc_pb2_grpc as pb2_grpc +import host.grpc_pb2 as pb2 + +DEFAULT_PORT = '5001' +DEFAULT_HOST = 'external.localhost' # Default DHCP2 server + + +class Client(): + """gRPC Client for the secondary DHCP server""" + def __init__(self, port=DEFAULT_PORT, host=DEFAULT_HOST): + self._port = port + self._host = host + + # Create a gRPC channel to connect to the server + self._channel = grpc.insecure_channel(self._host + ':' + self._port) + + # Create a gRPC stub + self._stub = pb2_grpc.HostNetworkModuleStub(self._channel) + + def check_interface_status(self, iface_name): + # Create a request message + request = pb2.CheckInterfaceStatusRequest() + request.iface_name = iface_name + + # Make the RPC call + response = self._stub.CheckInterfaceStatus(request) + + return response + + def set_iface_down(self, iface_name): + # Create a request message + request = pb2.SetIfaceRequest() + request.iface_name = iface_name + + # Make the RPC call + response = self._stub.SetIfaceDown(request) + + return response + + def set_iface_up(self, iface_name): + # Create a request message + request = pb2.SetIfaceRequest() + request.iface_name = iface_name + + # Make the RPC call + response = self._stub.SetIfaceUp(request) + + return response diff --git a/modules/test/conn/conf/module_config.json b/modules/test/conn/conf/module_config.json index 5289e7eb0..6583f881e 100644 --- a/modules/test/conn/conf/module_config.json +++ b/modules/test/conn/conf/module_config.json @@ -116,6 +116,12 @@ "Enable shared address space support in the DHCP client" ] }, + { + "name": "connection.dhcp_disconnect", + "test_description": "The device under test issues a new DHCPREQUEST packet after a port ph ysical disconnection and reconnection", + "expected_behavior": "A client SHOULD use DHCP to reacquire or verify its IP address and network parameters whenever the local network parameters may have changed; e.g., at system boot time or after a disconnection from the local network, as the local network configuration may change without the client's or user's knowledge. If a client has knowledge ofa previous network address and is unable to contact a local DHCP server, the client may continue to use the previous network addres until the lease for that address expires. If the lease expires before the client can contact a DHCP server, the client must immediately discontinue use of the previous network address and may inform local users of the problem.", + "required_result": "Required" + }, { "name": "connection.single_ip", "test_description": "The network switch port connected to the device reports only one IP address for the device under test.", diff --git a/modules/test/conn/python/src/connection_module.py b/modules/test/conn/python/src/connection_module.py index 88dd40393..cafd1ebef 100644 --- a/modules/test/conn/python/src/connection_module.py +++ b/modules/test/conn/python/src/connection_module.py @@ -15,10 +15,12 @@ import util import time import traceback +import os from scapy.all import rdpcap, DHCP, ARP, Ether, ICMP, IPv6, ICMPv6ND_NS from test_module import TestModule from dhcp1.client import Client as DHCPClient1 from dhcp2.client import Client as DHCPClient2 +from host.client import Client as HostClient from dhcp_util import DHCPUtil from port_stats_util import PortStatsUtil @@ -59,6 +61,7 @@ def __init__(self, self._port_stats = PortStatsUtil(logger=LOGGER) self.dhcp1_client = DHCPClient1() self.dhcp2_client = DHCPClient2() + self.host_client = HostClient() self._dhcp_util = DHCPUtil(self.dhcp1_client, self.dhcp2_client, LOGGER) self._lease_wait_time_sec = LEASE_WAIT_TIME_DEFAULT @@ -379,6 +382,73 @@ def _connection_ipaddr_dhcp_failover(self, config): result = None, 'Network is not ready for this test' return result + def _connection_dhcp_disconnect(self): + LOGGER.info('Running connection.dhcp.disconnect') + result = None + description = '' + dev_iface = os.getenv('DEV_IFACE') + iface_status = self.host_client.check_interface_status(dev_iface) + if iface_status.code == 200: + LOGGER.info('Successfully resolved iface status') + if iface_status.status: + lease = self._dhcp_util.get_cur_lease(mac_address=self._device_mac, + timeout=self._lease_wait_time_sec) + if lease is not None: + LOGGER.info('Current device lease resolved') + if self._dhcp_util.is_lease_active(lease): + + # Disable the device interface + iface_down = self.host_client.set_iface_down(dev_iface) + if iface_down: + LOGGER.info('Device interface set to down state') + + # Wait for the lease to expire + self._dhcp_util.wait_for_lease_expire(lease, + self._lease_wait_time_sec) + + # Wait an additonal 10 seconds to better test a true disconnect + # state + LOGGER.info('Waiting 10 seconds before bringing iface back up') + time.sleep(10) + + # Enable the device interface + iface_up = self.host_client.set_iface_up(dev_iface) + if iface_up: + LOGGER.info('Device interface set to up state') + + # Confirm device receives a new lease + if self._dhcp_util.get_cur_lease( + mac_address=self._device_mac, + timeout=self._lease_wait_time_sec): + if self._dhcp_util.is_lease_active(lease): + result = True + description = ( + 'Device received a DHCP lease after disconnect') + else: + result = False + description = ( + 'Could not confirm DHCP lease active after disconnect') + else: + result = False + description = ( + 'Device did not recieve a DHCP lease after disconnect') + else: + result = 'Error' + description = 'Failed to set device interface to up state' + else: + result = 'Error' + description = 'Failed to set device interface to down state' + else: + result = 'Error' + description = 'No active lease available for device' + else: + result = 'Error' + description = 'Device interface is down' + else: + result = 'Error' + description = 'Device interface could not be resolved' + return result, description + def _get_oui_manufacturer(self, mac_address): # Do some quick fixes on the format of the mac_address # to match the oui file pattern