From 28aa2b5ac0b4c079f61767a201dcb8ecb9b35581 Mon Sep 17 00:00:00 2001 From: Gary Snider <75227981+gsnider2195@users.noreply.github.com> Date: Wed, 11 Sep 2024 12:58:50 -0700 Subject: [PATCH 1/5] Added generate_app_test_data management command. --- .../commands/generate_app_test_data.py | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 nautobot_golden_config/management/commands/generate_app_test_data.py diff --git a/nautobot_golden_config/management/commands/generate_app_test_data.py b/nautobot_golden_config/management/commands/generate_app_test_data.py new file mode 100644 index 00000000..e45ae82f --- /dev/null +++ b/nautobot_golden_config/management/commands/generate_app_test_data.py @@ -0,0 +1,113 @@ +"""Generate test data for the Golden Config app.""" + +import random + +from django.core.management import call_command +from django.core.management.base import BaseCommand +from django.db import DEFAULT_DB_ALIAS +from nautobot.core.factory import get_random_instances +from nautobot.dcim.models import Platform +from netutils.lib_mapper import NETUTILSPARSER_LIB_MAPPER_REVERSE + +from nautobot_golden_config.models import ( + ComplianceFeature, + ComplianceRule, + ConfigCompliance, + GoldenConfig, +) + + +class Command(BaseCommand): + """Populate the database with various data as a baseline for testing (automated or manual).""" + + help = __doc__ + + def add_arguments(self, parser): # noqa: D102 + parser.add_argument( + "--flush", + action="store_true", + help="Flush any existing data from the database before generating new data.", + ) + parser.add_argument( + "--database", + default=DEFAULT_DB_ALIAS, + help='The database to generate the test data in. Defaults to the "default" database.', + ) + + def _generate_static_data(self): + platforms = get_random_instances( + Platform.objects.filter(devices__isnull=False).distinct(), + minimum=2, + maximum=4, + ) + devices = [p.devices.first() for p in platforms] + + # Ensure platform has a valid network_driver or compliance generation will fail + for platform in platforms: + if platform.network_driver not in NETUTILSPARSER_LIB_MAPPER_REVERSE: + platform.network_driver = random.choice(list(NETUTILSPARSER_LIB_MAPPER_REVERSE.keys())) # noqa: S311 + platform.save() + + # Create ComplianceFeatures + compliance_features = [] + message = "Creating 8 ComplianceFeatures..." + self.stdout.write(message) + for i in range(1, 9): + name = f"ComplianceFeature{i}" + compliance_features.append( + ComplianceFeature.objects.create(name=name, slug=name, description=f"Test ComplianceFeature {i}") + ) + + # Create ComplianceRules + count = len(compliance_features) * len(platforms) + message = f"Creating {count} ComplianceRules..." + self.stdout.write(message) + for feature in compliance_features: + for platform in platforms: + ComplianceRule.objects.create( + feature=feature, + platform=platform, + description=f"Test ComplianceRule for {feature.name} on {platform.name}", + match_config=f"match {feature.name} on {platform.name}", + ) + + # Create ConfigCompliances + count = len(devices) * len(compliance_features) + message = f"Creating {count} ConfigCompliances..." + self.stdout.write(message) + for device in devices: + for rule in ComplianceRule.objects.filter(platform=device.platform): + is_compliant = random.choice([True, False]) # noqa: S311 + ConfigCompliance.objects.create( + device=device, + rule=rule, + compliance=is_compliant, + compliance_int=int(is_compliant), + intended=rule.match_config, + actual=rule.match_config if is_compliant else f"mismatch {rule.feature.name}", + ) + + # Create GoldenConfigs + message = f"Creating {len(devices)} GoldenConfigs..." + self.stdout.write(message) + for device in devices: + GoldenConfig.objects.create( + device=device, + backup_config=f"backup config for {device.name}", + intended_config=f"intended config for {device.name}", + compliance_config=f"compliance config for {device.name}", + ) + + # TODO: Create ConfigRemoves + # TODO: Create ConfigReplaces + # TODO: Create RemediationSettings + # TODO: Create ConfigPlans + + def handle(self, *args, **options): + """Entry point to the management command.""" + # Call nautobot core's generate_test_data command to generate data for core models + call_command("generate_test_data", flush=options["flush"]) + + self._generate_static_data() + + self.stdout.write(self.style.SUCCESS(f"Database {options['database']} populated with app data successfully!")) From 5908d8a27af3957684bffa0e5b0b28cd9c49aab8 Mon Sep 17 00:00:00 2001 From: Gary Snider <75227981+gsnider2195@users.noreply.github.com> Date: Wed, 11 Sep 2024 15:12:17 -0700 Subject: [PATCH 2/5] docs and changelog --- changes/809.housekeeping | 1 + docs/dev/dev_environment.md | 14 ++++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 changes/809.housekeeping diff --git a/changes/809.housekeeping b/changes/809.housekeeping new file mode 100644 index 00000000..e63bd666 --- /dev/null +++ b/changes/809.housekeeping @@ -0,0 +1 @@ +Added management command `generate_app_test_data` to generate sample data for development environments. diff --git a/docs/dev/dev_environment.md b/docs/dev/dev_environment.md index 50408bcd..eb5891b9 100644 --- a/docs/dev/dev_environment.md +++ b/docs/dev/dev_environment.md @@ -482,3 +482,17 @@ invoke generate-app-config-schema ``` This command can only guess the schema, so it's up to the developer to manually update the schema as needed. + +### Test Data Generation + +To quickly generate test data for developing against this app, you can use the following command: + +!!! warning + The `--flush` flag will completely empty your database and replace it with test data. This command should never be run in a production environment. + +```bash +nautobot-server generate_app_test_data --flush +nautobot-server createsuperuser +``` + +This uses the [`generate_test_data`](https://docs.nautobot.com/projects/core/en/stable/user-guide/administration/tools/nautobot-server/#generate_test_data) management command from Nautobot core to generate the Statuses, Platforms, Device Types, Devices, etc. Nautobot version 2.2.0 is the minimum version required for devices to be generated. From 1b434342c1524a7e04e2aef742b0311a8cf83f65 Mon Sep 17 00:00:00 2001 From: Gary Snider <75227981+gsnider2195@users.noreply.github.com> Date: Wed, 11 Sep 2024 15:14:27 -0700 Subject: [PATCH 3/5] change annotation --- docs/dev/dev_environment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/dev_environment.md b/docs/dev/dev_environment.md index eb5891b9..e905f9d1 100644 --- a/docs/dev/dev_environment.md +++ b/docs/dev/dev_environment.md @@ -487,7 +487,7 @@ This command can only guess the schema, so it's up to the developer to manually To quickly generate test data for developing against this app, you can use the following command: -!!! warning +!!! danger The `--flush` flag will completely empty your database and replace it with test data. This command should never be run in a production environment. ```bash From 0fcc7e4338a09b86c28a4738a7d643033ceda299 Mon Sep 17 00:00:00 2001 From: Gary Snider <75227981+gsnider2195@users.noreply.github.com> Date: Thu, 12 Sep 2024 16:03:33 -0700 Subject: [PATCH 4/5] rename management command --- docs/dev/dev_environment.md | 13 ++++----- ..._test_data.py => generate_gc_test_data.py} | 27 +++++++------------ 2 files changed, 17 insertions(+), 23 deletions(-) rename nautobot_golden_config/management/commands/{generate_app_test_data.py => generate_gc_test_data.py} (80%) diff --git a/docs/dev/dev_environment.md b/docs/dev/dev_environment.md index e905f9d1..bb944e58 100644 --- a/docs/dev/dev_environment.md +++ b/docs/dev/dev_environment.md @@ -485,14 +485,15 @@ This command can only guess the schema, so it's up to the developer to manually ### Test Data Generation -To quickly generate test data for developing against this app, you can use the following command: - -!!! danger - The `--flush` flag will completely empty your database and replace it with test data. This command should never be run in a production environment. +To quickly generate test data for developing against this app, you can use the following commands: ```bash -nautobot-server generate_app_test_data --flush +nautobot-server generate_test_data --flush +nautobot-server generate_gc_test_data nautobot-server createsuperuser ``` -This uses the [`generate_test_data`](https://docs.nautobot.com/projects/core/en/stable/user-guide/administration/tools/nautobot-server/#generate_test_data) management command from Nautobot core to generate the Statuses, Platforms, Device Types, Devices, etc. Nautobot version 2.2.0 is the minimum version required for devices to be generated. +!!! danger + The `--flush` flag will completely empty your database and replace it with test data. This command should never be run in a production environment. + +This uses the [`generate_test_data`](https://docs.nautobot.com/projects/core/en/stable/user-guide/administration/tools/nautobot-server/#generate_test_data) management command from Nautobot core to generate the Statuses, Platforms, Device Types, Devices, etc. Nautobot version 2.2.0 is the minimum version required for devices to be generated. If using an older version of Nautobot, you'll need to create devices manually after running `nautobot-server generate_test_data`. diff --git a/nautobot_golden_config/management/commands/generate_app_test_data.py b/nautobot_golden_config/management/commands/generate_gc_test_data.py similarity index 80% rename from nautobot_golden_config/management/commands/generate_app_test_data.py rename to nautobot_golden_config/management/commands/generate_gc_test_data.py index e45ae82f..b2404142 100644 --- a/nautobot_golden_config/management/commands/generate_app_test_data.py +++ b/nautobot_golden_config/management/commands/generate_gc_test_data.py @@ -2,7 +2,6 @@ import random -from django.core.management import call_command from django.core.management.base import BaseCommand from django.db import DEFAULT_DB_ALIAS from nautobot.core.factory import get_random_instances @@ -23,20 +22,15 @@ class Command(BaseCommand): help = __doc__ def add_arguments(self, parser): # noqa: D102 - parser.add_argument( - "--flush", - action="store_true", - help="Flush any existing data from the database before generating new data.", - ) parser.add_argument( "--database", default=DEFAULT_DB_ALIAS, help='The database to generate the test data in. Defaults to the "default" database.', ) - def _generate_static_data(self): + def _generate_static_data(self, db): platforms = get_random_instances( - Platform.objects.filter(devices__isnull=False).distinct(), + Platform.objects.using(db).filter(devices__isnull=False).distinct(), minimum=2, maximum=4, ) @@ -55,7 +49,9 @@ def _generate_static_data(self): for i in range(1, 9): name = f"ComplianceFeature{i}" compliance_features.append( - ComplianceFeature.objects.create(name=name, slug=name, description=f"Test ComplianceFeature {i}") + ComplianceFeature.objects.using(db).create( + name=name, slug=name, description=f"Test ComplianceFeature {i}" + ) ) # Create ComplianceRules @@ -64,7 +60,7 @@ def _generate_static_data(self): self.stdout.write(message) for feature in compliance_features: for platform in platforms: - ComplianceRule.objects.create( + ComplianceRule.objects.using(db).create( feature=feature, platform=platform, description=f"Test ComplianceRule for {feature.name} on {platform.name}", @@ -76,9 +72,9 @@ def _generate_static_data(self): message = f"Creating {count} ConfigCompliances..." self.stdout.write(message) for device in devices: - for rule in ComplianceRule.objects.filter(platform=device.platform): + for rule in ComplianceRule.objects.using(db).filter(platform=device.platform): is_compliant = random.choice([True, False]) # noqa: S311 - ConfigCompliance.objects.create( + ConfigCompliance.objects.using(db).create( device=device, rule=rule, compliance=is_compliant, @@ -91,7 +87,7 @@ def _generate_static_data(self): message = f"Creating {len(devices)} GoldenConfigs..." self.stdout.write(message) for device in devices: - GoldenConfig.objects.create( + GoldenConfig.objects.using(db).create( device=device, backup_config=f"backup config for {device.name}", intended_config=f"intended config for {device.name}", @@ -105,9 +101,6 @@ def _generate_static_data(self): def handle(self, *args, **options): """Entry point to the management command.""" - # Call nautobot core's generate_test_data command to generate data for core models - call_command("generate_test_data", flush=options["flush"]) - - self._generate_static_data() + self._generate_static_data(db=options["database"]) self.stdout.write(self.style.SUCCESS(f"Database {options['database']} populated with app data successfully!")) From 0157327d13cede5c140c1355199eba58b9772b3f Mon Sep 17 00:00:00 2001 From: Gary Snider <75227981+gsnider2195@users.noreply.github.com> Date: Mon, 21 Oct 2024 13:04:52 -0700 Subject: [PATCH 5/5] Add GraphQLQuery and GoldenConfigSetting. Add --flush option. --- .../commands/generate_gc_test_data.py | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/nautobot_golden_config/management/commands/generate_gc_test_data.py b/nautobot_golden_config/management/commands/generate_gc_test_data.py index b2404142..f93f2cfe 100644 --- a/nautobot_golden_config/management/commands/generate_gc_test_data.py +++ b/nautobot_golden_config/management/commands/generate_gc_test_data.py @@ -2,10 +2,12 @@ import random +from django.contrib.contenttypes.models import ContentType from django.core.management.base import BaseCommand from django.db import DEFAULT_DB_ALIAS from nautobot.core.factory import get_random_instances from nautobot.dcim.models import Platform +from nautobot.extras.models import DynamicGroup, GraphQLQuery from netutils.lib_mapper import NETUTILSPARSER_LIB_MAPPER_REVERSE from nautobot_golden_config.models import ( @@ -13,6 +15,7 @@ ComplianceRule, ConfigCompliance, GoldenConfig, + GoldenConfigSetting, ) @@ -27,6 +30,11 @@ def add_arguments(self, parser): # noqa: D102 default=DEFAULT_DB_ALIAS, help='The database to generate the test data in. Defaults to the "default" database.', ) + parser.add_argument( + "--flush", + action="store_true", + help="Flush any existing golden config data from the database before generating new data.", + ) def _generate_static_data(self, db): platforms = get_random_instances( @@ -94,6 +102,112 @@ def _generate_static_data(self, db): compliance_config=f"compliance config for {device.name}", ) + # Create GraphQL query for GoldenConfigSetting.sot_agg_query + message = "Creating test GraphQLQuery for GoldenConfigSetting..." + self.stdout.write(message) + graphql_query_variables = {"device_id": ""} + graphql_sot_agg_query = """ + query ($device_id: ID!) { + device(id: $device_id) { + config_context + hostname: name + position + serial + primary_ip4 { + id + primary_ip4_for { + id + name + } + } + tenant { + name + } + tags { + name + } + role { + name + } + platform { + name + manufacturer { + name + } + network_driver + napalm_driver + } + location { + name + parent { + name + } + } + interfaces { + description + mac_address + enabled + name + ip_addresses { + address + tags { + id + } + } + connected_circuit_termination { + circuit { + cid + commit_rate + provider { + name + } + } + } + tagged_vlans { + id + } + untagged_vlan { + id + } + cable { + termination_a_type + status { + name + } + color + } + tags { + id + } + } + } + } + """ + gql_query = GraphQLQuery.objects.using(db).create( + name="GoldenConfigSetting.sot_agg_query", + variables=graphql_query_variables, + query=graphql_sot_agg_query, + ) + if not GoldenConfigSetting.objects.using(db).exists(): + dynamic_group, _ = DynamicGroup.objects.using(db).get_or_create( + name="GoldenConfigSetting Dynamic Group", + defaults={"content_type": ContentType.objects.using(db).get(app_label="dcim", model="device")}, + ) + GoldenConfigSetting.objects.using(db).create( + name="Default GoldenConfigSetting", + slug="default_goldenconfigsetting", + sot_agg_query=gql_query, + dynamic_group=dynamic_group, + ) + message = "Creating 1 GoldenConfigSetting..." + self.stdout.write(message) + else: + golden_config_setting = GoldenConfigSetting.objects.first() + message = f"Applying GraphQLQuery to GoldenConfigSetting '{golden_config_setting.name}'..." + self.stdout.write(message) + golden_config_setting.sot_agg_query = gql_query + golden_config_setting.save() + # TODO: Create ConfigRemoves # TODO: Create ConfigReplaces # TODO: Create RemediationSettings @@ -101,6 +215,15 @@ def _generate_static_data(self, db): def handle(self, *args, **options): """Entry point to the management command.""" + if options["flush"]: + self.stdout.write(self.style.WARNING("Flushing golden config objects from the database...")) + GoldenConfigSetting.objects.using(options["database"]).all().delete() + GoldenConfig.objects.using(options["database"]).all().delete() + ConfigCompliance.objects.using(options["database"]).all().delete() + ComplianceRule.objects.using(options["database"]).all().delete() + ComplianceFeature.objects.using(options["database"]).all().delete() + GraphQLQuery.objects.using(options["database"]).filter(name="GoldenConfigSetting.sot_agg_query").delete() + self._generate_static_data(db=options["database"]) self.stdout.write(self.style.SUCCESS(f"Database {options['database']} populated with app data successfully!"))