diff --git a/CHANGELOG.md b/CHANGELOG.md index 02f20257..9047662d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a Ch ## Unreleased ### New +- adding qualifier support for bootstrap roles ### Changes diff --git a/docs/source/bootstrapping.md b/docs/source/bootstrapping.md index 897b9abe..9c2354ae 100644 --- a/docs/source/bootstrapping.md +++ b/docs/source/bootstrapping.md @@ -14,7 +14,8 @@ Options: -p, --project TEXT Project identifier -t, --trusted-principal TEXT ARN of Principals trusted to assume the Toolchain Role - -b, --permissions-boundary TEXT ARN of a Managed Policy to set as the + -b, --permissions-boundary TEXT + ARN of a Managed Policy to set as the Permission Boundary on the Toolchain Role --as-target / --not-as-target Optionally also bootstrap the account as a Target account [default: not-as-target] @@ -22,6 +23,7 @@ Options: deploy [default: no-synth] --profile TEXT The AWS profile to initiate a session --region TEXT AWS region to use + --qualifier TEXT A qualifier to append to toolchain role (alpha-numeric char max length of 6) --debug / --no-debug Enable detail logging [default: no-debug] --help Show this message and exit. ``` @@ -33,11 +35,10 @@ The `permission-boundary` filed allows you to attach a policy to the role to act Typically, you can have the `toolchain account` act as the `target account`. The `as-target` field will bootstrap both in a single command. - ## Bootstrap Target Account ```bash -sage: seedfarmer bootstrap target [OPTIONS] +Usage: seedfarmer bootstrap target [OPTIONS] Bootstrap a Target account. @@ -46,12 +47,14 @@ Options: -t, --toolchain-account TEXT Account Id of the Toolchain account trusted to assume the Target account's Deployment Role [required] - -b, --permissions-boundary TEXT ARN of a Managed Policy to set as the + -b, --permissions-boundary TEXT + ARN of a Managed Policy to set as the Permission Boundary on the Toolchain Role --synth / --no-synth Synthesize a CFN template only...do not deploy [default: no-synth] --profile TEXT The AWS profile to initiate a session --region TEXT AWS region to use + --qualifier TEXT A qualifier to append to target role (alpha-numeric char max length of 6) --debug / --no-debug Enable detail logging [default: no-debug] --help Show this message and exit. ``` @@ -60,5 +63,12 @@ You must pass in the `toolchain-account` field so a trust-relationship can be se The `permission-boundary` filed allows you to attach a policy to the role to act as a [permissions boundary](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_boundaries.html) + +## Qualifiers for Toolchain Role and Target Roles +We have added suppprt for the use of a qualifier for the toolchain role and the target account deployment role(s). This is to help segregate target deployment when using a multi-account structure which has a central shared services (CICD account) as the toolchain account performing deployments across relevant environments (ex. DEV, INT, PROD). A `qualifier` can be used if you want to restrict the level of access/action a dev/tester/support team can perform on any target given environment. + +The qualifier post-pends a 6 char alpha-numeric string to the deployment role and toolchain role. The qualifier **MUST BE THE SAME ON THE TOOLCHAIN ROLE AND EACH TARGET ROLE.** + + ## Prepping the Account / Region `seedfarmer` leverages the AWS CDKv2. This must be bootstrapped in ech account/region combination to be used of each target account. \ No newline at end of file diff --git a/seedfarmer/__main__.py b/seedfarmer/__main__.py index 8d6ad437..c85487d8 100644 --- a/seedfarmer/__main__.py +++ b/seedfarmer/__main__.py @@ -57,6 +57,12 @@ def version() -> None: help="The AWS region to use for boto3.Sessions", required=False, ) +@click.option( + "--qualifier", + default=None, + help="A qualifier to append to toolchain / target roles", + required=False, +) @click.option( "--env-file", default=".env", @@ -102,6 +108,7 @@ def apply( spec: str, profile: Optional[str], region: Optional[str], + qualifier: Optional[str], env_file: str, debug: bool, dry_run: bool, @@ -124,6 +131,7 @@ def apply( deployment_manifest_path=spec, profile=profile, region_name=region, + qualifier=qualifier, dryrun=dry_run, show_manifest=show_manifest, enable_session_timeout=enable_session_timeout, @@ -161,6 +169,12 @@ def apply( help="The AWS region to use for toolchain", required=False, ) +@click.option( + "--qualifier", + default=None, + help="A qualifier to append to toolchain / target role", + required=False, +) @click.option( "--env-file", default=".env", @@ -193,6 +207,7 @@ def destroy( show_manifest: bool, profile: Optional[str], region: Optional[str], + qualifier: Optional[str], env_file: str, debug: bool, enable_session_timeout: bool, @@ -217,6 +232,7 @@ def destroy( deployment_name=deployment, profile=profile, region_name=region, + qualifier=qualifier, dryrun=dry_run, show_manifest=show_manifest, enable_session_timeout=enable_session_timeout, diff --git a/seedfarmer/cli_groups/_bootstrap_group.py b/seedfarmer/cli_groups/_bootstrap_group.py index cd55c539..506342a5 100644 --- a/seedfarmer/cli_groups/_bootstrap_group.py +++ b/seedfarmer/cli_groups/_bootstrap_group.py @@ -94,6 +94,12 @@ def bootstrap() -> None: help="AWS region to use", required=False, ) +@click.option( + "--qualifier", + default=None, + help="A qualifier to append to toolchain role (alpha-numeric char max length of 6)", + required=False, +) @click.option("--debug/--no-debug", default=False, help="Enable detail logging", show_default=True) def bootstrap_toolchain( project: Optional[str], @@ -101,6 +107,7 @@ def bootstrap_toolchain( permissions_boundary: Optional[str], profile: Optional[str], region: Optional[str], + qualifier: Optional[str], as_target: bool, synth: bool, debug: bool, @@ -115,6 +122,7 @@ def bootstrap_toolchain( principal_arns=trusted_principal, permissions_boundary_arn=permissions_boundary, profile=profile, + qualifier=qualifier, region_name=region, synthesize=synth, as_target=as_target, @@ -165,6 +173,12 @@ def bootstrap_toolchain( help="AWS region to use", required=False, ) +@click.option( + "--qualifier", + default=None, + help="A qualifier to append to target role (alpha-numeric char max length of 6)", + required=False, +) @click.option("--debug/--no-debug", default=False, help="Enable detail logging", show_default=True) def bootstrap_target( project: Optional[str], @@ -172,6 +186,7 @@ def bootstrap_target( permissions_boundary: Optional[str], profile: Optional[str], region: Optional[str], + qualifier: Optional[str], synth: bool, debug: bool, ) -> None: @@ -185,6 +200,7 @@ def bootstrap_target( project_name=project, profile=profile, region_name=region, + qualifier=qualifier, permissions_boundary_arn=permissions_boundary, synthesize=synth, ) diff --git a/seedfarmer/cli_groups/_list_group.py b/seedfarmer/cli_groups/_list_group.py index 20cc87e6..05180eac 100644 --- a/seedfarmer/cli_groups/_list_group.py +++ b/seedfarmer/cli_groups/_list_group.py @@ -102,6 +102,12 @@ def list() -> None: help="The AWS region of the toolchain", required=False, ) +@click.option( + "--qualifier", + default=None, + help="A qualifier to use with the seedfarmer roles", + required=False, +) @click.option( "--env-file", default=".env", @@ -121,6 +127,7 @@ def list_dependencies( project: Optional[str], profile: Optional[str], region: Optional[str], + qualifier: Optional[str], env_file: str, debug: bool, ) -> None: @@ -132,7 +139,7 @@ def list_dependencies( project = _load_project() load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) - SessionManager().get_or_create(project_name=project, profile=profile, region_name=region) + SessionManager().get_or_create(project_name=project, profile=profile, region_name=region, qualifier=qualifier) dep_manifest = du.generate_deployed_manifest(deployment_name=deployment, skip_deploy_spec=True) if dep_manifest: @@ -188,6 +195,12 @@ def list_dependencies( help="The AWS region of the toolchain", required=False, ) +@click.option( + "--qualifier", + default=None, + help="A qualifier to use with the seedfarmer roles", + required=False, +) @click.option( "--env-file", default=".env", @@ -207,6 +220,7 @@ def list_deployspec( project: Optional[str], profile: Optional[str], region: Optional[str], + qualifier: Optional[str], env_file: str, debug: bool, ) -> None: @@ -218,7 +232,9 @@ def list_deployspec( project = _load_project() load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) - session = SessionManager().get_or_create(project_name=project, profile=profile, region_name=region) + session = SessionManager().get_or_create( + project_name=project, profile=profile, region_name=region, qualifier=qualifier + ) dep_manifest = du.generate_deployed_manifest(deployment_name=deployment, skip_deploy_spec=True) if dep_manifest is None: @@ -280,6 +296,12 @@ def list_deployspec( help="The AWS region of the toolchain", required=False, ) +@click.option( + "--qualifier", + default=None, + help="A qualifier to use with the seedfarmer roles", + required=False, +) @click.option( "--export-local-env/--no-export-local-env", default=False, @@ -305,6 +327,7 @@ def list_module_metadata( project: Optional[str], profile: Optional[str], region: Optional[str], + qualifier: Optional[str], env_file: str, export_local_env: bool, debug: bool, @@ -317,7 +340,9 @@ def list_module_metadata( project = _load_project() load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) - session = SessionManager().get_or_create(project_name=project, profile=profile, region_name=region) + session = SessionManager().get_or_create( + project_name=project, profile=profile, region_name=region, qualifier=qualifier + ) dep_manifest = du.generate_deployed_manifest(deployment_name=deployment, skip_deploy_spec=True) if dep_manifest is None: @@ -374,6 +399,12 @@ def list_module_metadata( help="The AWS region of the toolchain", required=False, ) +@click.option( + "--qualifier", + default=None, + help="A qualifier to use with the seedfarmer roles", + required=False, +) @click.option( "--env-file", default=".env", @@ -391,6 +422,7 @@ def list_all_module_metadata( project: Optional[str], profile: Optional[str], region: Optional[str], + qualifier: Optional[str], env_file: str, debug: bool, ) -> None: @@ -402,7 +434,9 @@ def list_all_module_metadata( project = _load_project() load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) - session = SessionManager().get_or_create(project_name=project, profile=profile, region_name=region) + session = SessionManager().get_or_create( + project_name=project, profile=profile, region_name=region, qualifier=qualifier + ) dep_manifest = du.generate_deployed_manifest(deployment_name=deployment, skip_deploy_spec=True) if dep_manifest is None: @@ -458,6 +492,12 @@ def list_all_module_metadata( help="The AWS region of the toolchain", required=False, ) +@click.option( + "--qualifier", + default=None, + help="A qualifier to use with the seedfarmer roles", + required=False, +) @click.option( "--env-file", default=".env", @@ -475,6 +515,7 @@ def list_modules( project: Optional[str], profile: Optional[str], region: Optional[str], + qualifier: Optional[str], env_file: str, debug: bool, ) -> None: @@ -485,7 +526,7 @@ def list_modules( if project is None: project = _load_project() load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) - SessionManager().get_or_create(project_name=project, profile=profile, region_name=region) + SessionManager().get_or_create(project_name=project, profile=profile, region_name=region, qualifier=qualifier) dep_manifest = du.generate_deployed_manifest(deployment_name=deployment, skip_deploy_spec=True) if dep_manifest: @@ -512,6 +553,12 @@ def list_modules( help="The AWS region of the toolchain", required=False, ) +@click.option( + "--qualifier", + default=None, + help="A qualifier to use with the seedfarmer roles", + required=False, +) @click.option( "--debug/--no-debug", default=False, @@ -522,6 +569,7 @@ def list_deployments( project: Optional[str], profile: Optional[str], region: Optional[str], + qualifier: Optional[str], debug: bool, ) -> None: if debug: @@ -532,7 +580,7 @@ def list_deployments( deps = mi.get_all_deployments( session=SessionManager() - .get_or_create(project_name=project, profile=profile, region_name=region) + .get_or_create(project_name=project, profile=profile, region_name=region, qualifier=qualifier) .toolchain_session ) print_deployment_inventory(description="Deployment Names", dep=deps) @@ -585,6 +633,12 @@ def list_deployments( help="The AWS region of the toolchain", required=False, ) +@click.option( + "--qualifier", + default=None, + help="A qualifier to use with the seedfarmer roles", + required=False, +) @click.option( "--export-local-env/--no-export-local-env", default=False, @@ -611,6 +665,7 @@ def list_build_env_params( project: Optional[str], profile: Optional[str], region: Optional[str], + qualifier: Optional[str], env_file: str, export_local_env: str, debug: bool, @@ -625,7 +680,9 @@ def list_build_env_params( project = _load_project() load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) - session = SessionManager().get_or_create(project_name=project, profile=profile, region_name=region) + session = SessionManager().get_or_create( + project_name=project, profile=profile, region_name=region, qualifier=qualifier + ) dep_manifest = du.generate_deployed_manifest( deployment_name=deployment, skip_deploy_spec=True, ignore_deployed=True ) diff --git a/seedfarmer/cli_groups/_remove_group.py b/seedfarmer/cli_groups/_remove_group.py index 98c4d63c..d72b07ef 100644 --- a/seedfarmer/cli_groups/_remove_group.py +++ b/seedfarmer/cli_groups/_remove_group.py @@ -83,6 +83,12 @@ def remove() -> None: help="The AWS region of the toolchain", required=False, ) +@click.option( + "--qualifier", + default=None, + help="A qualifier to use with the seedfarmer roles", + required=False, +) @click.option( "--target-account-id", default=None, @@ -107,6 +113,7 @@ def remove_module_data( module: str, profile: Optional[str], region: Optional[str], + qualifier: Optional[str], project: Optional[str], target_account_id: Optional[str], target_region: Optional[str], @@ -127,7 +134,7 @@ def remove_module_data( elif target_account_id is not None and target_region is not None: session = ( SessionManager() - .get_or_create(project_name=project, profile=profile, region_name=region) + .get_or_create(project_name=project, profile=profile, region_name=region, qualifier=qualifier) .get_deployment_session(account_id=target_account_id, region_name=target_region) ) diff --git a/seedfarmer/cli_groups/_store_group.py b/seedfarmer/cli_groups/_store_group.py index 1acc1732..7e7ee4f7 100644 --- a/seedfarmer/cli_groups/_store_group.py +++ b/seedfarmer/cli_groups/_store_group.py @@ -92,6 +92,12 @@ def store() -> None: help="The AWS region of the toolchain", required=False, ) +@click.option( + "--qualifier", + default=None, + help="A qualifier to use with the seedfarmer roles", + required=False, +) @click.option( "--target-account-id", default=None, @@ -117,6 +123,7 @@ def store_deployspec( path: str, profile: Optional[str], region: Optional[str], + qualifier: Optional[str], project: Optional[str], target_account_id: Optional[str], target_region: Optional[str], @@ -141,7 +148,7 @@ def store_deployspec( elif target_account_id is not None and target_region is not None: session = ( SessionManager() - .get_or_create(project_name=project, profile=profile, region_name=region) + .get_or_create(project_name=project, profile=profile, region_name=region, qualifier=qualifier) .get_deployment_session(account_id=target_account_id, region_name=target_region) ) @@ -189,6 +196,12 @@ def store_deployspec( help="The AWS region of the toolchain", required=False, ) +@click.option( + "--qualifier", + default=None, + help="A qualifier to use with the seedfarmer roles", + required=False, +) @click.option( "--target-account-id", default=None, @@ -214,6 +227,7 @@ def store_module_metadata( project: Optional[str], profile: Optional[str], region: Optional[str], + qualifier: Optional[str], target_account_id: Optional[str], target_region: Optional[str], debug: bool, @@ -234,7 +248,7 @@ def store_module_metadata( elif target_account_id is not None and target_region is not None: session = ( SessionManager() - .get_or_create(project_name=project, profile=profile, region_name=region) + .get_or_create(project_name=project, profile=profile, region_name=region, qualifier=qualifier) .get_deployment_session(account_id=target_account_id, region_name=target_region) ) @@ -293,6 +307,12 @@ def store_module_metadata( help="The AWS region of the toolchain", required=False, ) +@click.option( + "--qualifier", + default=None, + help="A qualifier to use with the seedfarmer roles", + required=False, +) @click.option( "--target-account-id", default=None, @@ -321,6 +341,7 @@ def store_module_md5( target_region: Optional[str], profile: Optional[str], region: Optional[str], + qualifier: Optional[str], debug: bool, ) -> None: if debug: @@ -338,7 +359,7 @@ def store_module_md5( elif target_account_id is not None and target_region is not None: session = ( SessionManager() - .get_or_create(project_name=project, profile=profile, region_name=region) + .get_or_create(project_name=project, profile=profile, region_name=region, qualifier=qualifier) .get_deployment_session(account_id=target_account_id, region_name=target_region) ) diff --git a/seedfarmer/commands/_bootstrap_commands.py b/seedfarmer/commands/_bootstrap_commands.py index 4fcf9eaf..2d7306e5 100644 --- a/seedfarmer/commands/_bootstrap_commands.py +++ b/seedfarmer/commands/_bootstrap_commands.py @@ -16,7 +16,7 @@ import logging import os import sys -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple, cast import yaml from aws_codeseeder import services as cs_services @@ -24,9 +24,11 @@ from botocore.exceptions import WaiterError from jinja2 import Template +import seedfarmer.errors from seedfarmer import CLI_ROOT from seedfarmer.services import create_new_session, get_account_id, get_region from seedfarmer.services._iam import get_role +from seedfarmer.utils import get_deployment_role_name, get_toolchain_role_arn, get_toolchain_role_name, valid_qualifier _logger: logging.Logger = logging.getLogger(__name__) @@ -34,6 +36,7 @@ def get_toolchain_template( project_name: str, principal_arn: List[str], + role_name: str, permissions_boundary_arn: Optional[str] = None, ) -> Dict[Any, Any]: with open((os.path.join(CLI_ROOT, "resources/toolchain_role.template")), "r") as f: @@ -45,19 +48,22 @@ def get_toolchain_template( if permissions_boundary_arn: role["Resources"]["ToolchainRole"]["Properties"]["PermissionsBoundary"] = permissions_boundary_arn template = Template(json.dumps(role)) - t = template.render({"project_name": project_name}) + t = template.render({"project_name": project_name, "role_name": role_name}) return dict(json.loads(t)) def get_deployment_template( - toolchain_account_id: str, project_name: str, permissions_boundary_arn: Optional[str] = None + toolchain_role_arn: str, project_name: str, role_name: str, permissions_boundary_arn: Optional[str] = None ) -> Dict[Any, Any]: with open((os.path.join(CLI_ROOT, "resources/deployment_role.template")), "r") as f: role = yaml.safe_load(f) if permissions_boundary_arn: role["Resources"]["DeploymentRole"]["Properties"]["PermissionsBoundary"] = permissions_boundary_arn + template = Template(json.dumps(role)) - t = template.render({"toolchain_account_id": toolchain_account_id, "project_name": project_name}) + t = template.render( + {"toolchain_role_arn": toolchain_role_arn, "project_name": project_name, "role_name": role_name} + ) return dict(json.loads(t)) @@ -70,21 +76,32 @@ def bootstrap_toolchain_account( project_name: str, principal_arns: List[str], permissions_boundary_arn: Optional[str] = None, + qualifier: Optional[str] = None, profile: Optional[str] = None, region_name: Optional[str] = None, synthesize: bool = False, as_target: bool = False, ) -> Optional[Dict[Any, Any]]: - template = get_toolchain_template(project_name, principal_arns, permissions_boundary_arn) + + if qualifier and not valid_qualifier(qualifier): + raise seedfarmer.errors.InvalidConfigurationError("The Qualifier must be alphanumeric and 6 characters or less") + + role_stack_name = get_toolchain_role_name(project_name=project_name, qualifier=cast(str, qualifier)) + template = get_toolchain_template( + project_name=project_name, + role_name=role_stack_name, + principal_arn=principal_arns, + permissions_boundary_arn=permissions_boundary_arn, + ) _logger.debug((json.dumps(template, indent=4))) if not synthesize: session = create_new_session(profile=profile, region_name=region_name) - role_name, stack_name = f"seedfarmer-{project_name}-toolchain-role", f"seedfarmer-{project_name}-toolchain-role" - apply_deploy_logic(template=template, role_name=role_name, stack_name=stack_name, session=session) + apply_deploy_logic(template=template, role_name=role_stack_name, stack_name=role_stack_name, session=session) if as_target: bootstrap_target_account( toolchain_account_id=get_account_id(session), project_name=project_name, + qualifier=cast(str, qualifier), permissions_boundary_arn=permissions_boundary_arn, profile=profile, region_name=region_name, @@ -96,6 +113,7 @@ def bootstrap_toolchain_account( bootstrap_target_account( toolchain_account_id="123456789012", project_name=project_name, + qualifier=cast(str, qualifier), permissions_boundary_arn=permissions_boundary_arn, profile=profile, region_name=region_name, @@ -109,21 +127,33 @@ def bootstrap_target_account( toolchain_account_id: str, project_name: str, permissions_boundary_arn: Optional[str] = None, + qualifier: Optional[str] = None, profile: Optional[str] = None, region_name: Optional[str] = None, session: Optional[Session] = None, synthesize: bool = False, ) -> Optional[Dict[Any, Any]]: - template = get_deployment_template(toolchain_account_id, project_name, permissions_boundary_arn) + + if qualifier and not valid_qualifier(qualifier): + raise seedfarmer.errors.InvalidConfigurationError("The Qualifier must be alphanumeric and 6 characters or less") + + role_stack_name = get_deployment_role_name(project_name=project_name, qualifier=cast(str, qualifier)) + toolchain_role_arn = get_toolchain_role_arn( + toolchain_account_id=toolchain_account_id, project_name=project_name, qualifier=cast(str, qualifier) + ) + + template = get_deployment_template( + toolchain_role_arn=toolchain_role_arn, + project_name=project_name, + role_name=role_stack_name, + permissions_boundary_arn=permissions_boundary_arn, + ) _logger.debug((json.dumps(template, indent=4))) if not synthesize: if not session: session = create_new_session(profile=profile, region_name=region_name) - role_name, stack_name = ( - f"seedfarmer-{project_name}-deployment-role", - f"seedfarmer-{project_name}-deployment-role", - ) - apply_deploy_logic(template=template, role_name=role_name, stack_name=stack_name, session=session) + + apply_deploy_logic(template=template, role_name=role_stack_name, stack_name=role_stack_name, session=session) else: write_template(template=template) return template diff --git a/seedfarmer/commands/_deployment_commands.py b/seedfarmer/commands/_deployment_commands.py index cae9a450..126b2f9b 100644 --- a/seedfarmer/commands/_deployment_commands.py +++ b/seedfarmer/commands/_deployment_commands.py @@ -663,6 +663,7 @@ def apply( deployment_manifest_path: str, profile: Optional[str] = None, region_name: Optional[str] = None, + qualifier: Optional[str] = None, dryrun: bool = False, show_manifest: bool = False, enable_session_timeout: bool = False, @@ -683,6 +684,9 @@ def apply( If using an AWS Profile for deployment use it here region_name : str The name of the AWS region the deployment is based in for the toolchain + qualifier : str, optional + Any qualifier on the name of toolchain role + Defaults to None dryrun : bool, optional This flag indicates that the deployment manifest should be consumed and a DeploymentManifest object be created (for both apply and destroy) but DOES NOT @@ -717,6 +721,7 @@ def apply( session_manager = SessionManager().get_or_create( project_name=config.PROJECT, profile=profile, + qualifier=qualifier, toolchain_region=deployment_manifest.toolchain_region, region_name=region_name, enable_reaper=enable_session_timeout, @@ -784,6 +789,7 @@ def destroy( deployment_name: str, profile: Optional[str] = None, region_name: Optional[str] = None, + qualifier: Optional[str] = None, dryrun: bool = False, show_manifest: bool = False, retain_seedkit: bool = False, @@ -802,6 +808,9 @@ def destroy( If using an AWS Profile for deployment use it here region_name : str The name of the AWS region the deployment is based in for the toolchain + qualifier : str, optional + Any qualifier on the name of toolchain role + Defaults to None dryrun : bool, optional This flag indicates that the deployment WILL NOT enact any deployment changes. @@ -831,6 +840,7 @@ def destroy( project_name=project, profile=profile, region_name=region_name, + qualifier=qualifier, enable_reaper=enable_session_timeout, reaper_interval=session_timeout_interval, ) diff --git a/seedfarmer/resources/deployment_role.template b/seedfarmer/resources/deployment_role.template index 0bd09788..99576fce 100644 --- a/seedfarmer/resources/deployment_role.template +++ b/seedfarmer/resources/deployment_role.template @@ -13,7 +13,7 @@ Resources: - Action: sts:AssumeRole Effect: Allow Principal: - AWS: arn:aws:iam::{{ toolchain_account_id }}:role/seedfarmer-{{ project_name }}-toolchain-role + AWS: "{{ toolchain_role_arn }}" Path: / Policies: - PolicyName: InlineToolchain @@ -148,5 +148,5 @@ Resources: Resource: - Fn::Sub: "arn:aws:codeartifact:*:${AWS::AccountId}:domain/aws-codeseeder-{{ project_name }}" - Fn::Sub: "arn:aws:codeartifact:*:${AWS::AccountId}:repository/aws-codeseeder-{{ project_name }}*" - RoleName: seedfarmer-{{ project_name }}-deployment-role + RoleName: "{{ role_name }}" Type: AWS::IAM::Role diff --git a/seedfarmer/resources/toolchain_role.template b/seedfarmer/resources/toolchain_role.template index 6477d95a..7c1eee52 100644 --- a/seedfarmer/resources/toolchain_role.template +++ b/seedfarmer/resources/toolchain_role.template @@ -43,5 +43,5 @@ Resources: Effect: Allow Resource: Fn::Sub: "arn:aws:ssm:*:${AWS::AccountId}:*" - RoleName: seedfarmer-{{ project_name }}-toolchain-role + RoleName: "{{ role_name }}" Type: AWS::IAM::Role diff --git a/seedfarmer/services/session_manager.py b/seedfarmer/services/session_manager.py index 8903874a..8c66d814 100644 --- a/seedfarmer/services/session_manager.py +++ b/seedfarmer/services/session_manager.py @@ -3,12 +3,13 @@ from abc import abstractmethod, abstractproperty from threading import Thread from time import sleep -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple, cast from boto3 import Session import seedfarmer.errors from seedfarmer.services import boto3_client, create_new_session, create_new_session_with_creds, get_account_id +from seedfarmer.utils import get_deployment_role_arn, get_toolchain_role_arn, get_toolchain_role_name _logger: logging.Logger = logging.getLogger(__name__) @@ -42,6 +43,7 @@ def get_or_create( project_name: Optional[str] = None, region_name: Optional[str] = None, toolchain_region: Optional[str] = None, + qualifier: Optional[str] = None, profile: Optional[str] = None, enable_reaper: bool = False, **kwargs: Optional[Any], @@ -78,6 +80,7 @@ def get_or_create( region_name: Optional[str] = None, profile: Optional[str] = None, toolchain_region: Optional[str] = None, + qualifier: Optional[str] = None, reaper_interval: Optional[int] = None, enable_reaper: bool = False, **kwargs: Optional[Any], @@ -91,8 +94,9 @@ def get_or_create( self.config["region_name"] = region_name self.config["profile"] = profile self.config["toolchain_region"] = toolchain_region + self.config["qualifier"] = qualifier if qualifier else None self.config = {**self.config, **kwargs} - self.toolchain_role_name = f"seedfarmer-{project_name}-toolchain-role" + self.toolchain_role_name = get_toolchain_role_name(project_name, cast(str, qualifier)) if reaper_interval is not None: self.reaper_interval = reaper_interval @@ -116,13 +120,17 @@ def toolchain_session(self) -> Session: def get_deployment_session(self, account_id: str, region_name: str) -> Session: session_key = f"{account_id}-{region_name}" project_name = self.config["project_name"] + qualifier = self.config.get("qualifier") if self.config.get("qualifier") else None if not self.created: raise seedfarmer.errors.InvalidConfigurationError("The SessionManager object was never properly created...") if session_key not in self.sessions.keys(): _logger.info(f"Creating Session for {session_key}") self._check_for_toolchain() toolchain_role = self.sessions[self.TOOLCHAIN_KEY][self.ROLE] - deployment_role_arn = f"arn:aws:iam::{account_id}:role/seedfarmer-{project_name}-deployment-role" + deployment_role_arn = get_deployment_role_arn( + deployment_account_id=account_id, project_name=project_name, qualifier=cast(str, qualifier) + ) + _logger.debug(f"Got deployment role role - {deployment_role_arn}") # the boto sessions are not thread safe, so create a new one for the toolchain role every time to be sure sts_toolchain_client = boto3_client( service_name="sts", @@ -156,13 +164,18 @@ def _check_for_toolchain(self) -> None: def _get_toolchain(self) -> Tuple[Session, Dict[Any, Any]]: region_name = self.config.get("region_name") profile_name = self.config.get("profile") + project_name = self.config.get("project_name") + qualifier = self.config.get("qualifier") if self.config.get("qualifier") else None toolchain_region = self.config.get("toolchain_region") _logger.debug("Getting toolchain role") user_session = create_new_session(region_name=region_name, profile=profile_name) user_client = boto3_client(service_name="sts", session=user_session) - toolchain_account_id = get_account_id(user_session) - - toolchain_role_arn = f"arn:aws:iam::{toolchain_account_id}:role/{self.toolchain_role_name}" + toolchain_role_arn = get_toolchain_role_arn( + toolchain_account_id=get_account_id(user_session), + project_name=cast(str, project_name), + qualifier=cast(str, qualifier), + ) + _logger.debug(f"Using toolchain role - {toolchain_role_arn}") toolchain_role = user_client.assume_role( RoleArn=toolchain_role_arn, RoleSessionName="toolchainrole", diff --git a/seedfarmer/utils.py b/seedfarmer/utils.py index 5332b7b3..91faae14 100644 --- a/seedfarmer/utils.py +++ b/seedfarmer/utils.py @@ -126,3 +126,25 @@ def generate_codebuild_url(account_id: str, region: str, codebuild_id: str) -> s except Exception as e: _logger.error(f"Error...{account_id} - {region} - {codebuild_id} - {e} ") return "N/A" + + +def get_toolchain_role_name(project_name: str, qualifier: Optional[str] = None) -> str: + name = f"seedfarmer-{project_name}-toolchain-role" + return f"{name}-{qualifier}" if qualifier else name + + +def get_toolchain_role_arn(toolchain_account_id: str, project_name: str, qualifier: Optional[str] = None) -> str: + return f"arn:aws:iam::{toolchain_account_id}:role/{get_toolchain_role_name(project_name,qualifier)}" + + +def get_deployment_role_name(project_name: str, qualifier: Optional[str] = None) -> str: + name = f"seedfarmer-{project_name}-deployment-role" + return f"{name}-{qualifier}" if qualifier else name + + +def get_deployment_role_arn(deployment_account_id: str, project_name: str, qualifier: Optional[str] = None) -> str: + return f"arn:aws:iam::{deployment_account_id}:role/{get_deployment_role_name(project_name,qualifier)}" + + +def valid_qualifier(qualifer: str) -> bool: + return True if ((len(qualifer) <= 6) and qualifer.isalnum()) else False diff --git a/test/unit-test/mock_data/mock_deployment_manifest_for_destroy.py b/test/unit-test/mock_data/mock_deployment_manifest_for_destroy.py index 745d1c94..b7152cd4 100644 --- a/test/unit-test/mock_data/mock_deployment_manifest_for_destroy.py +++ b/test/unit-test/mock_data/mock_deployment_manifest_for_destroy.py @@ -9,41 +9,18 @@ { "name": "networking", "path": "modules/optionals/networking/", - - - - "parameters": [ - { - - "name": "internet-accessible", - "value": True - } - ], - + "parameters": [{"name": "internet-accessible", "value": True}], "target_account": "primary", "target_region": "us-east-1", - }, { "name": "datalake-buckets", "path": "modules/optionals/datalake-buckets", - - - - "parameters": [ - { - - "name": "encryption-type", - "value": "SSE" - } - ], - + "parameters": [{"name": "encryption-type", "value": "SSE"}], "target_account": "primary", "target_region": "us-east-1", - - } + }, ], - }, { "name": "core", @@ -52,47 +29,25 @@ { "name": "eks", "path": "modules/core/eks/", - - - "parameters": [ { "value_from": { - "module_metadata": { - "name": "networking", - "group": "optionals", - "key": "VpcId" - }, - - - - + "module_metadata": {"name": "networking", "group": "optionals", "key": "VpcId"}, }, "name": "vpc-id", - }, { "value_from": { "module_metadata": { "name": "networking", "group": "optionals", - "key": "PrivateSubnetIds" + "key": "PrivateSubnetIds", }, - - - - }, "name": "private-subnet-ids", - }, + {"name": "eks-admin-role-name", "value": "Admin"}, { - - "name": "eks-admin-role-name", - "value": "Admin" - }, - { - "name": "eks-compute", "value": { "eks_nodegroup_config": [ @@ -102,17 +57,14 @@ "eks_node_max_quantity": 6, "eks_node_min_quantity": 2, "eks_node_disk_size": 50, - "eks_node_instance_types": [ - "m5.large" - ] + "eks_node_instance_types": ["m5.large"], } ], "eks_version": 1.23, - "eks_node_spot": False - } + "eks_node_spot": False, + }, }, { - "name": "eks-addons", "value": { "deploy_aws_lb_controller": True, @@ -128,50 +80,29 @@ "deploy_cloudwatch_container_insights_logs": False, "cloudwatch_container_insights_logs_retention_days": 7, "deploy_amp": False, - "deploy_grafana_for_amp": False - } - } + "deploy_grafana_for_amp": False, + }, + }, ], - "target_account": "primary", "target_region": "us-east-1", - }, { "name": "efs", "path": "modules/core/efs", - - - "parameters": [ { "value_from": { - "module_metadata": { - "name": "networking", - "group": "optionals", - "key": "VpcId" - }, - - - - + "module_metadata": {"name": "networking", "group": "optionals", "key": "VpcId"}, }, "name": "vpc-id", - }, - { - - "name": "removal-policy", - "value": "DESTROY" - } + {"name": "removal-policy", "value": "DESTROY"}, ], - "target_account": "primary", "target_region": "us-east-1", - - } + }, ], - }, { "name": "platform", @@ -180,342 +111,209 @@ { "name": "kubeflow-platform", "path": "modules/mlops/kubeflow-platform/", - - - "parameters": [ { "value_from": { - "module_metadata": { - "name": "eks", - "group": "core", - "key": "EksClusterMasterRoleArn" - }, - - - - + "module_metadata": {"name": "eks", "group": "core", "key": "EksClusterMasterRoleArn"}, }, "name": "EksClusterMasterRoleArn", - }, { "value_from": { - "module_metadata": { - "name": "eks", - "group": "core", - "key": "EksClusterName" - }, - - - - + "module_metadata": {"name": "eks", "group": "core", "key": "EksClusterName"}, }, "name": "EksClusterName", - - }, - { - - "name": "InstallationOption", - "value": "kustomize" - }, - { - - "name": "DeploymentOption", - "value": "vanilla" - }, - { - - "name": "KubeflowReleaseVersion", - "value": "v1.6.1" }, - { - - "name": "AwsKubeflowBuild", - "value": "1.0.0" - } + {"name": "InstallationOption", "value": "kustomize"}, + {"name": "DeploymentOption", "value": "vanilla"}, + {"name": "KubeflowReleaseVersion", "value": "v1.6.1"}, + {"name": "AwsKubeflowBuild", "value": "1.0.0"}, ], - "target_account": "primary", "target_region": "us-east-1", - }, { "name": "efs-on-eks", "path": "modules/integration/efs-on-eks", - - - "parameters": [ { "value_from": { - "module_metadata": { - "name": "eks", - "group": "core", - "key": "EksClusterAdminRoleArn" - }, - - - - + "module_metadata": {"name": "eks", "group": "core", "key": "EksClusterAdminRoleArn"}, }, "name": "eks-cluster-admin-role-arn", - }, { "value_from": { - "module_metadata": { - "name": "eks", - "group": "core", - "key": "EksClusterName" - }, - - - - + "module_metadata": {"name": "eks", "group": "core", "key": "EksClusterName"}, }, "name": "eks-cluster-name", - }, { "value_from": { - "module_metadata": { - "name": "eks", - "group": "core", - "key": "EksOidcArn" - }, - - - - + "module_metadata": {"name": "eks", "group": "core", "key": "EksOidcArn"}, }, "name": "eks-oidc-arn", - }, { "value_from": { - "module_metadata": { - "name": "eks", - "group": "core", - "key": "EksClusterSecurityGroupId" - }, - - - - + "module_metadata": {"name": "eks", "group": "core", "key": "EksClusterSecurityGroupId"}, }, "name": "eks-cluster-security-group-id", - }, { "value_from": { - "module_metadata": { - "name": "efs", - "group": "core", - "key": "EFSFileSystemId" - }, - - - - + "module_metadata": {"name": "efs", "group": "core", "key": "EFSFileSystemId"}, }, "name": "efs-file-system-id", - }, { "value_from": { - "module_metadata": { - "name": "efs", - "group": "core", - "key": "EFSSecurityGroupId" - }, - - - - + "module_metadata": {"name": "efs", "group": "core", "key": "EFSSecurityGroupId"}, }, "name": "efs-security-group-id", - }, { "value_from": { - "module_metadata": { - "name": "efs", - "group": "core", - "key": "VpcId" - }, - - - - + "module_metadata": {"name": "efs", "group": "core", "key": "VpcId"}, }, "name": "vpc-id", - - } + }, ], - "target_account": "primary", "target_region": "us-east-1", - - } + }, ], - }, ], - "target_account_mappings": [ { "alias": "primary", "account_id": "123456789012", "default": True, - "parameters_global": { - "dockerCredentialsSecret": "aws-addf-docker-credentials" - }, + "parameters_global": {"dockerCredentialsSecret": "aws-addf-docker-credentials"}, "region_mappings": [ { "region": "us-east-1", "default": True, "parameters_regional": {}, - - } ], - } - ] + ], } destroy_manifest = { - "name": "mlops", - "toolchain_region": "us-east-1", - "target_account_mappings": [ - { - "alias": "primary", - "account_id": "123456789012", - "default": True, - "parameters_global": { - "dockerCredentialsSecret": "aws-addf-docker-credentials" - }, - "region_mappings": [ + "name": "mlops", + "toolchain_region": "us-east-1", + "target_account_mappings": [ { - "region": "us-east-1", - "default": True, - "parameters_regional": {}, + "alias": "primary", + "account_id": "123456789012", + "default": True, + "parameters_global": {"dockerCredentialsSecret": "aws-addf-docker-credentials"}, + "region_mappings": [ + { + "region": "us-east-1", + "default": True, + "parameters_regional": {}, + } + ], } - ] - } - ], - "groups": [ - { - "name": "users", - "modules": [ + ], + "groups": [ { - "name": "kubeflow-users", - "path": "modules/mlops/kubeflow-users", - "bundle_md5": "03c4cce1b534053ab2e9907c00ffef3e", - "manifest_md5": "d3e04a0cffa57cef83a57fb4f077ad50", - "deployspec_md5": "b13401fb18d61964e1e39e2a1474a205", - "parameters": [ - { - "value_from": { - "module_metadata": { - "name": "eks", - "group": "core", - "key": "EksClusterAdminRoleArn" - }, - }, - "name": "EksClusterAdminRoleArn", - }, - { - "value_from": { - "module_metadata": { - "name": "eks", - "group": "core", - "key": "EksClusterName" - } - }, - "name": "EksClusterName", - }, - { - "value_from": { - "module_metadata": { - "name": "eks", - "group": "core", - "key": "EksOidcArn" - } - }, - "name": "EksOidcArn", - }, - { - "value_from": { - "module_metadata": { - "name": "eks", - "group": "core", - "key": "EksClusterOpenIdConnectIssuer" - } - }, - "name": "EksClusterOpenIdConnectIssuer", - }, - { - "name": "KubeflowUsers", - "value": [ + "name": "users", + "modules": [ { - "policyArn": "arn:aws:iam::aws:policy/AdministratorAccess", - "secret": "addf-dataservice-users-kubeflow-users-kf-dgraeber" + "name": "kubeflow-users", + "path": "modules/mlops/kubeflow-users", + "bundle_md5": "03c4cce1b534053ab2e9907c00ffef3e", + "manifest_md5": "d3e04a0cffa57cef83a57fb4f077ad50", + "deployspec_md5": "b13401fb18d61964e1e39e2a1474a205", + "parameters": [ + { + "value_from": { + "module_metadata": {"name": "eks", "group": "core", "key": "EksClusterAdminRoleArn"}, + }, + "name": "EksClusterAdminRoleArn", + }, + { + "value_from": { + "module_metadata": {"name": "eks", "group": "core", "key": "EksClusterName"} + }, + "name": "EksClusterName", + }, + { + "value_from": {"module_metadata": {"name": "eks", "group": "core", "key": "EksOidcArn"}}, + "name": "EksOidcArn", + }, + { + "value_from": { + "module_metadata": { + "name": "eks", + "group": "core", + "key": "EksClusterOpenIdConnectIssuer", + } + }, + "name": "EksClusterOpenIdConnectIssuer", + }, + { + "name": "KubeflowUsers", + "value": [ + { + "policyArn": "arn:aws:iam::aws:policy/AdministratorAccess", + "secret": "addf-dataservice-users-kubeflow-users-kf-dgraeber", + } + ], + }, + ], + "deploy_spec": { + "deploy": { + "phases": { + "install": { + "commands": [ + "npm install -g aws-cdk@2.20.0", + "pip install -r requirements.txt", + "wget https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/v3.2.1/kustomize_kustomize.v3.2.1_linux_amd64", + "chmod +x kustomize_kustomize.v3.2.1_linux_amd64", + "mv kustomize_kustomize.v3.2.1_linux_amd64 /usr/local/bin/kustomize", + "kustomize version", + ] + }, + "pre_build": {"commands": []}, + "build": {"commands": ["echo 'Hi'"]}, + "post_build": {"commands": ['echo "Deploy successful"']}, + } + }, + "destroy": { + "phases": { + "install": { + "commands": ["npm install -g aws-cdk@2.20.0", "pip install -r requirements.txt"] + }, + "pre_build": {"commands": []}, + "build": { + "commands": [ + 'if [[ ${ADDF_PARAMETER_KUBEFLOW_USERS} ]]; then\n cdk destroy --force --app "python app.py";\nfi;\n' + ] + }, + "post_build": {"commands": []}, + } + }, + "build_type": "BUILD_GENERAL1_SMALL", + "publish_generic_env_variables": False, + }, + "target_account": "primary", + "target_region": "us-east-1", } - ] - } - ], - "deploy_spec": { - "deploy": { - "phases": { - "install": { - "commands": [ - "npm install -g aws-cdk@2.20.0", - "pip install -r requirements.txt", - "wget https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/v3.2.1/kustomize_kustomize.v3.2.1_linux_amd64", - "chmod +x kustomize_kustomize.v3.2.1_linux_amd64", - "mv kustomize_kustomize.v3.2.1_linux_amd64 /usr/local/bin/kustomize", - "kustomize version" - ] - }, - "pre_build": { "commands": [] }, - "build": { - "commands": [ - "echo 'Hi'" - ] - }, - "post_build": { "commands": ["echo \"Deploy successful\""] } - } - }, - "destroy": { - "phases": { - "install": { - "commands": [ - "npm install -g aws-cdk@2.20.0", - "pip install -r requirements.txt" - ] - }, - "pre_build": { "commands": [] }, - "build": { - "commands": [ - "if [[ ${ADDF_PARAMETER_KUBEFLOW_USERS} ]]; then\n cdk destroy --force --app \"python app.py\";\nfi;\n" - ] - }, - "post_build": { "commands": [] } - } - }, - "build_type": "BUILD_GENERAL1_SMALL", - "publish_generic_env_variables": False - }, - "target_account": "primary", - "target_region": "us-east-1" + ], } - ] - } - ] + ], } -module_dependencies={'optionals-networking': ['core-eks', 'core-efs'], 'core-eks': ['platform-kubeflow-platform', 'platform-efs-on-eks'], 'core-efs': ['platform-efs-on-eks']} \ No newline at end of file +module_dependencies = { + "optionals-networking": ["core-eks", "core-efs"], + "core-eks": ["platform-kubeflow-platform", "platform-efs-on-eks"], + "core-efs": ["platform-efs-on-eks"], +} diff --git a/test/unit-test/mock_data/mock_deployment_manifest_huge.py b/test/unit-test/mock_data/mock_deployment_manifest_huge.py index 6b6d4ad7..58e76782 100644 --- a/test/unit-test/mock_data/mock_deployment_manifest_huge.py +++ b/test/unit-test/mock_data/mock_deployment_manifest_huge.py @@ -9,41 +9,18 @@ { "name": "networking", "path": "modules/optionals/networking/", - - - - "parameters": [ - { - - "name": "internet-accessible", - "value": True - } - ], - + "parameters": [{"name": "internet-accessible", "value": True}], "target_account": "primary", "target_region": "us-east-1", - }, { "name": "datalake-buckets", "path": "modules/optionals/datalake-buckets", - - - - "parameters": [ - { - - "name": "encryption-type", - "value": "SSE" - } - ], - + "parameters": [{"name": "encryption-type", "value": "SSE"}], "target_account": "primary", "target_region": "us-east-1", - - } + }, ], - }, { "name": "core", @@ -52,47 +29,25 @@ { "name": "eks", "path": "modules/core/eks/", - - - "parameters": [ { "value_from": { - "module_metadata": { - "name": "networking", - "group": "optionals", - "key": "VpcId" - }, - - - - + "module_metadata": {"name": "networking", "group": "optionals", "key": "VpcId"}, }, "name": "vpc-id", - }, { "value_from": { "module_metadata": { "name": "networking", "group": "optionals", - "key": "PrivateSubnetIds" + "key": "PrivateSubnetIds", }, - - - - }, "name": "private-subnet-ids", - - }, - { - - "name": "eks-admin-role-name", - "value": "Admin" }, + {"name": "eks-admin-role-name", "value": "Admin"}, { - "name": "eks-compute", "value": { "eks_nodegroup_config": [ @@ -102,17 +57,14 @@ "eks_node_max_quantity": 6, "eks_node_min_quantity": 2, "eks_node_disk_size": 50, - "eks_node_instance_types": [ - "m5.large" - ] + "eks_node_instance_types": ["m5.large"], } ], "eks_version": 1.23, - "eks_node_spot": False - } + "eks_node_spot": False, + }, }, { - "name": "eks-addons", "value": { "deploy_aws_lb_controller": True, @@ -128,50 +80,29 @@ "deploy_cloudwatch_container_insights_logs": False, "cloudwatch_container_insights_logs_retention_days": 7, "deploy_amp": False, - "deploy_grafana_for_amp": False - } - } + "deploy_grafana_for_amp": False, + }, + }, ], - "target_account": "primary", "target_region": "us-east-1", - }, { "name": "efs", "path": "modules/core/efs", - - - "parameters": [ { "value_from": { - "module_metadata": { - "name": "networking", - "group": "optionals", - "key": "VpcId" - }, - - - - + "module_metadata": {"name": "networking", "group": "optionals", "key": "VpcId"}, }, "name": "vpc-id", - }, - { - - "name": "removal-policy", - "value": "DESTROY" - } + {"name": "removal-policy", "value": "DESTROY"}, ], - "target_account": "primary", "target_region": "us-east-1", - - } + }, ], - }, { "name": "platform", @@ -180,186 +111,78 @@ { "name": "kubeflow-platform", "path": "modules/mlops/kubeflow-platform/", - - - "parameters": [ { "value_from": { - "module_metadata": { - "name": "eks", - "group": "core", - "key": "EksClusterMasterRoleArn" - }, - - - - + "module_metadata": {"name": "eks", "group": "core", "key": "EksClusterMasterRoleArn"}, }, "name": "EksClusterMasterRoleArn", - }, { "value_from": { - "module_metadata": { - "name": "eks", - "group": "core", - "key": "EksClusterName" - }, - - - - + "module_metadata": {"name": "eks", "group": "core", "key": "EksClusterName"}, }, "name": "EksClusterName", - }, - { - - "name": "InstallationOption", - "value": "kustomize" - }, - { - - "name": "DeploymentOption", - "value": "vanilla" - }, - { - - "name": "KubeflowReleaseVersion", - "value": "v1.6.1" - }, - { - - "name": "AwsKubeflowBuild", - "value": "1.0.0" - } + {"name": "InstallationOption", "value": "kustomize"}, + {"name": "DeploymentOption", "value": "vanilla"}, + {"name": "KubeflowReleaseVersion", "value": "v1.6.1"}, + {"name": "AwsKubeflowBuild", "value": "1.0.0"}, ], - "target_account": "primary", "target_region": "us-east-1", - }, { "name": "efs-on-eks", "path": "modules/integration/efs-on-eks", - - - "parameters": [ { "value_from": { - "module_metadata": { - "name": "eks", - "group": "core", - "key": "EksClusterAdminRoleArn" - }, - - - - + "module_metadata": {"name": "eks", "group": "core", "key": "EksClusterAdminRoleArn"}, }, "name": "eks-cluster-admin-role-arn", - }, { "value_from": { - "module_metadata": { - "name": "eks", - "group": "core", - "key": "EksClusterName" - }, - - - - + "module_metadata": {"name": "eks", "group": "core", "key": "EksClusterName"}, }, "name": "eks-cluster-name", - }, { "value_from": { - "module_metadata": { - "name": "eks", - "group": "core", - "key": "EksOidcArn" - }, - - - - + "module_metadata": {"name": "eks", "group": "core", "key": "EksOidcArn"}, }, "name": "eks-oidc-arn", - }, { "value_from": { - "module_metadata": { - "name": "eks", - "group": "core", - "key": "EksClusterSecurityGroupId" - }, - - - - + "module_metadata": {"name": "eks", "group": "core", "key": "EksClusterSecurityGroupId"}, }, "name": "eks-cluster-security-group-id", - }, { "value_from": { - "module_metadata": { - "name": "efs", - "group": "core", - "key": "EFSFileSystemId" - }, - - - - + "module_metadata": {"name": "efs", "group": "core", "key": "EFSFileSystemId"}, }, "name": "efs-file-system-id", - }, { "value_from": { - "module_metadata": { - "name": "efs", - "group": "core", - "key": "EFSSecurityGroupId" - }, - - - - + "module_metadata": {"name": "efs", "group": "core", "key": "EFSSecurityGroupId"}, }, "name": "efs-security-group-id", - }, { "value_from": { - "module_metadata": { - "name": "efs", - "group": "core", - "key": "VpcId" - }, - - - - + "module_metadata": {"name": "efs", "group": "core", "key": "VpcId"}, }, "name": "vpc-id", - - } + }, ], - "target_account": "primary", "target_region": "us-east-1", - - } + }, ], - }, { "name": "users", @@ -368,109 +191,64 @@ { "name": "kubeflow-users", "path": "modules/mlops/kubeflow-users", - - - "parameters": [ { "value_from": { - "module_metadata": { - "name": "eks", - "group": "core", - "key": "EksClusterAdminRoleArn" - }, - - - - + "module_metadata": {"name": "eks", "group": "core", "key": "EksClusterAdminRoleArn"}, }, "name": "EksClusterAdminRoleArn", - }, { "value_from": { - "module_metadata": { - "name": "eks", - "group": "core", - "key": "EksClusterName" - }, - - - - + "module_metadata": {"name": "eks", "group": "core", "key": "EksClusterName"}, }, "name": "EksClusterName", - }, { "value_from": { - "module_metadata": { - "name": "eks", - "group": "core", - "key": "EksOidcArn" - }, - - - - + "module_metadata": {"name": "eks", "group": "core", "key": "EksOidcArn"}, }, "name": "EksOidcArn", - }, { "value_from": { "module_metadata": { "name": "eks", "group": "core", - "key": "EksClusterOpenIdConnectIssuer" + "key": "EksClusterOpenIdConnectIssuer", }, - - - - }, "name": "EksClusterOpenIdConnectIssuer", - }, { - "name": "KubeflowUsers", "value": [ { "policyArn": "arn:aws:iam::aws:policy/AdministratorAccess", - "secret": "addf-dataservice-users-kubeflow-users-kf-dgraeber" + "secret": "addf-dataservice-users-kubeflow-users-kf-dgraeber", } - ] - } + ], + }, ], - "target_account": "primary", "target_region": "us-east-1", - } ], - - } + }, ], - "target_account_mappings": [ { "alias": "primary", "account_id": "123456789012", "default": True, - "parameters_global": { - "dockerCredentialsSecret": "aws-addf-docker-credentials" - }, + "parameters_global": {"dockerCredentialsSecret": "aws-addf-docker-credentials"}, "region_mappings": [ { "region": "us-east-1", "default": True, "parameters_regional": {}, - - } ], - } - ] -} \ No newline at end of file + ], +} diff --git a/test/unit-test/mock_data/mock_deployspec.py b/test/unit-test/mock_data/mock_deployspec.py index 03806538..72dfaf8c 100644 --- a/test/unit-test/mock_data/mock_deployspec.py +++ b/test/unit-test/mock_data/mock_deployspec.py @@ -21,4 +21,5 @@ build: commands: - echo 'Look Ma....destroying' - """) \ No newline at end of file + """ +) diff --git a/test/unit-test/mock_data/mock_manifests.py b/test/unit-test/mock_data/mock_manifests.py index 5d605bbc..c22f0b1c 100644 --- a/test/unit-test/mock_data/mock_manifests.py +++ b/test/unit-test/mock_data/mock_manifests.py @@ -469,7 +469,8 @@ } ] } - """) + """ +) modules_manifest_duplicate = yaml.safe_load( """ @@ -689,7 +690,8 @@ """ ) -deployspec= yaml.safe_load(""" +deployspec = yaml.safe_load( + """ publishGenericEnvVariables: true deploy: phases: @@ -712,8 +714,11 @@ commands: # execute the CDK - echo 'Look Ma....destroying' - """) + """ +) -sample_metadata = {"GlueDBName": "addl-cicd-core-metadata-storage-vsidata", - "RosbagBagFilePartitionKey": "bag_file_prefix", - "RosbagBagFileTable": "addl-cicd-core-metadata-storage-Rosbag-BagFile-Metadata"} \ No newline at end of file +sample_metadata = { + "GlueDBName": "addl-cicd-core-metadata-storage-vsidata", + "RosbagBagFilePartitionKey": "bag_file_prefix", + "RosbagBagFileTable": "addl-cicd-core-metadata-storage-Rosbag-BagFile-Metadata", +} diff --git a/test/unit-test/mock_data/mock_module_info_huge.py b/test/unit-test/mock_data/mock_module_info_huge.py index 20595f31..961acf6e 100644 --- a/test/unit-test/mock_data/mock_module_info_huge.py +++ b/test/unit-test/mock_data/mock_module_info_huge.py @@ -5,46 +5,24 @@ "phases": { "build": { "commands": [ - "cdk deploy --require-approval never --progress events --app \"python app.py\" --outputs-file ./cdk-exports.json", - "export ADDF_MODULE_METADATA=$(python -c \"import json; file=open('cdk-exports.json'); print(json.load(file)['addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}']['metadata'])\")" - ] - }, - "install": { - "commands": [ - "npm install -g aws-cdk@2.49.1", - "pip install -r requirements.txt" + 'cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json', + "export ADDF_MODULE_METADATA=$(python -c \"import json; file=open('cdk-exports.json'); print(json.load(file)['addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}']['metadata'])\")", ] }, - "post_build": { - "commands": [] - }, - "pre_build": { - "commands": [] - } + "install": {"commands": ["npm install -g aws-cdk@2.49.1", "pip install -r requirements.txt"]}, + "post_build": {"commands": []}, + "pre_build": {"commands": []}, } }, "destroy": { "phases": { - "build": { - "commands": [ - "cdk destroy --force --app \"python app.py\"" - ] - }, - "install": { - "commands": [ - "npm install -g aws-cdk@2.49.1", - "pip install -r requirements.txt" - ] - }, - "post_build": { - "commands": [] - }, - "pre_build": { - "commands": [] - } + "build": {"commands": ['cdk destroy --force --app "python app.py"']}, + "install": {"commands": ["npm install -g aws-cdk@2.49.1", "pip install -r requirements.txt"]}, + "post_build": {"commands": []}, + "pre_build": {"commands": []}, } }, - "publish_generic_env_variables": False + "publish_generic_env_variables": False, }, "/addf/mlops/core/efs/manifest": { "bundle_md5": "fdabb0934b891dcb913ac412f90f9d7a", @@ -55,36 +33,26 @@ { "name": "vpc-id", "value_from": { - "module_metadata": { - "group": "optionals", - "key": "VpcId", - "name": "networking" - }, - } + "module_metadata": {"group": "optionals", "key": "VpcId", "name": "networking"}, + }, }, { "name": "removal-policy", "value": "DESTROY", - } + }, ], "path": "modules/core/efs", "target_account": "primary", - "target_region": "us-east-1" - }, - "/addf/mlops/core/efs/md5/bundle": { - "hash": "fdabb0934b891dcb913ac412f90f9d7a" - }, - "/addf/mlops/core/efs/md5/deployspec": { - "hash": "0e63e3c67f886a8cff15790485b1ae08" - }, - "/addf/mlops/core/efs/md5/manifest": { - "hash": "259ac4decc9055d838f90baf8722f06b" + "target_region": "us-east-1", }, + "/addf/mlops/core/efs/md5/bundle": {"hash": "fdabb0934b891dcb913ac412f90f9d7a"}, + "/addf/mlops/core/efs/md5/deployspec": {"hash": "0e63e3c67f886a8cff15790485b1ae08"}, + "/addf/mlops/core/efs/md5/manifest": {"hash": "259ac4decc9055d838f90baf8722f06b"}, "/addf/mlops/core/efs/metadata": { "EFSFileSystemArn": "arn:aws:elasticfilesystem:us-east-1:123456789012:file-system/fs-0fe786322349dc734", "EFSFileSystemId": "fs-0fe786322349dc734", "EFSSecurityGroupId": "sg-0275b1f7fe988476f", - "VPCId": "vpc-01e556d052f429282" + "VPCId": "vpc-01e556d052f429282", }, "/addf/mlops/core/eks/deployspec": { "build_type": "BUILD_GENERAL1_SMALL", @@ -92,33 +60,24 @@ "phases": { "build": { "commands": [ - "cdk deploy --require-approval never --progress events --app \"python app.py\" --outputs-file ./cdk-exports.json", + 'cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json', "export ADDF_MODULE_METADATA=$(python -c \"import json; file=open('cdk-exports.json'); print(json.load(file)['addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}']['metadata'])\")", - "export CNI_METRICS_ROLE_NAME=$(echo ${ADDF_MODULE_METADATA} | jq -r \".CNIMetricsHelperRoleName\")", + 'export CNI_METRICS_ROLE_NAME=$(echo ${ADDF_MODULE_METADATA} | jq -r ".CNIMetricsHelperRoleName")', "eval $(aws sts assume-role --role-arn arn:aws:iam::${AWS_ACCOUNT_ID}:role/addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}-${AWS_REGION}-masterrole --role-session-name aws-auth-ops | jq -r '.Credentials | \"export AWS_ACCESS_KEY_ID=\\(.AccessKeyId)\\nexport AWS_SECRET_ACCESS_KEY=\\(.SecretAccessKey)\\nexport AWS_SESSION_TOKEN=\\(.SessionToken)\\n\"')", "aws iam get-policy --policy-arn arn:aws:iam::${AWS_ACCOUNT_ID}:policy/AmazonEKSVPCCNIMetricsHelperPolicy || aws iam create-policy --policy-name AmazonEKSVPCCNIMetricsHelperPolicy --policy-document file://addons-iam-policies/cni-metrics-helper-policy.json", "eksctl create iamserviceaccount --name cni-metrics-helper --namespace kube-system --cluster addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}-cluster --role-name ${CNI_METRICS_ROLE_NAME} --attach-policy-arn arn:aws:iam::${AWS_ACCOUNT_ID}:policy/AmazonEKSVPCCNIMetricsHelperPolicy --approve", "curl -o cni-metrics-helper.yaml https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/v1.11.0/config/master/cni-metrics-helper.yaml", - "sed -i.bak -e \"s/us-west-2/$AWS_REGION/\" cni-metrics-helper.yaml", + 'sed -i.bak -e "s/us-west-2/$AWS_REGION/" cni-metrics-helper.yaml', "aws eks update-kubeconfig --name addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}-cluster --region ${AWS_REGION}", "kubectl apply -f cni-metrics-helper.yaml", "kubectl rollout restart deployment cni-metrics-helper -n kube-system", - "if [ -n \"$ADDF_PARAMETER_EKS_ADMIN_ROLE_NAME\" ] ; then\n eksctl get iamidentitymapping --cluster addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}-cluster --arn arn:aws:iam::${AWS_ACCOUNT_ID}:role/${ADDF_PARAMETER_EKS_ADMIN_ROLE_NAME} \\\n && echo \"IAM Identity Mapping already found\" \\\n || eksctl create iamidentitymapping --cluster addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}-cluster --arn arn:aws:iam::${AWS_ACCOUNT_ID}:role/${ADDF_PARAMETER_EKS_ADMIN_ROLE_NAME} --username addf-${ADDF_PARAMETER_EKS_ADMIN_ROLE_NAME} --group system:masters\nfi \n", - "unset AWS_ACCESS_KEY_ID && unset AWS_SECRET_ACCESS_KEY && unset AWS_SESSION_TOKEN" - ] - }, - "install": { - "commands": [ - "npm install -g aws-cdk@2.20.0", - "pip install -r requirements.txt" + 'if [ -n "$ADDF_PARAMETER_EKS_ADMIN_ROLE_NAME" ] ; then\n eksctl get iamidentitymapping --cluster addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}-cluster --arn arn:aws:iam::${AWS_ACCOUNT_ID}:role/${ADDF_PARAMETER_EKS_ADMIN_ROLE_NAME} \\\n && echo "IAM Identity Mapping already found" \\\n || eksctl create iamidentitymapping --cluster addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}-cluster --arn arn:aws:iam::${AWS_ACCOUNT_ID}:role/${ADDF_PARAMETER_EKS_ADMIN_ROLE_NAME} --username addf-${ADDF_PARAMETER_EKS_ADMIN_ROLE_NAME} --group system:masters\nfi \n', + "unset AWS_ACCESS_KEY_ID && unset AWS_SECRET_ACCESS_KEY && unset AWS_SESSION_TOKEN", ] }, - "post_build": { - "commands": [] - }, - "pre_build": { - "commands": [] - } + "install": {"commands": ["npm install -g aws-cdk@2.20.0", "pip install -r requirements.txt"]}, + "post_build": {"commands": []}, + "pre_build": {"commands": []}, } }, "destroy": { @@ -128,67 +87,37 @@ "eval $(aws sts assume-role --role-arn arn:aws:iam::${AWS_ACCOUNT_ID}:role/addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}-${AWS_REGION}-masterrole --role-session-name aws-auth-ops | jq -r '.Credentials | \"export AWS_ACCESS_KEY_ID=\\(.AccessKeyId)\\nexport AWS_SECRET_ACCESS_KEY=\\(.SecretAccessKey)\\nexport AWS_SESSION_TOKEN=\\(.SessionToken)\\n\"')", "eksctl delete iamserviceaccount --name cni-metrics-helper --cluster addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}-cluster --namespace kube-system", "unset AWS_ACCESS_KEY_ID && unset AWS_SECRET_ACCESS_KEY && unset AWS_SESSION_TOKEN", - "cdk destroy --force --app \"python app.py\"" + 'cdk destroy --force --app "python app.py"', ] }, - "install": { - "commands": [ - "npm install -g aws-cdk@2.20.0", - "pip install -r requirements.txt" - ] - }, - "post_build": { - "commands": [] - }, - "pre_build": { - "commands": [] - } + "install": {"commands": ["npm install -g aws-cdk@2.20.0", "pip install -r requirements.txt"]}, + "post_build": {"commands": []}, + "pre_build": {"commands": []}, } }, - "publish_generic_env_variables": False + "publish_generic_env_variables": False, }, "/addf/mlops/core/eks/manifest": { "bundle_md5": "b013f6f640b24eabd246b8f4747ee568", - - "deployspec_md5": "1536833a81c1a855654836b4e74ab289", "manifest_md5": "9991c994dcdf37e8f6a8c56c7173ae84", "name": "eks", "parameters": [ { "name": "vpc-id", - "value_from": { - - "module_metadata": { - "group": "optionals", - "key": "VpcId", - "name": "networking" - }, - - - - } + "module_metadata": {"group": "optionals", "key": "VpcId", "name": "networking"}, + }, }, { "name": "private-subnet-ids", - "value_from": { - - "module_metadata": { - "group": "optionals", - "key": "PrivateSubnetIds", - "name": "networking" - }, - - - - } + "module_metadata": {"group": "optionals", "key": "PrivateSubnetIds", "name": "networking"}, + }, }, { "name": "eks-admin-role-name", "value": "Admin", - }, { "name": "eks-compute", @@ -198,17 +127,14 @@ { "eks_ng_name": "ng1", "eks_node_disk_size": 50, - "eks_node_instance_types": [ - "m5.large" - ], + "eks_node_instance_types": ["m5.large"], "eks_node_max_quantity": 6, "eks_node_min_quantity": 2, - "eks_node_quantity": 3 + "eks_node_quantity": 3, } ], - "eks_version": 1.23 + "eks_version": 1.23, }, - }, { "name": "eks-addons", @@ -226,14 +152,13 @@ "deploy_external_secrets": False, "deploy_grafana_for_amp": False, "deploy_metrics_server": True, - "deploy_secretsmanager_csi": True + "deploy_secretsmanager_csi": True, }, - - } + }, ], "path": "modules/core/eks/", "target_account": "primary", - "target_region": "us-east-1" + "target_region": "us-east-1", }, "/addf/mlops/core/eks/metadata": { "CNIMetricsHelperRoleName": "addf-mlops-core-eks-CNIMetricsHelperRole", @@ -243,116 +168,48 @@ "EksClusterName": "addf-mlops-core-eks-cluster", "EksClusterOpenIdConnectIssuer": "oidc.eks.us-east-1.amazonaws.com/id/84FF84FA3B953B7AA8EEBD37E9D9C9E5", "EksClusterSecurityGroupId": "sg-0b72f310f50faab20", - "EksOidcArn": "arn:aws:iam::123456789012:oidc-provider/oidc.eks.us-east-1.amazonaws.com/id/84FF84FA3B953B7AA8EEBD37E9D9C9E5" + "EksOidcArn": "arn:aws:iam::123456789012:oidc-provider/oidc.eks.us-east-1.amazonaws.com/id/84FF84FA3B953B7AA8EEBD37E9D9C9E5", }, "/addf/mlops/manifest": { - "groups": [ - { - - "modules": [], - "name": "optionals", - "path": "manifests/mlops/optional-modules.yaml" - }, - { - - "modules": [], - "name": "core", - "path": "manifests/mlops/core-modules.yaml" - }, - { - - "modules": [], - "name": "platform", - "path": "manifests/mlops/kf-platform.yaml" - }, - { - - "modules": [], - "name": "users", - "path": "manifests/mlops/kf-users.yaml" - } + {"modules": [], "name": "optionals", "path": "manifests/mlops/optional-modules.yaml"}, + {"modules": [], "name": "core", "path": "manifests/mlops/core-modules.yaml"}, + {"modules": [], "name": "platform", "path": "manifests/mlops/kf-platform.yaml"}, + {"modules": [], "name": "users", "path": "manifests/mlops/kf-users.yaml"}, ], "name": "mlops", - "target_account_mappings": [ { "account_id": "123456789012", "alias": "primary", - "default": True, - "parameters_global": { - "dockerCredentialsSecret": "aws-addf-docker-credentials" - }, - "region_mappings": [ - { - - "default": True, - - "parameters_regional": {}, - "region": "us-east-1" - } - ] + "parameters_global": {"dockerCredentialsSecret": "aws-addf-docker-credentials"}, + "region_mappings": [{"default": True, "parameters_regional": {}, "region": "us-east-1"}], } ], - "toolchain_region": "us-east-1" - }, - "/addf/mlops/core/eks/md5/bundle": { - "hash": "b013f6f640b24eabd246b8f4747ee568" - }, - "/addf/mlops/core/eks/md5/deployspec": { - "hash": "1536833a81c1a855654836b4e74ab289" - }, - "/addf/mlops/core/eks/md5/manifest": { - "hash": "9991c994dcdf37e8f6a8c56c7173ae84" + "toolchain_region": "us-east-1", }, + "/addf/mlops/core/eks/md5/bundle": {"hash": "b013f6f640b24eabd246b8f4747ee568"}, + "/addf/mlops/core/eks/md5/deployspec": {"hash": "1536833a81c1a855654836b4e74ab289"}, + "/addf/mlops/core/eks/md5/manifest": {"hash": "9991c994dcdf37e8f6a8c56c7173ae84"}, "/addf/mlops/manifest/deployed": { - "groups": [ - { - - "name": "optionals", - "path": "manifests/mlops/optional-modules.yaml" - }, - { - - "name": "core", - "path": "manifests/mlops/core-modules.yaml" - }, - { - - "name": "platform", - "path": "manifests/mlops/kf-platform.yaml" - }, - { - - "name": "users", - "path": "manifests/mlops/kf-users.yaml" - } + {"name": "optionals", "path": "manifests/mlops/optional-modules.yaml"}, + {"name": "core", "path": "manifests/mlops/core-modules.yaml"}, + {"name": "platform", "path": "manifests/mlops/kf-platform.yaml"}, + {"name": "users", "path": "manifests/mlops/kf-users.yaml"}, ], "name": "mlops", - "target_account_mappings": [ { "account_id": "123456789012", "alias": "primary", - "default": True, - "parameters_global": { - "dockerCredentialsSecret": "aws-addf-docker-credentials" - }, - "region_mappings": [ - { - - "default": True, - - "parameters_regional": {}, - "region": "us-east-1" - } - ] + "parameters_global": {"dockerCredentialsSecret": "aws-addf-docker-credentials"}, + "region_mappings": [{"default": True, "parameters_regional": {}, "region": "us-east-1"}], } ], - "toolchain_region": "us-east-1" + "toolchain_region": "us-east-1", }, "/addf/mlops/optionals/datalake-buckets/deployspec": { "build_type": "BUILD_GENERAL1_SMALL", @@ -360,51 +217,27 @@ "phases": { "build": { "commands": [ - "cdk deploy --require-approval never --progress events --app \"python app.py\" --outputs-file ./cdk-exports.json", - "export ADDF_MODULE_METADATA=$(python -c \"import json; file=open('cdk-exports.json'); print(json.load(file)['addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}']['metadata'])\")" - ] - }, - "install": { - "commands": [ - "npm install -g aws-cdk@2.20.0", - "pip install -r requirements.txt" + 'cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json', + "export ADDF_MODULE_METADATA=$(python -c \"import json; file=open('cdk-exports.json'); print(json.load(file)['addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}']['metadata'])\")", ] }, - "post_build": { - "commands": [] - }, - "pre_build": { - "commands": [] - } + "install": {"commands": ["npm install -g aws-cdk@2.20.0", "pip install -r requirements.txt"]}, + "post_build": {"commands": []}, + "pre_build": {"commands": []}, } }, "destroy": { "phases": { - "build": { - "commands": [ - "cdk destroy --force --app \"python app.py\"" - ] - }, - "install": { - "commands": [ - "npm install -g aws-cdk@2.20.0", - "pip install -r requirements.txt" - ] - }, - "post_build": { - "commands": [] - }, - "pre_build": { - "commands": [] - } + "build": {"commands": ['cdk destroy --force --app "python app.py"']}, + "install": {"commands": ["npm install -g aws-cdk@2.20.0", "pip install -r requirements.txt"]}, + "post_build": {"commands": []}, + "pre_build": {"commands": []}, } }, - "publish_generic_env_variables": False + "publish_generic_env_variables": False, }, "/addf/mlops/optionals/datalake-buckets/manifest": { "bundle_md5": "a4e00e858e962dbd7a8436fa2e667498", - - "deployspec_md5": "8ab1e223db6e1e9193438b77e65e7233", "manifest_md5": "626d829d21af282c13d9d8ef56997c00", "name": "datalake-buckets", @@ -412,22 +245,15 @@ { "name": "encryption-type", "value": "SSE", - } ], "path": "modules/optionals/datalake-buckets", "target_account": "primary", - "target_region": "us-east-1" - }, - "/addf/mlops/optionals/datalake-buckets/md5/bundle": { - "hash": "a4e00e858e962dbd7a8436fa2e667498" - }, - "/addf/mlops/optionals/datalake-buckets/md5/deployspec": { - "hash": "8ab1e223db6e1e9193438b77e65e7233" - }, - "/addf/mlops/optionals/datalake-buckets/md5/manifest": { - "hash": "626d829d21af282c13d9d8ef56997c00" + "target_region": "us-east-1", }, + "/addf/mlops/optionals/datalake-buckets/md5/bundle": {"hash": "a4e00e858e962dbd7a8436fa2e667498"}, + "/addf/mlops/optionals/datalake-buckets/md5/deployspec": {"hash": "8ab1e223db6e1e9193438b77e65e7233"}, + "/addf/mlops/optionals/datalake-buckets/md5/manifest": {"hash": "626d829d21af282c13d9d8ef56997c00"}, "/addf/mlops/optionals/datalake-buckets/metadata": { "ArtifactsBucketName": "addf-mlops-artifacts-bucket-074ff5b4", "CuratedBucketName": "addf-mlops-curated-bucket-074ff5b4", @@ -435,7 +261,7 @@ "IntermediateBucketName": "addf-mlops-intermediate-bucket-074ff5b4", "LogsBucketName": "addf-mlops-logs-bucket-074ff5b4", "RawBucketName": "addf-mlops-raw-bucket-074ff5b4", - "ReadOnlyPolicyArn": "arn:aws:iam::123456789012:policy/addf-mlops-optionals-datalake-buckets-us-east-1-123456789012-readonly-access" + "ReadOnlyPolicyArn": "arn:aws:iam::123456789012:policy/addf-mlops-optionals-datalake-buckets-us-east-1-123456789012-readonly-access", }, "/addf/mlops/optionals/networking/deployspec": { "build_type": "BUILD_GENERAL1_SMALL", @@ -443,51 +269,27 @@ "phases": { "build": { "commands": [ - "cdk deploy --require-approval never --progress events --app \"python app.py\" --outputs-file ./cdk-exports.json", - "export ADDF_MODULE_METADATA=$(python -c \"import json; file=open('cdk-exports.json'); print(json.load(file)['addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}']['metadata'])\")" - ] - }, - "install": { - "commands": [ - "npm install -g aws-cdk@2.20.0", - "pip install -r requirements.txt" + 'cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json', + "export ADDF_MODULE_METADATA=$(python -c \"import json; file=open('cdk-exports.json'); print(json.load(file)['addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}']['metadata'])\")", ] }, - "post_build": { - "commands": [] - }, - "pre_build": { - "commands": [] - } + "install": {"commands": ["npm install -g aws-cdk@2.20.0", "pip install -r requirements.txt"]}, + "post_build": {"commands": []}, + "pre_build": {"commands": []}, } }, "destroy": { "phases": { - "build": { - "commands": [ - "cdk destroy --force --app \"python app.py\"" - ] - }, - "install": { - "commands": [ - "npm install -g aws-cdk@2.20.0", - "pip install -r requirements.txt" - ] - }, - "post_build": { - "commands": [] - }, - "pre_build": { - "commands": [] - } + "build": {"commands": ['cdk destroy --force --app "python app.py"']}, + "install": {"commands": ["npm install -g aws-cdk@2.20.0", "pip install -r requirements.txt"]}, + "post_build": {"commands": []}, + "pre_build": {"commands": []}, } }, - "publish_generic_env_variables": False + "publish_generic_env_variables": False, }, "/addf/mlops/optionals/networking/manifest": { "bundle_md5": "77f951ba81c10c5dcdf0240ec12ad6a3", - - "deployspec_md5": "063a842e34c45de927f20243bafda4bc", "manifest_md5": "82a3c7cae1c244c3308ca6619c93b144", "name": "networking", @@ -495,33 +297,20 @@ { "name": "internet-accessible", "value": True, - } ], "path": "modules/optionals/networking/", "target_account": "primary", - "target_region": "us-east-1" - }, - "/addf/mlops/optionals/networking/md5/bundle": { - "hash": "77f951ba81c10c5dcdf0240ec12ad6a3" - }, - "/addf/mlops/optionals/networking/md5/deployspec": { - "hash": "063a842e34c45de927f20243bafda4bc" - }, - "/addf/mlops/optionals/networking/md5/manifest": { - "hash": "82a3c7cae1c244c3308ca6619c93b144" + "target_region": "us-east-1", }, + "/addf/mlops/optionals/networking/md5/bundle": {"hash": "77f951ba81c10c5dcdf0240ec12ad6a3"}, + "/addf/mlops/optionals/networking/md5/deployspec": {"hash": "063a842e34c45de927f20243bafda4bc"}, + "/addf/mlops/optionals/networking/md5/manifest": {"hash": "82a3c7cae1c244c3308ca6619c93b144"}, "/addf/mlops/optionals/networking/metadata": { "IsolatedSubnetIds": [], - "PrivateSubnetIds": [ - "subnet-0758c0b5ba97e0fc9", - "subnet-0dc60fe4557261145" - ], - "PublicSubnetIds": [ - "subnet-089b632dada2c71e8", - "subnet-0296fff0ba0fa48c0" - ], - "VpcId": "vpc-01e556d052f429282" + "PrivateSubnetIds": ["subnet-0758c0b5ba97e0fc9", "subnet-0dc60fe4557261145"], + "PublicSubnetIds": ["subnet-089b632dada2c71e8", "subnet-0296fff0ba0fa48c0"], + "VpcId": "vpc-01e556d052f429282", }, "/addf/mlops/platform/efs-on-eks/deployspec": { "build_type": "BUILD_GENERAL1_SMALL", @@ -529,178 +318,85 @@ "phases": { "build": { "commands": [ - "cdk deploy --require-approval never --progress events --app \"python app.py\" --outputs-file ./cdk-exports.json", - "export ADDF_MODULE_METADATA=$(python -c \"import json; file=open('cdk-exports.json'); print(json.load(file)['addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}']['metadata'])\")" - ] - }, - "install": { - "commands": [ - "npm install -g aws-cdk@2.49.1", - "pip install -r requirements.txt" + 'cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json', + "export ADDF_MODULE_METADATA=$(python -c \"import json; file=open('cdk-exports.json'); print(json.load(file)['addf-${ADDF_DEPLOYMENT_NAME}-${ADDF_MODULE_NAME}']['metadata'])\")", ] }, - "post_build": { - "commands": [] - }, - "pre_build": { - "commands": [] - } + "install": {"commands": ["npm install -g aws-cdk@2.49.1", "pip install -r requirements.txt"]}, + "post_build": {"commands": []}, + "pre_build": {"commands": []}, } }, "destroy": { "phases": { - "build": { - "commands": [ - "cdk destroy --force --app \"python app.py\"" - ] - }, - "install": { - "commands": [ - "npm install -g aws-cdk@2.49.1", - "pip install -r requirements.txt" - ] - }, - "post_build": { - "commands": [] - }, - "pre_build": { - "commands": [] - } + "build": {"commands": ['cdk destroy --force --app "python app.py"']}, + "install": {"commands": ["npm install -g aws-cdk@2.49.1", "pip install -r requirements.txt"]}, + "post_build": {"commands": []}, + "pre_build": {"commands": []}, } }, - "publish_generic_env_variables": False + "publish_generic_env_variables": False, }, "/addf/mlops/platform/efs-on-eks/manifest": { "bundle_md5": "3be7473efa2e1699727f16e94d67c9ed", - - "deployspec_md5": "0e63e3c67f886a8cff15790485b1ae08", "manifest_md5": "b51b5eee187ceaddff2af366554a6895", "name": "efs-on-eks", "parameters": [ { "name": "eks-cluster-admin-role-arn", - "value_from": { - - "module_metadata": { - "group": "core", - "key": "EksClusterAdminRoleArn", - "name": "eks" - }, - - - - } + "module_metadata": {"group": "core", "key": "EksClusterAdminRoleArn", "name": "eks"}, + }, }, { "name": "eks-cluster-name", - "value_from": { - - "module_metadata": { - "group": "core", - "key": "EksClusterName", - "name": "eks" - }, - - - - } + "module_metadata": {"group": "core", "key": "EksClusterName", "name": "eks"}, + }, }, { "name": "eks-oidc-arn", - "value_from": { - - "module_metadata": { - "group": "core", - "key": "EksOidcArn", - "name": "eks" - }, - - - - } + "module_metadata": {"group": "core", "key": "EksOidcArn", "name": "eks"}, + }, }, { "name": "eks-cluster-security-group-id", - "value_from": { - - "module_metadata": { - "group": "core", - "key": "EksClusterSecurityGroupId", - "name": "eks" - }, - - - - } + "module_metadata": {"group": "core", "key": "EksClusterSecurityGroupId", "name": "eks"}, + }, }, { "name": "efs-file-system-id", - "value_from": { - - "module_metadata": { - "group": "core", - "key": "EFSFileSystemId", - "name": "efs" - }, - - - - } + "module_metadata": {"group": "core", "key": "EFSFileSystemId", "name": "efs"}, + }, }, { "name": "efs-security-group-id", - "value_from": { - - "module_metadata": { - "group": "core", - "key": "EFSSecurityGroupId", - "name": "efs" - }, - - - - } + "module_metadata": {"group": "core", "key": "EFSSecurityGroupId", "name": "efs"}, + }, }, { "name": "vpc-id", - "value_from": { - - "module_metadata": { - "group": "core", - "key": "VpcId", - "name": "efs" - }, - - - - } - } + "module_metadata": {"group": "core", "key": "VpcId", "name": "efs"}, + }, + }, ], "path": "modules/integration/efs-on-eks", "target_account": "primary", - "target_region": "us-east-1" - }, - "/addf/mlops/platform/efs-on-eks/md5/deployspec": { - "hash": "0e63e3c67f886a8cff15790485b1ae08" + "target_region": "us-east-1", }, + "/addf/mlops/platform/efs-on-eks/md5/deployspec": {"hash": "0e63e3c67f886a8cff15790485b1ae08"}, "/addf/mlops/platform/efs-on-eks/metadata": { "EFSStorageClassName": "platform-efs-on-eks-efs", - "EKSClusterName": "addf-mlops-core-eks-cluster" - }, - "/addf/mlops/platform/efs-on-eks/md5/bundle": { - "hash": "3be7473efa2e1699727f16e94d67c9ed" - }, - "/addf/mlops/platform/efs-on-eks/md5/manifest": { - "hash": "b51b5eee187ceaddff2af366554a6895" + "EKSClusterName": "addf-mlops-core-eks-cluster", }, + "/addf/mlops/platform/efs-on-eks/md5/bundle": {"hash": "3be7473efa2e1699727f16e94d67c9ed"}, + "/addf/mlops/platform/efs-on-eks/md5/manifest": {"hash": "b51b5eee187ceaddff2af366554a6895"}, "/addf/mlops/platform/kubeflow-platform/deployspec": { "build_type": "BUILD_GENERAL1_SMALL", "deploy": { @@ -734,20 +430,12 @@ "helm repo update", "helm upgrade -i nvdp nvdp/nvidia-device-plugin --version=${PLUGIN} --namespace nvidia-device-plugin --create-namespace --set-file config.map.config=gpu/nvidia-plugin-configmap.yaml || True", "unset AWS_ACCESS_KEY_ID && unset AWS_SECRET_ACCESS_KEY && unset AWS_SESSION_TOKEN", - "export ADDF_MODULE_METADATA=\"{'EksClusterName':'${ADDF_PARAMETER_EKS_CLUSTER_NAME}'}\"" + "export ADDF_MODULE_METADATA=\"{'EksClusterName':'${ADDF_PARAMETER_EKS_CLUSTER_NAME}'}\"", ] }, - "install": { - "commands": [ - "bash install_build.sh" - ] - }, - "post_build": { - "commands": [] - }, - "pre_build": { - "commands": [] - } + "install": {"commands": ["bash install_build.sh"]}, + "post_build": {"commands": []}, + "pre_build": {"commands": []}, } }, "destroy": { @@ -779,7 +467,7 @@ "aws eks update-kubeconfig --name ${CLUSTER_NAME}", "helm uninstall nvdp -n nvidia-device-plugin || True", "kubectl get profiles -o json | jq -r '.items[].metadata.name' >> profiles.out", - "for name in $(cat profiles.out); do kubectl patch profile $name --type json -p '{\"metadata\":{\"finalizers\":null}}' --type=merge; done || True", + 'for name in $(cat profiles.out); do kubectl patch profile $name --type json -p \'{"metadata":{"finalizers":null}}\' --type=merge; done || True', "make delete-kubeflow INSTALLATION_OPTION=$INSTALLATION_OPTION DEPLOYMENT_OPTION=$DEPLOYMENT_OPTION", "aws iam detach-role-policy --role-name kf-ack-sm-controller-role-${CLUSTER_NAME} --policy-arn arn:aws:iam::aws:policy/AmazonSageMakerFullAccess || True", "aws iam detach-role-policy --role-name kf-ack-sm-controller-role-${CLUSTER_NAME} --policy-arn arn:aws:iam::${AWS_ACCOUNT_ID}:policy/sm-studio-full-access-${CLUSTER_NAME} || True", @@ -787,99 +475,59 @@ "aws iam delete-role --role-name kf-ack-sm-controller-role-${CLUSTER_NAME} || True", "unset AWS_ACCESS_KEY_ID && unset AWS_SECRET_ACCESS_KEY && unset AWS_SESSION_TOKEN", "cd $ROOT_DIR", - "python manage_admin_user.py delete ${KF_POLICY_NAME} ${ADDF_PARAMETER_EKS_CLUSTER_MASTER_ROLE_ARN}" + "python manage_admin_user.py delete ${KF_POLICY_NAME} ${ADDF_PARAMETER_EKS_CLUSTER_MASTER_ROLE_ARN}", ] }, - "install": { - "commands": [ - "bash install_build.sh" - ] - }, - "post_build": { - "commands": [] - }, - "pre_build": { - "commands": [] - } + "install": {"commands": ["bash install_build.sh"]}, + "post_build": {"commands": []}, + "pre_build": {"commands": []}, } }, - "publish_generic_env_variables": False + "publish_generic_env_variables": False, }, "/addf/mlops/platform/kubeflow-platform/manifest": { "bundle_md5": "24618ab8e830d47166984b06f2df5e3f", - - "deployspec_md5": "d105924122e8abcaf7f560bc815ba602", "manifest_md5": "38324f55655e2915dbbf3efab77b57e2", "name": "kubeflow-platform", "parameters": [ { "name": "EksClusterMasterRoleArn", - "value_from": { - - "module_metadata": { - "group": "core", - "key": "EksClusterMasterRoleArn", - "name": "eks" - }, - - - - } + "module_metadata": {"group": "core", "key": "EksClusterMasterRoleArn", "name": "eks"}, + }, }, { "name": "EksClusterName", - "value_from": { - - "module_metadata": { - "group": "core", - "key": "EksClusterName", - "name": "eks" - }, - - - - } + "module_metadata": {"group": "core", "key": "EksClusterName", "name": "eks"}, + }, }, { "name": "InstallationOption", "value": "kustomize", - }, { "name": "DeploymentOption", "value": "vanilla", - }, { "name": "KubeflowReleaseVersion", "value": "v1.6.1", - }, { "name": "AwsKubeflowBuild", "value": "1.0.0", - - } + }, ], "path": "modules/mlops/kubeflow-platform/", "target_account": "primary", - "target_region": "us-east-1" - }, - "/addf/mlops/platform/kubeflow-platform/md5/bundle": { - "hash": "24618ab8e830d47166984b06f2df5e3f" - }, - "/addf/mlops/platform/kubeflow-platform/md5/deployspec": { - "hash": "d105924122e8abcaf7f560bc815ba602" - }, - "/addf/mlops/platform/kubeflow-platform/md5/manifest": { - "hash": "38324f55655e2915dbbf3efab77b57e2" - }, - "/addf/mlops/platform/kubeflow-platform/metadata": { - "EksClusterName": "addf-mlops-core-eks-cluster" + "target_region": "us-east-1", }, + "/addf/mlops/platform/kubeflow-platform/md5/bundle": {"hash": "24618ab8e830d47166984b06f2df5e3f"}, + "/addf/mlops/platform/kubeflow-platform/md5/deployspec": {"hash": "d105924122e8abcaf7f560bc815ba602"}, + "/addf/mlops/platform/kubeflow-platform/md5/manifest": {"hash": "38324f55655e2915dbbf3efab77b57e2"}, + "/addf/mlops/platform/kubeflow-platform/metadata": {"EksClusterName": "addf-mlops-core-eks-cluster"}, "/addf/mlops/users/kubeflow-users/deployspec": { "build_type": "BUILD_GENERAL1_SMALL", "deploy": { @@ -896,41 +544,26 @@ "wget https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/v3.2.1/kustomize_kustomize.v3.2.1_linux_amd64", "chmod +x kustomize_kustomize.v3.2.1_linux_amd64", "mv kustomize_kustomize.v3.2.1_linux_amd64 /usr/local/bin/kustomize", - "kustomize version" - ] - }, - "post_build": { - "commands": [ - "echo \"Deploy successful\"" + "kustomize version", ] }, - "pre_build": { - "commands": [] - } + "post_build": {"commands": ['echo "Deploy successful"']}, + "pre_build": {"commands": []}, } }, "destroy": { "phases": { "build": { "commands": [ - "if [[ ${ADDF_PARAMETER_KUBEFLOW_USERS} ]]; then\n cdk destroy --force --app \"python app.py\";\nfi;\n" - ] - }, - "install": { - "commands": [ - "npm install -g aws-cdk@2.20.0", - "pip install -r requirements.txt" + 'if [[ ${ADDF_PARAMETER_KUBEFLOW_USERS} ]]; then\n cdk destroy --force --app "python app.py";\nfi;\n' ] }, - "post_build": { - "commands": [] - }, - "pre_build": { - "commands": [] - } + "install": {"commands": ["npm install -g aws-cdk@2.20.0", "pip install -r requirements.txt"]}, + "post_build": {"commands": []}, + "pre_build": {"commands": []}, } }, - "publish_generic_env_variables": False + "publish_generic_env_variables": False, }, "/addf/mlops/users/kubeflow-users/metadata": { "EksClusterName": "addf-mlops-core-eks-cluster", @@ -938,100 +571,55 @@ { "policyArn": "arn:aws:iam::aws:policy/AdministratorAccess", "roleArn": "arn:aws:iam::123456789012:role/addf-mlops-users-kubeflow-users-us-east-1-0", - "secret": "addf-dataservice-users-kubeflow-users-kf-dgraeber" + "secret": "addf-dataservice-users-kubeflow-users-kf-dgraeber", } - ] + ], }, "/addf/mlops/users/kubeflow-users/manifest": { "bundle_md5": "03c4cce1b534053ab2e9907c00ffef3e", - - "deployspec_md5": "b13401fb18d61964e1e39e2a1474a205", "manifest_md5": "d3e04a0cffa57cef83a57fb4f077ad50", "name": "kubeflow-users", "parameters": [ { "name": "EksClusterAdminRoleArn", - "value_from": { - - "module_metadata": { - "group": "core", - "key": "EksClusterAdminRoleArn", - "name": "eks" - }, - - - - } + "module_metadata": {"group": "core", "key": "EksClusterAdminRoleArn", "name": "eks"}, + }, }, { "name": "EksClusterName", - "value_from": { - - "module_metadata": { - "group": "core", - "key": "EksClusterName", - "name": "eks" - }, - - - - } + "module_metadata": {"group": "core", "key": "EksClusterName", "name": "eks"}, + }, }, { "name": "EksOidcArn", - "value_from": { - - "module_metadata": { - "group": "core", - "key": "EksOidcArn", - "name": "eks" - }, - - - - } + "module_metadata": {"group": "core", "key": "EksOidcArn", "name": "eks"}, + }, }, { "name": "EksClusterOpenIdConnectIssuer", - "value_from": { - - "module_metadata": { - "group": "core", - "key": "EksClusterOpenIdConnectIssuer", - "name": "eks" - }, - - - - } + "module_metadata": {"group": "core", "key": "EksClusterOpenIdConnectIssuer", "name": "eks"}, + }, }, { "name": "KubeflowUsers", "value": [ { "policyArn": "arn:aws:iam::aws:policy/AdministratorAccess", - "secret": "addf-dataservice-users-kubeflow-users-kf-dgraeber" + "secret": "addf-dataservice-users-kubeflow-users-kf-dgraeber", } ], - - } + }, ], "path": "modules/mlops/kubeflow-users", "target_account": "primary", - "target_region": "us-east-1" - }, - "/addf/mlops/users/kubeflow-users/md5/bundle": { - "hash": "03c4cce1b534053ab2e9907c00ffef3e" - }, - "/addf/mlops/users/kubeflow-users/md5/deployspec": { - "hash": "b13401fb18d61964e1e39e2a1474a205" + "target_region": "us-east-1", }, - "/addf/mlops/users/kubeflow-users/md5/manifest": { - "hash": "d3e04a0cffa57cef83a57fb4f077ad50" - } -} \ No newline at end of file + "/addf/mlops/users/kubeflow-users/md5/bundle": {"hash": "03c4cce1b534053ab2e9907c00ffef3e"}, + "/addf/mlops/users/kubeflow-users/md5/deployspec": {"hash": "b13401fb18d61964e1e39e2a1474a205"}, + "/addf/mlops/users/kubeflow-users/md5/manifest": {"hash": "d3e04a0cffa57cef83a57fb4f077ad50"}, +} diff --git a/test/unit-test/test_cli_arg.py b/test/unit-test/test_cli_arg.py index e251ce41..cdb41b2e 100644 --- a/test/unit-test/test_cli_arg.py +++ b/test/unit-test/test_cli_arg.py @@ -15,21 +15,19 @@ import logging import os +import mock_data.mock_manifests as mock_manifests import pytest from _test_cli_helper_functions import _test_command +from moto import mock_sts from seedfarmer import config from seedfarmer.__main__ import apply, bootstrap, destroy, init from seedfarmer.__main__ import list as list from seedfarmer.__main__ import projectpolicy, remove, store, version +from seedfarmer.models._deploy_spec import DeploySpec +from seedfarmer.models.manifests import DeploymentManifest, ModulesManifest from seedfarmer.services._service_utils import boto3_client from seedfarmer.services.session_manager import SessionManager -from seedfarmer.models.manifests import DeploymentManifest, ModulesManifest -from seedfarmer.models._deploy_spec import DeploySpec -import mock_data.mock_manifests as mock_manifests - -from moto import mock_sts - # Override OPS_ROOT to reflect path of resource policy needed for some testing # _OPS_ROOT = config.OPS_ROOT @@ -39,7 +37,6 @@ _logger: logging.Logger = logging.getLogger(__name__) - @pytest.fixture(scope="function") def aws_credentials(): """Mocked AWS Credentials for moto.""" @@ -50,14 +47,16 @@ def aws_credentials(): os.environ["AWS_DEFAULT_REGION"] = "us-east-1" os.environ["MOTO_ACCOUNT_ID"] = "123456789012" + @pytest.fixture(scope="function") def sts_client(aws_credentials): with mock_sts(): yield boto3_client(service_name="sts", session=None) -@pytest.fixture(scope="function") + +@pytest.fixture(scope="function") def session_manager(sts_client): - SessionManager._instances={} + SessionManager._instances = {} SessionManager().get_or_create( project_name="test", region_name="us-east-1", @@ -66,7 +65,6 @@ def session_manager(sts_client): ) - # ------------------------------------------- # ----- Test the sub-command `init` ----- # ------------------------------------------- @@ -94,26 +92,18 @@ def test_init_create_group_module(mocker): module_name = "test-module" group_name = "group" - + mocker.patch("seedfarmer.cli_groups._init_group.minit.create_module_dir", return_value=None) # Creates a group and a module within the group - _test_command(sub_command=init, - options=["module", - "-g", group_name, - "-m", module_name, - "--debug"], - exit_code=0) + _test_command(sub_command=init, options=["module", "-g", group_name, "-m", module_name, "--debug"], exit_code=0) @pytest.mark.init def test_init_create_project(mocker): mocker.patch("seedfarmer.cli_groups._init_group.minit.create_project", return_value=None) - _test_command(sub_command=init, - options=["project"], - exit_code=0) - + _test_command(sub_command=init, options=["project"], exit_code=0) # # ------------------------------------------- @@ -185,7 +175,6 @@ def test_destroy_deployment(mocker): command_output = _test_command(sub_command=destroy, options=["myapp", "--debug"], exit_code=0) - @pytest.mark.bootstrap def test_bootstrap_toolchain_only(mocker): # Bootstrap an Account As Target @@ -198,6 +187,7 @@ def test_bootstrap_toolchain_only(mocker): exit_code=0, ) + @pytest.mark.bootstrap def test_bootstrap_target_account(mocker): # Bootstrap an Account As Target @@ -225,7 +215,6 @@ def test_apply_missing_deployment_group_name(): assert result.exception.args[0][0][0].exc.errors()[0]["msg"] == "none is not an allowed value" - @pytest.mark.apply def test_apply_missing_deployment_name(): deployment_manifest = f"{_TEST_ROOT}/manifests/test-missing-deployment-name/deployment.yaml" @@ -234,18 +223,19 @@ def test_apply_missing_deployment_name(): assert result.exception.args[0] == "One of 'name' or 'name_generator' is required" - # ------------------------------------------- # ----- Test the sub-command `list` ----- # ------------------------------------------- # Test Deployspec + @pytest.mark.list def test_error_messaging(): import seedfarmer.cli_groups._list_group as lg - - lg._error_messaging(deployment='test-dep',group='test-group',module='test-module') + + lg._error_messaging(deployment="test-dep", group="test-group", module="test-module") + @pytest.mark.list def test_list_help(): @@ -256,112 +246,162 @@ def test_list_help(): expected_output="List the relative data (module or deployment", ) + @pytest.mark.list @pytest.mark.list_deployspec @pytest.mark.parametrize("session", [None, boto3_client]) -def test_list_deployspec_deployed_error(session,mocker): +def test_list_deployspec_deployed_error(session, mocker): mocker.patch("seedfarmer.cli_groups._list_group.mi.get_deployspec", return_value=None) mocker.patch("seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", return_value=None) _test_command( list, - options=["deployspec", - "-d","crazystuff", - "-g","test-group", - "-m","test-module", - "-p","something", - "--debug", - ], - exit_code=1 + options=[ + "deployspec", + "-d", + "crazystuff", + "-g", + "test-group", + "-m", + "test-module", + "-p", + "something", + "--debug", + ], + exit_code=1, ) @pytest.mark.list @pytest.mark.list_deployspec -def test_list_deployspec_deployed_none(session_manager,mocker): +def test_list_deployspec_deployed_none(session_manager, mocker): mocker.patch("seedfarmer.cli_groups._list_group.mi.get_deployspec", return_value=None) mocker.patch("seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", return_value=None) _test_command( list, - options=["deployspec", - "-d","test", - "-g","test-group", - "-m","test-module", - "-p","myapp", - "--debug", - ], - exit_code=0 + options=[ + "deployspec", + "-d", + "test", + "-g", + "test-group", + "-m", + "test-module", + "-p", + "myapp", + "--debug", + ], + exit_code=0, ) + @pytest.mark.list @pytest.mark.list_deployspec -def test_list_deployspec_missing_session(session_manager,mocker): - mocker.patch("seedfarmer.cli_groups._list_group.mi.get_deployspec", return_value={"deploy":{"commands":"echo"}}) - mocker.patch("seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", return_value=(DeploymentManifest(**mock_manifests.deployment_manifest))) - mocker.patch("seedfarmer.cli_groups._list_group.mi.get_deployspec",return_value=DeploySpec(**mock_manifests.deployspec)) +def test_list_deployspec_missing_session(session_manager, mocker): + mocker.patch("seedfarmer.cli_groups._list_group.mi.get_deployspec", return_value={"deploy": {"commands": "echo"}}) + mocker.patch( + "seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", + return_value=(DeploymentManifest(**mock_manifests.deployment_manifest)), + ) + mocker.patch( + "seedfarmer.cli_groups._list_group.mi.get_deployspec", return_value=DeploySpec(**mock_manifests.deployspec) + ) _test_command( list, - options=["deployspec", - "-d","myapp", - "-g","messsedup", - "-m","networking", - "-p","myapp", - "--debug", - ], - exit_code=0 + options=[ + "deployspec", + "-d", + "myapp", + "-g", + "messsedup", + "-m", + "networking", + "-p", + "myapp", + "--debug", + ], + exit_code=0, ) + @pytest.mark.list @pytest.mark.list_deployspec -def test_list_deployspec(session_manager,mocker): - mocker.patch("seedfarmer.cli_groups._list_group.mi.get_deployspec", return_value={"deploy":{"commands":"echo"}}) - mocker.patch("seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", return_value=(DeploymentManifest(**mock_manifests.deployment_manifest))) - mocker.patch("seedfarmer.cli_groups._list_group.mi.get_deployspec",return_value=DeploySpec(**mock_manifests.deployspec)) +def test_list_deployspec(session_manager, mocker): + mocker.patch("seedfarmer.cli_groups._list_group.mi.get_deployspec", return_value={"deploy": {"commands": "echo"}}) + mocker.patch( + "seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", + return_value=(DeploymentManifest(**mock_manifests.deployment_manifest)), + ) + mocker.patch( + "seedfarmer.cli_groups._list_group.mi.get_deployspec", return_value=DeploySpec(**mock_manifests.deployspec) + ) _test_command( list, - options=["deployspec", - "-d","myapp", - "-g","optionals", - "-m","networking", - "-p","myapp", - "--debug", - ], - exit_code=0 - ) - + options=[ + "deployspec", + "-d", + "myapp", + "-g", + "optionals", + "-m", + "networking", + "-p", + "myapp", + "--debug", + ], + exit_code=0, + ) + + # Test list dependencies @pytest.mark.list @pytest.mark.list_dependencies -def test_list_dependencies_no_deployed_manifest(session_manager,mocker): +def test_list_dependencies_no_deployed_manifest(session_manager, mocker): mocker.patch("seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", return_value=None) mocker.patch("seedfarmer.cli_groups._list_group.du.generate_dependency_maps", return_value=None) _test_command( list, - options=["dependencies", - "-d","test", - "-g","test-group", - "-m","test-module", - "-p","myapp", - "--debug", - ], - exit_code=0 + options=[ + "dependencies", + "-d", + "test", + "-g", + "test-group", + "-m", + "test-module", + "-p", + "myapp", + "--debug", + ], + exit_code=0, ) + @pytest.mark.list @pytest.mark.list_dependencies -def test_list_dependencies(session_manager,mocker): - mocker.patch("seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", return_value=(DeploymentManifest(**mock_manifests.deployment_manifest))) +def test_list_dependencies(session_manager, mocker): + mocker.patch( + "seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", + return_value=(DeploymentManifest(**mock_manifests.deployment_manifest)), + ) mocker.patch("seedfarmer.cli_groups._list_group.du.generate_dependency_maps", return_value=None) _test_command( list, - options=["dependencies", - "-d","test", - "-g","test-group", - "-m","test-module", - "-p","myapp", - "--debug", - ], - exit_code=1 + options=[ + "dependencies", + "-d", + "test", + "-g", + "test-group", + "-m", + "test-module", + "-p", + "myapp", + "--debug", + ], + exit_code=1, ) + + # # Test `list deployments` # @@ -379,7 +419,7 @@ def test_list_deployments_help(): @pytest.mark.list @pytest.mark.list_deployments def test_list_deployments_extra_args(): - + _test_command( sub_command=list, options=[ @@ -393,11 +433,11 @@ def test_list_deployments_extra_args(): @pytest.mark.list @pytest.mark.list_deployments -def test_list_deployments(session_manager,mocker): +def test_list_deployments(session_manager, mocker): mocker.patch("seedfarmer.cli_groups._list_group.mi.get_all_deployments", return_value=None) _test_command( sub_command=list, - options=["deployments", "-p","myapp", "--debug"], + options=["deployments", "-p", "myapp", "--debug"], exit_code=1, ) @@ -432,81 +472,117 @@ def test_list_moduledata_missing_deployment_option(): @pytest.mark.list @pytest.mark.list_moduledata -def test_list_moduledata_no_dep_manifest(session_manager,mocker): +def test_list_moduledata_no_dep_manifest(session_manager, mocker): mocker.patch("seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", return_value=None) _test_command( sub_command=list, options=[ "moduledata", - "-d","test", - "-g","test-group", - "-m","test-module", - "-p","myapp", + "-d", + "test", + "-g", + "test-group", + "-m", + "test-module", + "-p", + "myapp", "--debug", - ], + ], exit_code=0, ) + @pytest.mark.list @pytest.mark.list_moduledata -def test_list_moduledata(session_manager,mocker): - mocker.patch("seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", return_value=(DeploymentManifest(**mock_manifests.deployment_manifest))) - mocker.patch("seedfarmer.cli_groups._list_group.mi.get_module_metadata",return_value=mock_manifests.sample_metadata) +def test_list_moduledata(session_manager, mocker): + mocker.patch( + "seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", + return_value=(DeploymentManifest(**mock_manifests.deployment_manifest)), + ) + mocker.patch( + "seedfarmer.cli_groups._list_group.mi.get_module_metadata", return_value=mock_manifests.sample_metadata + ) _test_command( sub_command=list, options=[ "moduledata", - "-d","test", - "-g","optionals", - "-m","networking", - "-p","myapp", + "-d", + "test", + "-g", + "optionals", + "-m", + "networking", + "-p", + "myapp", "--debug", - ], + ], exit_code=0, ) - + + @pytest.mark.list @pytest.mark.list_moduledata -def test_list_moduledata_export_envs(session_manager,mocker): - mocker.patch("seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", return_value=(DeploymentManifest(**mock_manifests.deployment_manifest))) - mocker.patch("seedfarmer.cli_groups._list_group.mi.get_module_metadata",return_value=mock_manifests.sample_metadata) - mocker.patch("seedfarmer.cli_groups._list_group.commands.generate_export_env_params",return_value=["export SOMETHING=yo"]) +def test_list_moduledata_export_envs(session_manager, mocker): + mocker.patch( + "seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", + return_value=(DeploymentManifest(**mock_manifests.deployment_manifest)), + ) + mocker.patch( + "seedfarmer.cli_groups._list_group.mi.get_module_metadata", return_value=mock_manifests.sample_metadata + ) + mocker.patch( + "seedfarmer.cli_groups._list_group.commands.generate_export_env_params", return_value=["export SOMETHING=yo"] + ) _test_command( sub_command=list, options=[ "moduledata", - "-d","test", - "-g","optionals", - "-m","networking", - "-p","myapp", + "-d", + "test", + "-g", + "optionals", + "-m", + "networking", + "-p", + "myapp", "--export-local-env", "--debug", - ], - exit_code=0 + ], + exit_code=0, ) - - + + @pytest.mark.list @pytest.mark.list_moduledata -def test_list_moduledata_mod_not_found(session_manager,mocker): - mocker.patch("seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", return_value=(DeploymentManifest(**mock_manifests.deployment_manifest))) - mocker.patch("seedfarmer.cli_groups._list_group.mi.get_module_metadata",return_value=mock_manifests.sample_metadata) +def test_list_moduledata_mod_not_found(session_manager, mocker): + mocker.patch( + "seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", + return_value=(DeploymentManifest(**mock_manifests.deployment_manifest)), + ) + mocker.patch( + "seedfarmer.cli_groups._list_group.mi.get_module_metadata", return_value=mock_manifests.sample_metadata + ) _test_command( sub_command=list, options=[ "moduledata", - "-d","test", - "-g","somethingcrazy", - "-m","networking", - "-p","myapp", + "-d", + "test", + "-g", + "somethingcrazy", + "-m", + "networking", + "-p", + "myapp", "--debug", - ], + ], exit_code=0, ) + @pytest.mark.list @pytest.mark.list_moduledata def test_list_moduledata_missing_deployment_arg(): @@ -525,8 +601,6 @@ def test_list_moduledata_missing_group_arg(): _test_command(sub_command=list, options=["moduledata", "-d", "test-deployment", "-g"], exit_code=2) - - @pytest.mark.list @pytest.mark.list_moduledata def test_list_moduledata_missing_module_arg(): @@ -548,21 +622,29 @@ def test_list_moduledata_non_existent_module(): @pytest.mark.list @pytest.mark.list_moduledata -def test_list_all_moduledata(session_manager,mocker): - mocker.patch("seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", return_value=(DeploymentManifest(**mock_manifests.deployment_manifest))) - mocker.patch("seedfarmer.cli_groups._list_group.mi.get_module_metadata",return_value=mock_manifests.sample_metadata) +def test_list_all_moduledata(session_manager, mocker): + mocker.patch( + "seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", + return_value=(DeploymentManifest(**mock_manifests.deployment_manifest)), + ) + mocker.patch( + "seedfarmer.cli_groups._list_group.mi.get_module_metadata", return_value=mock_manifests.sample_metadata + ) _test_command( sub_command=list, options=[ "allmoduledata", - "-d","test", - "-p","myapp", + "-d", + "test", + "-p", + "myapp", "--debug", - ], + ], exit_code=0, ) + # # Test `list modules` # @@ -604,130 +686,192 @@ def test_list_modules_non_existent_module(): @pytest.mark.list @pytest.mark.list_modules def test_list_modules(session_manager, mocker): - mocker.patch("seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", return_value=(DeploymentManifest(**mock_manifests.deployment_manifest))) + mocker.patch( + "seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", + return_value=(DeploymentManifest(**mock_manifests.deployment_manifest)), + ) _test_command( - sub_command=list, - options=["modules", - "-p", "myapp", - "-d", "example-test-dev", - "--debug"], - exit_code=0 + sub_command=list, options=["modules", "-p", "myapp", "-d", "example-test-dev", "--debug"], exit_code=0 ) - @pytest.mark.list @pytest.mark.list_build_env_params def test_list_build_env_params(session_manager, mocker): - mocker.patch("seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", return_value=(DeploymentManifest(**mock_manifests.deployment_manifest))) + mocker.patch( + "seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", + return_value=(DeploymentManifest(**mock_manifests.deployment_manifest)), + ) _test_command( sub_command=list, - options=["buildparams", - "-d","test", - "-g","optionals", - "-m","networking", - "--build-id","codebuild:12345", - "--export-local-env", - "-p","myapp", - "--debug", - ], - exit_code=0 + options=[ + "buildparams", + "-d", + "test", + "-g", + "optionals", + "-m", + "networking", + "--build-id", + "codebuild:12345", + "--export-local-env", + "-p", + "myapp", + "--debug", + ], + exit_code=0, ) + @pytest.mark.list @pytest.mark.list_build_env_params def test_list_build_env_params_no_dep_manifest(session_manager, mocker): mocker.patch("seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", return_value=None) _test_command( sub_command=list, - options=["buildparams", - "-d","test", - "-g","somethingcrazy", - "-m","networking", - "--build-id","codebuild:12345", - "--export-local-env", - "-p","myapp", - "--debug", - ], - exit_code=0 + options=[ + "buildparams", + "-d", + "test", + "-g", + "somethingcrazy", + "-m", + "networking", + "--build-id", + "codebuild:12345", + "--export-local-env", + "-p", + "myapp", + "--debug", + ], + exit_code=0, ) + @pytest.mark.list @pytest.mark.list_build_env_params def test_list_build_env_params_no_with_params(session_manager, mocker): mocker.patch("seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", return_value=None) - mocker.patch("seedfarmer.cli_groups._list_group.bi.get_build_env_params", return_value={"SEEDFARMER_PARAMETER":"AGreatName"}) + mocker.patch( + "seedfarmer.cli_groups._list_group.bi.get_build_env_params", return_value={"SEEDFARMER_PARAMETER": "AGreatName"} + ) _test_command( sub_command=list, - options=["buildparams", - "-d","test", - "-g","optionals", - "-m","networking", - "--build-id","codebuild:12345", - "--export-local-env", - "-p","myapp", - "--debug", - ], - exit_code=0 - ) - + options=[ + "buildparams", + "-d", + "test", + "-g", + "optionals", + "-m", + "networking", + "--build-id", + "codebuild:12345", + "--export-local-env", + "-p", + "myapp", + "--debug", + ], + exit_code=0, + ) + + @pytest.mark.list @pytest.mark.list_build_env_params def test_list_build_env_params_session_error(session_manager, mocker): - mocker.patch("seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", return_value=(DeploymentManifest(**mock_manifests.deployment_manifest))) - mocker.patch("seedfarmer.cli_groups._list_group.bi.get_build_env_params", return_value={"SEEDFARMER_PARAMETER":"AGreatName"}) + mocker.patch( + "seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", + return_value=(DeploymentManifest(**mock_manifests.deployment_manifest)), + ) + mocker.patch( + "seedfarmer.cli_groups._list_group.bi.get_build_env_params", return_value={"SEEDFARMER_PARAMETER": "AGreatName"} + ) _test_command( sub_command=list, - options=["buildparams", - "-d","test", - "-g","fsafasf", - "-m","networking", - "--build-id","codebuild:12345", - "--export-local-env", - "-p","myapp", - "--debug", - ], - exit_code=0 + options=[ + "buildparams", + "-d", + "test", + "-g", + "fsafasf", + "-m", + "networking", + "--build-id", + "codebuild:12345", + "--export-local-env", + "-p", + "myapp", + "--debug", + ], + exit_code=0, ) + @pytest.mark.list @pytest.mark.list_build_env_params def test_list_build_env_params_no_export_param(session_manager, mocker): - mocker.patch("seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", return_value=(DeploymentManifest(**mock_manifests.deployment_manifest))) - mocker.patch("seedfarmer.cli_groups._list_group.bi.get_build_env_params", return_value={"SEEDFARMER_PARAMETER":"AGreatName"}) + mocker.patch( + "seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", + return_value=(DeploymentManifest(**mock_manifests.deployment_manifest)), + ) + mocker.patch( + "seedfarmer.cli_groups._list_group.bi.get_build_env_params", return_value={"SEEDFARMER_PARAMETER": "AGreatName"} + ) _test_command( sub_command=list, - options=["buildparams", - "-d","test", - "-g","optionals", - "-m","networking", - "--build-id","codebuild:12345", - "-p","myapp", - "--debug", - ], - exit_code=0 + options=[ + "buildparams", + "-d", + "test", + "-g", + "optionals", + "-m", + "networking", + "--build-id", + "codebuild:12345", + "-p", + "myapp", + "--debug", + ], + exit_code=0, ) + @pytest.mark.list @pytest.mark.list_build_env_params def test_list_build_env_params_export_param(session_manager, mocker): - mocker.patch("seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", return_value=(DeploymentManifest(**mock_manifests.deployment_manifest))) - mocker.patch("seedfarmer.cli_groups._list_group.bi.get_build_env_params", return_value={"SEEDFARMER_PARAMETER":"AGreatName"}) - mocker.patch("seedfarmer.cli_groups._list_group.commands.generate_export_raw_env_params",return_value={"SEEDFARMER_PARAMETER":"AGreatName"}) + mocker.patch( + "seedfarmer.cli_groups._list_group.du.generate_deployed_manifest", + return_value=(DeploymentManifest(**mock_manifests.deployment_manifest)), + ) + mocker.patch( + "seedfarmer.cli_groups._list_group.bi.get_build_env_params", return_value={"SEEDFARMER_PARAMETER": "AGreatName"} + ) + mocker.patch( + "seedfarmer.cli_groups._list_group.commands.generate_export_raw_env_params", + return_value={"SEEDFARMER_PARAMETER": "AGreatName"}, + ) _test_command( sub_command=list, - options=["buildparams", - "-d","test", - "-g","optionals", - "-m","networking", - "--build-id","codebuild:12345", - "--export-local-env", - "-p","myapp", - "--debug", - ], - exit_code=0 + options=[ + "buildparams", + "-d", + "test", + "-g", + "optionals", + "-m", + "networking", + "--build-id", + "codebuild:12345", + "--export-local-env", + "-p", + "myapp", + "--debug", + ], + exit_code=0, ) + # ------------------------------------------- # ----- Test the sub-command `remove` ----- # ------------------------------------------- @@ -783,32 +927,48 @@ def test_remove_missing_module_arg(): @pytest.mark.remove -def test_remove_moduledata(session_manager,mocker): +def test_remove_moduledata(session_manager, mocker): mocker.patch("seedfarmer.cli_groups._list_group.mi.remove_module_info", return_value=None) _test_command( - sub_command=remove, options=["moduledata", - "-d", "deployment-name", - "-g", "group-name", - "-m", "mymodule", - "--target-account-id","123456789012", - "--target-region","us-east-1" - "--debug"], - exit_code=0 + sub_command=remove, + options=[ + "moduledata", + "-d", + "deployment-name", + "-g", + "group-name", + "-m", + "mymodule", + "--target-account-id", + "123456789012", + "--target-region", + "us-east-1" "--debug", + ], + exit_code=0, ) + @pytest.mark.remove -def test_remove_moduledata_bad_account_params(session_manager,mocker): +def test_remove_moduledata_bad_account_params(session_manager, mocker): mocker.patch("seedfarmer.cli_groups._list_group.mi.remove_module_info", return_value=None) _test_command( - sub_command=remove, options=["moduledata", - "-d", "deployment-name", - "-g", "group-name", - "-m", "mymodule", - "--target-account-id","123456789012", - "--debug"], - exit_code=1 + sub_command=remove, + options=[ + "moduledata", + "-d", + "deployment-name", + "-g", + "group-name", + "-m", + "mymodule", + "--target-account-id", + "123456789012", + "--debug", + ], + exit_code=1, ) + # ------------------------------------------- # ----- Test the sub-command `store` ----- # ------------------------------------------- @@ -864,14 +1024,7 @@ def test_store_md5_missing_module_option(): @pytest.mark.store @pytest.mark.store_md5 def test_store_md5_missing_module_arg(): - _test_command(sub_command=store, - options=["md5", - "-d", "deployment-name", - "-g", "group-name", - "-m"], - exit_code=2 - ) - + _test_command(sub_command=store, options=["md5", "-d", "deployment-name", "-g", "group-name", "-m"], exit_code=2) @pytest.mark.store @@ -975,34 +1128,49 @@ def test_store_moduledata_missing_module_arg(): @pytest.mark.store @pytest.mark.store_moduledata def test_store_moduledata(mocker, session_manager): - mocker.patch("seedfarmer.cli_groups._list_group.mi.write_metadata",return_value=None) - + mocker.patch("seedfarmer.cli_groups._list_group.mi.write_metadata", return_value=None) + _test_command( sub_command=store, - options=["moduledata", - "-d", "deployment-name", - "-g", "group-name", - "-m", "module-data", - "--project", "myapp", - "--debug"], + options=[ + "moduledata", + "-d", + "deployment-name", + "-g", + "group-name", + "-m", + "module-data", + "--project", + "myapp", + "--debug", + ], exit_code=0, ) + @pytest.mark.store @pytest.mark.store_moduledata def test_store_moduledata_with_account(mocker, session_manager): - mocker.patch("seedfarmer.cli_groups._list_group.mi.write_metadata",return_value=None) - + mocker.patch("seedfarmer.cli_groups._list_group.mi.write_metadata", return_value=None) + _test_command( sub_command=store, - options=["moduledata", - "-d", "deployment-name", - "-g", "group-name", - "-m", "module-data", - "--target-account-id","123456789012", - "--target-region","us-east-1", - "--project", "myapp", - "--debug"], + options=[ + "moduledata", + "-d", + "deployment-name", + "-g", + "group-name", + "-m", + "module-data", + "--target-account-id", + "123456789012", + "--target-region", + "us-east-1", + "--project", + "myapp", + "--debug", + ], exit_code=0, ) @@ -1067,13 +1235,18 @@ def test_store_deployspec_missing_acct(): sub_command=store, options=[ "deployspec", - "-d", "deployment-name", - "-g", "group-name", - "-m", "module", - "--project", "myapp", - "--path", "module/test/test" - "--target_region","us-east-1", - "--debug" + "-d", + "deployment-name", + "-g", + "group-name", + "-m", + "module", + "--project", + "myapp", + "--path", + "module/test/test" "--target_region", + "us-east-1", + "--debug", ], exit_code=2, ) @@ -1083,23 +1256,31 @@ def test_store_deployspec_missing_acct(): @pytest.mark.store_deployspec @pytest.mark.parametrize("session", [None]) def test_store_deployspec(session_manager, session): - + _test_command( sub_command=store, options=[ "deployspec", - "-d","deployment-name", - "-g","group-name", - "-m","module", - "--project","myapp", - "--path","module/test/test", - "--target-account-id","123456789012", - "--target-region","us-east-1", + "-d", + "deployment-name", + "-g", + "group-name", + "-m", + "module", + "--project", + "myapp", + "--path", + "module/test/test", + "--target-account-id", + "123456789012", + "--target-region", + "us-east-1", "--debug", ], exit_code=1, ) + @pytest.mark.projectpolicy def test_get_projectpolicy(): diff --git a/test/unit-test/test_commands_bootstrap.py b/test/unit-test/test_commands_bootstrap.py index 5c332274..f5f5ccbf 100644 --- a/test/unit-test/test_commands_bootstrap.py +++ b/test/unit-test/test_commands_bootstrap.py @@ -1,18 +1,18 @@ import logging import os + import boto3 +import pytest import yaml +from moto import mock_sts -import pytest import seedfarmer import seedfarmer.commands._bootstrap_commands as bc - +import seedfarmer.errors from seedfarmer.models.manifests import DeploymentManifest, ModuleManifest, ModulesManifest from seedfarmer.services._service_utils import boto3_client from seedfarmer.services.session_manager import SessionManager -from moto import mock_sts - @pytest.fixture(scope="function") def aws_credentials(): @@ -24,21 +24,24 @@ def aws_credentials(): os.environ["AWS_DEFAULT_REGION"] = "us-east-1" os.environ["MOTO_ACCOUNT_ID"] = "123456789012" + @pytest.fixture(scope="function") def sts_client(aws_credentials): with mock_sts(): yield boto3_client(service_name="sts", session=None) -@pytest.fixture(scope="function") + +@pytest.fixture(scope="function") def session_manager(sts_client): - SessionManager._instances={} + SessionManager._instances = {} SessionManager().get_or_create( project_name="test", region_name="us-east-1", toolchain_region="us-east-1", enable_reaper=False, ) - + + # deployment_yaml = yaml.safe_load( # """ # name: test @@ -57,101 +60,178 @@ def session_manager(sts_client): # someKey: someValue # """ # ) - - + + @pytest.mark.commands @pytest.mark.commands_bootstrap def test_deploy_template(session_manager, mocker): - mocker.patch("seedfarmer.commands._bootstrap_commands.role_deploy_status",return_value={"RoleName":"BLAHBLAH"}) - mocker.patch("seedfarmer.commands._bootstrap_commands.cs_services.cfn.deploy_template",return_value=None) - template = bc.get_toolchain_template(project_name="myapp",principal_arn=['arn:aws:iam::123456789012:role/AdminRole'] - ) - - bc.deploy_template(template=template,stack_name="UnitTest",session=None) + mocker.patch("seedfarmer.commands._bootstrap_commands.role_deploy_status", return_value={"RoleName": "BLAHBLAH"}) + mocker.patch("seedfarmer.commands._bootstrap_commands.cs_services.cfn.deploy_template", return_value=None) + template = bc.get_toolchain_template( + project_name="myapp", + role_name="seedfarmer-test-toolchain-role", + principal_arn=["arn:aws:iam::123456789012:role/AdminRole"], + ) + + bc.deploy_template(template=template, stack_name="UnitTest", session=None) + @pytest.mark.commands @pytest.mark.commands_bootstrap def test_apply_deploy_logic(session_manager, mocker): - mocker.patch("seedfarmer.commands._bootstrap_commands.role_deploy_status",return_value=({"RoleName":"BLAHBLAH"},["exists"])) - mocker.patch("seedfarmer.commands._bootstrap_commands.cs_services.cfn.deploy_template",return_value=None) - template = bc.get_toolchain_template(project_name="myapp",principal_arn=['arn:aws:iam::123456789012:role/AdminRole'] - ) - - bc.apply_deploy_logic(template=template, - role_name="toolchain-role", - stack_name="toolchain-stack", - session=None) + mocker.patch( + "seedfarmer.commands._bootstrap_commands.role_deploy_status", + return_value=({"RoleName": "BLAHBLAH"}, ["exists"]), + ) + mocker.patch("seedfarmer.commands._bootstrap_commands.cs_services.cfn.deploy_template", return_value=None) + template = bc.get_toolchain_template( + project_name="myapp", + role_name="seedfarmer-test-toolchain-role", + principal_arn=["arn:aws:iam::123456789012:role/AdminRole"], + ) + + bc.apply_deploy_logic(template=template, role_name="toolchain-role", stack_name="toolchain-stack", session=None) + @pytest.mark.commands @pytest.mark.commands_bootstrap def test_apply_deploy_logic_role_not_exists(session_manager, mocker): - mocker.patch("seedfarmer.commands._bootstrap_commands.role_deploy_status",return_value=(None,["exists"])) - mocker.patch("seedfarmer.commands._bootstrap_commands.cs_services.cfn.deploy_template",return_value=None) - template = bc.get_toolchain_template(project_name="myapp",principal_arn=['arn:aws:iam::123456789012:role/AdminRole'] - ) - - bc.apply_deploy_logic(template=template, - role_name="toolchain-role", - stack_name="toolchain-stack", - session=None) + mocker.patch("seedfarmer.commands._bootstrap_commands.role_deploy_status", return_value=(None, ["exists"])) + mocker.patch("seedfarmer.commands._bootstrap_commands.cs_services.cfn.deploy_template", return_value=None) + template = bc.get_toolchain_template( + project_name="myapp", + role_name="seedfarmer-test-toolchain-role", + principal_arn=["arn:aws:iam::123456789012:role/AdminRole"], + ) + + bc.apply_deploy_logic(template=template, role_name="toolchain-role", stack_name="toolchain-stack", session=None) + @pytest.mark.commands @pytest.mark.commands_bootstrap def test_apply_deploy_logic_stack_not_exists(session_manager, mocker): - mocker.patch("seedfarmer.commands._bootstrap_commands.role_deploy_status",return_value=(None,None)) - mocker.patch("seedfarmer.commands._bootstrap_commands.cs_services.cfn.deploy_template",return_value=None) - template = bc.get_toolchain_template(project_name="myapp",principal_arn=['arn:aws:iam::123456789012:role/AdminRole'] - ) - - bc.apply_deploy_logic(template=template, - role_name="toolchain-role", - stack_name="toolchain-stack", - session=None) - - - - - + mocker.patch("seedfarmer.commands._bootstrap_commands.role_deploy_status", return_value=(None, None)) + mocker.patch("seedfarmer.commands._bootstrap_commands.cs_services.cfn.deploy_template", return_value=None) + template = bc.get_toolchain_template( + project_name="myapp", + role_name="seedfarmer-test-toolchain-role", + principal_arn=["arn:aws:iam::123456789012:role/AdminRole"], + ) + + bc.apply_deploy_logic(template=template, role_name="toolchain-role", stack_name="toolchain-stack", session=None) + + @pytest.mark.commands @pytest.mark.commands_bootstrap @pytest.mark.parametrize("session", [boto3.Session()]) def test_bootstrap_toolchain_account(mocker, session): - #template = bc.get_toolchain_template(project_name="myapp",principal_arn=['arn:aws:iam::123456789012:role/AdminRole'] - - mocker.patch("seedfarmer.commands._bootstrap_commands.apply_deploy_logic",return_value="") - mocker.patch("seedfarmer.commands._bootstrap_commands.bootstrap_target_account",return_value="") - bc.bootstrap_toolchain_account(project_name="testing", - principal_arns=['arn:aws:iam::123456789012:role/AdminRole'], - permissions_boundary_arn=None, - region_name="us-east-1", - synthesize=False, - as_target=False) + + mocker.patch("seedfarmer.commands._bootstrap_commands.apply_deploy_logic", return_value="") + mocker.patch("seedfarmer.commands._bootstrap_commands.bootstrap_target_account", return_value="") + bc.bootstrap_toolchain_account( + project_name="testing", + principal_arns=["arn:aws:iam::123456789012:role/AdminRole"], + permissions_boundary_arn=None, + region_name="us-east-1", + synthesize=False, + as_target=False, + ) + @pytest.mark.commands @pytest.mark.commands_bootstrap @pytest.mark.parametrize("session", [boto3.Session()]) def test_bootstrap_toolchain_account_synth(mocker, session): - #template = bc.get_toolchain_template(project_name="myapp",principal_arn=['arn:aws:iam::123456789012:role/AdminRole'] - - mocker.patch("seedfarmer.commands._bootstrap_commands.apply_deploy_logic",return_value="") - bc.bootstrap_toolchain_account(project_name="testing", - principal_arns=['arn:aws:iam::123456789012:role/AdminRole'], - permissions_boundary_arn=None, - region_name="us-east-1", - synthesize=True, - as_target=False) - - + + mocker.patch("seedfarmer.commands._bootstrap_commands.apply_deploy_logic", return_value="") + bc.bootstrap_toolchain_account( + project_name="testing", + principal_arns=["arn:aws:iam::123456789012:role/AdminRole"], + permissions_boundary_arn=None, + region_name="us-east-1", + synthesize=True, + as_target=False, + ) + + +@pytest.mark.commands +@pytest.mark.commands_bootstrap +@pytest.mark.parametrize("session", [boto3.Session()]) +def test_bootstrap_toolchain_account_synth_with_qualifier(mocker, session): + + mocker.patch("seedfarmer.commands._bootstrap_commands.apply_deploy_logic", return_value="") + bc.bootstrap_toolchain_account( + project_name="testing", + principal_arns=["arn:aws:iam::123456789012:role/AdminRole"], + permissions_boundary_arn=None, + region_name="us-east-1", + qualifier="asdfgh", + synthesize=True, + as_target=False, + ) + + +@pytest.mark.commands +@pytest.mark.commands_bootstrap +@pytest.mark.parametrize("session", [boto3.Session()]) +def test_bootstrap_toolchain_account_synth_with_qualifier_fail(mocker, session): + mocker.patch("seedfarmer.commands._bootstrap_commands.apply_deploy_logic", return_value="") + + with pytest.raises(seedfarmer.errors.InvalidConfigurationError): + bc.bootstrap_toolchain_account( + project_name="testing", + principal_arns=["arn:aws:iam::123456789012:role/AdminRole"], + permissions_boundary_arn=None, + region_name="us-east-1", + qualifier="asdfghdd", + synthesize=True, + as_target=False, + ) + + @pytest.mark.commands @pytest.mark.commands_bootstrap @pytest.mark.parametrize("session", [boto3.Session()]) def test_bootstrap_target_account(mocker, session): - #template = bc.get_toolchain_template(project_name="myapp",principal_arn=['arn:aws:iam::123456789012:role/AdminRole'] - - mocker.patch("seedfarmer.commands._bootstrap_commands.apply_deploy_logic",return_value="") - bc.bootstrap_target_account(toolchain_account_id="123456789012", - project_name="testing", - permissions_boundary_arn=None, - region_name="us-east-1", - synthesize=False, - session=session) \ No newline at end of file + mocker.patch("seedfarmer.commands._bootstrap_commands.apply_deploy_logic", return_value="") + bc.bootstrap_target_account( + toolchain_account_id="123456789012", + project_name="testing", + permissions_boundary_arn=None, + region_name="us-east-1", + synthesize=False, + session=session, + ) + + +@pytest.mark.commands +@pytest.mark.commands_bootstrap +@pytest.mark.parametrize("session", [boto3.Session()]) +def test_bootstrap_target_account_with_qualifier(mocker, session): + mocker.patch("seedfarmer.commands._bootstrap_commands.apply_deploy_logic", return_value="") + bc.bootstrap_target_account( + toolchain_account_id="123456789012", + project_name="testing", + permissions_boundary_arn=None, + region_name="us-east-1", + qualifier="asdfgh", + synthesize=False, + session=session, + ) + + +@pytest.mark.commands +@pytest.mark.commands_bootstrap +@pytest.mark.parametrize("session", [boto3.Session()]) +def test_bootstrap_target_account_with_qualifier_fail(mocker, session): + mocker.patch("seedfarmer.commands._bootstrap_commands.apply_deploy_logic", return_value="") + with pytest.raises(seedfarmer.errors.InvalidConfigurationError): + bc.bootstrap_target_account( + toolchain_account_id="123456789012", + project_name="testing", + permissions_boundary_arn=None, + region_name="us-east-1", + qualifier="asdfghsds", + synthesize=False, + session=session, + ) diff --git a/test/unit-test/test_commands_deployment.py b/test/unit-test/test_commands_deployment.py index 5479d104..4f037b08 100644 --- a/test/unit-test/test_commands_deployment.py +++ b/test/unit-test/test_commands_deployment.py @@ -1,24 +1,24 @@ +import json import logging import os +from typing import Tuple, cast + import boto3 -import yaml, json -from typing import cast, Tuple +import mock_data.mock_deployment_manifest_for_destroy as mock_deployment_manifest_for_destroy +import mock_data.mock_deployment_manifest_huge as mock_deployment_manifest_huge +import mock_data.mock_deployspec as mock_deployspec +import mock_data.mock_manifests as mock_manifests +import mock_data.mock_module_info_huge as mock_module_info_huge import pytest +import yaml +from moto import mock_sts + import seedfarmer.commands._deployment_commands as dc import seedfarmer.errors - -from seedfarmer.models.manifests import DeploymentManifest, ModuleManifest, ModuleParameter, DataFile from seedfarmer.models._deploy_spec import DeploySpec +from seedfarmer.models.manifests import DataFile, DeploymentManifest, ModuleManifest, ModuleParameter from seedfarmer.services._service_utils import boto3_client from seedfarmer.services.session_manager import SessionManager -import mock_data.mock_deployment_manifest_for_destroy as mock_deployment_manifest_for_destroy -import mock_data.mock_deployment_manifest_huge as mock_deployment_manifest_huge -import mock_data.mock_module_info_huge as mock_module_info_huge -import mock_data.mock_manifests as mock_manifests -import mock_data.mock_deployspec as mock_deployspec - - -from moto import mock_sts @pytest.fixture(scope="function") @@ -31,26 +31,27 @@ def aws_credentials(): os.environ["AWS_DEFAULT_REGION"] = "us-east-1" os.environ["MOTO_ACCOUNT_ID"] = "123456789012" + @pytest.fixture(scope="function") def sts_client(aws_credentials): with mock_sts(): yield boto3_client(service_name="sts", session=None) -@pytest.fixture(scope="function") + +@pytest.fixture(scope="function") def session_manager(sts_client): - SessionManager._instances={} + SessionManager._instances = {} SessionManager().get_or_create( project_name="test", region_name="us-east-1", toolchain_region="us-east-1", enable_reaper=False, ) - @pytest.mark.commands @pytest.mark.commands_deployment -def test_apply_clean(session_manager,mocker): +def test_apply_clean(session_manager, mocker): mocker.patch("seedfarmer.commands._deployment_commands.write_deployment_manifest", return_value=None) mocker.patch("seedfarmer.commands._deployment_commands.prime_target_accounts", return_value=None) @@ -60,205 +61,238 @@ def test_apply_clean(session_manager,mocker): mocker.patch("seedfarmer.commands._deployment_commands.du.validate_module_dependencies", return_value=None) mocker.patch("seedfarmer.commands._deployment_commands.destroy_deployment", return_value=None) mocker.patch("seedfarmer.commands._deployment_commands.deploy_deployment", return_value=None) - dc.apply(deployment_manifest_path="test/unit-test/mock_data/manifests/module-test/deployment-hc.yaml", - dryrun=True) + dc.apply(deployment_manifest_path="test/unit-test/mock_data/manifests/module-test/deployment-hc.yaml", dryrun=True) + @pytest.mark.commands @pytest.mark.commands_deployment -def test_apply_violations(session_manager,mocker): +def test_apply_violations(session_manager, mocker): mocker.patch("seedfarmer.commands._deployment_commands.write_deployment_manifest", return_value=None) mocker.patch("seedfarmer.commands._deployment_commands.prime_target_accounts", return_value=None) mocker.patch("seedfarmer.commands._deployment_commands.du.populate_module_info_index", return_value=None) mocker.patch("seedfarmer.commands._deployment_commands.du.filter_deploy_destroy", return_value=None) mocker.patch("seedfarmer.commands._deployment_commands.write_deployment_manifest", return_value=None) - mocker.patch("seedfarmer.commands._deployment_commands.du.validate_module_dependencies", return_value=[{"module1":["moduleA","moduleB"]}]) + mocker.patch( + "seedfarmer.commands._deployment_commands.du.validate_module_dependencies", + return_value=[{"module1": ["moduleA", "moduleB"]}], + ) with pytest.raises(Exception): dc.apply(deployment_manifest_path="test/unit-test/mock_data/manifests/module-test/deployment-hc.yaml") - + @pytest.mark.commands @pytest.mark.commands_deployment -def test_destroy_clean(session_manager,mocker): +def test_destroy_clean(session_manager, mocker): - mocker.patch("seedfarmer.commands._deployment_commands.du.generate_deployed_manifest", - return_value=DeploymentManifest(**mock_deployment_manifest_for_destroy.destroy_manifest)) - mocker.patch("seedfarmer.commands._deployment_commands.destroy_deployment", return_value= None) + mocker.patch( + "seedfarmer.commands._deployment_commands.du.generate_deployed_manifest", + return_value=DeploymentManifest(**mock_deployment_manifest_for_destroy.destroy_manifest), + ) + mocker.patch("seedfarmer.commands._deployment_commands.destroy_deployment", return_value=None) - dc.destroy(deployment_name="myapp",dryrun=True, retain_seedkit=False) + dc.destroy(deployment_name="myapp", dryrun=True, retain_seedkit=False) @pytest.mark.commands @pytest.mark.commands_deployment -def test_destroy_not_found(session_manager,mocker): +def test_destroy_not_found(session_manager, mocker): mocker.patch("seedfarmer.commands._deployment_commands.du.generate_deployed_manifest", return_value=None) - mocker.patch("seedfarmer.commands._deployment_commands.destroy_deployment", return_value= None) + mocker.patch("seedfarmer.commands._deployment_commands.destroy_deployment", return_value=None) + + dc.destroy(deployment_name="myapp", dryrun=True, retain_seedkit=False) + - dc.destroy(deployment_name="myapp",dryrun=True, retain_seedkit=False) - - - # @pytest.mark.commands # @pytest.mark.commands_deployment # def test_tear_down_target_accounts(session_manager,mocker): # mocker.patch("seedfarmer.commands._deployment_commands.tear_down_target_accounts._teardown_accounts", return_value=None) -# # mocker.patch("seedfarmer.commands._deployment_commands.tear_down_target_accounts._teardown_accounts", +# # mocker.patch("seedfarmer.commands._deployment_commands.tear_down_target_accounts._teardown_accounts", # # return_value=None) # # with mocker.patch('seedfarmer.commands._deployment_commands.tear_down_target_accounts') as teardown: # # teardown. - ### COMMENT....I cannot get the nested threads to mock +### COMMENT....I cannot get the nested threads to mock # dc.tear_down_target_accounts(deployment_manifest=DeploymentManifest(**mock_deployment_manifest_for_destroy.destroy_manifest)) - - + + @pytest.mark.commands @pytest.mark.commands_deployment def test_clone_module_repo(mocker): - git_path_test = "git::https://github.com/awslabs/seedfarmer-modules.git//modules/dummy/blank?ref=release/1.0.0&depth=1" - return_dir =dc._clone_module_repo(git_path=git_path_test) + git_path_test = ( + "git::https://github.com/awslabs/seedfarmer-modules.git//modules/dummy/blank?ref=release/1.0.0&depth=1" + ) + return_dir = dc._clone_module_repo(git_path=git_path_test) # Rerun it so all methods are hit - return_dir =dc._clone_module_repo(git_path=git_path_test) - + return_dir = dc._clone_module_repo(git_path=git_path_test) + @pytest.mark.commands @pytest.mark.commands_deployment def test_process_data_files(mocker): - mocker.patch("seedfarmer.commands._deployment_commands._clone_module_repo",return_value=("git","path")) - mocker.patch("seedfarmer.commands._deployment_commands.du.validate_data_files",return_value=[]) - git_path_test = "git::https://github.com/awslabs/seedfarmer-modules.git//modules/dummy/blank?ref=release/1.0.0&depth=1" - datafile_list =[] + mocker.patch("seedfarmer.commands._deployment_commands._clone_module_repo", return_value=("git", "path")) + mocker.patch("seedfarmer.commands._deployment_commands.du.validate_data_files", return_value=[]) + git_path_test = ( + "git::https://github.com/awslabs/seedfarmer-modules.git//modules/dummy/blank?ref=release/1.0.0&depth=1" + ) + datafile_list = [] datafile_list.append(DataFile(file_path=git_path_test)) datafile_list.append(DataFile(file_path="")) - - dc._process_data_files(data_files=datafile_list, module_name="test",group_name="test") + + dc._process_data_files(data_files=datafile_list, module_name="test", group_name="test") @pytest.mark.commands @pytest.mark.commands_deployment def test_process_data_files_error(mocker): - mocker.patch("seedfarmer.commands._deployment_commands._clone_module_repo",return_value=("git","path")) - mocker.patch("seedfarmer.commands._deployment_commands.du.validate_data_files",return_value=["hey"]) - git_path_test = "git::https://github.com/awslabs/seedfarmer-modules.git//modules/dummy/blank?ref=release/1.0.0&depth=1" - datafile_list =[] + mocker.patch("seedfarmer.commands._deployment_commands._clone_module_repo", return_value=("git", "path")) + mocker.patch("seedfarmer.commands._deployment_commands.du.validate_data_files", return_value=["hey"]) + git_path_test = ( + "git::https://github.com/awslabs/seedfarmer-modules.git//modules/dummy/blank?ref=release/1.0.0&depth=1" + ) + datafile_list = [] datafile_list.append(DataFile(file_path=git_path_test)) datafile_list.append(DataFile(file_path="")) with pytest.raises(seedfarmer.errors.InvalidPathError): - dc._process_data_files(data_files=datafile_list, module_name="test",group_name="test") - - - + dc._process_data_files(data_files=datafile_list, module_name="test", group_name="test") + + @pytest.mark.commands @pytest.mark.commands_deployment -def test_execute_deploy_invalid_spec(session_manager,mocker): - mocker.patch("seedfarmer.commands._deployment_commands.load_parameter_values",return_value=None) - mocker.patch("seedfarmer.commands._deployment_commands.commands.deploy_module_stack", return_value=("stack_name","role_name")) - mocker.patch("seedfarmer.commands._deployment_commands.get_module_metadata",return_value=None) - mocker.patch("seedfarmer.commands._deployment_commands.du.prepare_ssm_for_deploy",return_value=None) +def test_execute_deploy_invalid_spec(session_manager, mocker): + mocker.patch("seedfarmer.commands._deployment_commands.load_parameter_values", return_value=None) + mocker.patch( + "seedfarmer.commands._deployment_commands.commands.deploy_module_stack", + return_value=("stack_name", "role_name"), + ) + mocker.patch("seedfarmer.commands._deployment_commands.get_module_metadata", return_value=None) + mocker.patch("seedfarmer.commands._deployment_commands.du.prepare_ssm_for_deploy", return_value=None) dep = DeploymentManifest(**mock_deployment_manifest_huge.deployment_manifest) group = dep.groups[0] with pytest.raises(seedfarmer.errors.InvalidManifestError): - dc._execute_deploy(group_name=group.name, - module_manifest=group.modules[0], - deployment_manifest = dep, - docker_credentials_secret=None, - permissions_boundary_arn=None, - codebuild_image=None) - + dc._execute_deploy( + group_name=group.name, + module_manifest=group.modules[0], + deployment_manifest=dep, + docker_credentials_secret=None, + permissions_boundary_arn=None, + codebuild_image=None, + ) + + @pytest.mark.commands @pytest.mark.commands_deployment -def test_execute_deploy(session_manager,mocker): - mocker.patch("seedfarmer.commands._deployment_commands.load_parameter_values",return_value=None) - mocker.patch("seedfarmer.commands._deployment_commands.commands.deploy_module_stack", return_value=("stack_name","role_name")) - mocker.patch("seedfarmer.commands._deployment_commands.get_module_metadata",return_value=None) - mocker.patch("seedfarmer.commands._deployment_commands.du.prepare_ssm_for_deploy",return_value=None) - mocker.patch("seedfarmer.commands._deployment_commands.commands.deploy_module",return_value=None) +def test_execute_deploy(session_manager, mocker): + mocker.patch("seedfarmer.commands._deployment_commands.load_parameter_values", return_value=None) + mocker.patch( + "seedfarmer.commands._deployment_commands.commands.deploy_module_stack", + return_value=("stack_name", "role_name"), + ) + mocker.patch("seedfarmer.commands._deployment_commands.get_module_metadata", return_value=None) + mocker.patch("seedfarmer.commands._deployment_commands.du.prepare_ssm_for_deploy", return_value=None) + mocker.patch("seedfarmer.commands._deployment_commands.commands.deploy_module", return_value=None) dep = DeploymentManifest(**mock_deployment_manifest_huge.deployment_manifest) group = dep.groups[0] - module_manifest=group.modules[0] - module_manifest.deploy_spec=DeploySpec(**mock_deployspec.dummy_deployspec) - dc._execute_deploy(group_name=group.name, - module_manifest=module_manifest, - deployment_manifest = dep, - docker_credentials_secret=None, - permissions_boundary_arn=None, - codebuild_image=None) - - - + module_manifest = group.modules[0] + module_manifest.deploy_spec = DeploySpec(**mock_deployspec.dummy_deployspec) + dc._execute_deploy( + group_name=group.name, + module_manifest=module_manifest, + deployment_manifest=dep, + docker_credentials_secret=None, + permissions_boundary_arn=None, + codebuild_image=None, + ) + + @pytest.mark.commands @pytest.mark.commands_deployment -def test_execute_destroy_invalid_spec(session_manager,mocker): +def test_execute_destroy_invalid_spec(session_manager, mocker): from seedfarmer.models.deploy_responses import ModuleDeploymentResponse + dep = DeploymentManifest(**mock_deployment_manifest_huge.deployment_manifest) group = dep.groups[0] - module_manifest=group.modules[0] - mod_resp = ModuleDeploymentResponse(deployment="myapp", group="optionals",module="metworking", status="SUCCESS") - - - mocker.patch("seedfarmer.commands._deployment_commands.get_module_metadata",return_value=None) - mocker.patch("seedfarmer.commands._deployment_commands.commands.get_module_stack_info", - return_value=("stack_name","role_name")) - mocker.patch("seedfarmer.commands._deployment_commands.commands.destroy_module",return_value=mod_resp) + module_manifest = group.modules[0] + mod_resp = ModuleDeploymentResponse(deployment="myapp", group="optionals", module="metworking", status="SUCCESS") + + mocker.patch("seedfarmer.commands._deployment_commands.get_module_metadata", return_value=None) + mocker.patch( + "seedfarmer.commands._deployment_commands.commands.get_module_stack_info", + return_value=("stack_name", "role_name"), + ) + mocker.patch("seedfarmer.commands._deployment_commands.commands.destroy_module", return_value=mod_resp) with pytest.raises(seedfarmer.errors.InvalidManifestError): - dc._execute_destroy(group_name=group.name, - module_manifest=module_manifest, - module_path="to/my/module", - deployment_manifest = dep, - docker_credentials_secret=None, - codebuild_image=None) + dc._execute_destroy( + group_name=group.name, + module_manifest=module_manifest, + module_path="to/my/module", + deployment_manifest=dep, + docker_credentials_secret=None, + codebuild_image=None, + ) + @pytest.mark.commands @pytest.mark.commands_deployment -def test_execute_destroy(session_manager,mocker): +def test_execute_destroy(session_manager, mocker): from seedfarmer.models.deploy_responses import ModuleDeploymentResponse + dep = DeploymentManifest(**mock_deployment_manifest_huge.deployment_manifest) group = dep.groups[0] - module_manifest=group.modules[0] - mod_resp = ModuleDeploymentResponse(deployment="myapp", group="optionals",module="metworking", status="SUCCESS") - mocker.patch("seedfarmer.commands._deployment_commands.get_module_metadata",return_value=None) - mocker.patch("seedfarmer.commands._deployment_commands.commands.get_module_stack_info", - return_value=("stack_name","role_name")) - mocker.patch("seedfarmer.commands._deployment_commands.commands.destroy_module",return_value=mod_resp) - mocker.patch("seedfarmer.commands._deployment_commands.commands.destroy_module_stack",return_value=None) - module_manifest.deploy_spec=DeploySpec(**mock_deployspec.dummy_deployspec) - dc._execute_destroy(group_name=group.name, - module_manifest=module_manifest, - module_path="to/my/module", - deployment_manifest = dep, - docker_credentials_secret=None, - codebuild_image=None) - - - + module_manifest = group.modules[0] + mod_resp = ModuleDeploymentResponse(deployment="myapp", group="optionals", module="metworking", status="SUCCESS") + mocker.patch("seedfarmer.commands._deployment_commands.get_module_metadata", return_value=None) + mocker.patch( + "seedfarmer.commands._deployment_commands.commands.get_module_stack_info", + return_value=("stack_name", "role_name"), + ) + mocker.patch("seedfarmer.commands._deployment_commands.commands.destroy_module", return_value=mod_resp) + mocker.patch("seedfarmer.commands._deployment_commands.commands.destroy_module_stack", return_value=None) + module_manifest.deploy_spec = DeploySpec(**mock_deployspec.dummy_deployspec) + dc._execute_destroy( + group_name=group.name, + module_manifest=module_manifest, + module_path="to/my/module", + deployment_manifest=dep, + docker_credentials_secret=None, + codebuild_image=None, + ) + + @pytest.mark.commands @pytest.mark.commands_deployment -def test_deploy_deployment(session_manager,mocker): +def test_deploy_deployment(session_manager, mocker): import hashlib - mock_hashlib = hashlib.md5(json.dumps({"hey":"yp"}, sort_keys=True).encode("utf-8")) - + + mock_hashlib = hashlib.md5(json.dumps({"hey": "yp"}, sort_keys=True).encode("utf-8")) + import seedfarmer.mgmt.deploy_utils as du - mocker.patch("seedfarmer.mgmt.deploy_utils.mi.get_parameter_data_cache",return_value=mock_module_info_huge.module_index_info_huge) - module_info_index=du.populate_module_info_index(deployment_manifest=DeploymentManifest(**mock_manifests.deployment_manifest)) - - - + + mocker.patch( + "seedfarmer.mgmt.deploy_utils.mi.get_parameter_data_cache", + return_value=mock_module_info_huge.module_index_info_huge, + ) + module_info_index = du.populate_module_info_index( + deployment_manifest=DeploymentManifest(**mock_manifests.deployment_manifest) + ) + dep = DeploymentManifest(**mock_deployment_manifest_huge.deployment_manifest) - mocker.patch("seedfarmer.commands._deployment_commands.print_manifest_inventory",return_value=None) - mocker.patch("seedfarmer.commands._deployment_commands.du.validate_group_parameters",return_value=None) - mocker.patch("seedfarmer.commands._deployment_commands.get_deployspec_path", - return_value='test/unit-test/mock_data/mock_deployspec.yaml') - mocker.patch("seedfarmer.commands._deployment_commands.checksum.get_module_md5",return_value="asfsadfsdfa") - mocker.patch("seedfarmer.commands._deployment_commands.hashlib.md5",return_value=mock_hashlib) - - - mocker.patch("seedfarmer.commands._deployment_commands._deploy_validated_deployment",return_value=None) - mocker.patch("seedfarmer.commands._deployment_commands.du.need_to_build", - return_value=None) + mocker.patch("seedfarmer.commands._deployment_commands.print_manifest_inventory", return_value=None) + mocker.patch("seedfarmer.commands._deployment_commands.du.validate_group_parameters", return_value=None) + mocker.patch( + "seedfarmer.commands._deployment_commands.get_deployspec_path", + return_value="test/unit-test/mock_data/mock_deployspec.yaml", + ) + mocker.patch("seedfarmer.commands._deployment_commands.checksum.get_module_md5", return_value="asfsadfsdfa") + mocker.patch("seedfarmer.commands._deployment_commands.hashlib.md5", return_value=mock_hashlib) + + mocker.patch("seedfarmer.commands._deployment_commands._deploy_validated_deployment", return_value=None) + mocker.patch("seedfarmer.commands._deployment_commands.du.need_to_build", return_value=None) # mocker.patch("seedfarmer.commands._deployment_commands.module_info_index.get_module_info", # return_value=None) - mocker.patch("seedfarmer.commands._deployment_commands.print_bolded",return_value=None) - module_upstream_dep = {"Nothing":[]} - - dc.deploy_deployment(deployment_manifest=dep, - module_info_index=module_info_index, - module_upstream_dep=module_upstream_dep) \ No newline at end of file + mocker.patch("seedfarmer.commands._deployment_commands.print_bolded", return_value=None) + module_upstream_dep = {"Nothing": []} + + dc.deploy_deployment( + deployment_manifest=dep, module_info_index=module_info_index, module_upstream_dep=module_upstream_dep + ) diff --git a/test/unit-test/test_commands_module.py b/test/unit-test/test_commands_module.py index d7802a88..30805e1d 100644 --- a/test/unit-test/test_commands_module.py +++ b/test/unit-test/test_commands_module.py @@ -1,19 +1,20 @@ +import json import logging import os +from typing import Tuple, cast + import boto3 -import yaml, json -from typing import cast, Tuple import pytest +import yaml +from moto import mock_sts + import seedfarmer.commands._module_commands as mc import seedfarmer.errors - +from seedfarmer.models._deploy_spec import DeploySpec from seedfarmer.models.manifests import DeploymentManifest, ModuleManifest, ModuleParameter -from seedfarmer.models._deploy_spec import DeploySpec from seedfarmer.services._service_utils import boto3_client from seedfarmer.services.session_manager import SessionManager -from moto import mock_sts - @pytest.fixture(scope="function") def aws_credentials(): @@ -25,27 +26,25 @@ def aws_credentials(): os.environ["AWS_DEFAULT_REGION"] = "us-east-1" os.environ["MOTO_ACCOUNT_ID"] = "123456789012" + @pytest.fixture(scope="function") def sts_client(aws_credentials): with mock_sts(): yield boto3_client(service_name="sts", session=None) -@pytest.fixture(scope="function") + +@pytest.fixture(scope="function") def session_manager(sts_client): - SessionManager._instances={} + SessionManager._instances = {} SessionManager().get_or_create( project_name="test", region_name="us-east-1", toolchain_region="us-east-1", enable_reaper=False, ) - -dummy_list_params = { - "vpc-id":"vpcid-12345", - "SubnetIds":["subnet-123","subnet-456"], - "Crazy_param_config":"123" - -} + + +dummy_list_params = {"vpc-id": "vpcid-12345", "SubnetIds": ["subnet-123", "subnet-456"], "Crazy_param_config": "123"} deployment_manifest_json = { "name": "mlops", @@ -58,35 +57,19 @@ def session_manager(sts_client): { "name": "networking", "path": "modules/optionals/networking/", - "parameters": [ - { - - "name": "internet-accessible", - "value": True - } - ], - + "parameters": [{"name": "internet-accessible", "value": True}], "target_account": "primary", "target_region": "us-east-1", - }, { "name": "datalake-buckets", "path": "modules/optionals/datalake-buckets", - "parameters": [ - { - - "name": "encryption-type", - "value": "SSE" - } - ], - + "parameters": [{"name": "encryption-type", "value": "SSE"}], "target_account": "primary", - "target_region": "us-east-1", - } - ] + "target_region": "us-east-1", + }, + ], }, - { "name": "core", "path": "manifests/mlops/core-modules.yaml", @@ -95,46 +78,21 @@ def session_manager(sts_client): "name": "efs", "path": "modules/core/efs", "parameters": [ - { - "name": "removal-policy", - "value": "DESTROY" - }, + {"name": "removal-policy", "value": "DESTROY"}, { "name": "vpc-id", "value_from": { - "module_metadata": { - "name": "networking", - "group": "optionals", - "key": "VpcId" - }, + "module_metadata": {"name": "networking", "group": "optionals", "key": "VpcId"}, }, }, - { - "name": "test-secrets-manager", - "value_from": { - "secretsManager": "my-secret-vpc-id" - } - }, - { - "name": "test-ssm-store", - "value_from": { - "parameterStore": "my-ssm-name" - } - }, - { - "name": "test-regional-param", - "value_from": { - "parameterValue": "testRegionalParam" - } - }, - + {"name": "test-secrets-manager", "value_from": {"secretsManager": "my-secret-vpc-id"}}, + {"name": "test-ssm-store", "value_from": {"parameterStore": "my-ssm-name"}}, + {"name": "test-regional-param", "value_from": {"parameterValue": "testRegionalParam"}}, ], "target_account": "primary", "target_region": "us-east-1", - } ], - }, ], "target_account_mappings": [ @@ -142,21 +100,16 @@ def session_manager(sts_client): "alias": "primary", "account_id": "123456789012", "default": True, - "parameters_global": { - "dockerCredentialsSecret": "aws-addf-docker-credentials" - }, + "parameters_global": {"dockerCredentialsSecret": "aws-addf-docker-credentials"}, "region_mappings": [ { "region": "us-east-1", "default": True, - "parameters_regional": { - "testRegionalParam": "somethingawesomehere" - }, + "parameters_regional": {"testRegionalParam": "somethingawesomehere"}, } ], - } - ] + ], } @@ -181,16 +134,17 @@ def session_manager(sts_client): build: commands: - echo 'Look Ma....destroying' - """) + """ +) resp_dict_str = { - "aws_region": "us-east-1", - "aws_account_id": "123456789012", - "codebuild_build_id": "somebuildid", - "codebuild_log_path": "somelogpath", - } - - + "aws_region": "us-east-1", + "aws_account_id": "123456789012", + "codebuild_build_id": "somebuildid", + "codebuild_log_path": "somelogpath", +} + + @pytest.mark.commands @pytest.mark.commands_modules def test_generate_export_env_params(): @@ -201,110 +155,130 @@ def test_generate_export_env_params(): @pytest.mark.commands_modules def test_generate_export_raw_env_params(): pass - + + @pytest.mark.commands @pytest.mark.commands_modules -def test_deploy_modules(session_manager,mocker): +def test_deploy_modules(session_manager, mocker): - mocker.patch("seedfarmer.commands._module_commands._execute_module_commands", return_value=(json.dumps(resp_dict_str),resp_dict_str)) + mocker.patch( + "seedfarmer.commands._module_commands._execute_module_commands", + return_value=(json.dumps(resp_dict_str), resp_dict_str), + ) dep = DeploymentManifest(**deployment_manifest_json) group_name = dep.groups[1].name module_manifest = dep.groups[1].modules[0] module_manifest.deploy_spec = DeploySpec(**dummy_deployspec) - mc.deploy_module(deployment_name=dep.name, - group_name=group_name, - module_manifest=module_manifest, - account_id="123456789012", - region='us-east-1', - parameters=module_manifest.parameters, - module_metadata=json.dumps(dummy_list_params), - docker_credentials_secret='aws-addf-docker-credentials', - permissions_boundary_arn="arn:aws:iam::123456789012:policy/boundary", - module_role_name="mlops-optionals-efs", + mc.deploy_module( + deployment_name=dep.name, + group_name=group_name, + module_manifest=module_manifest, + account_id="123456789012", + region="us-east-1", + parameters=module_manifest.parameters, + module_metadata=json.dumps(dummy_list_params), + docker_credentials_secret="aws-addf-docker-credentials", + permissions_boundary_arn="arn:aws:iam::123456789012:policy/boundary", + module_role_name="mlops-optionals-efs", ) @pytest.mark.commands @pytest.mark.commands_modules -def test_deploy_modules_error_deployspec(session_manager,mocker): +def test_deploy_modules_error_deployspec(session_manager, mocker): - mocker.patch("seedfarmer.commands._module_commands._execute_module_commands", return_value=(json.dumps(resp_dict_str),resp_dict_str)) + mocker.patch( + "seedfarmer.commands._module_commands._execute_module_commands", + return_value=(json.dumps(resp_dict_str), resp_dict_str), + ) dep = DeploymentManifest(**deployment_manifest_json) group_name = dep.groups[1].name module_manifest = dep.groups[1].modules[0] spec = DeploySpec(**dummy_deployspec) - spec.deploy=None + spec.deploy = None module_manifest.deploy_spec = spec with pytest.raises(seedfarmer.errors.InvalidConfigurationError): - mc.deploy_module(deployment_name=dep.name, - group_name=group_name, - module_manifest=module_manifest, - account_id="123456789012", - region='us-east-1', - parameters=module_manifest.parameters, - module_metadata=json.dumps(dummy_list_params), - docker_credentials_secret='aws-addf-docker-credentials', - permissions_boundary_arn="arn:aws:iam::123456789012:policy/boundary", - module_role_name="mlops-optionals-efs", + mc.deploy_module( + deployment_name=dep.name, + group_name=group_name, + module_manifest=module_manifest, + account_id="123456789012", + region="us-east-1", + parameters=module_manifest.parameters, + module_metadata=json.dumps(dummy_list_params), + docker_credentials_secret="aws-addf-docker-credentials", + permissions_boundary_arn="arn:aws:iam::123456789012:policy/boundary", + module_role_name="mlops-optionals-efs", ) + @pytest.mark.commands @pytest.mark.commands_modules -def test_destroy_modules(session_manager,mocker): +def test_destroy_modules(session_manager, mocker): - mocker.patch("seedfarmer.commands._module_commands._execute_module_commands", return_value=(json.dumps(resp_dict_str),resp_dict_str)) + mocker.patch( + "seedfarmer.commands._module_commands._execute_module_commands", + return_value=(json.dumps(resp_dict_str), resp_dict_str), + ) dep = DeploymentManifest(**deployment_manifest_json) group_name = dep.groups[1].name module_manifest = dep.groups[1].modules[0] module_manifest.deploy_spec = DeploySpec(**dummy_deployspec) - mc.destroy_module(deployment_name=dep.name, - group_name=group_name, - module_path="module/path/path", - module_manifest=module_manifest, - account_id="123456789012", - region='us-east-1', - parameters=module_manifest.parameters, - module_metadata=json.dumps(dummy_list_params), - module_role_name="mlops-optionals-efs", + mc.destroy_module( + deployment_name=dep.name, + group_name=group_name, + module_path="module/path/path", + module_manifest=module_manifest, + account_id="123456789012", + region="us-east-1", + parameters=module_manifest.parameters, + module_metadata=json.dumps(dummy_list_params), + module_role_name="mlops-optionals-efs", ) @pytest.mark.commands @pytest.mark.commands_modules -def test_destroy_modules_error_deployspec(session_manager,mocker): +def test_destroy_modules_error_deployspec(session_manager, mocker): - mocker.patch("seedfarmer.commands._module_commands._execute_module_commands", return_value=(json.dumps(resp_dict_str),resp_dict_str)) + mocker.patch( + "seedfarmer.commands._module_commands._execute_module_commands", + return_value=(json.dumps(resp_dict_str), resp_dict_str), + ) dep = DeploymentManifest(**deployment_manifest_json) group_name = dep.groups[1].name module_manifest = dep.groups[1].modules[0] - + with pytest.raises(seedfarmer.errors.InvalidConfigurationError): - mc.destroy_module(deployment_name=dep.name, - group_name=group_name, - module_path="module/path/path", - module_manifest=module_manifest, - account_id="123456789012", - region='us-east-1', - parameters=module_manifest.parameters, - module_metadata=json.dumps(dummy_list_params), - module_role_name="mlops-optionals-efs", + mc.destroy_module( + deployment_name=dep.name, + group_name=group_name, + module_path="module/path/path", + module_manifest=module_manifest, + account_id="123456789012", + region="us-east-1", + parameters=module_manifest.parameters, + module_metadata=json.dumps(dummy_list_params), + module_role_name="mlops-optionals-efs", ) - - + @pytest.mark.commands @pytest.mark.commands_modules -def test_execute_module_commands(session_manager,mocker): - mocker.patch("seedfarmer.commands._module_commands._execute_module_commands", return_value=(json.dumps(resp_dict_str),resp_dict_str)) +def test_execute_module_commands(session_manager, mocker): + mocker.patch( + "seedfarmer.commands._module_commands._execute_module_commands", + return_value=(json.dumps(resp_dict_str), resp_dict_str), + ) dep = DeploymentManifest(**deployment_manifest_json) - mc._execute_module_commands(deployment_name=dep.name, - group_name=dep.groups[1].name, - module_manifest_name=dep.groups[1].modules[0].name, - account_id="123456789012", - region='us-east-1', - metadata_env_variable= ".env" - ) - + mc._execute_module_commands( + deployment_name=dep.name, + group_name=dep.groups[1].name, + module_manifest_name=dep.groups[1].modules[0].name, + account_id="123456789012", + region="us-east-1", + metadata_env_variable=".env", + ) diff --git a/test/unit-test/test_commands_network_parameters.py b/test/unit-test/test_commands_network_parameters.py index 3427dd1f..3a760d4e 100644 --- a/test/unit-test/test_commands_network_parameters.py +++ b/test/unit-test/test_commands_network_parameters.py @@ -1,17 +1,18 @@ +import json import os +from typing import Any, Dict, cast + import boto3 -import yaml, json -from typing import cast, Dict, Any import pytest +import yaml +from moto import mock_sts + import seedfarmer.commands._network_parameter_commands as npc import seedfarmer.errors - from seedfarmer.models.manifests import DeploymentManifest, NetworkMapping from seedfarmer.services._service_utils import boto3_client from seedfarmer.services.session_manager import SessionManager -from moto import mock_sts - @pytest.fixture(scope="function") def aws_credentials(): @@ -23,14 +24,16 @@ def aws_credentials(): os.environ["AWS_DEFAULT_REGION"] = "us-east-1" os.environ["MOTO_ACCOUNT_ID"] = "123456789012" + @pytest.fixture(scope="function") def sts_client(aws_credentials): with mock_sts(): yield boto3_client(service_name="sts", session=None) -@pytest.fixture(scope="function") + +@pytest.fixture(scope="function") def session_manager(sts_client): - SessionManager._instances={} + SessionManager._instances = {} SessionManager().get_or_create( project_name="test", region_name="us-east-1", @@ -55,7 +58,6 @@ def session_manager(sts_client): "target_region": "us-east-1", } ], - } ], "description": "Testing", @@ -71,17 +73,9 @@ def session_manager(sts_client): "parameters_regional": { "dockerCredentialsSecret": "aws-addf-docker-credentials", "vpcId": "vpc-0c4cb9e06c9413222", - "privateSubnetIds": [ - "subnet-0c36d3d5808f67a02", - "subnet-00fa1e71cddcf57d3" - ], - "isolatedSubnetIds": [ - "subnet-XXXXXXXXX", - "subnet-XXXXXXXXX" - ], - "securityGroupIds": [ - "sg-049033188c114a3d2" - ] + "privateSubnetIds": ["subnet-0c36d3d5808f67a02", "subnet-00fa1e71cddcf57d3"], + "isolatedSubnetIds": ["subnet-XXXXXXXXX", "subnet-XXXXXXXXX"], + "securityGroupIds": ["sg-049033188c114a3d2"], }, # "network": { # "vpc_id": { @@ -109,259 +103,257 @@ def session_manager(sts_client): } ], } - ] + ], } @pytest.mark.commands @pytest.mark.commands_parameters -def test_no_values(session_manager,mocker): - deployment_manifest_json['target_account_mappings'][0]["region_mappings"][0]["network"] ={ - "vpc_id": None, - "private_subnet_ids": None, - "security_group_ids": None +def test_no_values(session_manager, mocker): + deployment_manifest_json["target_account_mappings"][0]["region_mappings"][0]["network"] = { + "vpc_id": None, + "private_subnet_ids": None, + "security_group_ids": None, } dep = DeploymentManifest(**deployment_manifest_json) dep.validate_and_set_module_defaults() target_account_region = dep.target_accounts_regions[0] - networkOut = npc.load_network_values( - cast(NetworkMapping, target_account_region["network"]), - cast(Dict[str, Any], target_account_region["parameters_regional"]), - target_account_region["account_id"], - target_account_region["region"]) - - + cast(NetworkMapping, target_account_region["network"]), + cast(Dict[str, Any], target_account_region["parameters_regional"]), + target_account_region["account_id"], + target_account_region["region"], + ) + + @pytest.mark.commands @pytest.mark.commands_parameters -def test_all_hardcoded_values(session_manager,mocker): - deployment_manifest_json['target_account_mappings'][0]["region_mappings"][0]["network"] = { - "vpc_id": "vpc-01e556d052f429282", - "private_subnet_ids":["subnet-0c36d3d5808f67a02","subnet-00fa1e71cddcf57d3"], - "security_group_ids":["sg-049033188c114a3d2"] - } +def test_all_hardcoded_values(session_manager, mocker): + deployment_manifest_json["target_account_mappings"][0]["region_mappings"][0]["network"] = { + "vpc_id": "vpc-01e556d052f429282", + "private_subnet_ids": ["subnet-0c36d3d5808f67a02", "subnet-00fa1e71cddcf57d3"], + "security_group_ids": ["sg-049033188c114a3d2"], + } dep = DeploymentManifest(**deployment_manifest_json) dep.validate_and_set_module_defaults() target_account_region = dep.target_accounts_regions[0] - networkOut = npc.load_network_values( - cast(NetworkMapping, target_account_region["network"]), - cast(Dict[str, Any], target_account_region["parameters_regional"]), - target_account_region["account_id"], - target_account_region["region"]) - - assert('vpc-01e556d052f429282' in networkOut.vpc_id) - assert('subnet-0c36d3d5808f67a02' in networkOut.private_subnet_ids) - assert('sg-049033188c114a3d2' in networkOut.security_group_ids) - + cast(NetworkMapping, target_account_region["network"]), + cast(Dict[str, Any], target_account_region["parameters_regional"]), + target_account_region["account_id"], + target_account_region["region"], + ) + + assert "vpc-01e556d052f429282" in networkOut.vpc_id + assert "subnet-0c36d3d5808f67a02" in networkOut.private_subnet_ids + assert "sg-049033188c114a3d2" in networkOut.security_group_ids + + @pytest.mark.commands @pytest.mark.commands_parameters -def test_all_from_env(session_manager,mocker): - os.environ['testenvvpcid']="vpc-01e556d052f429282" - os.environ['testsgs']='["sg-049033188c114a3d2"]' - os.environ['testsubnets']='["subnet-0c36d3d5808f67a02","subnet-00fa1e71cddcf57d3"]' - deployment_manifest_json['target_account_mappings'][0]["region_mappings"][0]["network"] = { - "vpc_id":{ - "value_from": { - "env_variable": "testenvvpcid", - } - }, - "private_subnet_ids": - { - "value_from": { - "env_variable": "testsubnets", - } - }, - "security_group_ids":{ - "value_from": { - "env_variable": "testsgs", - } - } - } +def test_all_from_env(session_manager, mocker): + os.environ["testenvvpcid"] = "vpc-01e556d052f429282" + os.environ["testsgs"] = '["sg-049033188c114a3d2"]' + os.environ["testsubnets"] = '["subnet-0c36d3d5808f67a02","subnet-00fa1e71cddcf57d3"]' + deployment_manifest_json["target_account_mappings"][0]["region_mappings"][0]["network"] = { + "vpc_id": { + "value_from": { + "env_variable": "testenvvpcid", + } + }, + "private_subnet_ids": { + "value_from": { + "env_variable": "testsubnets", + } + }, + "security_group_ids": { + "value_from": { + "env_variable": "testsgs", + } + }, + } dep = DeploymentManifest(**deployment_manifest_json) dep.validate_and_set_module_defaults() target_account_region = dep.target_accounts_regions[0] - networkOut = npc.load_network_values( - cast(NetworkMapping, target_account_region["network"]), - cast(Dict[str, Any], target_account_region["parameters_regional"]), - target_account_region["account_id"], - target_account_region["region"]) - - assert('vpc-01e556d052f429282' in networkOut.vpc_id) - assert('subnet-0c36d3d5808f67a02' in networkOut.private_subnet_ids) - assert('sg-049033188c114a3d2' in networkOut.security_group_ids) - + cast(NetworkMapping, target_account_region["network"]), + cast(Dict[str, Any], target_account_region["parameters_regional"]), + target_account_region["account_id"], + target_account_region["region"], + ) + + assert "vpc-01e556d052f429282" in networkOut.vpc_id + assert "subnet-0c36d3d5808f67a02" in networkOut.private_subnet_ids + assert "sg-049033188c114a3d2" in networkOut.security_group_ids + + @pytest.mark.commands @pytest.mark.commands_parameters -def test_vpc_from_ssm(session_manager,mocker): - mocker.patch("seedfarmer.commands._network_parameter_commands.ssm.get_parameter", - return_value="vpc-01e556d052f429282" - ) - deployment_manifest_json['target_account_mappings'][0]["region_mappings"][0]["network"] = { - "vpc_id":{ - "value_from": { - "parameter_store": "/idf/testing/vpcid", - } - }, - "private_subnet_ids":["subnet-0c36d3d5808f67a02","subnet-00fa1e71cddcf57d3"], - "security_group_ids":["sg-049033188c114a3d2"] - } +def test_vpc_from_ssm(session_manager, mocker): + mocker.patch( + "seedfarmer.commands._network_parameter_commands.ssm.get_parameter", return_value="vpc-01e556d052f429282" + ) + deployment_manifest_json["target_account_mappings"][0]["region_mappings"][0]["network"] = { + "vpc_id": { + "value_from": { + "parameter_store": "/idf/testing/vpcid", + } + }, + "private_subnet_ids": ["subnet-0c36d3d5808f67a02", "subnet-00fa1e71cddcf57d3"], + "security_group_ids": ["sg-049033188c114a3d2"], + } dep = DeploymentManifest(**deployment_manifest_json) dep.validate_and_set_module_defaults() target_account_region = dep.target_accounts_regions[0] - networkOut = npc.load_network_values( - cast(NetworkMapping, target_account_region["network"]), - cast(Dict[str, Any], target_account_region["parameters_regional"]), - target_account_region["account_id"], - target_account_region["region"]) - - assert('vpc-01e556d052f429282' in networkOut.vpc_id) - assert('subnet-0c36d3d5808f67a02' in networkOut.private_subnet_ids) - assert('sg-049033188c114a3d2' in networkOut.security_group_ids) - - + cast(NetworkMapping, target_account_region["network"]), + cast(Dict[str, Any], target_account_region["parameters_regional"]), + target_account_region["account_id"], + target_account_region["region"], + ) + + assert "vpc-01e556d052f429282" in networkOut.vpc_id + assert "subnet-0c36d3d5808f67a02" in networkOut.private_subnet_ids + assert "sg-049033188c114a3d2" in networkOut.security_group_ids + + @pytest.mark.commands @pytest.mark.commands_parameters -def test_vpc_from_regional_parameter(session_manager,mocker): - deployment_manifest_json['target_account_mappings'][0]["region_mappings"][0]["network"] = { - "vpc_id":{ - "value_from": { - "parameter_value": "vpcId", - } - }, - "private_subnet_ids":["subnet-0c36d3d5808f67a02","subnet-00fa1e71cddcf57d3"], - "security_group_ids":["sg-049033188c114a3d2"] - } +def test_vpc_from_regional_parameter(session_manager, mocker): + deployment_manifest_json["target_account_mappings"][0]["region_mappings"][0]["network"] = { + "vpc_id": { + "value_from": { + "parameter_value": "vpcId", + } + }, + "private_subnet_ids": ["subnet-0c36d3d5808f67a02", "subnet-00fa1e71cddcf57d3"], + "security_group_ids": ["sg-049033188c114a3d2"], + } dep = DeploymentManifest(**deployment_manifest_json) dep.validate_and_set_module_defaults() target_account_region = dep.target_accounts_regions[0] - networkOut = npc.load_network_values( - cast(NetworkMapping, target_account_region["network"]), - cast(Dict[str, Any], target_account_region["parameters_regional"]), - target_account_region["account_id"], - target_account_region["region"]) - - assert('vpc-0c4cb9e06c9413222' in networkOut.vpc_id) - assert('subnet-0c36d3d5808f67a02' in networkOut.private_subnet_ids) - assert('sg-049033188c114a3d2' in networkOut.security_group_ids) - + cast(NetworkMapping, target_account_region["network"]), + cast(Dict[str, Any], target_account_region["parameters_regional"]), + target_account_region["account_id"], + target_account_region["region"], + ) + + assert "vpc-0c4cb9e06c9413222" in networkOut.vpc_id + assert "subnet-0c36d3d5808f67a02" in networkOut.private_subnet_ids + assert "sg-049033188c114a3d2" in networkOut.security_group_ids + + @pytest.mark.commands @pytest.mark.commands_parameters -def test_sgs_subnets_from_ssm(session_manager,mocker): - mocker.patch("seedfarmer.commands._network_parameter_commands.ssm.get_parameter", - return_value=[ - "subnet-0c36d3d5808f67a02", - "subnet-00fa1e71cddcf57d3" - ], - ) - deployment_manifest_json['target_account_mappings'][0]["region_mappings"][0]["network"] = { - "vpc_id": "vpc-01e556d052f429282", - "private_subnet_ids": { - "value_from": { - "parameter_store": "/idf/testing/subnets", - } - }, - "security_group_ids": { - "value_from": { - "parameter_store": "/idf/testing/securitygroups", - } +def test_sgs_subnets_from_ssm(session_manager, mocker): + mocker.patch( + "seedfarmer.commands._network_parameter_commands.ssm.get_parameter", + return_value=["subnet-0c36d3d5808f67a02", "subnet-00fa1e71cddcf57d3"], + ) + deployment_manifest_json["target_account_mappings"][0]["region_mappings"][0]["network"] = { + "vpc_id": "vpc-01e556d052f429282", + "private_subnet_ids": { + "value_from": { + "parameter_store": "/idf/testing/subnets", + } + }, + "security_group_ids": { + "value_from": { + "parameter_store": "/idf/testing/securitygroups", } - + }, } - + dep = DeploymentManifest(**deployment_manifest_json) dep.validate_and_set_module_defaults() target_account_region = dep.target_accounts_regions[0] - networkOut = npc.load_network_values( - cast(NetworkMapping, target_account_region["network"]), - cast(Dict[str, Any], target_account_region["parameters_regional"]), - target_account_region["account_id"], - target_account_region["region"]) - - assert('vpc-01e556d052f429282' in networkOut.vpc_id) - assert('subnet-0c36d3d5808f67a02' in networkOut.private_subnet_ids) - assert('subnet-00fa1e71cddcf57d3' in networkOut.security_group_ids) - + cast(NetworkMapping, target_account_region["network"]), + cast(Dict[str, Any], target_account_region["parameters_regional"]), + target_account_region["account_id"], + target_account_region["region"], + ) + + assert "vpc-01e556d052f429282" in networkOut.vpc_id + assert "subnet-0c36d3d5808f67a02" in networkOut.private_subnet_ids + assert "subnet-00fa1e71cddcf57d3" in networkOut.security_group_ids + @pytest.mark.commands @pytest.mark.commands_parameters -def test_sgs_subnets_from_regional_parameter(session_manager,mocker): - mocker.patch("seedfarmer.commands._network_parameter_commands.ssm.get_parameter", - return_value=[ - "subnet-0c36d3d5808f67a02", - "subnet-00fa1e71cddcf57d3" - ], - ) - deployment_manifest_json['target_account_mappings'][0]["region_mappings"][0]["network"] = { - "vpc_id": "vpc-01e556d052f429282", - "private_subnet_ids": { - "value_from": { - "parameter_value": "privateSubnetIds", - } - }, - "security_group_ids": { - "value_from": { - "parameter_value": "securityGroupIds", - } +def test_sgs_subnets_from_regional_parameter(session_manager, mocker): + mocker.patch( + "seedfarmer.commands._network_parameter_commands.ssm.get_parameter", + return_value=["subnet-0c36d3d5808f67a02", "subnet-00fa1e71cddcf57d3"], + ) + deployment_manifest_json["target_account_mappings"][0]["region_mappings"][0]["network"] = { + "vpc_id": "vpc-01e556d052f429282", + "private_subnet_ids": { + "value_from": { + "parameter_value": "privateSubnetIds", } - + }, + "security_group_ids": { + "value_from": { + "parameter_value": "securityGroupIds", + } + }, } - + dep = DeploymentManifest(**deployment_manifest_json) dep.validate_and_set_module_defaults() target_account_region = dep.target_accounts_regions[0] - networkOut = npc.load_network_values( - cast(NetworkMapping, target_account_region["network"]), - cast(Dict[str, Any], target_account_region["parameters_regional"]), - target_account_region["account_id"], - target_account_region["region"]) - - assert('vpc-01e556d052f429282' in networkOut.vpc_id) - assert('subnet-0c36d3d5808f67a02' in networkOut.private_subnet_ids) - assert('sg-049033188c114a3d2' in networkOut.security_group_ids) - - + cast(NetworkMapping, target_account_region["network"]), + cast(Dict[str, Any], target_account_region["parameters_regional"]), + target_account_region["account_id"], + target_account_region["region"], + ) + + assert "vpc-01e556d052f429282" in networkOut.vpc_id + assert "subnet-0c36d3d5808f67a02" in networkOut.private_subnet_ids + assert "sg-049033188c114a3d2" in networkOut.security_group_ids + + @pytest.mark.commands @pytest.mark.commands_parameters -def test_too_many_subgroups(session_manager,mocker): - os.environ['testsgs']='["sg-049033188c114a3d2","sg-049033188c114a3d2","sg-049033188c114a3d2","sg-049033188c114a3d2","sg-049033188c114a3d2","sg-049033188c114a3d2"]' - - deployment_manifest_json['target_account_mappings'][0]["region_mappings"][0]["network"] = { - "vpc_id":"vpc-01e556d052f429282", - "private_subnet_ids":["subnet-0c36d3d5808f67a02","subnet-00fa1e71cddcf57d3"], - "security_group_ids":{ - "value_from": { - "env_variable": "testsgs", - } - } - } +def test_too_many_subgroups(session_manager, mocker): + os.environ[ + "testsgs" + ] = '["sg-049033188c114a3d2","sg-049033188c114a3d2","sg-049033188c114a3d2","sg-049033188c114a3d2","sg-049033188c114a3d2","sg-049033188c114a3d2"]' + + deployment_manifest_json["target_account_mappings"][0]["region_mappings"][0]["network"] = { + "vpc_id": "vpc-01e556d052f429282", + "private_subnet_ids": ["subnet-0c36d3d5808f67a02", "subnet-00fa1e71cddcf57d3"], + "security_group_ids": { + "value_from": { + "env_variable": "testsgs", + } + }, + } dep = DeploymentManifest(**deployment_manifest_json) dep.validate_and_set_module_defaults() target_account_region = dep.target_accounts_regions[0] with pytest.raises(seedfarmer.errors.InvalidConfigurationError): npc.load_network_values( - cast(NetworkMapping, target_account_region["network"]), - cast(Dict[str, Any], target_account_region["parameters_regional"]), - target_account_region["account_id"], - target_account_region["region"]) - \ No newline at end of file + cast(NetworkMapping, target_account_region["network"]), + cast(Dict[str, Any], target_account_region["parameters_regional"]), + target_account_region["account_id"], + target_account_region["region"], + ) diff --git a/test/unit-test/test_commands_parameters.py b/test/unit-test/test_commands_parameters.py index f3674191..9ed1cb07 100644 --- a/test/unit-test/test_commands_parameters.py +++ b/test/unit-test/test_commands_parameters.py @@ -1,17 +1,17 @@ import logging import os -import boto3 -import yaml from typing import cast + +import boto3 import pytest -import seedfarmer.commands._parameter_commands as pc +import yaml +from moto import mock_sts +import seedfarmer.commands._parameter_commands as pc from seedfarmer.models.manifests import DeploymentManifest, ModuleManifest, ModuleParameter from seedfarmer.services._service_utils import boto3_client from seedfarmer.services.session_manager import SessionManager -from moto import mock_sts - @pytest.fixture(scope="function") def aws_credentials(): @@ -23,27 +23,25 @@ def aws_credentials(): os.environ["AWS_DEFAULT_REGION"] = "us-east-1" os.environ["MOTO_ACCOUNT_ID"] = "123456789012" + @pytest.fixture(scope="function") def sts_client(aws_credentials): with mock_sts(): yield boto3_client(service_name="sts", session=None) -@pytest.fixture(scope="function") + +@pytest.fixture(scope="function") def session_manager(sts_client): - SessionManager._instances={} + SessionManager._instances = {} SessionManager().get_or_create( project_name="test", region_name="us-east-1", toolchain_region="us-east-1", enable_reaper=False, ) - -dummy_list_params = { - "vpc-id":"vpcid-12345", - "SubnetIds":["subnet-123","subnet-456"], - "Crazy_param_config":"123" - -} + + +dummy_list_params = {"vpc-id": "vpcid-12345", "SubnetIds": ["subnet-123", "subnet-456"], "Crazy_param_config": "123"} deployment_manifest_json = { "name": "mlops", @@ -56,35 +54,19 @@ def session_manager(sts_client): { "name": "networking", "path": "modules/optionals/networking/", - "parameters": [ - { - - "name": "internet-accessible", - "value": True - } - ], - + "parameters": [{"name": "internet-accessible", "value": True}], "target_account": "primary", "target_region": "us-east-1", - }, { "name": "datalake-buckets", "path": "modules/optionals/datalake-buckets", - "parameters": [ - { - - "name": "encryption-type", - "value": "SSE" - } - ], - + "parameters": [{"name": "encryption-type", "value": "SSE"}], "target_account": "primary", - "target_region": "us-east-1", - } - ] + "target_region": "us-east-1", + }, + ], }, - { "name": "core", "path": "manifests/mlops/core-modules.yaml", @@ -93,67 +75,26 @@ def session_manager(sts_client): "name": "efs", "path": "modules/core/efs", "parameters": [ - { - "name": "removal-policy", - "value": "DESTROY" - }, + {"name": "removal-policy", "value": "DESTROY"}, { "name": "vpc-id", "value_from": { - "module_metadata": { - "name": "networking", - "group": "optionals", - "key": "VpcId" - }, + "module_metadata": {"name": "networking", "group": "optionals", "key": "VpcId"}, }, }, - { - "name": "test-secrets-manager", - "value_from": { - "secretsManager": "my-secret-vpc-id" - } - }, - { - "name": "test-ssm-store", - "value_from": { - "parameterStore": "my-ssm-name" - } - }, - { - "name": "test-regional-param", - "value_from": { - "parameterValue": "testRegionalParam" - } - }, - { - "name": "booltest1", - "value" : False - }, - - { - "name": "booltest2", - "value" : "False" - }, - { - "name": "booltest3", - "value" : True - }, - { - "name": "booltest4", - "value" : "No" - }, - { - "name": "booltest5", - "value" : "True" - } - + {"name": "test-secrets-manager", "value_from": {"secretsManager": "my-secret-vpc-id"}}, + {"name": "test-ssm-store", "value_from": {"parameterStore": "my-ssm-name"}}, + {"name": "test-regional-param", "value_from": {"parameterValue": "testRegionalParam"}}, + {"name": "booltest1", "value": False}, + {"name": "booltest2", "value": "False"}, + {"name": "booltest3", "value": True}, + {"name": "booltest4", "value": "No"}, + {"name": "booltest5", "value": "True"}, ], "target_account": "primary", "target_region": "us-east-1", - } ], - }, ], "target_account_mappings": [ @@ -161,24 +102,19 @@ def session_manager(sts_client): "alias": "primary", "account_id": "123456789012", "default": True, - "parameters_global": { - "dockerCredentialsSecret": "aws-addf-docker-credentials" - }, + "parameters_global": {"dockerCredentialsSecret": "aws-addf-docker-credentials"}, "region_mappings": [ { "region": "us-east-1", "default": True, - "parameters_regional": { - "testRegionalParam": "somethingawesomehere" - }, + "parameters_regional": {"testRegionalParam": "somethingawesomehere"}, } ], - } - ] + ], } - - + + @pytest.mark.commands @pytest.mark.commands_parameters def test_generate_export_env_params(): @@ -189,31 +125,35 @@ def test_generate_export_env_params(): @pytest.mark.commands_parameters def test_generate_export_raw_env_params(): pc.generate_export_raw_env_params(dummy_list_params) - - + + @pytest.mark.commands @pytest.mark.commands_parameters -def test_load_parameter_values(session_manager,mocker): - mocker.patch("seedfarmer.commands._parameter_commands.get_module_metadata", - return_value={"IsolatedSubnetIds": [], - "PrivateSubnetIds": ["subnet-0758c0b5ba97e0fc9", "subnet-0dc60fe4557261145"], - "PublicSubnetIds": ["subnet-089b632dada2c71e8", "subnet-0296fff0ba0fa48c0"], - "VpcId": "vpc-01e556d052f429282"} - ) +def test_load_parameter_values(session_manager, mocker): + mocker.patch( + "seedfarmer.commands._parameter_commands.get_module_metadata", + return_value={ + "IsolatedSubnetIds": [], + "PrivateSubnetIds": ["subnet-0758c0b5ba97e0fc9", "subnet-0dc60fe4557261145"], + "PublicSubnetIds": ["subnet-089b632dada2c71e8", "subnet-0296fff0ba0fa48c0"], + "VpcId": "vpc-01e556d052f429282", + }, + ) dep = DeploymentManifest(**deployment_manifest_json) dep.validate_and_set_module_defaults() - - params = pc.load_parameter_values(deployment_name="mlops", - deployment_manifest=dep, - parameters=dep.groups[1].modules[0].parameters, - target_account="123456789012", - target_region="us-east-1", - ) + + params = pc.load_parameter_values( + deployment_name="mlops", + deployment_manifest=dep, + parameters=dep.groups[1].modules[0].parameters, + target_account="123456789012", + target_region="us-east-1", + ) names = [] print(params) for module_parameter in params: names.append(module_parameter.name) - assert ('removal-policy' in names) == True - assert ('vpc-id' in names) == True - assert ('test-secrets-manager' in names) == True - assert ('test-ssm-store' in names) == True \ No newline at end of file + assert ("removal-policy" in names) == True + assert ("vpc-id" in names) == True + assert ("test-secrets-manager" in names) == True + assert ("test-ssm-store" in names) == True diff --git a/test/unit-test/test_commands_stack.py b/test/unit-test/test_commands_stack.py index 3e85288b..399d4761 100644 --- a/test/unit-test/test_commands_stack.py +++ b/test/unit-test/test_commands_stack.py @@ -1,17 +1,16 @@ import logging import os + import boto3 +import pytest import yaml +from moto import mock_sts -import pytest import seedfarmer.commands._stack_commands as sc - from seedfarmer.models.manifests import DeploymentManifest, ModuleManifest, ModulesManifest from seedfarmer.services._service_utils import boto3_client from seedfarmer.services.session_manager import SessionManager -from moto import mock_sts - @pytest.fixture(scope="function") def aws_credentials(): @@ -23,21 +22,24 @@ def aws_credentials(): os.environ["AWS_DEFAULT_REGION"] = "us-east-1" os.environ["MOTO_ACCOUNT_ID"] = "123456789012" + @pytest.fixture(scope="function") def sts_client(aws_credentials): with mock_sts(): yield boto3_client(service_name="sts", session=None) -@pytest.fixture(scope="function") + +@pytest.fixture(scope="function") def session_manager(sts_client): - SessionManager._instances={} + SessionManager._instances = {} SessionManager().get_or_create( project_name="test", region_name="us-east-1", toolchain_region="us-east-1", enable_reaper=False, ) - + + deployment_yaml = yaml.safe_load( """ name: test @@ -56,123 +58,146 @@ def session_manager(sts_client): someKey: someValue """ ) - - + + @pytest.mark.commands @pytest.mark.commands_stack def test_deploy_managed_policy_stack_exists(session_manager, mocker): - mocker.patch("seedfarmer.commands._stack_commands.services.cfn.does_stack_exist",return_value=[True,{}]) + mocker.patch("seedfarmer.commands._stack_commands.services.cfn.does_stack_exist", return_value=[True, {}]) mocker.patch("seedfarmer.commands._stack_commands.services.cfn.deploy_template", return_value=None) - sc.deploy_managed_policy_stack(deployment_manifest=DeploymentManifest(**deployment_yaml), - account_id="123456789012", - region="us-east-1") - + sc.deploy_managed_policy_stack( + deployment_manifest=DeploymentManifest(**deployment_yaml), account_id="123456789012", region="us-east-1" + ) + + @pytest.mark.commands @pytest.mark.commands_stack def test_deploy_managed_policy_stack_not_exists(session_manager, mocker): - mocker.patch("seedfarmer.commands._stack_commands.services.cfn.does_stack_exist",return_value=[False,{}]) + mocker.patch("seedfarmer.commands._stack_commands.services.cfn.does_stack_exist", return_value=[False, {}]) mocker.patch("seedfarmer.commands._stack_commands.services.cfn.deploy_template", return_value=None) - sc.deploy_managed_policy_stack(deployment_manifest=DeploymentManifest(**deployment_yaml), - account_id="123456789012", - region="us-east-1") - + sc.deploy_managed_policy_stack( + deployment_manifest=DeploymentManifest(**deployment_yaml), account_id="123456789012", region="us-east-1" + ) + + @pytest.mark.commands @pytest.mark.commands_stack def test_destroy_managed_policy_stack_not_exists(session_manager, mocker): - mocker.patch("seedfarmer.commands._stack_commands.services.cfn.does_stack_exist",return_value=[False,{}]) + mocker.patch("seedfarmer.commands._stack_commands.services.cfn.does_stack_exist", return_value=[False, {}]) mocker.patch("seedfarmer.commands._stack_commands.services.cfn.destroy_stack", return_value=None) - sc.destroy_managed_policy_stack(account_id="123456789012",region="us-east-1") + sc.destroy_managed_policy_stack(account_id="123456789012", region="us-east-1") + - @pytest.mark.commands @pytest.mark.commands_stack def test_destroy_managed_policy_stack(session_manager, mocker): - mocker.patch("seedfarmer.commands._stack_commands.services.cfn.does_stack_exist",return_value=[True,{}]) + mocker.patch("seedfarmer.commands._stack_commands.services.cfn.does_stack_exist", return_value=[True, {}]) mocker.patch("seedfarmer.commands._stack_commands.services.cfn.destroy_stack", return_value=None) - sc.destroy_managed_policy_stack(account_id="123456789012",region="us-east-1") - - - - + sc.destroy_managed_policy_stack(account_id="123456789012", region="us-east-1") + + @pytest.mark.commands @pytest.mark.commands_stack def test_destroy_module_stack(session_manager, mocker): - mocker.patch("seedfarmer.commands._stack_commands.services.cfn.does_stack_exist",return_value=[True,{}]) + mocker.patch("seedfarmer.commands._stack_commands.services.cfn.does_stack_exist", return_value=[True, {}]) mocker.patch("seedfarmer.commands._stack_commands.services.cfn.destroy_stack", return_value=None) - mocker.patch("seedfarmer.commands._stack_commands.iam.delete_role",return_value=None) - mocker.patch("seedfarmer.commands._stack_commands.iam.detach_inline_policy_from_role",return_value=None) - sc.destroy_module_stack(deployment_name="myapp", - group_name="group", - module_name="module", - account_id="123456789012", - region="us-east-1", - docker_credentials_secret="fsfasdfsad") - + mocker.patch("seedfarmer.commands._stack_commands.iam.delete_role", return_value=None) + mocker.patch("seedfarmer.commands._stack_commands.iam.detach_inline_policy_from_role", return_value=None) + sc.destroy_module_stack( + deployment_name="myapp", + group_name="group", + module_name="module", + account_id="123456789012", + region="us-east-1", + docker_credentials_secret="fsfasdfsad", + ) + + @pytest.mark.commands @pytest.mark.commands_stack def test_deploy_module_stack(session_manager, mocker): - mocker.patch("seedfarmer.commands._stack_commands.services.cfn.does_stack_exist",return_value=(True,{"ProjectPolicyARN":"arn"})) - mocker.patch("seedfarmer.commands._stack_commands.iam.create_check_iam_role",return_value=None) - mocker.patch("seedfarmer.commands._stack_commands.services.cfn.deploy_template",return_value=None) - mocker.patch("seedfarmer.commands._stack_commands.commands.seedkit_deployed", return_value=(True,"stackname", {"SeedkitResourcesPolicyArn":"arn"})) - mocker.patch("seedfarmer.commands._stack_commands.iam.attach_policy_to_role", - return_value=["arn:aws:iam::aws:policy/AdministratorAccess","arn:aws:iam::aws:policy/AdministratorAccess2"]) - mocker.patch("seedfarmer.commands._stack_commands.iam.attach_inline_policy",return_value=None) - + mocker.patch( + "seedfarmer.commands._stack_commands.services.cfn.does_stack_exist", + return_value=(True, {"ProjectPolicyARN": "arn"}), + ) + mocker.patch("seedfarmer.commands._stack_commands.iam.create_check_iam_role", return_value=None) + mocker.patch("seedfarmer.commands._stack_commands.services.cfn.deploy_template", return_value=None) + mocker.patch( + "seedfarmer.commands._stack_commands.commands.seedkit_deployed", + return_value=(True, "stackname", {"SeedkitResourcesPolicyArn": "arn"}), + ) + mocker.patch( + "seedfarmer.commands._stack_commands.iam.attach_policy_to_role", + return_value=["arn:aws:iam::aws:policy/AdministratorAccess", "arn:aws:iam::aws:policy/AdministratorAccess2"], + ) + mocker.patch("seedfarmer.commands._stack_commands.iam.attach_inline_policy", return_value=None) + import mock_data.mock_deployment_manifest_huge as mock_deployment_manifest_huge + dep = DeploymentManifest(**mock_deployment_manifest_huge.deployment_manifest) - - sc.deploy_module_stack(module_stack_path="test/unit-test/mock_data/modules/module-test/modulestack.yaml", - deployment_name="myapp", - group_name="group", - module_name="module", - account_id="123456789012", - region="us-east-1", - parameters=dep.groups[1].modules[1].parameters, - docker_credentials_secret="fsfasdfsad") - + + sc.deploy_module_stack( + module_stack_path="test/unit-test/mock_data/modules/module-test/modulestack.yaml", + deployment_name="myapp", + group_name="group", + module_name="module", + account_id="123456789012", + region="us-east-1", + parameters=dep.groups[1].modules[1].parameters, + docker_credentials_secret="fsfasdfsad", + ) + + @pytest.mark.commands @pytest.mark.commands_stack -def test_destroy_seedkit(session_manager, mocker): - mocker.patch("seedfarmer.commands._stack_commands.commands.destroy_seedkit",return_value=None) - sc.destroy_seedkit(account_id="123456789012",region="us-east-1") +def test_destroy_seedkit(session_manager, mocker): + mocker.patch("seedfarmer.commands._stack_commands.commands.destroy_seedkit", return_value=None) + sc.destroy_seedkit(account_id="123456789012", region="us-east-1") + + # get_module_stack_info @pytest.mark.commands @pytest.mark.commands_stack def test_deploy_seedkit_exists(session_manager, mocker): - mocker.patch("seedfarmer.commands._stack_commands.commands.seedkit_deployed", return_value=(True,"stackname", {"CodeArtifactRepository":"asdfsfa"})) + mocker.patch( + "seedfarmer.commands._stack_commands.commands.seedkit_deployed", + return_value=(True, "stackname", {"CodeArtifactRepository": "asdfsfa"}), + ) mocker.patch("seedfarmer.commands._stack_commands.commands.deploy_seedkit", return_value=None) - sc.deploy_seedkit(account_id="123456789012", - region="us-east-1", - vpc_id="vpc-adfsfas", - private_subnet_ids=["subnet-234234","subnet-657657"], - security_group_ids=["sg-1","sg-2"], - + sc.deploy_seedkit( + account_id="123456789012", + region="us-east-1", + vpc_id="vpc-adfsfas", + private_subnet_ids=["subnet-234234", "subnet-657657"], + security_group_ids=["sg-1", "sg-2"], ) - - + + @pytest.mark.commands @pytest.mark.commands_stack def test_deploy_seedkit_no_exists(session_manager, mocker): - mocker.patch("seedfarmer.commands._stack_commands.commands.seedkit_deployed", return_value=(False,"stackname", {"CodeArtifactRepository":"asdfsfa"})) + mocker.patch( + "seedfarmer.commands._stack_commands.commands.seedkit_deployed", + return_value=(False, "stackname", {"CodeArtifactRepository": "asdfsfa"}), + ) mocker.patch("seedfarmer.commands._stack_commands.commands.deploy_seedkit", return_value=None) - sc.deploy_seedkit(account_id="123456789012", - region="us-east-1", - vpc_id="vpc-adfsfas", - private_subnet_ids=["subnet-234234","subnet-657657"], - security_group_ids=["sg-1","sg-2"], - + sc.deploy_seedkit( + account_id="123456789012", + region="us-east-1", + vpc_id="vpc-adfsfas", + private_subnet_ids=["subnet-234234", "subnet-657657"], + security_group_ids=["sg-1", "sg-2"], ) @pytest.mark.commands @pytest.mark.commands_stack def test_get_module_stack_info(session_manager, mocker): - mocker.patch("seedfarmer.commands._stack_commands.get_module_stack_names",return_value=("TheStackname","TheRoleName")) - sc.get_module_stack_info(deployment_name="myapp", - group_name="group", - module_name="module", - account_id="123456789012", - region="us-east-1") + mocker.patch( + "seedfarmer.commands._stack_commands.get_module_stack_names", return_value=("TheStackname", "TheRoleName") + ) + sc.get_module_stack_info( + deployment_name="myapp", group_name="group", module_name="module", account_id="123456789012", region="us-east-1" + ) diff --git a/test/unit-test/test_mgmt_deploy_utils.py b/test/unit-test/test_mgmt_deploy_utils.py index 83dac1fc..dc78f208 100644 --- a/test/unit-test/test_mgmt_deploy_utils.py +++ b/test/unit-test/test_mgmt_deploy_utils.py @@ -14,20 +14,20 @@ import logging import os -import boto3 -import pytest -import seedfarmer.mgmt.deploy_utils as du -import seedfarmer.errors -from seedfarmer.models.manifests import DeploymentManifest, ModulesManifest, DataFile -from seedfarmer.services._service_utils import boto3_client -from seedfarmer.services.session_manager import SessionManager +import boto3 +import mock_data.mock_deployment_manifest_for_destroy as mock_deployment_manifest_for_destroy +import mock_data.mock_deployment_manifest_huge as mock_deployment_manifest_huge import mock_data.mock_manifests as mock_manifests import mock_data.mock_module_info_huge as mock_module_info_huge -import mock_data.mock_deployment_manifest_huge as mock_deployment_manifest_huge -import mock_data.mock_deployment_manifest_for_destroy as mock_deployment_manifest_for_destroy +import pytest from moto import mock_sts +import seedfarmer.errors +import seedfarmer.mgmt.deploy_utils as du +from seedfarmer.models.manifests import DataFile, DeploymentManifest, ModulesManifest +from seedfarmer.services._service_utils import boto3_client +from seedfarmer.services.session_manager import SessionManager _logger: logging.Logger = logging.getLogger(__name__) @@ -48,9 +48,10 @@ def sts_client(aws_credentials): with mock_sts(): yield boto3_client(service_name="sts", session=None) -@pytest.fixture(scope="function") + +@pytest.fixture(scope="function") def session_manager(sts_client): - SessionManager._instances={} + SessionManager._instances = {} SessionManager().get_or_create( project_name="test", region_name="us-east-1", @@ -58,241 +59,308 @@ def session_manager(sts_client): enable_reaper=False, ) - - - - @pytest.mark.mgmt @pytest.mark.mgmt_deployment_utils def test_dependency_maps(): - module_depends_on, module_dependencies = du.generate_dependency_maps(DeploymentManifest(**mock_manifests.deployment_manifest)) - assert 'optionals-networking' in list(module_depends_on['core-eks']) - assert 'core-eks' in list(module_dependencies['optionals-networking']) - + module_depends_on, module_dependencies = du.generate_dependency_maps( + DeploymentManifest(**mock_manifests.deployment_manifest) + ) + assert "optionals-networking" in list(module_depends_on["core-eks"]) + assert "core-eks" in list(module_dependencies["optionals-networking"]) + + @pytest.mark.mgmt @pytest.mark.mgmt_deployment_utils def test_validate_group_parameters(): manifest = DeploymentManifest(**mock_manifests.deployment_manifest) du.validate_group_parameters(manifest.groups[1]) - + + @pytest.mark.mgmt @pytest.mark.mgmt_deployment_utils def test_validate_group_parameters_failure(): manifest = ModulesManifest(**mock_manifests.modules_manifest_duplicate) with pytest.raises(seedfarmer.errors.InvalidConfigurationError): - du.validate_group_parameters(manifest) - + du.validate_group_parameters(manifest) + + @pytest.mark.mgmt @pytest.mark.mgmt_deployment_utils -def test_need_to_build_no(mocker,session_manager): +def test_need_to_build_no(mocker, session_manager): manifest = DeploymentManifest(**mock_manifests.deployment_manifest) - module_manifest= manifest.groups[1].modules[0] + module_manifest = manifest.groups[1].modules[0] mocker.patch("seedfarmer.mgmt.deploy_utils.mi.does_md5_match", return_value=True) - needed = du.need_to_build(deployment_name='test', - group_name='group', - active_modules=[], - module_upstream_dep={"",None}, - module_manifest=module_manifest,deployment_params_cache=None) + needed = du.need_to_build( + deployment_name="test", + group_name="group", + active_modules=[], + module_upstream_dep={"", None}, + module_manifest=module_manifest, + deployment_params_cache=None, + ) assert needed == False - + + @pytest.mark.mgmt @pytest.mark.mgmt_deployment_utils -def test_need_to_build_yes(mocker,session_manager): +def test_need_to_build_yes(mocker, session_manager): manifest = DeploymentManifest(**mock_manifests.deployment_manifest) - module_manifest= manifest.groups[1].modules[0] + module_manifest = manifest.groups[1].modules[0] mocker.patch("seedfarmer.mgmt.deploy_utils.mi.does_md5_match", return_value=False) - needed = du.need_to_build(deployment_name='test', - group_name='group', - active_modules=[], - module_upstream_dep={"",None}, - module_manifest=module_manifest,deployment_params_cache=None) - assert needed == True + needed = du.need_to_build( + deployment_name="test", + group_name="group", + active_modules=[], + module_upstream_dep={"", None}, + module_manifest=module_manifest, + deployment_params_cache=None, + ) + assert needed == True + @pytest.mark.mgmt @pytest.mark.mgmt_deployment_utils -def test_need_to_build_with_force_deploy_yes(mocker,session_manager): +def test_need_to_build_with_force_deploy_yes(mocker, session_manager): manifest = DeploymentManifest(**mock_manifests.deployment_manifest) - module_manifest= manifest.groups[1].modules[0] + module_manifest = manifest.groups[1].modules[0] mocker.patch("seedfarmer.mgmt.deploy_utils.mi.does_md5_match", return_value=True) - needed = du.need_to_build(deployment_name='test', - group_name='core', - active_modules=["optionals-networking"], - module_upstream_dep={"core-eks":["optionals-networking"]}, - force_redeploy_flag=True, - module_manifest=module_manifest,deployment_params_cache=None) - assert needed == True + needed = du.need_to_build( + deployment_name="test", + group_name="core", + active_modules=["optionals-networking"], + module_upstream_dep={"core-eks": ["optionals-networking"]}, + force_redeploy_flag=True, + module_manifest=module_manifest, + deployment_params_cache=None, + ) + assert needed == True + @pytest.mark.mgmt @pytest.mark.mgmt_deployment_utils -def test_need_to_build_without_force_deploy_yes(mocker,session_manager): +def test_need_to_build_without_force_deploy_yes(mocker, session_manager): manifest = DeploymentManifest(**mock_manifests.deployment_manifest) - module_manifest= manifest.groups[1].modules[0] + module_manifest = manifest.groups[1].modules[0] mocker.patch("seedfarmer.mgmt.deploy_utils.mi.does_md5_match", return_value=False) - needed = du.need_to_build(deployment_name='test', - group_name='core', - active_modules=["optionals-networking"], - module_upstream_dep={"core-eks":["optionals-networking"]}, - force_redeploy_flag=False, - module_manifest=module_manifest,deployment_params_cache=None) - assert needed == True - - -#--------------------------------- + needed = du.need_to_build( + deployment_name="test", + group_name="core", + active_modules=["optionals-networking"], + module_upstream_dep={"core-eks": ["optionals-networking"]}, + force_redeploy_flag=False, + module_manifest=module_manifest, + deployment_params_cache=None, + ) + assert needed == True + + +# --------------------------------- # Test Write / SSM methods in deploy_utils -#--------------------------------- +# --------------------------------- @pytest.mark.mgmt @pytest.mark.mgmt_deployment_utils -def test_write_deployed_deployment_manifest(mocker,session_manager): +def test_write_deployed_deployment_manifest(mocker, session_manager): mocker.patch("seedfarmer.mgmt.deploy_utils.mi.write_deployed_deployment_manifest", return_value=None) du.write_deployed_deployment_manifest(DeploymentManifest(**mock_manifests.deployment_manifest)) @pytest.mark.mgmt @pytest.mark.mgmt_deployment_utils -def test_write_group_manifest(mocker,session_manager): +def test_write_group_manifest(mocker, session_manager): mocker.patch("seedfarmer.mgmt.deploy_utils.mi.write_group_manifest", return_value=None) - du.write_group_manifest(DeploymentManifest(**mock_manifests.deployment_manifest), ModulesManifest(**mock_manifests.modules_manifest)) + du.write_group_manifest( + DeploymentManifest(**mock_manifests.deployment_manifest), ModulesManifest(**mock_manifests.modules_manifest) + ) + - @pytest.mark.mgmt @pytest.mark.mgmt_deployment_utils -def test_prepare_ssm_for_deploy(mocker,session_manager): +def test_prepare_ssm_for_deploy(mocker, session_manager): mocker.patch("seedfarmer.mgmt.deploy_utils.mi.write_module_manifest", return_value=None) mocker.patch("seedfarmer.mgmt.deploy_utils.mi.write_deployspec", return_value=None) mocker.patch("seedfarmer.mgmt.deploy_utils.mi.write_module_md5", return_value=None) mocker.patch("seedfarmer.mgmt.deploy_utils.mi.remove_module_md5", return_value=None) manifest = DeploymentManifest(**mock_manifests.deployment_manifest) - module_manifest= manifest.groups[1].modules[0] - du.prepare_ssm_for_deploy(deployment_name='test', - group_name='group', - module_manifest=module_manifest, - account_id='123456789012', - region='us-east-1') - - + module_manifest = manifest.groups[1].modules[0] + du.prepare_ssm_for_deploy( + deployment_name="test", + group_name="group", + module_manifest=module_manifest, + account_id="123456789012", + region="us-east-1", + ) @pytest.mark.mgmt @pytest.mark.mgmt_deployment_utils -def test_generate_deployed_manifest_already_deployed(mocker,session_manager): - mocker.patch("seedfarmer.mgmt.deploy_utils.mi.get_deployed_deployment_manifest", return_value=mock_manifests.deployment_manifest) - mocker.patch("seedfarmer.mgmt.deploy_utils.mi.get_deployment_manifest", return_value=mock_manifests.deployment_manifest) +def test_generate_deployed_manifest_already_deployed(mocker, session_manager): + mocker.patch( + "seedfarmer.mgmt.deploy_utils.mi.get_deployed_deployment_manifest", + return_value=mock_manifests.deployment_manifest, + ) + mocker.patch( + "seedfarmer.mgmt.deploy_utils.mi.get_deployment_manifest", return_value=mock_manifests.deployment_manifest + ) mocker.patch("seedfarmer.mgmt.deploy_utils.populate_module_info_index", return_value=None) - mocker.patch("seedfarmer.mgmt.deploy_utils._populate_group_modules_from_index",return_value=mock_manifests.deployment_manifest['groups']) - du.generate_deployed_manifest(deployment_name="myapp", - skip_deploy_spec=True, - ignore_deployed=True) - + mocker.patch( + "seedfarmer.mgmt.deploy_utils._populate_group_modules_from_index", + return_value=mock_manifests.deployment_manifest["groups"], + ) + du.generate_deployed_manifest(deployment_name="myapp", skip_deploy_spec=True, ignore_deployed=True) + + @pytest.mark.mgmt @pytest.mark.mgmt_deployment_utils -def test_generate_deployed_manifest(mocker,session_manager): +def test_generate_deployed_manifest(mocker, session_manager): mocker.patch("seedfarmer.mgmt.deploy_utils.mi.get_deployed_deployment_manifest", return_value=None) - mocker.patch("seedfarmer.mgmt.deploy_utils.mi.get_deployment_manifest", return_value=mock_manifests.deployment_manifest) + mocker.patch( + "seedfarmer.mgmt.deploy_utils.mi.get_deployment_manifest", return_value=mock_manifests.deployment_manifest + ) mocker.patch("seedfarmer.mgmt.deploy_utils.populate_module_info_index", return_value=None) - mocker.patch("seedfarmer.mgmt.deploy_utils._populate_group_modules_from_index",return_value=mock_manifests.deployment_manifest['groups']) - du.generate_deployed_manifest(deployment_name="myapp", - skip_deploy_spec=True, - ignore_deployed=False) - - + mocker.patch( + "seedfarmer.mgmt.deploy_utils._populate_group_modules_from_index", + return_value=mock_manifests.deployment_manifest["groups"], + ) + du.generate_deployed_manifest(deployment_name="myapp", skip_deploy_spec=True, ignore_deployed=False) + + @pytest.mark.mgmt @pytest.mark.mgmt_deployment_utils -def test_get_deployed_group_ordering_not_deployed(mocker,session_manager): +def test_get_deployed_group_ordering_not_deployed(mocker, session_manager): mocker.patch("seedfarmer.mgmt.deploy_utils.mi.get_deployed_deployment_manifest", return_value=None) - mocker.patch("seedfarmer.mgmt.deploy_utils.mi.get_deployment_manifest", return_value=mock_manifests.deployment_manifest) + mocker.patch( + "seedfarmer.mgmt.deploy_utils.mi.get_deployment_manifest", return_value=mock_manifests.deployment_manifest + ) mocker.patch("seedfarmer.mgmt.deploy_utils.populate_module_info_index", return_value=None) - mocker.patch("seedfarmer.mgmt.deploy_utils._populate_group_modules_from_index",return_value=mock_manifests.deployment_manifest['groups']) - du.get_deployed_group_ordering(deployment_name="myapp") - + mocker.patch( + "seedfarmer.mgmt.deploy_utils._populate_group_modules_from_index", + return_value=mock_manifests.deployment_manifest["groups"], + ) + du.get_deployed_group_ordering(deployment_name="myapp") + + @pytest.mark.mgmt @pytest.mark.mgmt_deployment_utils -def test_get_deployed_group_ordering_deployed(mocker,session_manager): - mocker.patch("seedfarmer.mgmt.deploy_utils.mi.get_deployed_deployment_manifest", return_value=mock_manifests.deployment_manifest) - mocker.patch("seedfarmer.mgmt.deploy_utils.mi.get_deployment_manifest", return_value=mock_manifests.deployment_manifest) +def test_get_deployed_group_ordering_deployed(mocker, session_manager): + mocker.patch( + "seedfarmer.mgmt.deploy_utils.mi.get_deployed_deployment_manifest", + return_value=mock_manifests.deployment_manifest, + ) + mocker.patch( + "seedfarmer.mgmt.deploy_utils.mi.get_deployment_manifest", return_value=mock_manifests.deployment_manifest + ) mocker.patch("seedfarmer.mgmt.deploy_utils.populate_module_info_index", return_value=None) - mocker.patch("seedfarmer.mgmt.deploy_utils._populate_group_modules_from_index",return_value=mock_manifests.deployment_manifest['groups']) + mocker.patch( + "seedfarmer.mgmt.deploy_utils._populate_group_modules_from_index", + return_value=mock_manifests.deployment_manifest["groups"], + ) du.get_deployed_group_ordering(deployment_name="myapp") - - - + + @pytest.mark.mgmt @pytest.mark.mgmt_deployment_utils @pytest.mark.parametrize("session", [None, boto3.Session()]) -def test_update_deployspec(mocker,session): +def test_update_deployspec(mocker, session): mocker.patch("seedfarmer.mgmt.deploy_utils.mi.write_deployspec", return_value=mock_manifests.deployment_manifest) - du.update_deployspec(deployment="myapp", - group="test-group", - module="test-module", - module_path="test/unit-test/mock_data/modules/module-test", - session=session - ) - - + du.update_deployspec( + deployment="myapp", + group="test-group", + module="test-module", + module_path="test/unit-test/mock_data/modules/module-test", + session=session, + ) + + @pytest.mark.mgmt -@pytest.mark.mgmt_deployment_utils -def test_populate_module_info_index(session_manager,mocker): +@pytest.mark.mgmt_deployment_utils +def test_populate_module_info_index(session_manager, mocker): import json - mocker.patch("seedfarmer.mgmt.deploy_utils.mi.get_parameter_data_cache",return_value=mock_module_info_huge.module_index_info_huge) - module_info_index=du.populate_module_info_index(deployment_manifest=DeploymentManifest(**mock_manifests.deployment_manifest)) - - -#----------------------- + + mocker.patch( + "seedfarmer.mgmt.deploy_utils.mi.get_parameter_data_cache", + return_value=mock_module_info_huge.module_index_info_huge, + ) + module_info_index = du.populate_module_info_index( + deployment_manifest=DeploymentManifest(**mock_manifests.deployment_manifest) + ) + + +# ----------------------- # Test Filtering for Deploy / Destroy -#----------------------- +# ----------------------- @pytest.mark.mgmt -@pytest.mark.mgmt_deployment_utils_filter -def test_filter_destroy_deploy(session_manager,mocker): +@pytest.mark.mgmt_deployment_utils_filter +def test_filter_destroy_deploy(session_manager, mocker): import json - mocker.patch("seedfarmer.mgmt.deploy_utils.mi.get_parameter_data_cache",return_value=mock_module_info_huge.module_index_info_huge) - module_info_index=du.populate_module_info_index(deployment_manifest=DeploymentManifest(**mock_deployment_manifest_huge.deployment_manifest)) + + mocker.patch( + "seedfarmer.mgmt.deploy_utils.mi.get_parameter_data_cache", + return_value=mock_module_info_huge.module_index_info_huge, + ) + module_info_index = du.populate_module_info_index( + deployment_manifest=DeploymentManifest(**mock_deployment_manifest_huge.deployment_manifest) + ) deployment_manifest = DeploymentManifest(**mock_deployment_manifest_huge.deployment_manifest) - du.filter_deploy_destroy(apply_manifest=deployment_manifest,module_info_index=module_info_index) + du.filter_deploy_destroy(apply_manifest=deployment_manifest, module_info_index=module_info_index) + @pytest.mark.mgmt -@pytest.mark.mgmt_deployment_utils_filter -def test_filter_destroy_deploy_with_destroy(session_manager,mocker): +@pytest.mark.mgmt_deployment_utils_filter +def test_filter_destroy_deploy_with_destroy(session_manager, mocker): import json - mocker.patch("seedfarmer.mgmt.deploy_utils.mi.get_parameter_data_cache",return_value=mock_module_info_huge.module_index_info_huge) - module_info_index=du.populate_module_info_index(deployment_manifest=DeploymentManifest(**mock_deployment_manifest_huge.deployment_manifest)) - deployment_manifest_destroy = DeploymentManifest(**mock_deployment_manifest_for_destroy.deployment_manifest) - du.filter_deploy_destroy(apply_manifest=deployment_manifest_destroy, - module_info_index=module_info_index) + mocker.patch( + "seedfarmer.mgmt.deploy_utils.mi.get_parameter_data_cache", + return_value=mock_module_info_huge.module_index_info_huge, + ) + module_info_index = du.populate_module_info_index( + deployment_manifest=DeploymentManifest(**mock_deployment_manifest_huge.deployment_manifest) + ) + deployment_manifest_destroy = DeploymentManifest(**mock_deployment_manifest_for_destroy.deployment_manifest) + du.filter_deploy_destroy(apply_manifest=deployment_manifest_destroy, module_info_index=module_info_index) @pytest.mark.mgmt -@pytest.mark.mgmt_deployment_utils_filter -def test_populate_groups_to_remove(session_manager,mocker): +@pytest.mark.mgmt_deployment_utils_filter +def test_populate_groups_to_remove(session_manager, mocker): import json - mocker.patch("seedfarmer.mgmt.deploy_utils.mi.get_parameter_data_cache",return_value=mock_module_info_huge.module_index_info_huge) - module_info_index=du.populate_module_info_index(deployment_manifest=DeploymentManifest(**mock_deployment_manifest_huge.deployment_manifest)) + + mocker.patch( + "seedfarmer.mgmt.deploy_utils.mi.get_parameter_data_cache", + return_value=mock_module_info_huge.module_index_info_huge, + ) + module_info_index = du.populate_module_info_index( + deployment_manifest=DeploymentManifest(**mock_deployment_manifest_huge.deployment_manifest) + ) deployment_manifest_destroy = DeploymentManifest(**mock_deployment_manifest_for_destroy.deployment_manifest) - modules_to_destory_list=du._populate_groups_to_remove(deployment_name=deployment_manifest_destroy.name, - apply_groups=deployment_manifest_destroy.groups, - module_info_index=module_info_index) + modules_to_destory_list = du._populate_groups_to_remove( + deployment_name=deployment_manifest_destroy.name, + apply_groups=deployment_manifest_destroy.groups, + module_info_index=module_info_index, + ) found_for_delete = False - false_found_for_delete= False + false_found_for_delete = False for module in modules_to_destory_list: - if module.name in ['users']: + if module.name in ["users"]: found_for_delete = True if module.name in ["networking"]: - false_found_for_delete =True - assert found_for_delete ==True + false_found_for_delete = True + assert found_for_delete == True assert false_found_for_delete == False - - - + + @pytest.mark.mgmt -@pytest.mark.mgmt_deployment_utils_filter +@pytest.mark.mgmt_deployment_utils_filter def test_validate_module_dependencies(): - du.validate_module_dependencies(module_dependencies=mock_deployment_manifest_for_destroy.module_dependencies, - destroy_manifest=DeploymentManifest(**mock_deployment_manifest_for_destroy.destroy_manifest) - + du.validate_module_dependencies( + module_dependencies=mock_deployment_manifest_for_destroy.module_dependencies, + destroy_manifest=DeploymentManifest(**mock_deployment_manifest_for_destroy.destroy_manifest), ) - + + @pytest.mark.mgmt -@pytest.mark.mgmt_deployment_utils_filter +@pytest.mark.mgmt_deployment_utils_filter def test_validate_datafiles(): files = DataFile(file_path="") diff --git a/test/unit-test/test_mgmt_module_info.py b/test/unit-test/test_mgmt_module_info.py index dd932877..0d1d3b3c 100644 --- a/test/unit-test/test_mgmt_module_info.py +++ b/test/unit-test/test_mgmt_module_info.py @@ -14,9 +14,8 @@ import logging import os -import boto3 - +import boto3 import pytest _logger: logging.Logger = logging.getLogger(__name__) @@ -41,325 +40,288 @@ def session(aws_credentials): ### Test Model Init @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_get_all_deployments(aws_credentials,session,mocker): +def test_get_all_deployments(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi - mocker.patch("seedfarmer.mgmt.module_info.ssm.list_parameters_with_filter", - return_value=["/myapp/test/hey"]) + + mocker.patch("seedfarmer.mgmt.module_info.ssm.list_parameters_with_filter", return_value=["/myapp/test/hey"]) mi.get_all_deployments(session=session) + @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_get_all_deployments_with_manifest(aws_credentials,session,mocker): +def test_get_all_deployments_with_manifest(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi - mocker.patch("seedfarmer.mgmt.module_info.ssm.list_parameters_with_filter", - return_value=["/myapp/test/manifest"]) + + mocker.patch("seedfarmer.mgmt.module_info.ssm.list_parameters_with_filter", return_value=["/myapp/test/manifest"]) mi.get_all_deployments(session=session) + @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_get_all_deployments_with_nothing(aws_credentials,session,mocker): +def test_get_all_deployments_with_nothing(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi - mocker.patch("seedfarmer.mgmt.module_info.ssm.list_parameters_with_filter", - return_value=[]) + + mocker.patch("seedfarmer.mgmt.module_info.ssm.list_parameters_with_filter", return_value=[]) mi.get_all_deployments(session=session) @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_get_all_groups(aws_credentials,session,mocker): +def test_get_all_groups(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi - mocker.patch("seedfarmer.mgmt.module_info.ssm.list_parameters_with_filter", - return_value=["/myapp/test/manifest","/myapp/test/optionals","/myapp/test/core"]) + + mocker.patch( + "seedfarmer.mgmt.module_info.ssm.list_parameters_with_filter", + return_value=["/myapp/test/manifest", "/myapp/test/optionals", "/myapp/test/core"], + ) mi.get_all_groups(deployment="myapp", session=session) @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_get_deployed_modules(aws_credentials,session,mocker): +def test_get_deployed_modules(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi - mocker.patch("seedfarmer.mgmt.module_info.ssm.list_parameters_with_filter", - return_value=["/myapp/test/manifest","/myapp/test/optionals","/myapp/test/core"]) - mi.get_deployed_modules(deployment="myapp", - group='test', - session=session) + + mocker.patch( + "seedfarmer.mgmt.module_info.ssm.list_parameters_with_filter", + return_value=["/myapp/test/manifest", "/myapp/test/optionals", "/myapp/test/core"], + ) + mi.get_deployed_modules(deployment="myapp", group="test", session=session) + @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_get_module_md5(aws_credentials,session,mocker): +def test_get_module_md5(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi - mocker.patch("seedfarmer.mgmt.module_info.ssm.get_parameter_if_exists", - return_value={"hash":"12345678"}) - mi.get_module_md5(deployment="myapp", - group='test', - module="mymodule", - type=mi.ModuleConst.METADATA, - session=session) + mocker.patch("seedfarmer.mgmt.module_info.ssm.get_parameter_if_exists", return_value={"hash": "12345678"}) + mi.get_module_md5( + deployment="myapp", group="test", module="mymodule", type=mi.ModuleConst.METADATA, session=session + ) - mi.get_module_md5(deployment="myapp", - group='test', - module="mymodule", - type=mi.ModuleConst.BUNDLE, - session=session) + mi.get_module_md5(deployment="myapp", group="test", module="mymodule", type=mi.ModuleConst.BUNDLE, session=session) - mi.get_module_md5(deployment="myapp", - group='test', - module="mymodule", - type=mi.ModuleConst.DEPLOYSPEC, - session=session) - mi.get_module_md5(deployment="myapp", - group='test', - module="mymodule", - type=mi.ModuleConst.METADATA, - session=session) + mi.get_module_md5( + deployment="myapp", group="test", module="mymodule", type=mi.ModuleConst.DEPLOYSPEC, session=session + ) + mi.get_module_md5( + deployment="myapp", group="test", module="mymodule", type=mi.ModuleConst.METADATA, session=session + ) @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_get_module_metadata(aws_credentials,session,mocker): +def test_get_module_metadata(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi - mocker.patch("seedfarmer.mgmt.module_info._fetch_helper", - return_value={"hey":"yo"}) - mi.get_module_metadata(deployment="myapp", - group='test', - module="mymodule", - session=session) + + mocker.patch("seedfarmer.mgmt.module_info._fetch_helper", return_value={"hey": "yo"}) + mi.get_module_metadata(deployment="myapp", group="test", module="mymodule", session=session) + @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_get_module_manifest(aws_credentials,session,mocker): +def test_get_module_manifest(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi - mocker.patch("seedfarmer.mgmt.module_info._fetch_helper", - return_value={"hey":"yo"}) - mi.get_module_manifest(deployment="myapp", - group='test', - module="mymodule", - session=session) + + mocker.patch("seedfarmer.mgmt.module_info._fetch_helper", return_value={"hey": "yo"}) + mi.get_module_manifest(deployment="myapp", group="test", module="mymodule", session=session) @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_get_deployment_manifest(aws_credentials,session,mocker): +def test_get_deployment_manifest(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi - mocker.patch("seedfarmer.mgmt.module_info._fetch_helper", - return_value={"hey":"yo"}) + + mocker.patch("seedfarmer.mgmt.module_info._fetch_helper", return_value={"hey": "yo"}) mi.get_deployment_manifest(deployment="myapp", session=session) + @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_get_deployed_deployment_manifest(aws_credentials,session,mocker): +def test_get_deployed_deployment_manifest(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi - mocker.patch("seedfarmer.mgmt.module_info._fetch_helper", - return_value={"hey":"yo"}) - mi.get_deployed_deployment_manifest(deployment="myapp",session=session) + + mocker.patch("seedfarmer.mgmt.module_info._fetch_helper", return_value={"hey": "yo"}) + mi.get_deployed_deployment_manifest(deployment="myapp", session=session) + @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_get_deployspec(aws_credentials,session,mocker): +def test_get_deployspec(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi - mocker.patch("seedfarmer.mgmt.module_info._fetch_helper", - return_value={"hey":"yo"}) - mi.get_deployspec(deployment="myapp", - group='test', - module="mymodule", - session=session) + + mocker.patch("seedfarmer.mgmt.module_info._fetch_helper", return_value={"hey": "yo"}) + mi.get_deployspec(deployment="myapp", group="test", module="mymodule", session=session) + @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_get_group_manifest(aws_credentials,session,mocker): +def test_get_group_manifest(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi - mocker.patch("seedfarmer.mgmt.module_info._fetch_helper", - return_value={"hey":"yo"}) - mi.get_group_manifest(deployment="myapp", - group='test', - session=session) + + mocker.patch("seedfarmer.mgmt.module_info._fetch_helper", return_value={"hey": "yo"}) + mi.get_group_manifest(deployment="myapp", group="test", session=session) + @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_does_module_exist(aws_credentials,session,mocker): +def test_does_module_exist(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi - mocker.patch("seedfarmer.mgmt.module_info.ssm.does_parameter_exist", - return_value=True) - mi.does_module_exist(deployment="myapp", - group='test', - module='mymodule', - session=session) + + mocker.patch("seedfarmer.mgmt.module_info.ssm.does_parameter_exist", return_value=True) + mi.does_module_exist(deployment="myapp", group="test", module="mymodule", session=session) @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_does_md5_match(aws_credentials,session,mocker): +def test_does_md5_match(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi - mocker.patch("seedfarmer.mgmt.module_info.ssm.get_parameter_if_exists", - return_value={"hash":"12345678"}) - mi.does_md5_match(deployment="myapp", - group='test', - module="mymodule", - hash="12345678", - type=mi.ModuleConst.BUNDLE, - session=session) - mi.does_md5_match(deployment="myapp", - group='test', - module="mymodule", - hash="blah", - type=mi.ModuleConst.BUNDLE, - session=session) + mocker.patch("seedfarmer.mgmt.module_info.ssm.get_parameter_if_exists", return_value={"hash": "12345678"}) + mi.does_md5_match( + deployment="myapp", + group="test", + module="mymodule", + hash="12345678", + type=mi.ModuleConst.BUNDLE, + session=session, + ) + + mi.does_md5_match( + deployment="myapp", group="test", module="mymodule", hash="blah", type=mi.ModuleConst.BUNDLE, session=session + ) + @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_does_md5_match_no(aws_credentials,session,mocker): +def test_does_md5_match_no(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi - mocker.patch("seedfarmer.mgmt.module_info.ssm.get_parameter_if_exists", - return_value=None) - mi.does_md5_match(deployment="myapp", - group='test', - module="mymodule", - hash="blah", - type=mi.ModuleConst.BUNDLE, - session=session) + mocker.patch("seedfarmer.mgmt.module_info.ssm.get_parameter_if_exists", return_value=None) + + mi.does_md5_match( + deployment="myapp", group="test", module="mymodule", hash="blah", type=mi.ModuleConst.BUNDLE, session=session + ) @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_write_metadata(aws_credentials,session,mocker): +def test_write_metadata(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi + mocker.patch("seedfarmer.mgmt.module_info.ssm.put_parameter", return_value=True) - mi.write_metadata(deployment="myapp", - group='test', - module='mymodule', - data={"Hey","Yo"}, - session=session) + mi.write_metadata(deployment="myapp", group="test", module="mymodule", data={"Hey", "Yo"}, session=session) + @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_write_group_manifest(aws_credentials,session,mocker): +def test_write_group_manifest(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi + mocker.patch("seedfarmer.mgmt.module_info.ssm.put_parameter", return_value=True) - mi.write_group_manifest(deployment="myapp", - group='test', - data={"Hey","Yo"}, - session=session) + mi.write_group_manifest(deployment="myapp", group="test", data={"Hey", "Yo"}, session=session) + @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_write_module_manifest(aws_credentials,session,mocker): +def test_write_module_manifest(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi + mocker.patch("seedfarmer.mgmt.module_info.ssm.put_parameter", return_value=True) - mi.write_module_manifest(deployment="myapp", - group='test', - module='mymodule', - data={"Hey","Yo"}, - session=session) + mi.write_module_manifest(deployment="myapp", group="test", module="mymodule", data={"Hey", "Yo"}, session=session) + @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_write_deployspec(aws_credentials,session,mocker): +def test_write_deployspec(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi + mocker.patch("seedfarmer.mgmt.module_info.ssm.put_parameter", return_value=True) - mi.write_deployspec(deployment="myapp", - group='test', - module='mymodule', - data={"Hey","Yo"}, - session=session) + mi.write_deployspec(deployment="myapp", group="test", module="mymodule", data={"Hey", "Yo"}, session=session) + @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_write_module_md5(aws_credentials,session,mocker): +def test_write_module_md5(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi + mocker.patch("seedfarmer.mgmt.module_info.ssm.put_parameter", return_value=True) - mi.write_module_md5(deployment="myapp", - group='test', - module='mymodule', - hash="12345", - type=mi.ModuleConst.BUNDLE, - session=session) + mi.write_module_md5( + deployment="myapp", group="test", module="mymodule", hash="12345", type=mi.ModuleConst.BUNDLE, session=session + ) @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_write_deployment_manifest(aws_credentials,session,mocker): +def test_write_deployment_manifest(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi + mocker.patch("seedfarmer.mgmt.module_info.ssm.put_parameter", return_value=True) - mi.write_deployment_manifest(deployment="myapp", - data={"Hey","Yo"}, - session=session) + mi.write_deployment_manifest(deployment="myapp", data={"Hey", "Yo"}, session=session) + @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_write_deployed_deployment_manifest(aws_credentials,session,mocker): +def test_write_deployed_deployment_manifest(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi + mocker.patch("seedfarmer.mgmt.module_info.ssm.put_parameter", return_value=True) - mi.write_deployed_deployment_manifest(deployment="myapp", - data={"Hey","Yo"}, - session=session) + mi.write_deployed_deployment_manifest(deployment="myapp", data={"Hey", "Yo"}, session=session) @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_remove_all_info(aws_credentials,session,mocker): +def test_remove_all_info(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi - mocker.patch("seedfarmer.mgmt.module_info.ssm.delete_parameters", return_value=True) - mi.remove_deployed_deployment_manifest(deployment="myapp",session=session) - mi.remove_deployment_manifest(deployment="myapp",session=session) - mi.remove_group_info(deployment="myapp", - group="mygroup", - session=session) - mi.remove_module_info(deployment="myapp", - group="mygroup", - module='mymodule', - session=session) - mi.remove_module_md5(deployment="myapp", - group="mygroup", - module='mymodule', - type=mi.ModuleConst.BUNDLE, - session=session) - - - + mocker.patch("seedfarmer.mgmt.module_info.ssm.delete_parameters", return_value=True) + mi.remove_deployed_deployment_manifest(deployment="myapp", session=session) + mi.remove_deployment_manifest(deployment="myapp", session=session) + mi.remove_group_info(deployment="myapp", group="mygroup", session=session) + mi.remove_module_info(deployment="myapp", group="mygroup", module="mymodule", session=session) + mi.remove_module_md5( + deployment="myapp", group="mygroup", module="mymodule", type=mi.ModuleConst.BUNDLE, session=session + ) @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_get_module_stack_names(mocker,session): +def test_get_module_stack_names(mocker, session): import seedfarmer.mgmt.module_info as mi + mocker.patch("seedfarmer.mgmt.module_info.generate_hash", return_value="1234") mocker.patch("seedfarmer.mgmt.module_info.generate_session_hash", return_value="1234dade") - mi.get_module_stack_names(deployment_name="myapp", - group_name='test', - module_name="mymodule", - session=session) + mi.get_module_stack_names(deployment_name="myapp", group_name="test", module_name="mymodule", session=session) @pytest.mark.mgmt @pytest.mark.mgmt_module_info def test_get_modulestack_path(): import seedfarmer.mgmt.module_info as mi + mi.get_modulestack_path(module_path="modules/basic/something") @@ -367,149 +329,146 @@ def test_get_modulestack_path(): @pytest.mark.mgmt_module_info def test_get_deployspec_path(): import seedfarmer.mgmt.module_info as mi + with pytest.raises(Exception): mi.get_deployspec_path(module_path="modules/basic/something") @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_fetch_helper(aws_credentials,session,mocker): +def test_fetch_helper(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi + mocker.patch("seedfarmer.mgmt.module_info.ssm.get_parameter_if_exists", return_value={}) mi._fetch_helper(name="myapp", session=session) - mi._fetch_helper(name="myapp",params_cache={"myapp":"yo"}, session=session) - + mi._fetch_helper(name="myapp", params_cache={"myapp": "yo"}, session=session) + secrets_manager_mock_data = { - "Versions": [ - { - "VersionId": "17853545-211c-461b-938c-6f9bf36652ce", - "VersionStages": [ - "AWSPREVIOUS", - "WTFTESTING" - ], - "LastAccessedDate": "2023-04-17 20:00:00-04:00", - "CreatedDate": "2023-04-17 20:52:11.327000-04:00", - "KmsKeyIds": [ - "DefaultEncryptionKey" - ] - }, - { - "VersionId": "3ae24b7a-a4dc-4ee3-ba47-ef4969c1e687", - "VersionStages": [ - "USEME", - "AWSCURRENT" - ], - "LastAccessedDate": "2023-04-17 20:00:00-04:00", - "CreatedDate": "2023-04-17 20:53:26.644000-04:00", - "KmsKeyIds": [ - "DefaultEncryptionKey" - ] - } - ], - "ARN": "arn:aws:secretsmanager:us-east-1:123456789012:secret:testderekaddf-QZHkSe", - "Name": "testderekaddf", - "ResponseMetadata": { - "RequestId": "078d54ce-4005-43ad-af49-722dd1016e36", - "HTTPStatusCode": 200, - "HTTPHeaders": { - "x-amzn-requestid": "078d54ce-4005-43ad-af49-722dd1016e36", - "content-type": "application/x-amz-json-1.1", - "content-length": "505", - "date": "Tue, 18 Apr 2023 02:06:29 GMT" - }, - "RetryAttempts": 0 - } - } + "Versions": [ + { + "VersionId": "17853545-211c-461b-938c-6f9bf36652ce", + "VersionStages": ["AWSPREVIOUS", "WTFTESTING"], + "LastAccessedDate": "2023-04-17 20:00:00-04:00", + "CreatedDate": "2023-04-17 20:52:11.327000-04:00", + "KmsKeyIds": ["DefaultEncryptionKey"], + }, + { + "VersionId": "3ae24b7a-a4dc-4ee3-ba47-ef4969c1e687", + "VersionStages": ["USEME", "AWSCURRENT"], + "LastAccessedDate": "2023-04-17 20:00:00-04:00", + "CreatedDate": "2023-04-17 20:53:26.644000-04:00", + "KmsKeyIds": ["DefaultEncryptionKey"], + }, + ], + "ARN": "arn:aws:secretsmanager:us-east-1:123456789012:secret:testderekaddf-QZHkSe", + "Name": "testderekaddf", + "ResponseMetadata": { + "RequestId": "078d54ce-4005-43ad-af49-722dd1016e36", + "HTTPStatusCode": 200, + "HTTPHeaders": { + "x-amzn-requestid": "078d54ce-4005-43ad-af49-722dd1016e36", + "content-type": "application/x-amz-json-1.1", + "content-length": "505", + "date": "Tue, 18 Apr 2023 02:06:29 GMT", + }, + "RetryAttempts": 0, + }, +} + + +@pytest.mark.mgmt +@pytest.mark.mgmt_module_info +def test_get_secrets_version_with_version_id(aws_credentials, session, mocker): + import seedfarmer.mgmt.module_info as mi + + mocker.patch( + "seedfarmer.mgmt.module_info.secrets.list_secret_version_ids", + return_value=secrets_manager_mock_data["Versions"], + ) + + val = mi.get_secrets_version( + secret_name="sometest", version_ref="17853545-211c-461b-938c-6f9bf36652ce", session=session + ) - -@pytest.mark.mgmt -@pytest.mark.mgmt_module_info -def test_get_secrets_version_with_version_id(aws_credentials,session,mocker): - import seedfarmer.mgmt.module_info as mi - - mocker.patch("seedfarmer.mgmt.module_info.secrets.list_secret_version_ids", - return_value=secrets_manager_mock_data['Versions']) - - val = mi.get_secrets_version(secret_name="sometest", - version_ref="17853545-211c-461b-938c-6f9bf36652ce", - session=session) - assert val == "17853545-211c-461b-938c-6f9bf36652ce" - + + @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_get_secrets_version_with_no_ref(aws_credentials,session,mocker): +def test_get_secrets_version_with_no_ref(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi - - mocker.patch("seedfarmer.mgmt.module_info.secrets.list_secret_version_ids", - return_value=secrets_manager_mock_data['Versions']) - - val = mi.get_secrets_version(secret_name="sometest", - session=session) - + + mocker.patch( + "seedfarmer.mgmt.module_info.secrets.list_secret_version_ids", + return_value=secrets_manager_mock_data["Versions"], + ) + + val = mi.get_secrets_version(secret_name="sometest", session=session) + assert val == "3ae24b7a-a4dc-4ee3-ba47-ef4969c1e687" @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_get_secrets_version_with_status(aws_credentials,session,mocker): +def test_get_secrets_version_with_status(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi - - mocker.patch("seedfarmer.mgmt.module_info.secrets.list_secret_version_ids", - return_value=secrets_manager_mock_data['Versions']) - - val = mi.get_secrets_version(secret_name="sometest:username", - version_ref="USEME", - session=session) - + + mocker.patch( + "seedfarmer.mgmt.module_info.secrets.list_secret_version_ids", + return_value=secrets_manager_mock_data["Versions"], + ) + + val = mi.get_secrets_version(secret_name="sometest:username", version_ref="USEME", session=session) + assert val == "3ae24b7a-a4dc-4ee3-ba47-ef4969c1e687" - - + + @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_get_ssm_parameter_version(aws_credentials,session,mocker): +def test_get_ssm_parameter_version(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi + test_json = { - "Parameters": [ - { + "Parameters": [ + { "Name": "testingversioning", "Type": "String", "LastModifiedUser": "arn:aws:sts::123456789012:assumed-role/Admin/someone-Isengard", "Version": 5, "Tier": "Standard", "Policies": [], - "DataType": "text" - } - ], - "ResponseMetadata": { - "RequestId": "693a5834-1802-4aa1-8254-879f942e9f5b", - "HTTPStatusCode": 200, - "HTTPHeaders": { + "DataType": "text", + } + ], + "ResponseMetadata": { + "RequestId": "693a5834-1802-4aa1-8254-879f942e9f5b", + "HTTPStatusCode": 200, + "HTTPHeaders": { "server": "Server", "date": "Mon, 17 Apr 2023 23:45:53 GMT", "content-type": "application/x-amz-json-1.1", "content-length": "243", "connection": "keep-alive", - "x-amzn-requestid": "693a5834-1802-4aa1-8254-879f942e9f5b" - }, - "RetryAttempts": 0 - } - } - - - + "x-amzn-requestid": "693a5834-1802-4aa1-8254-879f942e9f5b", + }, + "RetryAttempts": 0, + }, + } + mocker.patch("seedfarmer.mgmt.module_info.ssm.describe_parameter", return_value=test_json) - val = mi.get_ssm_parameter_version("sometest",session=session) - + val = mi.get_ssm_parameter_version("sometest", session=session) + assert val == 5 - + + @pytest.mark.mgmt @pytest.mark.mgmt_module_info -def test_get_ssm_parameter_version_failure(aws_credentials,session,mocker): +def test_get_ssm_parameter_version_failure(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi + mocker.patch("seedfarmer.mgmt.module_info.ssm.describe_parameter", return_value=None) with pytest.raises(Exception): - mi.get_ssm_parameter_version("sometest",session=session) + mi.get_ssm_parameter_version("sometest", session=session) diff --git a/test/unit-test/test_models.py b/test/unit-test/test_models.py index 2fa7045f..9f2d2d0d 100644 --- a/test/unit-test/test_models.py +++ b/test/unit-test/test_models.py @@ -19,8 +19,8 @@ import pytest import yaml -from seedfarmer.models.deploy_responses import CodeSeederMetadata, ModuleDeploymentResponse import seedfarmer.errors +from seedfarmer.models.deploy_responses import CodeSeederMetadata, ModuleDeploymentResponse from seedfarmer.models.manifests import DeploymentManifest, ModuleManifest from seedfarmer.models.manifests._module_manifest import DeploySpec diff --git a/test/unit-test/test_services.py b/test/unit-test/test_services.py index 0d4f464a..e54bc0c3 100644 --- a/test/unit-test/test_services.py +++ b/test/unit-test/test_services.py @@ -15,9 +15,10 @@ import botocore import pytest from moto import mock_codebuild, mock_iam, mock_secretsmanager, mock_ssm, mock_sts + +from seedfarmer.services import _service_utils from seedfarmer.services._service_utils import boto3_client from seedfarmer.services.session_manager import SessionManager -from seedfarmer.services import _service_utils _logger: logging.Logger = logging.getLogger(__name__) @@ -44,9 +45,9 @@ def sts_client(aws_credentials): yield boto3_client(service_name="sts", session=None) -@pytest.fixture(scope="function") +@pytest.fixture(scope="function") def session_manager(sts_client): - SessionManager._instances={} + SessionManager._instances = {} SessionManager().get_or_create( project_name="test", region_name="us-east-1", @@ -54,11 +55,13 @@ def session_manager(sts_client): enable_reaper=False, ) + @pytest.fixture(scope="function") def secretsmanager_client(aws_credentials, session_manager): with mock_sts(): yield boto3_client(service_name="secretsmanager", session=None) + @pytest.mark.parametrize("session", [None, boto3.Session()]) def test_utils_boto3_client(aws_credentials, session): _service_utils.boto3_client("s3", session) diff --git a/test/unit-test/test_session_manager.py b/test/unit-test/test_session_manager.py index 25d4f42b..941c82ad 100644 --- a/test/unit-test/test_session_manager.py +++ b/test/unit-test/test_session_manager.py @@ -17,9 +17,9 @@ import pytest from moto import mock_sts +import seedfarmer.errors from seedfarmer.services._service_utils import boto3_client from seedfarmer.services.session_manager import SessionManager -import seedfarmer.errors @pytest.fixture(scope="function") @@ -67,4 +67,6 @@ def test_singleton(session_manager, sts_client): @pytest.mark.session_manager def test_deployment_session(session_manager, sts_client): - SessionManager().get_or_create(project_name="test").get_deployment_session(account_id="111111111111", region_name="us-east-1") + SessionManager().get_or_create(project_name="test").get_deployment_session( + account_id="111111111111", region_name="us-east-1" + )