diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b714ea4..ed7cfc20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,11 +11,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### **Added** +- added `mlflow-image` and `mlflow-fargate` modules - added `sagemaker-studio` module - added `sagemaker-endpoint` module ### **Changed** ### **Removed** - -======= \ No newline at end of file diff --git a/README.md b/README.md index c0c9d3d4..16d543c6 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ The modules in this repository are decoupled from each other and can be aggregat The modules in this repository are / must be generic for reuse without affiliation to any one particular project in Machine Learning Operations domain. -All modules in this repository adhere to the module strutucture defined in the the [SeedFarmer Guide](https://seed-farmer.readthedocs.io/en/latest) +All modules in this repository adhere to the module structure defined in the the [SeedFarmer Guide](https://seed-farmer.readthedocs.io/en/latest) - [Project Structure](https://seed-farmer.readthedocs.io/en/latest/project_development.html) - [Module Development](https://seed-farmer.readthedocs.io/en/latest/module_development.html) @@ -24,6 +24,13 @@ All modules in this repository adhere to the module strutucture defined in the t | [SageMaker Studio Module](modules/sagemaker/sagemaker-studio/README.md) | Creates SageMaker Studio Domain. | +### Mlflow Modules + +| Type | Description | +|-------------------------------------------------------------------------|---------------------------------| +| [Mlflow Image Module](modules/mlflow/mlflow-image/README.md) | Creates Mlflow container image. | +| [Mlflow on AWS Fargate Module](modules/mlflow/mlflow-fargate/README.md) | Runs Mlflow on AWS Fargate. | + ### Industry Data Framework (IDF) Modules The modules in this repository are compatible with [Industry Data Framework (IDF) Modules](https://github.com/awslabs/idf-modules) and can be used together within the same deployment. Refer to `examples/manifests` for examples. diff --git a/manifests/deployment.yaml b/manifests/deployment.yaml index 6806f3ef..33946fb4 100644 --- a/manifests/deployment.yaml +++ b/manifests/deployment.yaml @@ -4,8 +4,14 @@ forceDependencyRedeploy: true groups: - name: networking path: manifests/networking-modules.yaml + - name: storage + path: manifests/storage-modules.yaml - name: sagemaker-studio path: manifests/sagemaker-studio-modules.yaml + - name: images + path: manifests/images-modules.yaml + - name: mlflow + path: manifests/mlflow-modules.yaml targetAccountMappings: - alias: primary accountId: diff --git a/manifests/images-modules.yaml b/manifests/images-modules.yaml new file mode 100644 index 00000000..dc85af23 --- /dev/null +++ b/manifests/images-modules.yaml @@ -0,0 +1,9 @@ +name: mlflow-image +path: modules/mlflow/mlflow-image +parameters: + - name: ecr-repository-name + valueFrom: + moduleMetadata: + group: storage + name: ecr-mlflow + key: EcrRepositoryName \ No newline at end of file diff --git a/manifests/mlflow-modules.yaml b/manifests/mlflow-modules.yaml new file mode 100644 index 00000000..17d9c571 --- /dev/null +++ b/manifests/mlflow-modules.yaml @@ -0,0 +1,27 @@ +name: mlflow-fargate +path: modules/mlflow/mlflow-fargate +parameters: + - name: vpc-id + valueFrom: + moduleMetadata: + group: networking + name: networking + key: VpcId + - name: subnet-ids + valueFrom: + moduleMetadata: + group: networking + name: networking + key: PrivateSubnetIds + - name: ecr-repository-name + valueFrom: + moduleMetadata: + group: storage + name: ecr-mlflow + key: EcrRepositoryName + - name: artifacts-bucket-name + valueFrom: + moduleMetadata: + group: storage + name: buckets + key: ArtifactsBucketName \ No newline at end of file diff --git a/manifests/sagemaker-studio-modules.yaml b/manifests/sagemaker-studio-modules.yaml index 4404c2c4..06e7e43d 100644 --- a/manifests/sagemaker-studio-modules.yaml +++ b/manifests/sagemaker-studio-modules.yaml @@ -3,7 +3,7 @@ path: modules/sagemaker/sagemaker-studio targetAccount: primary parameters: - name: studio_domain_name - value: mlops + value: mlops-studio - name: vpc_id valueFrom: moduleMetadata: diff --git a/manifests/storage-modules.yaml b/manifests/storage-modules.yaml new file mode 100644 index 00000000..88c18e64 --- /dev/null +++ b/manifests/storage-modules.yaml @@ -0,0 +1,15 @@ +name: ecr-mlflow +path: git::https://github.com/awslabs/idf-modules.git//modules/storage/ecr?ref=release/1.3.0&depth=1 +parameters: + - name: repository_name + value: "ecr-mlflow" +--- +name: buckets +path: git::https://github.com/awslabs/idf-modules.git//modules/storage/buckets?ref=release/1.3.0&depth=1 +targetAccount: primary +targetRegion: us-east-1 +parameters: + - name: encryption-type + value: SSE + - name: retention-type + value: RETAIN \ No newline at end of file diff --git a/modules/mlflow/mlflow-fargate/README.md b/modules/mlflow/mlflow-fargate/README.md new file mode 100644 index 00000000..c317e76c --- /dev/null +++ b/modules/mlflow/mlflow-fargate/README.md @@ -0,0 +1,83 @@ +# Mlflow on Fargate module + +## Description + +This module runs Mlflow on AWS Fargate. + +By default, uses EFS for backend storage. + +### Architecture + +![Mlflow on AWS Fargate Module Architecture](docs/_static/mlflow-fargate-module-architecture.png "Mlflow on AWS Fargate Module Architecture") + +## Inputs/Outputs + +### Input Parameters + +#### Required + +- `vpc-id`: The VPC-ID that the ECS cluster will be created in. +- `subnet-ids`: The subnets that the Fargate task will use. +- `ecr-repository-name`: The name of the ECR repository to pull the image from. +- `artifacts-bucket-name`: Name of the artifacts store bucket + +#### Optional + +- `ecs-cluster-name`: Name of the ECS cluster. +- `service-name`: Name of the service. +- `task-cpu-units`: The number of cpu units used by the Fargate task. +- `task-memory-limit-mb`: The amount (in MiB) of memory used by the Fargate task. +- `lb-access-logs-bucket-name`: Name of the bucket to store load balancer access logs +- `lb-access-logs-bucket-prefix`: Prefix for load balancer access logs + +### Sample manifest declaration + +```yaml +name: mlflow-fargate +path: modules/mlflow/mlflow-fargate +parameters: + - name: vpc-id + valueFrom: + moduleMetadata: + group: networking + name: networking + key: VpcId + - name: subnet-ids + valueFrom: + moduleMetadata: + group: networking + name: networking + key: PrivateSubnetIds + - name: ecr-repository-name + valueFrom: + moduleMetadata: + group: storage + name: ecr-mlflow + key: EcrRepositoryName + - name: artifacts-bucket-name + valueFrom: + moduleMetadata: + group: storage + name: buckets + key: ArtifactsBucketName +``` + +### Module Metadata Outputs + +- `ECSClusterName`: Name of the ECS cluster. +- `ServiceName`: Name of the service. +- `LoadBalancerDNSName`: Load balancer DNS name. +- `LoadBalancerAccessLogsBucketArn`: Load balancer access logs bucket arn +- `EFSFileSystemId`: EFS file system id. + +#### Output Example + +``` +{ + "ECSClusterName": "mlops-mlops-mlflow-mlflow-fargate-EcsCluster97242B84-xxxxxxxxxxxx", + "ServiceName": "mlops-mlops-mlflow-mlflow-fargate-MlflowLBServiceEBACC043-xxxxxxxxxxxx", + "LoadBalancerDNSName": "xxxxxxxxxxxx.elb.us-east-1.amazonaws.com", + "LoadBalancerAccessLogsBucketArn": "arn:aws:s3:::xxxxxxxxxxxx", + "EFSFileSystemId": "fs-xxxxxxxxxxx", +} +``` diff --git a/modules/mlflow/mlflow-fargate/app.py b/modules/mlflow/mlflow-fargate/app.py new file mode 100644 index 00000000..0596144b --- /dev/null +++ b/modules/mlflow/mlflow-fargate/app.py @@ -0,0 +1,93 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +import json +import os + +import aws_cdk + +from stack import MlflowFargateStack + + +def _param(name: str) -> str: + return f"SEEDFARMER_PARAMETER_{name}" + + +project_name = os.getenv("SEEDFARMER_PROJECT_NAME", "") +deployment_name = os.getenv("SEEDFARMER_DEPLOYMENT_NAME", "") +module_name = os.getenv("SEEDFARMER_MODULE_NAME", "") +app_prefix = f"{project_name}-{deployment_name}-{module_name}" + +DEFAULT_ECS_CLUSTER_NAME = None +DEFAULT_SERVICE_NAME = None +DEFAULT_TASK_CPU_UNITS = 4 * 1024 +DEFAULT_TASK_MEMORY_LIMIT_MB = 8 * 1024 +DEFAULT_AUTOSCALE_MAX_CAPACITY = 2 +DEFAULT_LB_ACCESS_LOGS_BUCKET_NAME = None +DEFAULT_LB_ACCESS_LOGS_BUCKET_PREFIX = None + +environment = aws_cdk.Environment( + account=os.environ["CDK_DEFAULT_ACCOUNT"], + region=os.environ["CDK_DEFAULT_REGION"], +) + +vpc_id = os.getenv(_param("VPC_ID")) +subnet_ids = json.loads(os.getenv(_param("SUBNET_IDS"), "[]")) +ecs_cluster_name = os.getenv(_param("ECS_CLUSTER_NAME"), DEFAULT_ECS_CLUSTER_NAME) +service_name = os.getenv(_param("SERVICE_NAME"), DEFAULT_SERVICE_NAME) +ecr_repo_name = os.getenv(_param("ECR_REPOSITORY_NAME")) +task_cpu_units = os.getenv(_param("TASK_CPU_UNITS"), DEFAULT_TASK_CPU_UNITS) +task_memory_limit_mb = os.getenv(_param("TASK_MEMORY_LIMIT_MB"), DEFAULT_TASK_MEMORY_LIMIT_MB) +autoscale_max_capacity = os.getenv(_param("AUTOSCALE_MAX_CAPACITY"), DEFAULT_AUTOSCALE_MAX_CAPACITY) +artifacts_bucket_name = os.getenv(_param("ARTIFACTS_BUCKET_NAME")) +lb_access_logs_bucket_name = os.getenv(_param("LB_ACCESS_LOGS_BUCKET_NAME"), DEFAULT_LB_ACCESS_LOGS_BUCKET_NAME) +lb_access_logs_bucket_prefix = os.getenv(_param("LB_ACCESS_LOGS_BUCKET_PREFIX"), DEFAULT_LB_ACCESS_LOGS_BUCKET_PREFIX) + +if not vpc_id: + raise ValueError("Missing input parameter vpc-id") + +if not ecr_repo_name: + raise ValueError("Missing input parameter ecr-repository-name") + +if not artifacts_bucket_name: + raise ValueError("Missing input parameter artifacts-bucket-name") + + +app = aws_cdk.App() +stack = MlflowFargateStack( + scope=app, + id=app_prefix, + app_prefix=app_prefix, + vpc_id=vpc_id, + subnet_ids=subnet_ids, + ecs_cluster_name=ecs_cluster_name, + service_name=service_name, + ecr_repo_name=ecr_repo_name, + task_cpu_units=int(task_cpu_units), + task_memory_limit_mb=int(task_memory_limit_mb), + autoscale_max_capacity=int(autoscale_max_capacity), + artifacts_bucket_name=artifacts_bucket_name, + lb_access_logs_bucket_name=lb_access_logs_bucket_name, + lb_access_logs_bucket_prefix=lb_access_logs_bucket_prefix, + env=aws_cdk.Environment( + account=os.environ["CDK_DEFAULT_ACCOUNT"], + region=os.environ["CDK_DEFAULT_REGION"], + ), +) + + +aws_cdk.CfnOutput( + scope=stack, + id="metadata", + value=stack.to_json_string( + { + "ECSClusterName": stack.cluster.cluster_name, + "ServiceName": stack.service.service.service_name, + "LoadBalancerDNSName": stack.service.load_balancer.load_balancer_dns_name, + "LoadBalancerAccessLogsBucketArn": stack.lb_access_logs_bucket.bucket_arn, + "EFSFileSystemId": stack.fs.file_system_id, + } + ), +) + +app.synth() diff --git a/modules/mlflow/mlflow-fargate/coverage.ini b/modules/mlflow/mlflow-fargate/coverage.ini new file mode 100644 index 00000000..c3878739 --- /dev/null +++ b/modules/mlflow/mlflow-fargate/coverage.ini @@ -0,0 +1,3 @@ +[run] +omit = + tests/* \ No newline at end of file diff --git a/modules/mlflow/mlflow-fargate/deployspec.yaml b/modules/mlflow/mlflow-fargate/deployspec.yaml new file mode 100644 index 00000000..2c6a998e --- /dev/null +++ b/modules/mlflow/mlflow-fargate/deployspec.yaml @@ -0,0 +1,35 @@ +publishGenericEnvVariables: true +deploy: + phases: + install: + commands: + - npm install -g aws-cdk@2.128.0 + - pip install -r requirements.txt + pre_build: + commands: + - echo "Prebuild stage" + build: + commands: + - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json + # Export metadata + - seedfarmer metadata convert -f cdk-exports.json || true + post_build: + commands: + - echo "Build successful" +destroy: + phases: + install: + commands: + - npm install -g aws-cdk@2.128.0 + - pip install -r requirements.txt + pre_build: + commands: + - echo "Prebuild stage" + build: + commands: + - cdk destroy --force --app "python app.py" + post_build: + commands: + - echo "Destroy successful" + +build_type: BUILD_GENERAL1_SMALL \ No newline at end of file diff --git a/modules/mlflow/mlflow-fargate/docs/_static/mlflow-fargate-module-architecture.png b/modules/mlflow/mlflow-fargate/docs/_static/mlflow-fargate-module-architecture.png new file mode 100644 index 00000000..c76a4f48 Binary files /dev/null and b/modules/mlflow/mlflow-fargate/docs/_static/mlflow-fargate-module-architecture.png differ diff --git a/modules/mlflow/mlflow-fargate/docs/_static/mlflow-fargate-module-architecture.xml b/modules/mlflow/mlflow-fargate/docs/_static/mlflow-fargate-module-architecture.xml new file mode 100644 index 00000000..6d350b19 --- /dev/null +++ b/modules/mlflow/mlflow-fargate/docs/_static/mlflow-fargate-module-architecture.xml @@ -0,0 +1 @@ +7V1bd5s4EP41eVSOBOL26NhJN7vtbk/Spu1TjgBh02DwARzb+fU74mIDxsRp7BgS2pzEGoSuo5lPnwZ8Jg+ny08hm02+BDb3ziRsL8/k0ZkkGSqF30KwSgWqkgnGoWunIrIR3LpPPBPiTDp3bR6VMsZB4MXurCy0At/nVlySsTAMFuVsTuCVa52xMd8S3FrM25b+cO14kkp1SdvI/+LueJLXTFQjvTJleeasJ9GE2cGiIJIvz+RhGARx+mm6HHJPjF0+Ltl9Pl/G4sq1fce8edYsbVPCDY+CeWjxEY+s0J3FQQg3hZkwzX1/Jg/yVvDQZZ77xGI38NEjDyP4m+Z6zLKwbMjCmoKzOm/5lPmxa41YzIaBHzPX5+E+pad3x6Hrjz+7MQ+Zl85dzP241OtZGMx4GGdaM4ljMd+DM+kKfiB74AXj1XnErXnoxqtzNmVPgX9u80e47ARz305aAAnbZeOQTdGjG83XLQM5kyRFMRQVWbLtIGpJFjKpbkASa46qM9My0im+Stt8fXO9NawvahVosTv2ketHM9BTMZZXVjCdBT70PIKETpmOTcVBikolRBnRkKHoCuKmY5uq4lCTW4cdmmgVxXyKpmLJwnyABFNFNhRDQ1TSZUQdVUE6NjBiXNUcw5a5QY3ioMCHek3Ir9boZn4pU/d61V+vurCsFy9ReNIRhSe9wn84hQ/M38JZSdhjJvjLpF+ZuoJiWK73bTXLxnzMoXzXQuuxS/PlK8WTb0gw+jL8m+PlN/TzaWSZt08oLa28tqJ4lTu1UIwKFyXgM/liMQHFvJ2xZJYX4MVBNomnolkEPjqu5w1hjKEnIx9aACKbRZPkdnEdlB6Gg3kDD6YbZGYQx8EULqQNFZf5cueyzkSh7Xyfgd6mrSq48CjisRiMu+LSkgu9+8SDKY/DFWTOa1GzsVyVocRi47+pqqeyScF3U1KxB+N10RsN2NP4NE1LO2zS0dc3dRRiGMRBjmypsL4prG8TY4SJrKiYmYbk0F3re92GxWJxvpDPg1CsVijOEItWzIYkIdAZFK2g30vkQ4VyLNbMIS1YyXiMBbxFURzOrXgectEOk3LOLB1hyhVEOdguXTXArNqK6pgG1UzNbHcH36OJbvS1Xx4upYfRX1b0+/6HvLgYo8V//3x+G/+LFctSZGIhGwwNooYlQz9soS2ypmmWQiXV2bsfzealbVDiUF5TND91nA1OVPVi4aFc6I06jhM3lIrMXHD3dZjLoESzmi/peOHuWq/8OLPKfvje1snMlH8N9PnNz8d/RnEUDu+RojT64VngioUGn5UL+AHjPMRnClwZitS5pFQE1bRWFpDtlCijLKimtbKAVIsnlfpJtYEFwVaqVDyu1I8LDYQf+SKYx7Do+XC9lRfoRIACFxS/AkEKAKUOvzigYxmZQKQ8nQ28KBU248lUTpeJXT9ni0jC52MARrOkzmsrcZk1l+/F1EMBcRg88LxRYJwlquuE1uOlKkSKA1ELy1Ied4SmRdADseKT1EjGWasLVQwGF9qFXsRf+JQwyyBlmKVu4yw5h1RFnAVjdTCc1bTs2oGzGu14c/PbZsd7vFGDN3oc3Xkc3bhEE7TwksoljLGoHKzjVVKzNQEDAtWmMGXfer/7D36w8L8KfrqVAPVNUV0zNTJhgBi8fQCZqjYCMm6PeQ4TgjCeBOPAZ97lRnpRpk4KKIQv3finEAPCSVO/skzi82hZTKwKia/QBfB+YtxSmQ+e8GdeqEj82pQqkpuiktSqmKoW9pvH8SrDQWweBwJkrbv1ORAwJKknHTrR++cQRNEI1O9C8lMaFo5543aFaAcEJSH3QJEfeaWEIwGMVI06CzDy5vcAowMAo9te4tjgQVWIoqkSRho1YUg1zKEpho1Mw3QU1VQtkx4XPFx4zH/4F6b3evQCJvjAhz7b42ITx8KqypGsq0LTOUa6jClSLe5QQIyYKfgA3dCO3Q+VaCYHz4QsBxYrZZggpik20gm3ZcDCkpLtJtvJNV0Ob+GmoTePhFN+LevkgE9lMS8DnQYFexnxVEs+1RFQtSTUNhFVypZQQzU1VIV1Mm1bSLaz5WzStrBOVkedVe8mNXeTyt27iasKeQP/r8R8bBFacO1K0y8xLVwbuSEUlHp7XyC2Cq8E94ywMoTlV8NEOcm/Kk2Uc1CfhQJ/DSI3K359XNdMUlnQKrGKSieDz5BsLPHCkHDcpWjHDtYtNzYp53YByVr2LVf+0zFecoZWcsZL0tQtxkuvIbz0Y58rdiXU4U18YWfAqIU1QH+cIolpAARVxhEzFQUxaKSjU1OyjAODuhawXZ3mgire/ASsUIcOMFuHxG5ejb+4Fe6DvaQee/XY611hL6H4p8NdqoZLuKvupPEkuCsnpruJu1oaLXJ03AUKZRiORBCTCEYUawb4MIkh1bEZAC/i8N0Haj3uOg3uuunRVuvQ1hpD3YL5xgPwYA5L8t5C4bwebNXCqkjeB1VpParqDqpS8UCWtZehKknTCFE/EKoCvW8PqJL0lpBZWqdB1YEPRDoDqjin1JQNhixqmgCqwOWbhkqQ5FCdW8SQea44PahqB6ja4bV7mNUamFULlhqDnuptktGInF4T85THKq0DkDoWq5Q9BbZHrNIBHfCxQpUaJr+z7tT4oO703QUqdcjG9yFLTbay8yFLx+5GByKWNodiHougXsj1OWA2/IHdMfOtXaFK9YdiaRn3HpRwbyb3J/bjeVyi94xOdxid/pxsn3Oy+rVwOpYH/p0rZZ6nLUFLeqeB6YG9YWeAqa0bkuMwDak6xojqWEO6YRCkKAqWiGzz9eFse4Dpx+Z5Gnx8z/V0mOvZ8ezw0R5wO8mzZNo2QbOj31IHGJqmKeuAJ2xu/odzhe+Oo+kfJusmM3P8p7AOTc002/APzc00xTBf5c9/4W8senh1OHMEbjWez+7jpLDn+RrSHILz7O69ACb23MiXeYPsbToVkgHji8srqW7n/gSTfl7qpBjeDWsBU5ewFqfanFMiJXROYXdONHUtKmzQlZoNOtXynf3xnivqdizGoe1ij0zavEnv36PTahqi6rt67uH9cQ/N+OBjBJo0Q9vnI006zGN0BS00N79HCx1ACz2P0Wke41T7/7cJMfnYNMbB0ERzeEiPJvZBEyd5xV5y6yAM2aqQIQvn2ZT8VQg2ZIiacxx5nAIurcRn88uSWlH5tAUbPV535QBQpyvBC83N76FOD3V6qNNDnT+HOv37/5pPbFKuC18PvsDvm8Db8Vj0Cw5twqSQPeCT1AifXhUOWn8QM9SITK52ndq0LtAzASQ8vHzkaZgx2R38GSR48uxEx0SKfPLXnzQbsa6ioAPb4B4F9cdD/fHQHx4P1TjKE5wQtR5Od5PJocdick5Cvsj7ki/6KciXw3h22m3PfuD3d/Wevec3en7jRfyG/D74jaM/9NxxfuN7dICvNWBW5i2eBRJF57gNJPLd+3zqDZIi9yAakv5eMOthnICMukdU67mIAgqp5VJOxRYQUjlHqXmvl1zDFqhHZwvkbmOKA1u01mCKZ9w50y0F27aNGDN0RHWLI0aYhRRmKA7smSnXDjwyLSALOr2VTm1yv3vu3O75bnJ39/D34NMP5XYJeOW/H4M7+i9Sj/f6rtqvLMyv/MFXFrY/nmLHENc8Ztrh7yxs0qMO+N/m5rfN/z63r2jWuK7vj9r3BXDvfkPek0jvjkRq2ab+8kp8zdFj4M2nr49X4E50H8UMBi2094I7/XveO/RWsP497/u8Fay4BE759YVq5V1g8psHkjSjoa5C0/7Joe4ghT6Q5H2zX2Xw0rNgb4Y2xagFQVzI+kmozhexVEH4Pw== \ No newline at end of file diff --git a/modules/mlflow/mlflow-fargate/pyproject.toml b/modules/mlflow/mlflow-fargate/pyproject.toml new file mode 100644 index 00000000..1f856efe --- /dev/null +++ b/modules/mlflow/mlflow-fargate/pyproject.toml @@ -0,0 +1,35 @@ +[tool.black] +line-length = 120 +target-version = ["py36", "py37", "py38"] +exclude = ''' +/( + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | \.env + | _build + | buck-out + | build + | dist + | codeseeder.out +)/ +''' + +[tool.isort] +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +ensure_newline_before_comments = true +line_length = 120 +py_version = 36 +skip_gitignore = false + +[tool.pytest.ini_options] +addopts = "-v --cov=. --cov-report term --cov-config=coverage.ini --cov-fail-under=80" +pythonpath = [ + "." +] \ No newline at end of file diff --git a/modules/mlflow/mlflow-fargate/requirements-dev.in b/modules/mlflow/mlflow-fargate/requirements-dev.in new file mode 100644 index 00000000..6412f987 --- /dev/null +++ b/modules/mlflow/mlflow-fargate/requirements-dev.in @@ -0,0 +1,14 @@ +awscli +black +cdk-nag +cfn-lint +check-manifest +flake8 +isort +mypy +pip-tools +pydot +pyroma +pytest +types-PyYAML +types-setuptools diff --git a/modules/mlflow/mlflow-fargate/requirements-dev.txt b/modules/mlflow/mlflow-fargate/requirements-dev.txt new file mode 100644 index 00000000..1864c0bb --- /dev/null +++ b/modules/mlflow/mlflow-fargate/requirements-dev.txt @@ -0,0 +1,254 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --output-file=requirements-dev.txt requirements-dev.in +# +annotated-types==0.6.0 + # via pydantic +attrs==23.2.0 + # via + # cattrs + # jschema-to-python + # jsii + # jsonschema + # referencing + # sarif-om +aws-cdk-asset-awscli-v1==2.2.202 + # via aws-cdk-lib +aws-cdk-asset-kubectl-v20==2.1.2 + # via aws-cdk-lib +aws-cdk-asset-node-proxy-agent-v6==2.0.1 + # via aws-cdk-lib +aws-cdk-lib==2.127.0 + # via cdk-nag +aws-sam-translator==1.84.0 + # via cfn-lint +awscli==1.32.41 + # via -r requirements-dev.in +black==24.2.0 + # via -r requirements-dev.in +boto3==1.34.41 + # via aws-sam-translator +botocore==1.34.41 + # via + # awscli + # boto3 + # s3transfer +build==1.0.3 + # via + # check-manifest + # pip-tools + # pyroma +cattrs==23.2.3 + # via jsii +cdk-nag==2.28.34 + # via -r requirements-dev.in +certifi==2024.2.2 + # via requests +cfn-lint==0.85.1 + # via -r requirements-dev.in +charset-normalizer==3.3.2 + # via requests +check-manifest==0.49 + # via -r requirements-dev.in +click==8.1.7 + # via + # black + # pip-tools +colorama==0.4.4 + # via awscli +constructs==10.3.0 + # via + # aws-cdk-lib + # cdk-nag +docutils==0.16 + # via + # awscli + # pyroma +exceptiongroup==1.2.0 + # via + # cattrs + # pytest +flake8==7.0.0 + # via -r requirements-dev.in +idna==3.6 + # via requests +importlib-metadata==7.0.1 + # via build +importlib-resources==6.1.1 + # via jsii +iniconfig==2.0.0 + # via pytest +isort==5.13.2 + # via -r requirements-dev.in +jmespath==1.0.1 + # via + # boto3 + # botocore +jschema-to-python==1.2.3 + # via cfn-lint +jsii==1.94.0 + # via + # aws-cdk-asset-awscli-v1 + # aws-cdk-asset-kubectl-v20 + # aws-cdk-asset-node-proxy-agent-v6 + # aws-cdk-lib + # cdk-nag + # constructs +jsonpatch==1.33 + # via cfn-lint +jsonpickle==3.0.2 + # via jschema-to-python +jsonpointer==2.4 + # via jsonpatch +jsonschema==4.21.1 + # via + # aws-sam-translator + # cfn-lint +jsonschema-specifications==2023.12.1 + # via jsonschema +junit-xml==1.9 + # via cfn-lint +mccabe==0.7.0 + # via flake8 +mpmath==1.3.0 + # via sympy +mypy==1.8.0 + # via -r requirements-dev.in +mypy-extensions==1.0.0 + # via + # black + # mypy +networkx==3.2.1 + # via cfn-lint +packaging==23.2 + # via + # black + # build + # pyroma + # pytest +pathspec==0.12.1 + # via black +pbr==6.0.0 + # via + # jschema-to-python + # sarif-om +pip-tools==7.3.0 + # via -r requirements-dev.in +platformdirs==4.2.0 + # via black +pluggy==1.4.0 + # via pytest +publication==0.0.3 + # via + # aws-cdk-asset-awscli-v1 + # aws-cdk-asset-kubectl-v20 + # aws-cdk-asset-node-proxy-agent-v6 + # aws-cdk-lib + # cdk-nag + # constructs + # jsii +pyasn1==0.5.1 + # via rsa +pycodestyle==2.11.1 + # via flake8 +pydantic==2.6.1 + # via aws-sam-translator +pydantic-core==2.16.2 + # via pydantic +pydot==2.0.0 + # via -r requirements-dev.in +pyflakes==3.2.0 + # via flake8 +pygments==2.17.2 + # via pyroma +pyparsing==3.1.1 + # via pydot +pyproject-hooks==1.0.0 + # via build +pyroma==4.2 + # via -r requirements-dev.in +pytest==8.0.0 + # via -r requirements-dev.in +python-dateutil==2.8.2 + # via + # botocore + # jsii +pyyaml==6.0.1 + # via + # awscli + # cfn-lint +referencing==0.33.0 + # via + # jsonschema + # jsonschema-specifications +regex==2023.12.25 + # via cfn-lint +requests==2.31.0 + # via pyroma +rpds-py==0.18.0 + # via + # jsonschema + # referencing +rsa==4.7.2 + # via awscli +s3transfer==0.10.0 + # via + # awscli + # boto3 +sarif-om==1.0.4 + # via cfn-lint +six==1.16.0 + # via + # junit-xml + # python-dateutil +sympy==1.12 + # via cfn-lint +tomli==2.0.1 + # via + # black + # build + # check-manifest + # mypy + # pip-tools + # pyproject-hooks + # pytest +trove-classifiers==2024.1.31 + # via pyroma +typeguard==2.13.3 + # via + # aws-cdk-asset-awscli-v1 + # aws-cdk-asset-kubectl-v20 + # aws-cdk-asset-node-proxy-agent-v6 + # aws-cdk-lib + # cdk-nag + # constructs + # jsii +types-pyyaml==6.0.12.12 + # via -r requirements-dev.in +types-setuptools==69.0.0.20240125 + # via -r requirements-dev.in +typing-extensions==4.9.0 + # via + # aws-sam-translator + # black + # cattrs + # jsii + # mypy + # pydantic + # pydantic-core +urllib3==1.26.18 + # via + # botocore + # requests +wheel==0.42.0 + # via pip-tools +zipp==3.17.0 + # via + # importlib-metadata + # importlib-resources + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/modules/mlflow/mlflow-fargate/requirements.in b/modules/mlflow/mlflow-fargate/requirements.in new file mode 100644 index 00000000..14ffea02 --- /dev/null +++ b/modules/mlflow/mlflow-fargate/requirements.in @@ -0,0 +1,4 @@ +aws-cdk-lib==2.128.0 +cdk-nag==2.28.27 +boto3==1.34.35 +cdk_ecr_deployment==2.5.30 \ No newline at end of file diff --git a/modules/mlflow/mlflow-fargate/requirements.txt b/modules/mlflow/mlflow-fargate/requirements.txt new file mode 100644 index 00000000..263a2cdf --- /dev/null +++ b/modules/mlflow/mlflow-fargate/requirements.txt @@ -0,0 +1,90 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --output-file=requirements.txt requirements.in +# +attrs==23.1.0 + # via + # cattrs + # jsii +aws-cdk-asset-awscli-v1==2.2.202 + # via aws-cdk-lib +aws-cdk-asset-kubectl-v20==2.1.2 + # via aws-cdk-lib +aws-cdk-asset-node-proxy-agent-v6==2.0.1 + # via aws-cdk-lib +aws-cdk-lib==2.128.0 + # via + # -r requirements.in + # cdk-ecr-deployment + # cdk-nag +boto3==1.34.35 + # via -r requirements.in +botocore==1.34.43 + # via + # boto3 + # s3transfer +cattrs==23.1.2 + # via jsii +cdk-ecr-deployment==2.5.30 + # via -r requirements.in +cdk-nag==2.28.27 + # via -r requirements.in +constructs==10.0.91 + # via + # aws-cdk-lib + # cdk-ecr-deployment + # cdk-nag +exceptiongroup==1.1.3 + # via cattrs +importlib-resources==6.1.0 + # via jsii +jmespath==1.0.1 + # via + # boto3 + # botocore +jsii==1.94.0 + # via + # aws-cdk-asset-awscli-v1 + # aws-cdk-asset-kubectl-v20 + # aws-cdk-asset-node-proxy-agent-v6 + # aws-cdk-lib + # cdk-ecr-deployment + # cdk-nag + # constructs +publication==0.0.3 + # via + # aws-cdk-asset-awscli-v1 + # aws-cdk-asset-kubectl-v20 + # aws-cdk-asset-node-proxy-agent-v6 + # aws-cdk-lib + # cdk-ecr-deployment + # cdk-nag + # constructs + # jsii +python-dateutil==2.8.2 + # via + # botocore + # jsii +s3transfer==0.10.0 + # via boto3 +six==1.16.0 + # via python-dateutil +typeguard==2.13.3 + # via + # aws-cdk-asset-awscli-v1 + # aws-cdk-asset-kubectl-v20 + # aws-cdk-asset-node-proxy-agent-v6 + # aws-cdk-lib + # cdk-ecr-deployment + # cdk-nag + # jsii +typing-extensions==4.8.0 + # via + # cattrs + # jsii +urllib3==1.26.18 + # via botocore +zipp==3.17.0 + # via importlib-resources diff --git a/modules/mlflow/mlflow-fargate/setup.cfg b/modules/mlflow/mlflow-fargate/setup.cfg new file mode 100644 index 00000000..6136e2bb --- /dev/null +++ b/modules/mlflow/mlflow-fargate/setup.cfg @@ -0,0 +1,28 @@ +[metadata] +license_files = + LICENSE + NOTICE + VERSION + +[flake8] +max-line-length = 120 +extend-ignore = E203, W503 +exclude = + .git, + __pycache__, + docs/source/conf.py, + old, + build, + dist, + .venv, + codeseeder.out, + bundle + +[mypy] +python_version = 3.7 +strict = True +ignore_missing_imports = True +allow_untyped_decorators = True +exclude = + codeseeder.out/|example/|tests/ +warn_unused_ignores = False \ No newline at end of file diff --git a/modules/mlflow/mlflow-fargate/stack.py b/modules/mlflow/mlflow-fargate/stack.py new file mode 100644 index 00000000..6c4fe1ae --- /dev/null +++ b/modules/mlflow/mlflow-fargate/stack.py @@ -0,0 +1,212 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from typing import Any, List, Optional, cast + +from aws_cdk import Aspects, Duration, Stack, Tags +from aws_cdk import aws_ec2 as ec2 +from aws_cdk import aws_ecr as ecr +from aws_cdk import aws_ecs as ecs +from aws_cdk import aws_ecs_patterns as ecs_patterns +from aws_cdk import aws_efs as efs +from aws_cdk import aws_iam as iam +from aws_cdk import aws_s3 as s3 +from cdk_nag import AwsSolutionsChecks, NagPackSuppression, NagSuppressions +from constructs import Construct, IConstruct + + +class MlflowFargateStack(Stack): # type: ignore + def __init__( + self, + scope: Construct, + id: str, + app_prefix: str, + vpc_id: str, + subnet_ids: List[str], + ecs_cluster_name: Optional[str], + service_name: Optional[str], + ecr_repo_name: str, + task_cpu_units: int, + task_memory_limit_mb: int, + autoscale_max_capacity: int, + artifacts_bucket_name: str, + lb_access_logs_bucket_name: Optional[str], + lb_access_logs_bucket_prefix: Optional[str], + **kwargs: Any, + ) -> None: + super().__init__(scope, id, **kwargs) + + Tags.of(scope=cast(IConstruct, self)).add(key="Deployment", value=app_prefix[:64]) + + role = iam.Role( + self, + "TaskRole", + assumed_by=iam.ServicePrincipal(service="ecs-tasks.amazonaws.com"), + managed_policies=[ + iam.ManagedPolicy.from_aws_managed_policy_name("AmazonECS_FullAccess"), + ], + ) + + # Grant artifacts bucket read-write permissions + model_bucket = s3.Bucket.from_bucket_name(self, "ArtifactsBucket", bucket_name=artifacts_bucket_name) + model_bucket.grant_read_write(role) + + vpc = ec2.Vpc.from_lookup(self, "Vpc", vpc_id=vpc_id) + subnets = [ec2.Subnet.from_subnet_id(self, f"Sub{subnet_id}", subnet_id) for subnet_id in subnet_ids] + + cluster = ecs.Cluster( + self, + "EcsCluster", + cluster_name=ecs_cluster_name, + vpc=vpc, + container_insights=True, + ) + self.cluster = cluster + + task_definition = ecs.FargateTaskDefinition( + self, + "MlflowTask", + task_role=role, + cpu=task_cpu_units, + memory_limit_mib=task_memory_limit_mb, + ) + + container = task_definition.add_container( + "ContainerDef", + image=ecs.ContainerImage.from_ecr_repository( + repository=ecr.Repository.from_repository_name( + self, + "ECRRepo", + repository_name=ecr_repo_name, + ), + ), + environment={ + "BUCKET": f"s3://{artifacts_bucket_name}", + }, + logging=ecs.LogDriver.aws_logs(stream_prefix="mlflow"), + ) + port_mapping = ecs.PortMapping(container_port=5000, host_port=5000, protocol=ecs.Protocol.TCP) + container.add_port_mappings(port_mapping) + + # Add EFS + fs = efs.FileSystem( + self, + "EfsFileSystem", + vpc=vpc, + encrypted=True, + throughput_mode=efs.ThroughputMode.ELASTIC, + performance_mode=efs.PerformanceMode.GENERAL_PURPOSE, + file_system_policy=iam.PolicyDocument( + statements=[ + iam.PolicyStatement( + actions=[ + "elasticfilesystem:ClientMount", + "elasticfilesystem:ClientWrite", + "elasticfilesystem:ClientRootAccess", + ], + principals=[iam.AnyPrincipal()], + resources=["*"], + conditions={"Bool": {"elasticfilesystem:AccessedViaMountTarget": "true"}}, + ), + ] + ), + ) + self.fs = fs + + # Attach and mount volume + task_definition.add_volume( + name="efs-volume", + efs_volume_configuration=ecs.EfsVolumeConfiguration( + file_system_id=fs.file_system_id, + transit_encryption="ENABLED", + ), + ) + container.add_mount_points( + ecs.MountPoint( + container_path="./mlruns", + source_volume="efs-volume", + read_only=False, + ) + ) + + # Create ECS Service + service = ecs_patterns.NetworkLoadBalancedFargateService( + self, + "MlflowLBService", + service_name=service_name, + cluster=cluster, + task_definition=task_definition, + task_subnets=ec2.SubnetSelection(subnets=subnets), + circuit_breaker=ecs.DeploymentCircuitBreaker(rollback=True), + ) + self.service = service + + # Enable access logs + if lb_access_logs_bucket_name: + lb_access_logs_bucket = s3.Bucket.from_bucket_name( + self, "LBAccessLogsBucket", bucket_name=lb_access_logs_bucket_name + ) + else: + lb_access_logs_bucket = s3.Bucket( + self, + "LBAccessLogsBucket", + encryption=s3.BucketEncryption.S3_MANAGED, + block_public_access=s3.BlockPublicAccess.BLOCK_ALL, + enforce_ssl=True, + ) + service.load_balancer.log_access_logs(bucket=lb_access_logs_bucket, prefix=lb_access_logs_bucket_prefix) + self.lb_access_logs_bucket = lb_access_logs_bucket + + # Allow access to EFS from Fargate service + fs.grant_root_access(service.task_definition.task_role.grant_principal) + fs.connections.allow_default_port_from(service.service.connections) + + # Setup security group + service.service.connections.security_groups[0].add_ingress_rule( + peer=ec2.Peer.ipv4(vpc.vpc_cidr_block), + connection=ec2.Port.tcp(5000), + description="Allow inbound from VPC for mlflow", + ) + + # Setup autoscaling policy + scaling = service.service.auto_scale_task_count(max_capacity=autoscale_max_capacity) + scaling.scale_on_cpu_utilization( + id="AutoscalingPolicy", + target_utilization_percent=70, + scale_in_cooldown=Duration.seconds(60), + scale_out_cooldown=Duration.seconds(60), + ) + + # Add CDK nag solutions checks + Aspects.of(self).add(AwsSolutionsChecks()) + + NagSuppressions.add_stack_suppressions( + self, + apply_to_nested_stacks=True, + suppressions=[ + NagPackSuppression( + **{ + "id": "AwsSolutions-IAM4", + "reason": "Managed Policies are for src account roles only", + } + ), + NagPackSuppression( + **{ + "id": "AwsSolutions-IAM5", + "reason": "Resource access restricted to resources", + } + ), + NagPackSuppression( + **{ + "id": "AwsSolutions-ECS2", + "reason": "Not passing secrets via env variables", + } + ), + NagPackSuppression( + **{ + "id": "AwsSolutions-S1", + "reason": "Access logs not required for access logs bucket", + } + ), + ], + ) diff --git a/modules/mlflow/mlflow-fargate/tests/__init__.py b/modules/mlflow/mlflow-fargate/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/modules/mlflow/mlflow-fargate/tests/test_app.py b/modules/mlflow/mlflow-fargate/tests/test_app.py new file mode 100644 index 00000000..d0615f84 --- /dev/null +++ b/modules/mlflow/mlflow-fargate/tests/test_app.py @@ -0,0 +1,46 @@ +import os +import sys + +import pytest + + +@pytest.fixture(scope="function") +def stack_defaults(): + os.environ["SEEDFARMER_PROJECT_NAME"] = "test-project" + os.environ["SEEDFARMER_DEPLOYMENT_NAME"] = "test-deployment" + os.environ["SEEDFARMER_MODULE_NAME"] = "test-module" + os.environ["CDK_DEFAULT_ACCOUNT"] = "111111111111" + os.environ["CDK_DEFAULT_REGION"] = "us-east-1" + + os.environ["SEEDFARMER_PARAMETER_VPC_ID"] = "vpc-12345" + os.environ["SEEDFARMER_PARAMETER_ECR_REPOSITORY_NAME"] = "repo5" + os.environ["SEEDFARMER_PARAMETER_ARTIFACTS_BUCKET_NAME"] = "bucket" + + # Unload the app import so that subsequent tests don't reuse + if "app" in sys.modules: + del sys.modules["app"] + + +def test_app(stack_defaults): + import app # noqa: F401 + + +def test_vpc_id(stack_defaults): + del os.environ["SEEDFARMER_PARAMETER_VPC_ID"] + + with pytest.raises(Exception, match="Missing input parameter vpc-id"): + import app # noqa: F401 + + +def test_ecr_repository_name(stack_defaults): + del os.environ["SEEDFARMER_PARAMETER_ECR_REPOSITORY_NAME"] + + with pytest.raises(Exception, match="Missing input parameter ecr-repository-name"): + import app # noqa: F401 + + +def test_artifacts_bucket_name(stack_defaults): + del os.environ["SEEDFARMER_PARAMETER_ARTIFACTS_BUCKET_NAME"] + + with pytest.raises(Exception, match="Missing input parameter artifacts-bucket-name"): + import app # noqa: F401 diff --git a/modules/mlflow/mlflow-fargate/tests/test_stack.py b/modules/mlflow/mlflow-fargate/tests/test_stack.py new file mode 100644 index 00000000..88784514 --- /dev/null +++ b/modules/mlflow/mlflow-fargate/tests/test_stack.py @@ -0,0 +1,60 @@ +import os +import sys + +import aws_cdk as cdk +import pytest +from aws_cdk.assertions import Template + + +@pytest.fixture(scope="function") +def stack_defaults() -> None: + os.environ["CDK_DEFAULT_ACCOUNT"] = "111111111111" + os.environ["CDK_DEFAULT_REGION"] = "us-east-1" + + # Unload the app import so that subsequent tests don't reuse + if "stack" in sys.modules: + del sys.modules["stack"] + + +def test_synthesize_stack() -> None: + import stack + + app = cdk.App() + + project_name = "test-project" + dep_name = "test-deployment" + mod_name = "test-module" + app_prefix = f"{project_name}-{dep_name}-{mod_name}" + + vpc_id = "vpc-123" + subnet_ids = [] + ecr_repo_name = "repo" + task_cpu_units = 4 * 1024 + task_memory_limit_mb = 8 * 1024 + autoscale_max_capacity = 2 + artifacts_bucket_name = "bucket" + + stack = stack.MlflowFargateStack( + scope=app, + id=app_prefix, + app_prefix=app_prefix, + vpc_id=vpc_id, + subnet_ids=subnet_ids, + ecs_cluster_name=None, + service_name=None, + ecr_repo_name=ecr_repo_name, + task_cpu_units=task_cpu_units, + task_memory_limit_mb=task_memory_limit_mb, + autoscale_max_capacity=autoscale_max_capacity, + artifacts_bucket_name=artifacts_bucket_name, + lb_access_logs_bucket_name=None, + lb_access_logs_bucket_prefix=None, + env=cdk.Environment( + account=os.environ["CDK_DEFAULT_ACCOUNT"], + region=os.environ["CDK_DEFAULT_REGION"], + ), + ) + + template = Template.from_stack(stack) + template.resource_count_is("AWS::ECS::Cluster", 1) + template.resource_count_is("AWS::ECS::TaskDefinition", 1) diff --git a/modules/mlflow/mlflow-image/README.md b/modules/mlflow/mlflow-image/README.md new file mode 100644 index 00000000..6adff988 --- /dev/null +++ b/modules/mlflow/mlflow-image/README.md @@ -0,0 +1,39 @@ +# Mlflow image module + +## Description + +This module creates an mlflow container image and pushes to the specified ECR. + +## Inputs/Outputs + +### Input Parameters + +#### Required + +- `ecr-repository-name`: The name of the ECR repository to push the image to. + +### Sample manifest declaration + +```yaml +name: mlflow-image +path: modules/mlflow/mlflow-image +parameters: + - name: ecr-repository-name + valueFrom: + moduleMetadata: + group: storage + name: ecr-mlflow + key: EcrRepositoryName +``` + +### Module Metadata Outputs + +- `MlflowImageUri`: Mlflow image URI + +#### Output Example + +```json +{ + "MlflowImageUri": "xxxxxxxxxxxx.dkr.ecr.us-east-1.amazonaws.com/ecr-mlflow:latest" +} +``` diff --git a/modules/mlflow/mlflow-image/app.py b/modules/mlflow/mlflow-image/app.py new file mode 100644 index 00000000..6d8cddfb --- /dev/null +++ b/modules/mlflow/mlflow-image/app.py @@ -0,0 +1,54 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os + +import aws_cdk + +from stack import MlflowImagePublishingStack + + +def _param(name: str) -> str: + return f"SEEDFARMER_PARAMETER_{name}" + + +project_name = os.getenv("SEEDFARMER_PROJECT_NAME", "") +deployment_name = os.getenv("SEEDFARMER_DEPLOYMENT_NAME", "") +module_name = os.getenv("SEEDFARMER_MODULE_NAME", "") +app_prefix = f"{project_name}-{deployment_name}-{module_name}" + +environment = aws_cdk.Environment( + account=os.environ["CDK_DEFAULT_ACCOUNT"], + region=os.environ["CDK_DEFAULT_REGION"], +) + +ecr_repo_name = os.getenv(_param("ECR_REPOSITORY_NAME")) + +if not ecr_repo_name: + raise ValueError("Missing input parameter ecr-repository-name") + + +app = aws_cdk.App() +stack = MlflowImagePublishingStack( + scope=app, + id=app_prefix, + app_prefix=app_prefix, + ecr_repo_name=ecr_repo_name, + env=aws_cdk.Environment( + account=os.environ["CDK_DEFAULT_ACCOUNT"], + region=os.environ["CDK_DEFAULT_REGION"], + ), +) + + +aws_cdk.CfnOutput( + scope=stack, + id="metadata", + value=stack.to_json_string( + { + "MlflowImageUri": stack.image_uri, + } + ), +) + +app.synth() diff --git a/modules/mlflow/mlflow-image/coverage.ini b/modules/mlflow/mlflow-image/coverage.ini new file mode 100644 index 00000000..c3878739 --- /dev/null +++ b/modules/mlflow/mlflow-image/coverage.ini @@ -0,0 +1,3 @@ +[run] +omit = + tests/* \ No newline at end of file diff --git a/modules/mlflow/mlflow-image/deployspec.yaml b/modules/mlflow/mlflow-image/deployspec.yaml new file mode 100644 index 00000000..dc6cd86f --- /dev/null +++ b/modules/mlflow/mlflow-image/deployspec.yaml @@ -0,0 +1,35 @@ +publishGenericEnvVariables: true +deploy: + phases: + install: + commands: + - npm install -g aws-cdk@2.126.0 + - pip install -r requirements.txt + pre_build: + commands: + - echo "Prebuild stage" + build: + commands: + - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json + # Export metadata + - seedfarmer metadata convert -f cdk-exports.json || true + post_build: + commands: + - echo "Build successful" +destroy: + phases: + install: + commands: + - npm install -g aws-cdk@2.126.0 + - pip install -r requirements.txt + pre_build: + commands: + - echo "Prebuild stage" + build: + commands: + - cdk destroy --force --app "python app.py" + post_build: + commands: + - echo "Destroy successful" + +build_type: BUILD_GENERAL1_SMALL \ No newline at end of file diff --git a/modules/mlflow/mlflow-image/pyproject.toml b/modules/mlflow/mlflow-image/pyproject.toml new file mode 100644 index 00000000..1f856efe --- /dev/null +++ b/modules/mlflow/mlflow-image/pyproject.toml @@ -0,0 +1,35 @@ +[tool.black] +line-length = 120 +target-version = ["py36", "py37", "py38"] +exclude = ''' +/( + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | \.env + | _build + | buck-out + | build + | dist + | codeseeder.out +)/ +''' + +[tool.isort] +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +ensure_newline_before_comments = true +line_length = 120 +py_version = 36 +skip_gitignore = false + +[tool.pytest.ini_options] +addopts = "-v --cov=. --cov-report term --cov-config=coverage.ini --cov-fail-under=80" +pythonpath = [ + "." +] \ No newline at end of file diff --git a/modules/mlflow/mlflow-image/requirements-dev.in b/modules/mlflow/mlflow-image/requirements-dev.in new file mode 100644 index 00000000..6412f987 --- /dev/null +++ b/modules/mlflow/mlflow-image/requirements-dev.in @@ -0,0 +1,14 @@ +awscli +black +cdk-nag +cfn-lint +check-manifest +flake8 +isort +mypy +pip-tools +pydot +pyroma +pytest +types-PyYAML +types-setuptools diff --git a/modules/mlflow/mlflow-image/requirements-dev.txt b/modules/mlflow/mlflow-image/requirements-dev.txt new file mode 100644 index 00000000..96ce3195 --- /dev/null +++ b/modules/mlflow/mlflow-image/requirements-dev.txt @@ -0,0 +1,273 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --output-file=requirements-dev.txt requirements-dev.in +# +aiohttp==3.9.2 + # via black +aiosignal==1.3.1 + # via aiohttp +annotated-types==0.6.0 + # via pydantic +async-timeout==4.0.3 + # via aiohttp +attrs==23.1.0 + # via + # aiohttp + # cattrs + # jschema-to-python + # jsii + # jsonschema + # referencing + # sarif-om +aws-cdk-asset-awscli-v1==2.2.201 + # via aws-cdk-lib +aws-cdk-asset-kubectl-v20==2.1.2 + # via aws-cdk-lib +aws-cdk-asset-node-proxy-agent-v6==2.0.1 + # via aws-cdk-lib +aws-cdk-lib==2.115.0 + # via cdk-nag +aws-sam-translator==1.82.0 + # via cfn-lint +awscli==1.32.0 + # via -r requirements-dev.in +black==23.12.0 + # via -r requirements-dev.in +boto3==1.34.0 + # via aws-sam-translator +botocore==1.34.0 + # via + # awscli + # boto3 + # s3transfer +build==1.0.3 + # via + # check-manifest + # pip-tools + # pyroma +cattrs==23.2.3 + # via jsii +cdk-nag==2.27.216 + # via -r requirements-dev.in +certifi==2023.11.17 + # via requests +cfn-lint==0.83.5 + # via -r requirements-dev.in +charset-normalizer==3.3.2 + # via requests +check-manifest==0.49 + # via -r requirements-dev.in +click==8.1.7 + # via + # black + # pip-tools +colorama==0.4.4 + # via awscli +constructs==10.3.0 + # via + # aws-cdk-lib + # cdk-nag +docutils==0.16 + # via + # awscli + # pyroma +exceptiongroup==1.2.0 + # via + # cattrs + # pytest +flake8==6.1.0 + # via -r requirements-dev.in +frozenlist==1.4.0 + # via + # aiohttp + # aiosignal +idna==3.6 + # via + # requests + # yarl +importlib-metadata==7.0.0 + # via build +importlib-resources==6.1.1 + # via jsii +iniconfig==2.0.0 + # via pytest +isort==5.13.2 + # via -r requirements-dev.in +jmespath==1.0.1 + # via + # boto3 + # botocore +jschema-to-python==1.2.3 + # via cfn-lint +jsii==1.93.0 + # via + # aws-cdk-asset-awscli-v1 + # aws-cdk-asset-kubectl-v20 + # aws-cdk-asset-node-proxy-agent-v6 + # aws-cdk-lib + # cdk-nag + # constructs +jsonpatch==1.33 + # via cfn-lint +jsonpickle==3.0.2 + # via jschema-to-python +jsonpointer==2.4 + # via jsonpatch +jsonschema==4.20.0 + # via + # aws-sam-translator + # cfn-lint +jsonschema-specifications==2023.11.2 + # via jsonschema +junit-xml==1.9 + # via cfn-lint +mccabe==0.7.0 + # via flake8 +mpmath==1.3.0 + # via sympy +multidict==6.0.4 + # via + # aiohttp + # yarl +mypy==1.7.1 + # via -r requirements-dev.in +mypy-extensions==1.0.0 + # via + # black + # mypy +networkx==3.2.1 + # via cfn-lint +packaging==23.2 + # via + # black + # build + # pyroma + # pytest +pathspec==0.12.1 + # via black +pbr==6.0.0 + # via + # jschema-to-python + # sarif-om +pip-tools==7.3.0 + # via -r requirements-dev.in +platformdirs==4.1.0 + # via black +pluggy==1.3.0 + # via pytest +publication==0.0.3 + # via + # aws-cdk-asset-awscli-v1 + # aws-cdk-asset-kubectl-v20 + # aws-cdk-asset-node-proxy-agent-v6 + # aws-cdk-lib + # cdk-nag + # constructs + # jsii +pyasn1==0.5.1 + # via rsa +pycodestyle==2.11.1 + # via flake8 +pydantic==2.5.2 + # via aws-sam-translator +pydantic-core==2.14.5 + # via pydantic +pydot==1.4.2 + # via -r requirements-dev.in +pyflakes==3.1.0 + # via flake8 +pygments==2.17.2 + # via pyroma +pyparsing==3.1.1 + # via pydot +pyproject-hooks==1.0.0 + # via build +pyroma==4.2 + # via -r requirements-dev.in +pytest==7.4.3 + # via -r requirements-dev.in +python-dateutil==2.8.2 + # via + # botocore + # jsii +pyyaml==6.0.1 + # via + # awscli + # cfn-lint +referencing==0.32.0 + # via + # jsonschema + # jsonschema-specifications +regex==2023.10.3 + # via cfn-lint +requests==2.31.0 + # via pyroma +rpds-py==0.13.2 + # via + # jsonschema + # referencing +rsa==4.7.2 + # via awscli +s3transfer==0.9.0 + # via + # awscli + # boto3 +sarif-om==1.0.4 + # via cfn-lint +six==1.16.0 + # via + # junit-xml + # python-dateutil +sympy==1.12 + # via cfn-lint +tomli==2.0.1 + # via + # black + # build + # check-manifest + # mypy + # pip-tools + # pyproject-hooks + # pytest +trove-classifiers==2023.11.29 + # via pyroma +typeguard==2.13.3 + # via + # aws-cdk-asset-awscli-v1 + # aws-cdk-asset-kubectl-v20 + # aws-cdk-asset-node-proxy-agent-v6 + # aws-cdk-lib + # cdk-nag + # constructs + # jsii +types-pyyaml==6.0.12.12 + # via -r requirements-dev.in +types-setuptools==69.0.0.0 + # via -r requirements-dev.in +typing-extensions==4.9.0 + # via + # aws-sam-translator + # black + # cattrs + # jsii + # mypy + # pydantic + # pydantic-core +urllib3==1.26.18 + # via + # botocore + # requests +wheel==0.42.0 + # via pip-tools +yarl==1.9.4 + # via aiohttp +zipp==3.17.0 + # via + # importlib-metadata + # importlib-resources + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/modules/mlflow/mlflow-image/requirements.in b/modules/mlflow/mlflow-image/requirements.in new file mode 100644 index 00000000..fcacd9ac --- /dev/null +++ b/modules/mlflow/mlflow-image/requirements.in @@ -0,0 +1,3 @@ +aws-cdk-lib==2.126.0 +cdk-nag==2.28.27 +boto3==1.34.35 \ No newline at end of file diff --git a/modules/mlflow/mlflow-image/requirements.txt b/modules/mlflow/mlflow-image/requirements.txt new file mode 100644 index 00000000..fa7b5c65 --- /dev/null +++ b/modules/mlflow/mlflow-image/requirements.txt @@ -0,0 +1,91 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --output-file=requirements.txt requirements.in +# +attrs==23.2.0 + # via + # cattrs + # jsii +aws-cdk-asset-awscli-v1==2.2.202 + # via aws-cdk-lib +aws-cdk-asset-kubectl-v20==2.1.2 + # via aws-cdk-lib +aws-cdk-asset-node-proxy-agent-v6==2.0.1 + # via aws-cdk-lib +aws-cdk-lib==2.126.0 + # via + # -r requirements.in + # cdk-ecr-deployment + # cdk-nag +boto3==1.34.35 + # via -r requirements.in +botocore==1.34.40 + # via + # boto3 + # s3transfer +cattrs==23.2.3 + # via jsii +cdk-ecr-deployment==2.5.30 + # via -r requirements.in +cdk-nag==2.28.27 + # via -r requirements.in +constructs==10.3.0 + # via + # aws-cdk-lib + # cdk-ecr-deployment + # cdk-nag +exceptiongroup==1.2.0 + # via cattrs +importlib-resources==6.1.1 + # via jsii +jmespath==1.0.1 + # via + # boto3 + # botocore +jsii==1.94.0 + # via + # aws-cdk-asset-awscli-v1 + # aws-cdk-asset-kubectl-v20 + # aws-cdk-asset-node-proxy-agent-v6 + # aws-cdk-lib + # cdk-ecr-deployment + # cdk-nag + # constructs +publication==0.0.3 + # via + # aws-cdk-asset-awscli-v1 + # aws-cdk-asset-kubectl-v20 + # aws-cdk-asset-node-proxy-agent-v6 + # aws-cdk-lib + # cdk-ecr-deployment + # cdk-nag + # constructs + # jsii +python-dateutil==2.8.2 + # via + # botocore + # jsii +s3transfer==0.10.0 + # via boto3 +six==1.16.0 + # via python-dateutil +typeguard==2.13.3 + # via + # aws-cdk-asset-awscli-v1 + # aws-cdk-asset-kubectl-v20 + # aws-cdk-asset-node-proxy-agent-v6 + # aws-cdk-lib + # cdk-ecr-deployment + # cdk-nag + # constructs + # jsii +typing-extensions==4.9.0 + # via + # cattrs + # jsii +urllib3==1.26.18 + # via botocore +zipp==3.17.0 + # via importlib-resources diff --git a/modules/mlflow/mlflow-image/setup.cfg b/modules/mlflow/mlflow-image/setup.cfg new file mode 100644 index 00000000..6136e2bb --- /dev/null +++ b/modules/mlflow/mlflow-image/setup.cfg @@ -0,0 +1,28 @@ +[metadata] +license_files = + LICENSE + NOTICE + VERSION + +[flake8] +max-line-length = 120 +extend-ignore = E203, W503 +exclude = + .git, + __pycache__, + docs/source/conf.py, + old, + build, + dist, + .venv, + codeseeder.out, + bundle + +[mypy] +python_version = 3.7 +strict = True +ignore_missing_imports = True +allow_untyped_decorators = True +exclude = + codeseeder.out/|example/|tests/ +warn_unused_ignores = False \ No newline at end of file diff --git a/modules/mlflow/mlflow-image/src/Dockerfile b/modules/mlflow/mlflow-image/src/Dockerfile new file mode 100644 index 00000000..9b08a8b6 --- /dev/null +++ b/modules/mlflow/mlflow-image/src/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.10.12 + +RUN pip install \ + mlflow==2.10.2 \ + boto3==1.34.45 && \ + mkdir /mlflow/ + +EXPOSE 5000 + +CMD mlflow server \ + --host 0.0.0.0 \ + --port 5000 \ + --default-artifact-root ${BUCKET} diff --git a/modules/mlflow/mlflow-image/stack.py b/modules/mlflow/mlflow-image/stack.py new file mode 100644 index 00000000..3557eb93 --- /dev/null +++ b/modules/mlflow/mlflow-image/stack.py @@ -0,0 +1,64 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os +from typing import Any, cast + +from aws_cdk import Aspects, Stack, Tags +from aws_cdk import aws_ecr as ecr +from aws_cdk.aws_ecr_assets import DockerImageAsset +from cdk_ecr_deployment import DockerImageName, ECRDeployment +from cdk_nag import AwsSolutionsChecks, NagPackSuppression, NagSuppressions +from constructs import Construct, IConstruct + + +class MlflowImagePublishingStack(Stack): # type: ignore + def __init__( + self, + scope: Construct, + id: str, + app_prefix: str, + ecr_repo_name: str, + **kwargs: Any, + ) -> None: + super().__init__(scope, id, **kwargs) + + Tags.of(scope=cast(IConstruct, self)).add(key="Deployment", value=app_prefix[:64]) + + repo = ecr.Repository.from_repository_name(self, "ECR", repository_name=ecr_repo_name) + + local_image = DockerImageAsset( + self, + "ImageAsset", + directory=os.path.join(os.path.dirname(os.path.abspath(__file__)), "src"), + ) + + self.image_uri = f"{repo.repository_uri}:latest" + ECRDeployment( + self, + "ECRDeployment", + src=DockerImageName(local_image.image_uri), + dest=DockerImageName(self.image_uri), + ) + + # Add CDK nag solutions checks + Aspects.of(self).add(AwsSolutionsChecks()) + + NagSuppressions.add_stack_suppressions( + self, + apply_to_nested_stacks=True, + suppressions=[ + NagPackSuppression( + **{ + "id": "AwsSolutions-IAM4", + "reason": "Managed Policies are for src account roles only", + } + ), + NagPackSuppression( + **{ + "id": "AwsSolutions-IAM5", + "reason": "Resource access restricted to resources", + } + ), + ], + ) diff --git a/modules/mlflow/mlflow-image/tests/__init__.py b/modules/mlflow/mlflow-image/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/modules/mlflow/mlflow-image/tests/test_app.py b/modules/mlflow/mlflow-image/tests/test_app.py new file mode 100644 index 00000000..4a2d61c2 --- /dev/null +++ b/modules/mlflow/mlflow-image/tests/test_app.py @@ -0,0 +1,30 @@ +import os +import sys + +import pytest + + +@pytest.fixture(scope="function") +def stack_defaults(): + os.environ["SEEDFARMER_PROJECT_NAME"] = "test-project" + os.environ["SEEDFARMER_DEPLOYMENT_NAME"] = "test-deployment" + os.environ["SEEDFARMER_MODULE_NAME"] = "test-module" + os.environ["CDK_DEFAULT_ACCOUNT"] = "111111111111" + os.environ["CDK_DEFAULT_REGION"] = "us-east-1" + + os.environ["SEEDFARMER_PARAMETER_ECR_REPOSITORY_NAME"] = "repo" + + # Unload the app import so that subsequent tests don't reuse + if "app" in sys.modules: + del sys.modules["app"] + + +def test_app(stack_defaults): + import app # noqa: F401 + + +def test_vpc_id(stack_defaults): + del os.environ["SEEDFARMER_PARAMETER_ECR_REPOSITORY_NAME"] + + with pytest.raises(Exception, match="Missing input parameter ecr-repository-name"): + import app # noqa: F401 diff --git a/modules/mlflow/mlflow-image/tests/test_stack.py b/modules/mlflow/mlflow-image/tests/test_stack.py new file mode 100644 index 00000000..d60ac344 --- /dev/null +++ b/modules/mlflow/mlflow-image/tests/test_stack.py @@ -0,0 +1,43 @@ +import os +import sys + +import aws_cdk as cdk +import pytest +from aws_cdk.assertions import Template + + +@pytest.fixture(scope="function") +def stack_defaults() -> None: + os.environ["CDK_DEFAULT_ACCOUNT"] = "111111111111" + os.environ["CDK_DEFAULT_REGION"] = "us-east-1" + + # Unload the app import so that subsequent tests don't reuse + if "stack" in sys.modules: + del sys.modules["stack"] + + +def test_synthesize_stack() -> None: + import stack + + app = cdk.App() + + project_name = "test-project" + dep_name = "test-deployment" + mod_name = "test-module" + app_prefix = f"{project_name}-{dep_name}-{mod_name}" + + ecr_repo_name = "repo" + + stack = stack.MlflowImagePublishingStack( + scope=app, + id=app_prefix, + app_prefix=app_prefix, + ecr_repo_name=ecr_repo_name, + env=cdk.Environment( + account=os.environ["CDK_DEFAULT_ACCOUNT"], + region=os.environ["CDK_DEFAULT_REGION"], + ), + ) + + template = Template.from_stack(stack) + template.resource_count_is("Custom::CDKBucketDeployment", 1)