From cc19b15e7e552167930af0dcd940310611221864 Mon Sep 17 00:00:00 2001 From: "jake@prefect.io" Date: Wed, 12 Jul 2023 16:59:35 -0400 Subject: [PATCH 1/5] dont break on debug logs --- prefect_aws/workers/ecs_worker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/prefect_aws/workers/ecs_worker.py b/prefect_aws/workers/ecs_worker.py index 64d206e5..da2382e7 100644 --- a/prefect_aws/workers/ecs_worker.py +++ b/prefect_aws/workers/ecs_worker.py @@ -675,7 +675,7 @@ def _create_task_and_wait_for_start( _TASK_DEFINITION_CACHE[flow_run.deployment_id] = task_definition_arn logger.info(f"Using ECS task definition {task_definition_arn!r}...") - logger.debug(f"Task definition {json.dumps(task_definition, indent=2)}") + logger.debug(f"Task definition {json.dumps(task_definition, indent=2, default=str)}") # Prepare the task run request task_run_request = self._prepare_task_run_request( @@ -686,7 +686,7 @@ def _create_task_and_wait_for_start( ) logger.info("Creating ECS task run...") - logger.debug(f"Task run request {json.dumps(task_run_request, indent=2)}") + logger.debug(f"Task run request {json.dumps(task_run_request, indent=2, default=str)}") try: task = self._create_task_run(ecs_client, task_run_request) task_arn = task["taskArn"] @@ -861,7 +861,7 @@ def _register_task_definition( Returns the ARN. """ logger.info("Registering ECS task definition...") - logger.debug(f"Task definition request {json.dumps(task_definition, indent=2)}") + logger.debug(f"Task definition request {json.dumps(task_definition, indent=2, default=str)}") response = ecs_client.register_task_definition(**task_definition) return response["taskDefinition"]["taskDefinitionArn"] From 232ae2f182d987a9ccd3be826c9995ed6a6575b3 Mon Sep 17 00:00:00 2001 From: "jake@prefect.io" Date: Wed, 12 Jul 2023 17:14:30 -0400 Subject: [PATCH 2/5] black --- prefect_aws/workers/ecs_worker.py | 33 ++++++++++++++++++------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/prefect_aws/workers/ecs_worker.py b/prefect_aws/workers/ecs_worker.py index da2382e7..b37bfa1f 100644 --- a/prefect_aws/workers/ecs_worker.py +++ b/prefect_aws/workers/ecs_worker.py @@ -364,15 +364,15 @@ class ECSVariables(BaseVariables): "field will be slugified to match AWS character requirements." ), ) - launch_type: Optional[Literal["FARGATE", "EC2", "EXTERNAL", "FARGATE_SPOT"]] = ( - Field( - default=ECS_DEFAULT_LAUNCH_TYPE, - description=( - "The type of ECS task run infrastructure that should be used. Note that" - " 'FARGATE_SPOT' is not a formal ECS launch type, but we will configure" - " the proper capacity provider stategy if set here." - ), - ) + launch_type: Optional[ + Literal["FARGATE", "EC2", "EXTERNAL", "FARGATE_SPOT"] + ] = Field( + default=ECS_DEFAULT_LAUNCH_TYPE, + description=( + "The type of ECS task run infrastructure that should be used. Note that" + " 'FARGATE_SPOT' is not a formal ECS launch type, but we will configure" + " the proper capacity provider stategy if set here." + ), ) image: Optional[str] = Field( default=None, @@ -675,7 +675,9 @@ def _create_task_and_wait_for_start( _TASK_DEFINITION_CACHE[flow_run.deployment_id] = task_definition_arn logger.info(f"Using ECS task definition {task_definition_arn!r}...") - logger.debug(f"Task definition {json.dumps(task_definition, indent=2, default=str)}") + logger.debug( + f"Task definition {json.dumps(task_definition, indent=2, default=str)}" + ) # Prepare the task run request task_run_request = self._prepare_task_run_request( @@ -686,7 +688,9 @@ def _create_task_and_wait_for_start( ) logger.info("Creating ECS task run...") - logger.debug(f"Task run request {json.dumps(task_run_request, indent=2, default=str)}") + logger.debug( + f"Task run request {json.dumps(task_run_request, indent=2, default=str)}" + ) try: task = self._create_task_run(ecs_client, task_run_request) task_arn = task["taskArn"] @@ -861,7 +865,9 @@ def _register_task_definition( Returns the ARN. """ logger.info("Registering ECS task definition...") - logger.debug(f"Task definition request {json.dumps(task_definition, indent=2, default=str)}") + logger.debug( + f"Task definition request {json.dumps(task_definition, indent=2, default=str)}" + ) response = ecs_client.register_task_definition(**task_definition) return response["taskDefinition"]["taskDefinitionArn"] @@ -1255,8 +1261,7 @@ def _load_vpc_network_config( ) raise ValueError( f"Failed to find {vpc_message}. " - "Network configuration cannot be inferred. " - + help_message + "Network configuration cannot be inferred. " + help_message ) vpc_id = vpcs[0]["VpcId"] From f83f10a765adca64f19f487346128671534826df Mon Sep 17 00:00:00 2001 From: "jake@prefect.io" Date: Wed, 12 Jul 2023 17:30:01 -0400 Subject: [PATCH 3/5] mypy gods... --- prefect_aws/workers/ecs_worker.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/prefect_aws/workers/ecs_worker.py b/prefect_aws/workers/ecs_worker.py index b37bfa1f..c769d515 100644 --- a/prefect_aws/workers/ecs_worker.py +++ b/prefect_aws/workers/ecs_worker.py @@ -364,8 +364,7 @@ class ECSVariables(BaseVariables): "field will be slugified to match AWS character requirements." ), ) - launch_type: Optional[ - Literal["FARGATE", "EC2", "EXTERNAL", "FARGATE_SPOT"] + launch_type: Optional[Literal["FARGATE", "EC2", "EXTERNAL", "FARGATE_SPOT"] ] = Field( default=ECS_DEFAULT_LAUNCH_TYPE, description=( @@ -866,7 +865,8 @@ def _register_task_definition( """ logger.info("Registering ECS task definition...") logger.debug( - f"Task definition request {json.dumps(task_definition, indent=2, default=str)}" + "Task definition request" + f"{json.dumps(task_definition, indent=2, default=str)}" ) response = ecs_client.register_task_definition(**task_definition) return response["taskDefinition"]["taskDefinitionArn"] @@ -1261,7 +1261,8 @@ def _load_vpc_network_config( ) raise ValueError( f"Failed to find {vpc_message}. " - "Network configuration cannot be inferred. " + help_message + "Network configuration cannot be inferred. " + + help_message ) vpc_id = vpcs[0]["VpcId"] From 9bb086809eb671cec5c7e70e776070c784110a75 Mon Sep 17 00:00:00 2001 From: "jake@prefect.io" Date: Fri, 14 Jul 2023 14:04:28 -0400 Subject: [PATCH 4/5] appease flake8 --- prefect_aws/workers/ecs_worker.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/prefect_aws/workers/ecs_worker.py b/prefect_aws/workers/ecs_worker.py index c769d515..27544e6e 100644 --- a/prefect_aws/workers/ecs_worker.py +++ b/prefect_aws/workers/ecs_worker.py @@ -364,14 +364,15 @@ class ECSVariables(BaseVariables): "field will be slugified to match AWS character requirements." ), ) - launch_type: Optional[Literal["FARGATE", "EC2", "EXTERNAL", "FARGATE_SPOT"] - ] = Field( - default=ECS_DEFAULT_LAUNCH_TYPE, - description=( - "The type of ECS task run infrastructure that should be used. Note that" - " 'FARGATE_SPOT' is not a formal ECS launch type, but we will configure" - " the proper capacity provider stategy if set here." - ), + launch_type: Optional[Literal["FARGATE", "EC2", "EXTERNAL", "FARGATE_SPOT"]] = ( + Field( + default=ECS_DEFAULT_LAUNCH_TYPE, + description=( + "The type of ECS task run infrastructure that should be used. Note that" + " 'FARGATE_SPOT' is not a formal ECS launch type, but we will configure" + " the proper capacity provider stategy if set here." + ), + ) ) image: Optional[str] = Field( default=None, From 70d4eebd52ace8073cbfca0d53e7e83dff7b4347 Mon Sep 17 00:00:00 2001 From: "jake@prefect.io" Date: Fri, 14 Jul 2023 14:27:11 -0400 Subject: [PATCH 5/5] update changelog --- CHANGELOG.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 023cb879..a7b63fe9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,12 +11,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Fixed `S3Bucket.stream_from` path resolution - [#291](https://github.com/PrefectHQ/prefect-aws/pull/291) - ### Deprecated ### Removed +## 0.3.5 + +### Fixed + +- Fixed `S3Bucket.stream_from` path resolution - [#291](https://github.com/PrefectHQ/prefect-aws/pull/291) +- Fixed `ECSWorker` debug logs from failing to parse json - [#296](https://github.com/PrefectHQ/prefect-aws/pull/296) + ## 0.3.4 Released on June 15th, 2023.