Merge pull request #3420 from aws/tmp/1700154710/main

Merge main to develop

docs/globals.rst

@@ -74,6 +74,7 @@ Currently, the following resources and properties are being supported:
       Architectures:
       EphemeralStorage:
       RuntimeManagementConfig:
+      LoggingConfig:
 
     Api:
       # Properties of AWS::Serverless::Api

integration/combination/test_function_with_msk.py

@@ -28,6 +28,15 @@ def test_function_with_msk_trigger_using_manage_policy(self):
         parameters.append(self.generate_parameter("MskClusterName2", cluster_name))
         self._common_validations_for_MSK("combination/function_with_msk_using_managed_policy", parameters)
 
+    def test_function_with_msk_trigger_and_s3_onfailure_events_destinations(self):
+        companion_stack_outputs = self.companion_stack_outputs
+        parameters = self.get_parameters(companion_stack_outputs)
+        cluster_name = "MskCluster3-" + generate_suffix()
+        parameters.append(self.generate_parameter("MskClusterName3", cluster_name))
+        self._common_validations_for_MSK(
+            "combination/function_with_msk_trigger_and_s3_onfailure_events_destinations", parameters
+        )
+
     def _common_validations_for_MSK(self, file_name, parameters):
         self.create_and_verify_stack(file_name, parameters)
 

integration/resources/expected/combination/function_with_msk_trigger_and_s3_onfailure_events_destinations.json

@@ -0,0 +1,22 @@
+[
+  {
+    "LogicalResourceId": "MyMskStreamProcessor",
+    "ResourceType": "AWS::Lambda::Function"
+  },
+  {
+    "LogicalResourceId": "MyLambdaExecutionRole",
+    "ResourceType": "AWS::IAM::Role"
+  },
+  {
+    "LogicalResourceId": "MyMskCluster",
+    "ResourceType": "AWS::MSK::Cluster"
+  },
+  {
+    "LogicalResourceId": "MyMskStreamProcessorMyMskEvent",
+    "ResourceType": "AWS::Lambda::EventSourceMapping"
+  },
+  {
+    "LogicalResourceId": "PreCreatedS3Bucket",
+    "ResourceType": "AWS::S3::Bucket"
+  }
+]

integration/resources/templates/combination/function_with_msk_trigger_and_s3_onfailure_events_destinations.yaml

@@ -0,0 +1,80 @@
+Parameters:
+  PreCreatedSubnetOne:
+    Type: String
+  PreCreatedSubnetTwo:
+    Type: String
+  MskClusterName3:
+    Type: String
+
+Resources:
+  MyLambdaExecutionRole:
+    Type: AWS::IAM::Role
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+        - Action: [sts:AssumeRole]
+          Effect: Allow
+          Principal:
+            Service: [lambda.amazonaws.com]
+      Policies:
+      - PolicyName: IntegrationTestExecution
+        PolicyDocument:
+          Statement:
+          - Action: [kafka:DescribeCluster, kafka:GetBootstrapBrokers, ec2:CreateNetworkInterface,
+              ec2:DescribeNetworkInterfaces, ec2:DescribeVpcs, ec2:DeleteNetworkInterface,
+              ec2:DescribeSubnets, ec2:DescribeSecurityGroups, logs:CreateLogGroup,
+              logs:CreateLogStream, logs:PutLogEvents]
+            Effect: Allow
+            Resource: '*'
+      ManagedPolicyArns: [arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole]
+      Tags:
+      - {Value: SAM, Key: lambda:createdBy}
+
+  MyMskCluster:
+    Type: AWS::MSK::Cluster
+    Properties:
+      BrokerNodeGroupInfo:
+        ClientSubnets:
+        - Ref: PreCreatedSubnetOne
+        - Ref: PreCreatedSubnetTwo
+        InstanceType: kafka.t3.small
+        StorageInfo:
+          EBSStorageInfo:
+            VolumeSize: 1
+      ClusterName:
+        Ref: MskClusterName3
+      KafkaVersion: 2.4.1.1
+      NumberOfBrokerNodes: 2
+
+  MyMskStreamProcessor:
+    Type: AWS::Serverless::Function
+    Properties:
+      Runtime: nodejs18.x
+      Handler: index.handler
+      CodeUri: ${codeuri}
+      Role:
+        Fn::GetAtt: [MyLambdaExecutionRole, Arn]
+      Events:
+        MyMskEvent:
+          Type: MSK
+          Properties:
+            StartingPosition: LATEST
+            Stream:
+              Ref: MyMskCluster
+            Topics:
+            - MyDummyTestTopic
+            DestinationConfig:
+              OnFailure:
+                Type: S3
+                Destination:
+                  Fn::GetAtt:
+                  - PreCreatedS3Bucket
+                  - Arn
+
+  PreCreatedS3Bucket:
+    Type: AWS::S3::Bucket
+    DeletionPolicy: Delete
+
+Metadata:
+  SamTransformTest: true

samtranslator/__init__.py

@@ -1 +1 @@
-__version__ = "1.79.0"
+__version__ = "1.80.0"

samtranslator/internal/schema_source/aws_serverless_function.py

@@ -412,6 +412,7 @@ class MSKEventProperties(BaseModel):
     Stream: PassThroughProp = mskeventproperties("Stream")
     Topics: PassThroughProp = mskeventproperties("Topics")
     SourceAccessConfigurations: Optional[PassThroughProp] = mskeventproperties("SourceAccessConfigurations")
+    DestinationConfig: Optional[PassThroughProp]  # TODO: add documentation
 
 
 class MSKEvent(BaseModel):
@@ -504,6 +505,7 @@ class ScheduleV2Event(BaseModel):
 EphemeralStorage = Optional[PassThroughProp]
 SnapStart = Optional[PassThroughProp]  # TODO: check the type
 RuntimeManagementConfig = Optional[PassThroughProp]  # TODO: check the type
+LoggingConfig = Optional[PassThroughProp]  # TODO: add documentation
 
 
 class Properties(BaseModel):
@@ -629,6 +631,7 @@ class Properties(BaseModel):
     Tracing: Optional[Tracing] = prop("Tracing")
     VersionDescription: Optional[PassThroughProp] = prop("VersionDescription")
     VpcConfig: Optional[VpcConfig] = prop("VpcConfig")
+    LoggingConfig: Optional[PassThroughProp]  # TODO: add documentation
 
 
 class Globals(BaseModel):
@@ -686,6 +689,7 @@ class Globals(BaseModel):
     )
     SnapStart: Optional[SnapStart] = prop("SnapStart")
     RuntimeManagementConfig: Optional[RuntimeManagementConfig] = prop("RuntimeManagementConfig")
+    LoggingConfig: Optional[PassThroughProp]  # TODO: add documentation
 
 
 class Resource(ResourceAttributes):

samtranslator/model/eventsources/pull.py

@@ -172,13 +172,15 @@ def to_cloudformation(self, **kwargs):  # type: ignore[no-untyped-def] # noqa: P
             # `Type` property is for sam to attach the right policies
             destination_type = on_failure.get("Type")
 
-            # SAM attaches the policies for SQS or SNS only if 'Type' is given
+            # SAM attaches the policies for SQS, SNS or S3 only if 'Type' is given
             if destination_type:
                 # delete this field as its used internally for SAM to determine the policy
                 del on_failure["Type"]
-                # the values 'SQS' and 'SNS' are allowed. No intrinsics are allowed
-                if destination_type not in ["SQS", "SNS"]:
-                    raise InvalidEventException(self.logical_id, "The only valid values for 'Type' are 'SQS' and 'SNS'")
+                # the values 'SQS', 'SNS', and 'S3' are allowed. No intrinsics are allowed
+                if destination_type not in ["SQS", "SNS", "S3"]:
+                    raise InvalidEventException(
+                        self.logical_id, "The only valid values for 'Type' are 'SQS', 'SNS', and 'S3'"
+                    )
                 if destination_type == "SQS":
                     queue_arn = on_failure.get("Destination")
                     destination_config_policy = IAMRolePolicies().sqs_send_message_role_policy(
@@ -189,6 +191,11 @@ def to_cloudformation(self, **kwargs):  # type: ignore[no-untyped-def] # noqa: P
                     destination_config_policy = IAMRolePolicies().sns_publish_role_policy(
                         sns_topic_arn, self.logical_id
                     )
+                elif destination_type == "S3":
+                    s3_arn = on_failure.get("Destination")
+                    destination_config_policy = IAMRolePolicies().s3_send_event_payload_role_policy(
+                        s3_arn, self.logical_id
+                    )
 
             lambda_eventsourcemapping.DestinationConfig = self.DestinationConfig
 

samtranslator/model/iam.py

@@ -126,6 +126,19 @@ def sns_publish_role_policy(cls, topic_arn: Any, logical_id: str) -> Dict[str, A
             "PolicyDocument": {"Statement": [{"Action": "sns:publish", "Effect": "Allow", "Resource": topic_arn}]},
         }
 
+    @classmethod
+    def s3_send_event_payload_role_policy(cls, s3_arn: Any, logical_id: str) -> Dict[str, Any]:
+        s3_arn_with_wild_card = {"Fn::Join": ["/", [s3_arn, "*"]]}
+        return {
+            "PolicyName": logical_id + "S3Policy",
+            "PolicyDocument": {
+                "Statement": [
+                    {"Action": "s3:PutObject", "Effect": "Allow", "Resource": s3_arn_with_wild_card},
+                    {"Action": "s3:ListBucket", "Effect": "Allow", "Resource": s3_arn},
+                ]
+            },
+        }
+
     @classmethod
     def event_bus_put_events_role_policy(cls, event_bus_arn: Any, logical_id: str) -> Dict[str, Any]:
         return {

samtranslator/model/lambda_.py

@@ -34,6 +34,7 @@ class LambdaFunction(Resource):
         "SnapStart": GeneratedProperty(),
         "EphemeralStorage": GeneratedProperty(),
         "RuntimeManagementConfig": GeneratedProperty(),
+        "LoggingConfig": GeneratedProperty(),
     }
 
     Code: Dict[str, Any]
@@ -60,6 +61,7 @@ class LambdaFunction(Resource):
     SnapStart: Optional[Dict[str, Any]]
     EphemeralStorage: Optional[Dict[str, Any]]
     RuntimeManagementConfig: Optional[Dict[str, Any]]
+    LoggingConfig: Optional[Dict[str, Any]]
 
     runtime_attrs = {"name": lambda self: ref(self.logical_id), "arn": lambda self: fnGetAtt(self.logical_id, "Arn")}
 

samtranslator/model/sam_resources.py

@@ -177,6 +177,7 @@ class SamFunction(SamResourceMacro):
         "SnapStart": PropertyType(False, IS_DICT),
         "FunctionUrlConfig": PropertyType(False, IS_DICT),
         "RuntimeManagementConfig": PassThroughProperty(False),
+        "LoggingConfig": PassThroughProperty(False),
     }
 
     FunctionName: Optional[Intrinsicable[str]]
@@ -218,6 +219,7 @@ class SamFunction(SamResourceMacro):
     Architectures: Optional[List[Any]]
     SnapStart: Optional[Dict[str, Any]]
     FunctionUrlConfig: Optional[Dict[str, Any]]
+    LoggingConfig: Optional[Dict[str, Any]]
 
     event_resolver = ResourceTypeResolver(
         samtranslator.model.eventsources,
@@ -601,6 +603,7 @@ def _construct_lambda_function(self, intrinsics_resolver: IntrinsicsResolver) ->
         lambda_function.CodeSigningConfigArn = self.CodeSigningConfigArn
 
         lambda_function.RuntimeManagementConfig = self.RuntimeManagementConfig  # type: ignore[attr-defined]
+        lambda_function.LoggingConfig = self.LoggingConfig
         self._validate_package_type(lambda_function)
         return lambda_function
 

samtranslator/plugins/globals/globals.py

@@ -53,6 +53,7 @@ class Globals:
             "EphemeralStorage",
             "FunctionUrlConfig",
             "RuntimeManagementConfig",
+            "LoggingConfig",
         ],
         # Everything except
         #   DefinitionBody: because its hard to reason about merge of Swagger dictionaries

samtranslator/schema/schema.json

@@ -255167,6 +255167,9 @@
           "markdownDescription": "A string that configures how events will be read from Kafka topics\\.  \n*Type*: String  \n*Required*: No  \n*AWS CloudFormation compatibility*: This property is passed directly to the [`AmazonManagedKafkaConfiguration`](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-lambda-eventsourcemapping.html) property of an `AWS::Lambda::EventSourceMapping` resource\\.",
           "title": "ConsumerGroupId"
         },
+        "DestinationConfig": {
+          "$ref": "#/definitions/PassThroughProp"
+        },
         "FilterCriteria": {
           "allOf": [
             {
@@ -257744,6 +257747,9 @@
           "markdownDescription": "The list of `LayerVersion` ARNs that this function should use\\. The order specified here is the order in which they will be imported when running the Lambda function\\.  \n*Type*: List  \n*Required*: No  \n*AWS CloudFormation compatibility*: This property is passed directly to the [`Layers`](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-lambda-function.html#cfn-lambda-function-layers) property of an `AWS::Lambda::Function` resource\\.",
           "title": "Layers"
         },
+        "LoggingConfig": {
+          "$ref": "#/definitions/PassThroughProp"
+        },
         "MemorySize": {
           "allOf": [
             {
@@ -258090,6 +258096,9 @@
           "markdownDescription": "The list of `LayerVersion` ARNs that this function should use\\. The order specified here is the order in which they will be imported when running the Lambda function\\.  \n*Type*: List  \n*Required*: No  \n*AWS CloudFormation compatibility*: This property is passed directly to the [`Layers`](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-lambda-function.html#cfn-lambda-function-layers) property of an `AWS::Lambda::Function` resource\\.",
           "title": "Layers"
         },
+        "LoggingConfig": {
+          "$ref": "#/definitions/PassThroughProp"
+        },
         "MemorySize": {
           "allOf": [
             {

schema_source/sam.schema.json

@@ -2380,6 +2380,9 @@
           "markdownDescription": "A string that configures how events will be read from Kafka topics\\.  \n*Type*: String  \n*Required*: No  \n*AWS CloudFormation compatibility*: This property is passed directly to the [`AmazonManagedKafkaConfiguration`](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-lambda-eventsourcemapping.html) property of an `AWS::Lambda::EventSourceMapping` resource\\.",
           "title": "ConsumerGroupId"
         },
+        "DestinationConfig": {
+          "$ref": "#/definitions/PassThroughProp"
+        },
         "FilterCriteria": {
           "allOf": [
             {
@@ -5391,6 +5394,9 @@
           "markdownDescription": "The list of `LayerVersion` ARNs that this function should use\\. The order specified here is the order in which they will be imported when running the Lambda function\\.  \n*Type*: List  \n*Required*: No  \n*AWS CloudFormation compatibility*: This property is passed directly to the [`Layers`](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-lambda-function.html#cfn-lambda-function-layers) property of an `AWS::Lambda::Function` resource\\.",
           "title": "Layers"
         },
+        "LoggingConfig": {
+          "$ref": "#/definitions/PassThroughProp"
+        },
         "MemorySize": {
           "allOf": [
             {
@@ -5914,6 +5920,9 @@
           "markdownDescription": "The list of `LayerVersion` ARNs that this function should use\\. The order specified here is the order in which they will be imported when running the Lambda function\\.  \n*Type*: List  \n*Required*: No  \n*AWS CloudFormation compatibility*: This property is passed directly to the [`Layers`](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-lambda-function.html#cfn-lambda-function-layers) property of an `AWS::Lambda::Function` resource\\.",
           "title": "Layers"
         },
+        "LoggingConfig": {
+          "$ref": "#/definitions/PassThroughProp"
+        },
         "MemorySize": {
           "allOf": [
             {

tests/translator/input/function_with_msk_s3_onfailure_dest.yaml

@@ -0,0 +1,20 @@
+Resources:
+  MyMskStreamProcessor:
+    Type: AWS::Serverless::Function
+    Properties:
+      Runtime: nodejs12.x
+      Handler: index.handler
+      CodeUri: s3://sam-demo-bucket/kafka.zip
+      Events:
+        MyMskEvent:
+          Type: MSK
+          Properties:
+            StartingPosition: LATEST
+            Stream: !Sub arn:${AWS::Partition}:kafka:${AWS::Region}:012345678901:cluster/mycluster/6cc0432b-8618-4f44-bccc-e1fbd8fb7c4d-2
+            Topics:
+            - MyDummyTestTopic
+            ConsumerGroupId: consumergroup1
+            DestinationConfig:
+              OnFailure:
+                Type: S3
+                Destination: !Sub arn:${AWS::Partition}:s3:::my-s3-arn

tests/translator/input/globals_for_function.yaml

@@ -30,6 +30,10 @@ Globals:
       Size: 1024
     RuntimeManagementConfig:
       UpdateRuntimeOn: Auto
+    LoggingConfig:
+      LogGroup: myJsonStructuredLogs
+
+
 
 Resources:
   MinimalFunction: