[Rule Tuning] Add Investigation Fields to Specific AWS Rules (#4261)

* adding investigation fields to specific aws rules

* updated patch

* removing min-stack requirements

* removed user.name redundancy

* adjusted order of investigation fields

* adding source address

Removed changes from:
- rules/integrations/aws/discovery_ec2_multiple_discovery_api_calls_via_cli.toml
- rules/integrations/aws/impact_aws_s3_bucket_enumeration_or_brute_force.toml
- rules/integrations/aws/persistence_iam_user_created_access_keys_for_another_user.toml
- rules/integrations/aws/privilege_escalation_iam_administratoraccess_policy_attached_to_user.toml
- rules/integrations/aws_bedrock/aws_bedrock_multiple_attempts_to_use_denied_models_by_user.toml
- rules/integrations/aws_bedrock/aws_bedrock_multiple_validation_exception_errors_by_single_user.toml

(selectively cherry picked from commit ef453d8)

detection_rules/etc/non-ecs-schema.json

@@ -152,7 +152,7 @@
     "aws.cloudtrail.flattened.request_parameters.fromPort": "keyword",
     "aws.cloudtrail.flattened.request_parameters.roleArn": "keyword",
     "aws.cloudtrail.flattened.request_parameters.roleName": "keyword",
-    "aws.cloudtrail.flattened.request_paramters.policyArn": "keyword",
+    "aws.cloudtrail.flattened.request_parameters.policyArn": "keyword",
     "aws.cloudtrail.flattened.request_parameters.serialNumber": "keyword"
   },
   "logs-azure.signinlogs-*": {

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "detection_rules"
-version = "0.1.5"
+version = "0.1.6"
 description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine."
 readme = "README.md"
 requires-python = ">=3.12"

rules/cross-platform/execution_aws_ssm_sendcommand_with_command_parameters.toml

@@ -2,7 +2,7 @@
 creation_date = "2022/09/03"
 integration = ["endpoint", "auditd_manager"]
 maturity = "production"
-updated_date = "2024/09/23"
+updated_date = "2024/11/07"
 
 [rule]
 author = ["Elastic"]
@@ -87,6 +87,25 @@ and process.args: (
 )
 '''
 
+[rule.investigation_fields]
+field_names = [
+    "@timestamp",
+    "process.user.name",
+    "process.entry_leader.group.name",
+    "process.entry_leader.real_user.name",
+    "event.action",
+    "event.type",
+    "host.os.type",
+    "host.os.kernel",
+    "process.entry_leader.executable",
+    "process.entry_leader.working_directory",
+    "process.parent.executable",
+    "process.executable",
+    "process.hash.sha256",
+    "process.parent.command_line",
+    "process.command_line",
+    "process.args"
+]
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"

rules/integrations/aws/defense_evasion_cloudtrail_logging_deleted.toml

@@ -2,7 +2,7 @@
 creation_date = "2020/05/26"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2024/05/21"
+updated_date = "2024/11/07"
 
 [rule]
 author = ["Elastic"]
@@ -86,9 +86,26 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.dataset:aws.cloudtrail and event.provider:cloudtrail.amazonaws.com and event.action:DeleteTrail and event.outcome:success
+event.dataset:aws.cloudtrail
+    and event.provider:cloudtrail.amazonaws.com
+    and event.action:DeleteTrail
+    and event.outcome:success
 '''
 
+[rule.investigation_fields]
+field_names = [
+    "@timestamp",
+    "user.name",
+    "aws.cloudtrail.user_identity.arn",
+    "aws.cloudtrail.user_identity.type",
+    "source.address",
+    "user_agent.original",
+    "aws.cloudtrail.flattened.request_parameters.name",
+    "event.action",
+    "event.outcome",
+    "cloud.region",
+    "aws.cloudtrail.request_parameters"
+]
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"

rules/integrations/aws/discovery_ec2_multi_region_describe_instances.toml

@@ -2,7 +2,7 @@
 creation_date = "2024/08/26"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2024/11/05"
+updated_date = "2024/11/07"
 
 [rule]
 author = ["Elastic"]
@@ -107,6 +107,13 @@ from logs-aws.cloudtrail-*
 | sort target_time_window desc
 '''
 
+[rule.investigation_fields]
+field_names = [
+    "aws.cloudtrail.user_identity.arn",
+    "target_time_window",
+    "region_count",
+    "window_count"
+]
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"

rules/integrations/aws/discovery_new_terms_sts_getcalleridentity.toml

@@ -2,19 +2,19 @@
 creation_date = "2024/05/24"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2024/09/30"
+updated_date = "2024/11/07"
 
 [rule]
 author = ["Elastic"]
 description = """
-An adversary with access to a set of compromised credentials may attempt to verify that the credentials are valid and 
-determine what account they are using. This rule looks for the first time an identity has called the 
-STS `GetCallerIdentity` API operation in the last 15 days, which may be an indicator of compromised credentials. 
+An adversary with access to a set of compromised credentials may attempt to verify that the credentials are valid and
+determine what account they are using. This rule looks for the first time an identity has called the
+STS `GetCallerIdentity` API operation in the last 15 days, which may be an indicator of compromised credentials.
 A legitimate user would not need to call this operation as they should know the account they are using.
 """
 false_positives = [
     """
-    Verify whether the user identity should be using the STS `GetCallerIdentity` API operation. 
+    Verify whether the user identity should be using the STS `GetCallerIdentity` API operation.
     If known behavior is causing false positives, it can be exempted from the rule.
     """,
 ]
@@ -29,7 +29,7 @@ note = """## Triage and analysis
 ### Investigating AWS GetCallerIdentity API Called for the First Time
 
 AWS Security Token Service (AWS STS) is a service that enables you to request temporary, limited-privilege credentials for users.
-The `GetCallerIdentity` function returns details about the IAM user or role owning the credentials used to call the operation. 
+The `GetCallerIdentity` function returns details about the IAM user or role owning the credentials used to call the operation.
 No permissions are required to run this operation and the same information is returned even when access is denied.
 This rule looks for use of the `GetCallerIdentity` operation. This is a [New Terms](https://www.elastic.co/guide/en/security/master/rules-ui-create.html#create-new-terms-rule) rule indicating this is the first time a specific user identity has called this operation within the last 15 days.
 
@@ -93,10 +93,26 @@ timestamp_override = "event.ingested"
 type = "new_terms"
 
 query = '''
-event.dataset: "aws.cloudtrail" and event.provider: "sts.amazonaws.com" and event.action: "GetCallerIdentity"
-and not aws.cloudtrail.user_identity.type: "AssumedRole"
+event.dataset: "aws.cloudtrail"
+    and event.provider: "sts.amazonaws.com"
+    and event.action: "GetCallerIdentity"
+    and event.outcome: "success"
+    and not aws.cloudtrail.user_identity.type: "AssumedRole"
 '''
 
+[rule.investigation_fields]
+field_names = [
+    "@timestamp",
+    "user.name",
+    "source.address",
+    "aws.cloudtrail.user_identity.type",
+    "aws.cloudtrail.user_identity.arn",
+    "user_agent.original",
+    "event.action",
+    "event.outcome",
+    "cloud.region",
+    "aws.cloudtrail.request_parameters"
+]
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"

rules/integrations/aws/execution_ssm_command_document_created_by_rare_user.toml

@@ -2,25 +2,27 @@
 creation_date = "2024/11/01"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2024/11/01"
+updated_date = "2024/11/07"
 
 [rule]
 author = ["Elastic"]
 description = """
-Identifies when an AWS Systems Manager (SSM) command document is created by a user who does not typically perform this action. Adversaries may create SSM command documents to execute commands on managed instances, potentially leading to unauthorized access, command and control, data exfiltration and more.
+Identifies when an AWS Systems Manager (SSM) command document is created by a user who does not typically perform this
+action. Adversaries may create SSM command documents to execute commands on managed instances, potentially leading to
+unauthorized access, command and control, data exfiltration and more.
 """
 false_positives = [
     """
-    Legitimate users may create SSM command documents for legitimate purposes. Ensure that the document is authorized and the user is known before taking action.
+    Legitimate users may create SSM command documents for legitimate purposes. Ensure that the document is authorized
+    and the user is known before taking action.
     """,
 ]
 from = "now-9m"
 index = ["filebeat-*", "logs-aws.cloudtrail-*"]
 language = "kuery"
 license = "Elastic License v2"
 name = "AWS SSM Command Document Created by Rare User"
-note = """
-## Triage and Analysis
+note = """## Triage and Analysis
 
 ### Investigating AWS SSM Command Document Created by Rare User
 
@@ -55,7 +57,7 @@ For further guidance on managing and securing AWS Systems Manager in your enviro
 """
 references = [
     "https://docs.aws.amazon.com/systems-manager/latest/APIReference/API_CreateDocument.html",
-    "https://docs.aws.amazon.com/systems-manager/latest/userguide/documents.html"
+    "https://docs.aws.amazon.com/systems-manager/latest/userguide/documents.html",
 ]
 risk_score = 21
 rule_id = "50a2bdea-9876-11ef-89db-f661ea17fbcd"
@@ -68,7 +70,7 @@ tags = [
     "Data Source: AWS Systems Manager",
     "Resources: Investigation Guide",
     "Use Case: Threat Detection",
-    "Tactic: Execution"
+    "Tactic: Execution",
 ]
 timestamp_override = "event.ingested"
 type = "new_terms"
@@ -81,6 +83,21 @@ event.dataset: "aws.cloudtrail"
     and aws.cloudtrail.response_elements: *documentType=Command*
 '''
 
+[rule.investigation_fields]
+field_names = [
+    "@timestamp",
+    "user.name",
+    "source.address",
+    "aws.cloudtrail.user_identity.arn",
+    "aws.cloudtrail.user_identity.type",
+    "user_agent.original",
+    "event.action",
+    "event.outcome",
+    "cloud.region",
+    "aws.cloudtrail.request_parameters",
+    "aws.cloudtrail.response_elements",
+]
+
 [[rule.threat]]
 framework = "MITRE ATT&CK"
 

rules/integrations/aws/exfiltration_sns_email_subscription_by_rare_user.toml

@@ -2,7 +2,7 @@
 creation_date = "2024/11/01"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2024/11/01"
+updated_date = "2024/11/07"
 
 [rule]
 author = ["Elastic"]
@@ -81,6 +81,22 @@ event.dataset: "aws.cloudtrail"
     and aws.cloudtrail.request_parameters: *protocol=email*
 '''
 
+[rule.investigation_fields]
+field_names = [
+    "@timestamp",
+    "user.name",
+    "source.address",
+    "aws.cloudtrail.user_identity.arn",
+    "aws.cloudtrail.user_identity.type",
+    "user_agent.original",
+    "event.action",
+    "event.outcome",
+    "cloud.region",
+    "aws.cloudtrail.flattened.request_parameters.protocol",
+    "aws.cloudtrail.flattened.request_parameters.topicArn",
+    "aws.cloudtrail.flattened.response_elements.subscriptionArn",
+    "aws.cloudtrail.request_parameters"
+]
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"

rules/integrations/aws/persistence_ec2_security_group_configuration_change_detection.toml

@@ -2,7 +2,7 @@
 creation_date = "2021/05/05"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2024/11/05"
+updated_date = "2024/11/07"
 
 [rule]
 author = ["Elastic", "Austin Songer"]
@@ -100,6 +100,22 @@ event.dataset: "aws.cloudtrail"
     and event.outcome: "success"
 '''
 
+[rule.investigation_fields]
+field_names = [
+   "@timestamp",
+   "user.name",
+   "aws.cloudtrail.user_identity.arn",
+   "aws.cloudtrail.user_identity.type",
+   "user_agent.original",
+   "aws.cloudtrail.flattened.request_parameters.instanceId",
+   "event.action",
+   "event.outcome",
+   "cloud.region",
+   "event.provider",
+   "aws.cloudtrail.request_parameters",
+   "aws.cloudtrail.response_elements"
+]
+
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"

rules/integrations/aws/persistence_iam_create_user_via_assumed_role_on_ec2_instance.toml

@@ -2,7 +2,7 @@
 creation_date = "2024/11/04"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2024/11/04"
+updated_date = "2024/11/07"
 
 [rule]
 author = ["Elastic"]
@@ -96,6 +96,20 @@ event.dataset: "aws.cloudtrail"
     and aws.cloudtrail.user_identity.arn: *i-*
 '''
 
+[rule.investigation_fields]
+field_names = [
+    "@timestamp",
+    "user.name",
+    "source.address",
+    "aws.cloudtrail.user_identity.arn",
+    "aws.cloudtrail.user_identity.type",
+    "user_agent.original",
+    "event.action",
+    "event.outcome",
+    "cloud.region",
+    "aws.cloudtrail.request_parameters",
+    "aws.cloudtrail.response_elements"
+]
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"

rules/integrations/aws/privilege_escalation_iam_customer_managed_policy_attached_to_role.toml

@@ -2,7 +2,7 @@
 creation_date = "2024/11/04"
 integration = ["aws"]
 maturity = "production"
-updated_date = "2024/11/04"
+updated_date = "2024/11/07"
 
 [rule]
 author = ["Elastic"]
@@ -94,9 +94,25 @@ event.dataset: "aws.cloudtrail"
     and event.provider: "iam.amazonaws.com"
     and event.action: "AttachRolePolicy"
     and event.outcome: "success"
-    and not aws.cloudtrail.flattened.request_paramters.policyArn: arn\:aws\:iam\:\:aws\:policy*
+    and not aws.cloudtrail.flattened.request_parameters.policyArn: arn\:aws\:iam\:\:aws\:policy*
 '''
 
+[rule.investigation_fields]
+field_names = [
+    "@timestamp",
+    "user.name",
+    "source.address",
+    "aws.cloudtrail.user_identity.arn",
+    "aws.cloudtrail.user_identity.type",
+    "user_agent.original",
+    "aws.cloudtrail.flattened.request_parameters.policyArn",
+    "aws.cloudtrail.flattened.request_parameters.roleName",
+    "event.action",
+    "event.outcome",
+    "cloud.region",
+    "event.provider",
+    "aws.cloudtrail.request_parameters"
+]
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"