From 97d9101df99e3514fd98a4aec26a509ec66ab40e Mon Sep 17 00:00:00 2001 From: Subhobrata Dey Date: Tue, 11 Jul 2023 16:57:33 -0700 Subject: [PATCH] [backport 2.x] Logtypes pr v2 (#482) Signed-off-by: Subhobrata Dey Co-authored-by: Petar Dzepina (cherry picked from commit 978878100cacf91f5e610eee427387407d3fe73e) --- .../SecurityAnalyticsPlugin.java | 72 +- .../action/GetAlertsRequest.java | 19 +- .../action/GetFindingsRequest.java | 21 +- .../action/IndexRuleRequest.java | 8 - .../alerts/AlertsService.java | 8 +- .../monitors/DetectorMonitorConfig.java | 116 +- .../index/codec/CorrelationCodecService.java | 5 +- .../findings/FindingsService.java | 4 +- .../DetectorIndexManagementService.java | 88 +- .../logtype/BuiltinLogTypeLoader.java | 37 +- .../logtype/LogTypeService.java | 581 +++- .../logtype/MappingSchema.java | 15 + .../mapper/MapperService.java | 392 ++- .../mapper/MapperTopicStore.java | 75 - .../securityanalytics/model/Detector.java | 39 +- .../model/FieldMappingDoc.java | 150 + .../securityanalytics/model/LogType.java | 4 +- .../securityanalytics/model/RuleCategory.java | 33 - .../resthandler/RestGetAlertsAction.java | 2 +- .../resthandler/RestGetFindingsAction.java | 2 +- .../settings/SecurityAnalyticsSettings.java | 7 + .../transport/TransportGetAlertsAction.java | 6 +- .../TransportGetAllRuleCategoriesAction.java | 16 +- .../transport/TransportGetFindingsAction.java | 24 +- .../TransportIndexDetectorAction.java | 333 +- .../transport/TransportIndexRuleAction.java | 95 +- .../securityanalytics/util/RuleIndices.java | 4 +- .../util/RuleTopicIndices.java | 68 +- .../resources/OSMapping/NetFlowMapping.json | 29 - .../resources/OSMapping/ad_ldap/mappings.json | 84 - .../OSMapping/apache_access/fieldmappings.yml | 8 - .../OSMapping/apache_access/mappings.json | 16 - .../OSMapping/azure/fieldmappings.yml | 31 - .../resources/OSMapping/azure/mappings.json | 104 - .../OSMapping/cloudtrail/fieldmappings.yml | 17 - .../OSMapping/cloudtrail/mappings.json | 120 - .../OSMapping/cloudtrail_logtype.json | 183 +- .../resources/OSMapping/dns/fieldmappings.yml | 5 - .../resources/OSMapping/dns/mappings.json | 20 - src/main/resources/OSMapping/dns_logtype.json | 102 +- .../OSMapping/github/fieldmappings.yml | 2 - .../resources/OSMapping/github/mappings.json | 8 - .../OSMapping/gworkspace/fieldmappings.yml | 4 - .../OSMapping/gworkspace/mappings.json | 16 - .../OSMapping/linux/fieldmappings.yml | 15 - .../resources/OSMapping/linux/mappings.json | 56 - .../OSMapping/m365/fieldmappings.yml | 5 - .../resources/OSMapping/m365/mappings.json | 20 - .../resources/OSMapping/mapper_topics.json | 24 - .../resources/OSMapping/netflow_logtype.json | 8 +- .../OSMapping/network/NetFlowMapping.json | 32 - .../OSMapping/network/fieldmappings.yml | 18 - .../resources/OSMapping/network/mappings.json | 92 - .../OSMapping/okta/fieldmappings.yml | 3 - .../resources/OSMapping/okta/mappings.json | 12 - .../others_application/fieldmappings.yml | 4 - .../others_application/mappings.json | 12 - .../OSMapping/others_apt/fieldmappings.yml | 4 - .../OSMapping/others_apt/mappings.json | 12 - .../OSMapping/others_cloud/fieldmappings.yml | 24 - .../OSMapping/others_cloud/mappings.json | 76 - .../others_compliance/fieldmappings.yml | 7 - .../OSMapping/others_compliance/mappings.json | 24 - .../OSMapping/others_macos/fieldmappings.yml | 7 - .../OSMapping/others_macos/mappings.json | 12 - .../OSMapping/others_proxy/fieldmappings.yml | 7 - .../OSMapping/others_proxy/mappings.json | 32 - .../OSMapping/others_web/fieldmappings.yml | 9 - .../OSMapping/others_web/mappings.json | 32 - .../resources/OSMapping/s3/fieldmappings.yml | 4 - src/main/resources/OSMapping/s3/mappings.json | 56 - .../OSMapping/test_windows/fieldmappings.yml | 12 - .../OSMapping/test_windows/mappings.json | 32 - .../resources/OSMapping/vpcflow_logtype.json | 132 + .../OSMapping/windows/fieldmappings.yml | 65 - .../resources/OSMapping/windows/mappings.json | 196 -- .../mappings/log_type_config_mapping.json | 35 + .../aws_create_load_balancer_layer.yml | 22 + src/main/resources/rules/rule_categories.json | 4 + .../LogTypeServiceTests.java | 151 + .../SecurityAnalyticsRestTestCase.java | 2 +- .../securityanalytics/TestHelpers.java | 4 +- .../action/IndexDetectorRequestTests.java | 2 +- .../action/IndexDetectorResponseTests.java | 2 +- .../alerts/AlertingServiceTests.java | 4 +- .../CorrelationEngineRestApiIT.java | 1 - .../findings/FindingServiceTests.java | 4 +- .../mapper/MapperRestApiIT.java | 73 +- .../mapper/MapperServiceTests.java | 4 +- .../mapper/MapperUtilsTests.java | 5 +- .../resthandler/DetectorRestApiIT.java | 4 +- .../resthandler/OCSFDetectorRestApiIT.java | 2767 +++++++++++++++++ .../resthandler/RuleRestApiIT.java | 36 +- .../{Writable => writable}/LogTypeTests.java | 4 +- src/test/resources/ad_ldap-sample.json | 2 +- src/test/resources/azure-sample.json | 2 +- src/test/resources/cloudtrail-sample.json | 2 +- src/test/resources/dns-sample.json | 72 +- src/test/resources/s3-sample.json | 2 +- 99 files changed, 5041 insertions(+), 2180 deletions(-) create mode 100644 src/main/java/org/opensearch/securityanalytics/logtype/MappingSchema.java delete mode 100644 src/main/java/org/opensearch/securityanalytics/mapper/MapperTopicStore.java create mode 100644 src/main/java/org/opensearch/securityanalytics/model/FieldMappingDoc.java delete mode 100644 src/main/resources/OSMapping/NetFlowMapping.json delete mode 100644 src/main/resources/OSMapping/ad_ldap/mappings.json delete mode 100644 src/main/resources/OSMapping/apache_access/fieldmappings.yml delete mode 100644 src/main/resources/OSMapping/apache_access/mappings.json delete mode 100644 src/main/resources/OSMapping/azure/fieldmappings.yml delete mode 100644 src/main/resources/OSMapping/azure/mappings.json delete mode 100644 src/main/resources/OSMapping/cloudtrail/fieldmappings.yml delete mode 100644 src/main/resources/OSMapping/cloudtrail/mappings.json delete mode 100644 src/main/resources/OSMapping/dns/fieldmappings.yml delete mode 100644 src/main/resources/OSMapping/dns/mappings.json delete mode 100644 src/main/resources/OSMapping/github/fieldmappings.yml delete mode 100644 src/main/resources/OSMapping/github/mappings.json delete mode 100644 src/main/resources/OSMapping/gworkspace/fieldmappings.yml delete mode 100644 src/main/resources/OSMapping/gworkspace/mappings.json delete mode 100644 src/main/resources/OSMapping/linux/fieldmappings.yml delete mode 100644 src/main/resources/OSMapping/linux/mappings.json delete mode 100644 src/main/resources/OSMapping/m365/fieldmappings.yml delete mode 100644 src/main/resources/OSMapping/m365/mappings.json delete mode 100644 src/main/resources/OSMapping/mapper_topics.json delete mode 100644 src/main/resources/OSMapping/network/NetFlowMapping.json delete mode 100644 src/main/resources/OSMapping/network/fieldmappings.yml delete mode 100644 src/main/resources/OSMapping/network/mappings.json delete mode 100644 src/main/resources/OSMapping/okta/fieldmappings.yml delete mode 100644 src/main/resources/OSMapping/okta/mappings.json delete mode 100644 src/main/resources/OSMapping/others_application/fieldmappings.yml delete mode 100644 src/main/resources/OSMapping/others_application/mappings.json delete mode 100644 src/main/resources/OSMapping/others_apt/fieldmappings.yml delete mode 100644 src/main/resources/OSMapping/others_apt/mappings.json delete mode 100644 src/main/resources/OSMapping/others_cloud/fieldmappings.yml delete mode 100644 src/main/resources/OSMapping/others_cloud/mappings.json delete mode 100644 src/main/resources/OSMapping/others_compliance/fieldmappings.yml delete mode 100644 src/main/resources/OSMapping/others_compliance/mappings.json delete mode 100644 src/main/resources/OSMapping/others_macos/fieldmappings.yml delete mode 100644 src/main/resources/OSMapping/others_macos/mappings.json delete mode 100644 src/main/resources/OSMapping/others_proxy/fieldmappings.yml delete mode 100644 src/main/resources/OSMapping/others_proxy/mappings.json delete mode 100644 src/main/resources/OSMapping/others_web/fieldmappings.yml delete mode 100644 src/main/resources/OSMapping/others_web/mappings.json delete mode 100644 src/main/resources/OSMapping/s3/fieldmappings.yml delete mode 100644 src/main/resources/OSMapping/s3/mappings.json delete mode 100644 src/main/resources/OSMapping/test_windows/fieldmappings.yml delete mode 100644 src/main/resources/OSMapping/test_windows/mappings.json create mode 100644 src/main/resources/OSMapping/vpcflow_logtype.json delete mode 100644 src/main/resources/OSMapping/windows/fieldmappings.yml delete mode 100644 src/main/resources/OSMapping/windows/mappings.json create mode 100644 src/main/resources/mappings/log_type_config_mapping.json create mode 100644 src/main/resources/rules/cloudtrail/aws_create_load_balancer_layer.yml create mode 100644 src/test/java/org/opensearch/securityanalytics/LogTypeServiceTests.java create mode 100644 src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java rename src/test/java/org/opensearch/securityanalytics/{Writable => writable}/LogTypeTests.java (98%) diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index a199a2155..9b57436eb 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -10,10 +10,16 @@ import java.util.Map; import java.util.Optional; import java.util.function.Supplier; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.ActionListener; import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionResponse; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.component.LifecycleComponent; @@ -31,7 +37,9 @@ import org.opensearch.index.codec.CodecServiceFactory; import org.opensearch.index.engine.EngineFactory; import org.opensearch.index.mapper.Mapper; +import org.opensearch.index.query.QueryBuilders; import org.opensearch.plugins.ActionPlugin; +import org.opensearch.plugins.ClusterPlugin; import org.opensearch.plugins.EnginePlugin; import org.opensearch.plugins.MapperPlugin; import org.opensearch.plugins.Plugin; @@ -40,11 +48,13 @@ import org.opensearch.rest.RestController; import org.opensearch.rest.RestHandler; import org.opensearch.script.ScriptService; +import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.securityanalytics.action.*; import org.opensearch.securityanalytics.correlation.index.codec.CorrelationCodecService; import org.opensearch.securityanalytics.correlation.index.mapper.CorrelationVectorFieldMapper; import org.opensearch.securityanalytics.correlation.index.query.CorrelationQueryBuilder; import org.opensearch.securityanalytics.indexmanagment.DetectorIndexManagementService; +import org.opensearch.securityanalytics.logtype.BuiltinLogTypeLoader; import org.opensearch.securityanalytics.logtype.LogTypeService; import org.opensearch.securityanalytics.mapper.IndexTemplateManager; import org.opensearch.securityanalytics.mapper.MapperService; @@ -62,7 +72,9 @@ import org.opensearch.threadpool.ThreadPool; import org.opensearch.watcher.ResourceWatcherService; -public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin { +public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin { + + private static final Logger log = LogManager.getLogger(SecurityAnalyticsPlugin.class); public static final String PLUGINS_BASE_URI = "/_plugins/_security_analytics"; public static final String MAPPER_BASE_URI = PLUGINS_BASE_URI + "/mappings"; @@ -91,8 +103,12 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map private IndexTemplateManager indexTemplateManager; + private BuiltinLogTypeLoader builtinLogTypeLoader; + private LogTypeService logTypeService; + private Client client; + @Override public Collection createComponents(Client client, ClusterService clusterService, @@ -105,21 +121,26 @@ public Collection createComponents(Client client, NamedWriteableRegistry namedWriteableRegistry, IndexNameExpressionResolver indexNameExpressionResolver, Supplier repositoriesServiceSupplier) { - logTypeService = new LogTypeService(); + builtinLogTypeLoader = new BuiltinLogTypeLoader(); + logTypeService = new LogTypeService(client, clusterService, xContentRegistry, builtinLogTypeLoader); detectorIndices = new DetectorIndices(client.admin(), clusterService, threadPool); - ruleTopicIndices = new RuleTopicIndices(client, clusterService); + ruleTopicIndices = new RuleTopicIndices(client, clusterService, logTypeService); correlationIndices = new CorrelationIndices(client, clusterService); indexTemplateManager = new IndexTemplateManager(client, clusterService, indexNameExpressionResolver, xContentRegistry); - mapperService = new MapperService(client, clusterService, indexNameExpressionResolver, indexTemplateManager); + mapperService = new MapperService(client, clusterService, indexNameExpressionResolver, indexTemplateManager, logTypeService); ruleIndices = new RuleIndices(logTypeService, client, clusterService, threadPool); correlationRuleIndices = new CorrelationRuleIndices(client, clusterService); + this.client = client; - return List.of(detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, ruleIndices, mapperService, indexTemplateManager); + return List.of( + detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, ruleIndices, + mapperService, indexTemplateManager, builtinLogTypeLoader + ); } @Override public Collection> getGuiceServiceClasses() { - return Collections.singletonList(DetectorIndexManagementService.class); + return List.of(DetectorIndexManagementService.class, BuiltinLogTypeLoader.class); } @Override @@ -180,7 +201,7 @@ public Optional getEngineFactory(IndexSettings indexSettings) { @Override public Optional getCustomCodecServiceFactory(IndexSettings indexSettings) { if (indexSettings.getValue(SecurityAnalyticsSettings.IS_CORRELATION_INDEX_SETTING)) { - return Optional.of(CorrelationCodecService::new); + return Optional.of(config -> new CorrelationCodecService(config, indexSettings)); } return Optional.empty(); } @@ -208,7 +229,8 @@ public List> getSettings() { SecurityAnalyticsSettings.FINDING_HISTORY_ROLLOVER_PERIOD, SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD, SecurityAnalyticsSettings.IS_CORRELATION_INDEX_SETTING, - SecurityAnalyticsSettings.CORRELATION_TIME_WINDOW + SecurityAnalyticsSettings.CORRELATION_TIME_WINDOW, + SecurityAnalyticsSettings.DEFAULT_MAPPING_SCHEMA ); } @@ -239,4 +261,38 @@ public List> getSettings() { new ActionPlugin.ActionHandler<>(SearchCorrelationRuleAction.INSTANCE, TransportSearchCorrelationRuleAction.class) ); } + + @Override + public void onNodeStarted(DiscoveryNode localNode) { +// Trigger initialization of log types + logTypeService.ensureConfigIndexIsInitialized(new ActionListener<>() { + @Override + public void onResponse(Void unused) { + log.info("LogType config index successfully created and builtin log types loaded"); + } + + @Override + public void onFailure(Exception e) { + log.warn("Failed to initialize LogType config index and builtin log types"); + } + }); + // Trigger initialization of prepackaged rules by calling SearchRule API + SearchRequest searchRequest = new SearchRequest(Rule.PRE_PACKAGED_RULES_INDEX); + searchRequest.source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).size(0)); + client.execute( + SearchRuleAction.INSTANCE, + new SearchRuleRequest(true, searchRequest), + new ActionListener<>() { + @Override + public void onResponse(SearchResponse searchResponse) { + log.info("Successfully initialized prepackaged rules"); + } + + @Override + public void onFailure(Exception e) { + log.warn("Failed initializing prepackaged rules", e); + } + } + ); + } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/action/GetAlertsRequest.java b/src/main/java/org/opensearch/securityanalytics/action/GetAlertsRequest.java index bf48aa197..5ed416deb 100644 --- a/src/main/java/org/opensearch/securityanalytics/action/GetAlertsRequest.java +++ b/src/main/java/org/opensearch/securityanalytics/action/GetAlertsRequest.java @@ -19,7 +19,7 @@ public class GetAlertsRequest extends ActionRequest { private String detectorId; - private Detector.DetectorType detectorType; + private String logType; private Table table; private String severityLevel; private String alertState; @@ -28,14 +28,14 @@ public class GetAlertsRequest extends ActionRequest { public GetAlertsRequest( String detectorId, - Detector.DetectorType detectorType, + String logType, Table table, String severityLevel, String alertState ) { super(); this.detectorId = detectorId; - this.detectorType = detectorType; + this.logType = logType; this.table = table; this.severityLevel = severityLevel; this.alertState = alertState; @@ -43,7 +43,7 @@ public GetAlertsRequest( public GetAlertsRequest(StreamInput sin) throws IOException { this( sin.readOptionalString(), - sin.readBoolean() ? sin.readEnum(Detector.DetectorType.class) : null, + sin.readOptionalString(), Table.readFrom(sin), sin.readString(), sin.readString() @@ -53,7 +53,7 @@ public GetAlertsRequest(StreamInput sin) throws IOException { @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; - if ((detectorId == null || detectorId.length() == 0) && detectorType == null) { + if ((detectorId == null || detectorId.length() == 0) && logType == null) { validationException = addValidationError(String.format(Locale.getDefault(), "At least one of detector type or detector id needs to be passed", DETECTOR_ID), validationException); @@ -64,10 +64,7 @@ public ActionRequestValidationException validate() { @Override public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(detectorId); - if (detectorType != null) { - out.writeBoolean(true); - out.writeEnum(detectorType); - } else out.writeBoolean(false); + out.writeOptionalString(logType); table.writeTo(out); out.writeString(severityLevel); out.writeString(alertState); @@ -89,7 +86,7 @@ public String getAlertState() { return alertState; } - public Detector.DetectorType getDetectorType() { - return detectorType; + public String getLogType() { + return logType; } } diff --git a/src/main/java/org/opensearch/securityanalytics/action/GetFindingsRequest.java b/src/main/java/org/opensearch/securityanalytics/action/GetFindingsRequest.java index a941b89fe..cc4c5d7eb 100644 --- a/src/main/java/org/opensearch/securityanalytics/action/GetFindingsRequest.java +++ b/src/main/java/org/opensearch/securityanalytics/action/GetFindingsRequest.java @@ -18,7 +18,7 @@ public class GetFindingsRequest extends ActionRequest { - private Detector.DetectorType detectorType; + private String logType; private String detectorId; private Table table; @@ -31,21 +31,21 @@ public GetFindingsRequest(String detectorId) { public GetFindingsRequest(StreamInput sin) throws IOException { this( sin.readOptionalString(), - sin.readBoolean() ? sin.readEnum(Detector.DetectorType.class) : null, + sin.readOptionalString(), Table.readFrom(sin) ); } - public GetFindingsRequest(String detectorId, Detector.DetectorType detectorType, Table table) { + public GetFindingsRequest(String detectorId, String logType, Table table) { this.detectorId = detectorId; - this.detectorType = detectorType; + this.logType = logType; this.table = table; } @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; - if ((detectorId == null || detectorId.length() == 0) && detectorType == null) { + if ((detectorId == null || detectorId.length() == 0) && logType == null) { validationException = addValidationError(String.format(Locale.getDefault(), "At least one of detector type or detector id needs to be passed", DETECTOR_ID), validationException); @@ -56,12 +56,7 @@ public ActionRequestValidationException validate() { @Override public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(detectorId); - if (detectorType != null) { - out.writeBoolean(true); - out.writeEnum(detectorType); - } else { - out.writeBoolean(false); - } + out.writeOptionalString(logType); table.writeTo(out); } @@ -69,8 +64,8 @@ public String getDetectorId() { return detectorId; } - public Detector.DetectorType getDetectorType() { - return detectorType; + public String getLogType() { + return logType; } public Table getTable() { diff --git a/src/main/java/org/opensearch/securityanalytics/action/IndexRuleRequest.java b/src/main/java/org/opensearch/securityanalytics/action/IndexRuleRequest.java index 0702b7ac2..309789fc3 100644 --- a/src/main/java/org/opensearch/securityanalytics/action/IndexRuleRequest.java +++ b/src/main/java/org/opensearch/securityanalytics/action/IndexRuleRequest.java @@ -85,14 +85,6 @@ public ActionRequestValidationException validate() { if (logType == null || logType.length() == 0) { validationException = addValidationError("rule categoty is missing", validationException); - } else { - Optional found = - Arrays.stream(Detector.DetectorType.values()) - .filter(e -> e.getDetectorType().equals(logType)) - .findFirst(); - if (found.isPresent() == false) { - validationException = addValidationError("Invalid rule category", validationException); - } } return validationException; } diff --git a/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java b/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java index 19078f4f1..705f127a5 100644 --- a/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java +++ b/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java @@ -174,7 +174,7 @@ void setIndicesAdminClient(Client client) { public void getAlerts( List detectors, - Detector.DetectorType detectorType, + String logType, Table table, String severityLevel, String alertState, @@ -200,7 +200,7 @@ public void getAlerts( AlertsService.this.getAlertsByMonitorIds( monitorToDetectorMapping, allMonitorIds, - DetectorMonitorConfig.getAllAlertsIndicesPattern(detectorType.getDetectorType()), + DetectorMonitorConfig.getAllAlertsIndicesPattern(logType), table, severityLevel, alertState, @@ -254,8 +254,8 @@ public void getAlerts(List alertIds, null, DetectorMonitorConfig.getAllAlertsIndicesPattern(detector.getDetectorType()), null, - alertIds, - null); + null, + alertIds); AlertingPluginInterface.INSTANCE.getAlerts( (NodeClient) client, request, actionListener); diff --git a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java index fc79c6454..459f523b7 100644 --- a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java @@ -6,129 +6,54 @@ import java.util.List; import java.util.stream.Collectors; +import org.opensearch.common.inject.Inject; +import org.opensearch.securityanalytics.logtype.LogTypeService; import org.opensearch.securityanalytics.model.Detector; import java.util.Arrays; import java.util.HashMap; import java.util.Locale; import java.util.Map; +import org.opensearch.securityanalytics.model.LogType; public class DetectorMonitorConfig { - public static final String OPENSEARCH_DEFAULT_RULE_INDEX = ".opensearch-sap-detectors-queries-default"; - public static final String OPENSEARCH_DEFAULT_ALERT_INDEX = ".opensearch-sap-alerts-default"; - public static final String OPENSEARCH_DEFAULT_ALL_ALERT_INDICES_PATTERN = ".opensearch-sap-alerts-default*"; - public static final String OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX = ".opensearch-sap-alerts-history-default"; - public static final String OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX_PATTERN = "<.opensearch-sap-alerts-history-default-{now/d}-1>"; - public static final String OPENSEARCH_DEFAULT_FINDINGS_INDEX = ".opensearch-sap-findings-default"; - public static final String OPENSEARCH_DEFAULT_ALL_FINDINGS_INDICES_PATTERN = ".opensearch-sap-findings-default*"; - public static final String OPENSEARCH_DEFAULT_FINDINGS_INDEX_PATTERN = "<.opensearch-sap-findings-default-{now/d}-1>"; - public static final String OPENSEARCH_SAP_RULE_INDEX_TEMPLATE = ".opensearch-sap-detectors-queries-index-template"; - private static Map detectorTypeToIndicesMapping; - - static { - detectorTypeToIndicesMapping = new HashMap<>(); - Arrays.stream(Detector.DetectorType.values()).forEach( - detectorType -> { - String ruleIndex = String.format( - Locale.getDefault(), ".opensearch-sap-%s-detectors-queries", detectorType.getDetectorType()); - String alertsIndex = String.format( - Locale.getDefault(), ".opensearch-sap-%s-alerts", detectorType.getDetectorType()); - String alertsHistoryIndex = String.format( - Locale.getDefault(), ".opensearch-sap-%s-alerts-history", detectorType.getDetectorType()); - String alertsHistoryIndexPattern = String.format( - Locale.getDefault(), "<.opensearch-sap-%s-alerts-history-{now/d}-1>", detectorType.getDetectorType()); - String allAlertsIndicesPattern = String.format( - Locale.getDefault(), ".opensearch-sap-%s-alerts*", detectorType.getDetectorType()); - String findingsIndex = String.format( - Locale.getDefault(), ".opensearch-sap-%s-findings", detectorType.getDetectorType()); - String allFindingsIndicesPattern = String.format( - Locale.getDefault(), ".opensearch-sap-%s-findings*", detectorType.getDetectorType()); - String findingsIndexPattern = String.format( - Locale.getDefault(), "<.opensearch-sap-%s-findings-{now/d}-1>", detectorType.getDetectorType()); - - MonitorConfig monitor = new MonitorConfig( - alertsIndex, alertsHistoryIndex, alertsHistoryIndexPattern, allAlertsIndicesPattern, - findingsIndex, findingsIndexPattern, allFindingsIndicesPattern, - ruleIndex - ); - detectorTypeToIndicesMapping.put(detectorType.getDetectorType(), monitor); - }); - } - - public static String getRuleIndex(String detectorType) { - return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ? - detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getRuleIndex() : - OPENSEARCH_DEFAULT_RULE_INDEX; - } - - public static String getAlertsIndex(String detectorType) { - return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ? - detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getAlertsIndex() : - OPENSEARCH_DEFAULT_ALERT_INDEX; + public static String getRuleIndex(String logType) { + return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries", logType); } - public static String getAlertsHistoryIndex(String detectorType) { - return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ? - detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getAlertsHistoryIndex() : - OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX; + public static String getAlertsIndex(String logType) { + return String.format(Locale.getDefault(), ".opensearch-sap-%s-alerts", logType); } - public static String getAlertsHistoryIndexPattern(String detectorType) { - return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ? - detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getAlertsHistoryIndexPattern() : - OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX_PATTERN; + public static String getAlertsHistoryIndex(String logType) { + return String.format(Locale.getDefault(), ".opensearch-sap-%s-alerts-history", logType); } - public static String getAllAlertsIndicesPattern(String detectorType) { - return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ? - detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getAllAlertsIndicesPattern() : - OPENSEARCH_DEFAULT_ALL_ALERT_INDICES_PATTERN; + public static String getAlertsHistoryIndexPattern(String logType) { + return String.format(Locale.getDefault(), "<.opensearch-sap-%s-alerts-history-{now/d}-1>", logType); } - public static List getAllAlertsIndicesPatternForAllTypes() { - return detectorTypeToIndicesMapping.entrySet() - .stream() - .map(e -> e.getValue().getAllAlertsIndicesPattern()) - .collect(Collectors.toList()); + public static String getAllAlertsIndicesPattern(String logType) { + return String.format(Locale.getDefault(), ".opensearch-sap-%s-alerts*", logType); } - public static String getFindingsIndex(String detectorType) { - return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ? - detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getFindingsIndex() : - OPENSEARCH_DEFAULT_FINDINGS_INDEX; + public static String getFindingsIndexPattern(String logType) { + return String.format(Locale.getDefault(), "<.opensearch-sap-%s-findings-{now/d}-1>", logType); } - public static String getAllFindingsIndicesPattern(String detectorType) { - return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ? - detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getAllFindingsIndicesPattern() : - OPENSEARCH_DEFAULT_ALL_FINDINGS_INDICES_PATTERN; + public static String getFindingsIndex(String logType) { + return String.format(Locale.getDefault(), ".opensearch-sap-%s-findings", logType); } - public static List getAllFindingsIndicesPatternForAllTypes() { - return detectorTypeToIndicesMapping.entrySet() - .stream() - .map(e -> e.getValue().getAllFindingsIndicesPattern()) - .collect(Collectors.toList()); + public static String getAllFindingsIndicesPattern(String logType) { + return String.format(Locale.getDefault(), ".opensearch-sap-%s-findings*", logType); } - public static List getAllRuleIndices() { - return detectorTypeToIndicesMapping.entrySet() - .stream() - .map(e -> e.getValue().getRuleIndex()) - .collect(Collectors.toList()); - } - - public static String getFindingsIndexPattern(String detectorType) { - return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ? - detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getFindingsIndexPattern() : - OPENSEARCH_DEFAULT_FINDINGS_INDEX_PATTERN; - } - - public static Map> getRuleIndexMappingsByType(String detectorType) { + public static Map> getRuleIndexMappingsByType() { HashMap properties = new HashMap<>(); properties.put("analyzer", "rule_analyzer"); HashMap> fieldMappingProperties = new HashMap<>(); @@ -199,4 +124,3 @@ public String getRuleIndex() { } } - diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/CorrelationCodecService.java b/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/CorrelationCodecService.java index 9b29dd106..77ca1afc2 100644 --- a/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/CorrelationCodecService.java +++ b/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/CorrelationCodecService.java @@ -5,6 +5,7 @@ package org.opensearch.securityanalytics.correlation.index.codec; import org.apache.lucene.codecs.Codec; +import org.opensearch.index.IndexSettings; import org.opensearch.index.codec.CodecService; import org.opensearch.index.codec.CodecServiceConfig; import org.opensearch.index.mapper.MapperService; @@ -13,8 +14,8 @@ public class CorrelationCodecService extends CodecService { private final MapperService mapperService; - public CorrelationCodecService(CodecServiceConfig codecServiceConfig) { - super(codecServiceConfig.getMapperService(), codecServiceConfig.getLogger()); + public CorrelationCodecService(CodecServiceConfig codecServiceConfig, IndexSettings indexSettings) { + super(codecServiceConfig.getMapperService(), indexSettings, codecServiceConfig.getLogger()); mapperService = codecServiceConfig.getMapperService(); } diff --git a/src/main/java/org/opensearch/securityanalytics/findings/FindingsService.java b/src/main/java/org/opensearch/securityanalytics/findings/FindingsService.java index dc5880ec6..5047c0268 100644 --- a/src/main/java/org/opensearch/securityanalytics/findings/FindingsService.java +++ b/src/main/java/org/opensearch/securityanalytics/findings/FindingsService.java @@ -162,7 +162,7 @@ void setIndicesAdminClient(Client client) { public void getFindings( List detectors, - Detector.DetectorType detectorType, + String logType, Table table, ActionListener listener ) { @@ -186,7 +186,7 @@ public void getFindings( FindingsService.this.getFindingsByMonitorIds( monitorToDetectorMapping, allMonitorIds, - DetectorMonitorConfig.getAllFindingsIndicesPattern(detectorType.getDetectorType()), + DetectorMonitorConfig.getAllFindingsIndicesPattern(logType), table, new ActionListener<>() { @Override diff --git a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java index 8d726d18f..444df5042 100644 --- a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java @@ -4,16 +4,15 @@ */ package org.opensearch.securityanalytics.indexmanagment; -import com.carrotsearch.hppc.cursors.ObjectCursor; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.ActionListener; @@ -35,11 +34,10 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; -import org.opensearch.securityanalytics.model.Detector; +import org.opensearch.securityanalytics.logtype.LogTypeService; import org.opensearch.threadpool.Scheduler; import org.opensearch.threadpool.ThreadPool; - import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_ENABLED; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_INDEX_MAX_AGE; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS; @@ -58,6 +56,7 @@ public class DetectorIndexManagementService extends AbstractLifecycleComponent i private final Client client; private final ThreadPool threadPool; private final ClusterService clusterService; + private final LogTypeService logTypeService; private Settings settings; private volatile Boolean alertHistoryEnabled; @@ -84,11 +83,18 @@ public class DetectorIndexManagementService extends AbstractLifecycleComponent i List findingHistoryIndices = new ArrayList<>(); @Inject - public DetectorIndexManagementService(Settings settings, Client client, ThreadPool threadPool, ClusterService clusterService) { + public DetectorIndexManagementService( + Settings settings, + Client client, + ThreadPool threadPool, + ClusterService clusterService, + LogTypeService logTypeService + ) { this.settings = settings; this.client = client; this.threadPool = threadPool; this.clusterService = clusterService; + this.logTypeService = logTypeService; clusterService.addListener(this); @@ -131,16 +137,18 @@ public DetectorIndexManagementService(Settings settings, Client client, ThreadPo clusterService.getClusterSettings().addSettingsUpdateConsumer(FINDING_HISTORY_RETENTION_PERIOD, this::setFindingHistoryRetentionPeriod); initFromClusterSettings(); - - initAllIndexLists(); } - private void initAllIndexLists() { - Arrays.stream(Detector.DetectorType.values()).forEach( - detectorType -> { + private void populateAllIndexLists(List logTypes) { + + alertHistoryIndices.clear(); + findingHistoryIndices.clear(); - String alertsHistoryIndex = DetectorMonitorConfig.getAlertsHistoryIndex(detectorType.getDetectorType()); - String alertsHistoryIndexPattern = DetectorMonitorConfig.getAlertsHistoryIndexPattern(detectorType.getDetectorType()); + logTypes.forEach( + logType -> { + + String alertsHistoryIndex = DetectorMonitorConfig.getAlertsHistoryIndex(logType); + String alertsHistoryIndexPattern = DetectorMonitorConfig.getAlertsHistoryIndexPattern(logType); alertHistoryIndices.add(new HistoryIndexInfo( alertsHistoryIndex, @@ -148,11 +156,11 @@ private void initAllIndexLists() { alertMapping(), alertHistoryMaxDocs, alertHistoryMaxAge, - false + clusterService.state().metadata().hasAlias(alertsHistoryIndex) )); - String findingsIndex = DetectorMonitorConfig.getFindingsIndex(detectorType.getDetectorType()); - String findingsIndexPattern = DetectorMonitorConfig.getFindingsIndexPattern(detectorType.getDetectorType()); + String findingsIndex = DetectorMonitorConfig.getFindingsIndex(logType); + String findingsIndexPattern = DetectorMonitorConfig.getFindingsIndexPattern(logType); findingHistoryIndices.add(new HistoryIndexInfo( findingsIndex, @@ -160,7 +168,7 @@ private void initAllIndexLists() { findingMapping(), findingHistoryMaxDocs, findingHistoryMaxAge, - false + clusterService.state().metadata().hasAlias(findingsIndex) )); }); } @@ -202,8 +210,10 @@ public void clusterChanged(ClusterChangedEvent event) { private void onMaster() { try { // try to rollover immediately as we might be restarting the cluster - rolloverAlertHistoryIndices(); - rolloverFindingHistoryIndices(); + threadPool.schedule(() -> { + rolloverAndDeleteAlertHistoryIndices(); + rolloverAndDeleteFindingHistoryIndices(); + }, TimeValue.timeValueSeconds(1), executorName()); // schedule the next rollover for approx MAX_AGE later scheduledAlertsRollover = threadPool .scheduleWithFixedDelay(() -> rolloverAndDeleteAlertHistoryIndices(), alertHistoryRolloverPeriod, executorName()); @@ -233,7 +243,6 @@ private String executorName() { } private void deleteOldIndices(String tag, String... indices) { - logger.error("info deleteOldIndices"); ClusterStateRequest clusterStateRequest = new ClusterStateRequest() .clear() .indices(indices) @@ -360,16 +369,47 @@ public void onFailure(Exception e) { } private void rolloverAndDeleteAlertHistoryIndices() { - if (alertHistoryEnabled) rolloverAlertHistoryIndices(); - deleteOldIndices("Alert", DetectorMonitorConfig.getAllAlertsIndicesPatternForAllTypes().toArray(new String[0])); + logTypeService.getAllLogTypes(ActionListener.wrap(logTypes -> { + if (logTypes == null || logTypes.isEmpty()) { + return; + } + // We have to do this every time to account for newly added log types + populateAllIndexLists(logTypes); + + if (alertHistoryEnabled) rolloverAlertHistoryIndices(); + deleteOldIndices("Alert", getAllAlertsIndicesPatternForAllTypes(logTypes).toArray(new String[0])); + }, e -> {})); } private void rolloverAndDeleteFindingHistoryIndices() { - if (findingHistoryEnabled) rolloverFindingHistoryIndices(); - deleteOldIndices("Finding", DetectorMonitorConfig.getAllFindingsIndicesPatternForAllTypes().toArray(new String[0])); + logTypeService.getAllLogTypes(ActionListener.wrap(logTypes -> { + if (logTypes == null || logTypes.isEmpty()) { + return; + } + // We have to do this every time to account for newly added log types + populateAllIndexLists(logTypes); + + if (findingHistoryEnabled) rolloverFindingHistoryIndices(); + deleteOldIndices("Finding", getAllFindingsIndicesPatternForAllTypes(logTypes).toArray(new String[0])); + }, e -> {})); + } + + private List getAllAlertsIndicesPatternForAllTypes(List logTypes) { + return logTypes + .stream() + .map(logType -> DetectorMonitorConfig.getAllAlertsIndicesPattern(logType)) + .collect(Collectors.toList()); } - private void rolloverIndex( + private List getAllFindingsIndicesPatternForAllTypes(List logTypes) { + return logTypes + .stream() + .map(logType -> DetectorMonitorConfig.getAllFindingsIndicesPattern(logType)) + .collect(Collectors.toList()); + } + + + private void rolloverIndex( Boolean initialized, String index, String pattern, diff --git a/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java b/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java index 3783143cc..604744486 100644 --- a/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java +++ b/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java @@ -19,13 +19,14 @@ import java.util.stream.Stream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.common.component.AbstractLifecycleComponent; import org.opensearch.common.settings.SettingsException; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.securityanalytics.model.LogType; import org.opensearch.securityanalytics.util.FileUtils; -public class BuiltinLogTypeLoader { +public class BuiltinLogTypeLoader extends AbstractLifecycleComponent { private static final Logger logger = LogManager.getLogger(BuiltinLogTypeLoader.class); @@ -33,30 +34,25 @@ public class BuiltinLogTypeLoader { private static final String LOG_TYPE_FILE_SUFFIX = "_logtype.json"; - private static List logTypes; - private static Map logTypeMap; + private List logTypes; + private Map logTypeMap; - - static { - ensureLogTypesLoaded(); - } - - public static List getAllLogTypes() { + public List getAllLogTypes() { ensureLogTypesLoaded(); return logTypes; } - public static LogType getLogTypeByName(String logTypeName) { + public LogType getLogTypeByName(String logTypeName) { ensureLogTypesLoaded(); return logTypeMap.get(logTypeName); } - public static boolean logTypeExists(String logTypeName) { + public boolean logTypeExists(String logTypeName) { ensureLogTypesLoaded(); return logTypeMap.containsKey(logTypeName); } - private static void ensureLogTypesLoaded() { + public void ensureLogTypesLoaded() { try { if (logTypes != null) { return; @@ -69,7 +65,7 @@ private static void ensureLogTypesLoaded() { } } - private static List loadBuiltinLogTypes() throws URISyntaxException, IOException { + private List loadBuiltinLogTypes() throws URISyntaxException, IOException { List logTypes = new ArrayList<>(); final String url = Objects.requireNonNull(BuiltinLogTypeLoader.class.getClassLoader().getResource(BASE_PATH)).toURI().toString(); @@ -106,4 +102,19 @@ private static List loadBuiltinLogTypes() throws URISyntaxException, IO return logTypes; } + + @Override + protected void doStart() { + ensureLogTypesLoaded(); + } + + @Override + protected void doStop() { + + } + + @Override + protected void doClose() throws IOException { + + } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java index 6523ba92b..8fb978144 100644 --- a/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java +++ b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java @@ -4,15 +4,59 @@ */ package org.opensearch.securityanalytics.logtype; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.ExceptionsHelper; +import org.opensearch.ResourceAlreadyExistsException; +import org.opensearch.action.ActionListener; +import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.support.PlainActionFuture; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.Client; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.MappingMetadata; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.io.Streams; +import org.opensearch.common.util.set.Sets; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.SearchHit; +import org.opensearch.search.aggregations.bucket.terms.Terms; +import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.securityanalytics.model.FieldMappingDoc; import org.opensearch.securityanalytics.model.LogType; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import static org.opensearch.action.support.ActiveShardCount.ALL; +import static org.opensearch.securityanalytics.model.FieldMappingDoc.LOG_TYPES; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.DEFAULT_MAPPING_SCHEMA; + /** * @@ -21,39 +65,548 @@ public class LogTypeService { private static final Logger logger = LogManager.getLogger(LogTypeService.class); + public static final String LOG_TYPE_INDEX = ".opensearch-sap-log-types-config"; + + public static final String LOG_TYPE_INDEX_MAPPING_FILE = "mappings/log_type_config_mapping.json"; + + public static final String LOG_TYPE_MAPPING_VERSION_META_FIELD = "schema_version"; + + public static final int LOG_TYPE_MAPPING_VERSION = 1; // must match version in log_type_config_mapping.json + + public static final int MAX_LOG_TYPE_COUNT = 100; + + private static volatile boolean isConfigIndexInitialized; + + private final Client client; + + private final ClusterService clusterService; + + private final NamedXContentRegistry xContentRegistry; + private BuiltinLogTypeLoader builtinLogTypeLoader; - public LogTypeService() { - this.builtinLogTypeLoader = new BuiltinLogTypeLoader(); + private String defaultSchemaField; + + @Inject + public LogTypeService(Client client, ClusterService clusterService, NamedXContentRegistry xContentRegistry, BuiltinLogTypeLoader builtinLogTypeLoader) { + this.client = client; + this.clusterService = clusterService; + this.xContentRegistry = xContentRegistry; + this.builtinLogTypeLoader = builtinLogTypeLoader; + + this.defaultSchemaField = DEFAULT_MAPPING_SCHEMA.get(clusterService.getSettings()); + clusterService.getClusterSettings().addSettingsUpdateConsumer( + DEFAULT_MAPPING_SCHEMA, + newDefaultSchema -> this.defaultSchemaField = newDefaultSchema + ); + } + + public void getAllLogTypes(ActionListener> listener) { + ensureConfigIndexIsInitialized(ActionListener.wrap(e -> { + + SearchRequest searchRequest = new SearchRequest(LOG_TYPE_INDEX); + searchRequest.source(new SearchSourceBuilder().aggregation( + new TermsAggregationBuilder("logTypes") + .field(LOG_TYPES) + .size(MAX_LOG_TYPE_COUNT) + )); + searchRequest.preference("_primary"); + client.search( + searchRequest, + ActionListener.delegateFailure( + listener, + (delegatedListener, searchResponse) -> { + List logTypes = new ArrayList<>(); + Terms termsAgg = searchResponse.getAggregations().get("logTypes"); + for(Terms.Bucket bucket : termsAgg.getBuckets()) { + logTypes.add(bucket.getKeyAsString()); + } + delegatedListener.onResponse(logTypes); + } + ) + ); + }, listener::onFailure)); + } + + private void doIndexFieldMappings(List fieldMappingDocs, ActionListener listener) { + if (fieldMappingDocs.isEmpty()) { + listener.onResponse(null); + } + getAllFieldMappings(ActionListener.wrap(existingFieldMappings -> { + + List mergedFieldMappings = mergeFieldMappings(existingFieldMappings, fieldMappingDocs); + + BulkRequest bulkRequest = new BulkRequest(); + mergedFieldMappings.stream() + .filter(e -> e.isDirty()) + .forEach(fieldMappingDoc -> { + + IndexRequest indexRequest = new IndexRequest(LOG_TYPE_INDEX); + try { + indexRequest.id(fieldMappingDoc.getId() == null ? generateFieldMappingDocId(fieldMappingDoc) : fieldMappingDoc.getId()); + indexRequest.source(fieldMappingDoc.toXContent(XContentFactory.jsonBuilder(), null)); + indexRequest.opType(DocWriteRequest.OpType.INDEX); + bulkRequest.add(indexRequest); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + } catch (IOException ex) { + logger.error("Failed converting FieldMappingDoc to XContent!", ex); + } + }); + // Index all fieldMapping docs + logger.info("Indexing [" + bulkRequest.numberOfActions() + "] fieldMappingDocs"); + client.bulk( + bulkRequest, + ActionListener.delegateFailure(listener, (l, r) -> { + if (r.hasFailures()) { + logger.error("FieldMappingDoc Bulk Index had failures:\n ", r.buildFailureMessage()); + listener.onFailure(new IllegalStateException(r.buildFailureMessage())); + } else { + logger.info("Loaded [" + r.getItems().length + "] field mapping docs successfully!"); + listener.onResponse(null); + } + }) + ); + + }, listener::onFailure)); + } + + private String generateFieldMappingDocId(FieldMappingDoc fieldMappingDoc) { + String generatedId = fieldMappingDoc.getRawField() + "|"; + if (fieldMappingDoc.getSchemaFields().containsKey(defaultSchemaField)) { + generatedId = generatedId + fieldMappingDoc.getSchemaFields().get(defaultSchemaField); + } + return generatedId; + } + + public void indexFieldMappings(List fieldMappingDocs, ActionListener listener) { + ensureConfigIndexIsInitialized(ActionListener.wrap(e -> { + doIndexFieldMappings(fieldMappingDocs, listener); + }, listener::onFailure)); + } + + private List mergeFieldMappings(List existingFieldMappings, List fieldMappingDocs) { + // Insert new fieldMappings + List newFieldMappings = new ArrayList<>(); + fieldMappingDocs.forEach( newFieldMapping -> { + Optional foundFieldMappingDoc = existingFieldMappings + .stream() + .filter( + e -> e.getRawField().equals(newFieldMapping.getRawField()) && ( + e.get(defaultSchemaField) != null && newFieldMapping.get(defaultSchemaField) != null && + e.get(defaultSchemaField).equals(newFieldMapping.get(defaultSchemaField)) + ) || ( + e.get(defaultSchemaField) == null && newFieldMapping.get(defaultSchemaField) == null + ) + ) + .findFirst(); + if (foundFieldMappingDoc.isEmpty()) { + newFieldMapping.setIsDirty(true); + newFieldMappings.add(newFieldMapping); + } else { + // Merge new with existing by merging schema field mappings and log type arrays + foundFieldMappingDoc.get().getSchemaFields().putAll(newFieldMapping.getSchemaFields()); + foundFieldMappingDoc.get().getLogTypes().addAll(newFieldMapping.getLogTypes()); + foundFieldMappingDoc.get().setIsDirty(true); + } + }); + existingFieldMappings.addAll(newFieldMappings); + return existingFieldMappings; + } + + public void getAllFieldMappings(ActionListener> listener) { + SearchRequest searchRequest = new SearchRequest(LOG_TYPE_INDEX); + searchRequest.source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).size(10000)); + client.search( + searchRequest, + ActionListener.delegateFailure( + listener, + (delegatedListener, searchResponse) -> { + List fieldMappingDocs = new ArrayList<>(); + for(SearchHit hit : searchResponse.getHits().getHits()) { + try { + fieldMappingDocs.add(FieldMappingDoc.parse(hit, xContentRegistry)); + } catch (IOException e) { + logger.error("Failed parsing FieldMapping document", e); + delegatedListener.onFailure(e); + return; + } + } + delegatedListener.onResponse(fieldMappingDocs); + } + ) + ); + } + + public void getFieldMappingsByLogType(String logType, ActionListener> listener) { + ensureConfigIndexIsInitialized(ActionListener.wrap(() -> + getFieldMappingsByLogTypes(List.of(logType), listener) + )); + } + + public void getFieldMappingsByLogTypes(List logTypes, ActionListener> listener) { + SearchRequest searchRequest = new SearchRequest(LOG_TYPE_INDEX); + searchRequest.source(new SearchSourceBuilder().query( + QueryBuilders.termsQuery(LOG_TYPES, logTypes.toArray(new String[0]))) + .size(10000) + ); + client.search( + searchRequest, + ActionListener.delegateFailure( + listener, + (delegatedListener, searchResponse) -> { + List fieldMappingDocs = new ArrayList<>(); + for(SearchHit hit : searchResponse.getHits().getHits()) { + try { + fieldMappingDocs.add(FieldMappingDoc.parse(hit, xContentRegistry)); + } catch (IOException e) { + logger.error("Failed parsing FieldMapping document", e); + delegatedListener.onFailure(e); + return; + } + } + delegatedListener.onResponse(fieldMappingDocs); + } + ) + ); + } + /** + * if isConfigIndexInitialized is false does following: + * 1. Creates log type config index with proper mappings/settings + * 2. Loads builtin log types into index + * 3. sets isConfigIndexInitialized to true + * */ + public void ensureConfigIndexIsInitialized(ActionListener listener) { + + ClusterState state = clusterService.state(); + + if (state.routingTable().hasIndex(LOG_TYPE_INDEX) == false) { + isConfigIndexInitialized = false; + CreateIndexRequest createIndexRequest = new CreateIndexRequest(); + createIndexRequest.settings(logTypeIndexSettings()); + createIndexRequest.index(LOG_TYPE_INDEX); + createIndexRequest.mapping(logTypeIndexMapping()); + createIndexRequest.cause("auto(sap-logtype api)"); + client.admin().indices().create(createIndexRequest, new ActionListener<>() { + @Override + public void onResponse(CreateIndexResponse result) { + loadBuiltinLogTypes(ActionListener.delegateFailure( + listener, + (delegatedListener, unused) -> { + isConfigIndexInitialized = true; + delegatedListener.onResponse(null); + }) + ); + } + + @Override + public void onFailure(Exception e) { + isConfigIndexInitialized = false; + if (ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException) { + loadBuiltinLogTypes(ActionListener.delegateFailure( + listener, + (delegatedListener, unused) -> { + isConfigIndexInitialized = true; + delegatedListener.onResponse(null); + }) + ); + } else { + logger.error("Failed creating LOG_TYPE_INDEX", e); + listener.onFailure(e); + } + } + }); + } else { + IndexMetadata metadata = state.getMetadata().index(LOG_TYPE_INDEX); + if (getConfigIndexMappingVersion(metadata) < LOG_TYPE_MAPPING_VERSION) { + // The index already exists but doesn't have our mapping + client.admin() + .indices() + .preparePutMapping(LOG_TYPE_INDEX) + .setSource(logTypeIndexMapping(), XContentType.JSON) + .execute(ActionListener.delegateFailure(listener, (l, r) -> { + loadBuiltinLogTypes(ActionListener.delegateFailure( + listener, + (delegatedListener, unused) -> { + isConfigIndexInitialized = true; + delegatedListener.onResponse(null); + }) + ); + })); + } else { + if (isConfigIndexInitialized) { + listener.onResponse(null); + return; + } + loadBuiltinLogTypes(ActionListener.delegateFailure( + listener, + (delegatedListener, unused) -> { + isConfigIndexInitialized = true; + delegatedListener.onResponse(null); + }) + ); + } + } + } + + public void loadBuiltinLogTypes(ActionListener listener) { + logger.info("Loading builtin types!"); + List logTypes = builtinLogTypeLoader.getAllLogTypes(); + if (logTypes == null || logTypes.size() == 0) { + logger.error("Failed loading builtin log types from disk!"); + listener.onFailure(SecurityAnalyticsException.wrap( + new IllegalStateException("Failed loading builtin log types from disk!")) + ); + return; + } + List fieldMappingDocs = createFieldMappingDocs(logTypes); + logger.info("Indexing [" + fieldMappingDocs.size() + "] fieldMappingDocs from logTypes: " + logTypes.size()); + doIndexFieldMappings(fieldMappingDocs, listener); } + /** + * Loops through all builtin LogTypes and creates collection of FieldMappingDocs + * */ + private List createFieldMappingDocs(List logTypes) { + Map fieldMappingMap = new HashMap<>(); + logTypes.stream() + .filter(e -> e.getMappings() != null) + .forEach( logType -> logType.getMappings().forEach(mapping -> { + // key is rawField + defaultSchemaField(ecs) + String key = mapping.getRawField() + "|" + mapping.getEcs(); + FieldMappingDoc existingDoc = fieldMappingMap.get(key); + if (existingDoc == null) { + // create new doc + Map schemaFields = new HashMap<>(); + if (mapping.getEcs() != null) { + schemaFields.put("ecs", mapping.getEcs()); + } + if (mapping.getOcsf() != null) { + schemaFields.put("ocsf", mapping.getOcsf()); + } + fieldMappingMap.put( + key, + new FieldMappingDoc( + mapping.getRawField(), + schemaFields, + Sets.newHashSet(logType.getName()) + ) + ); + } else { + // merge with existing doc + existingDoc.getSchemaFields().put("ocsf", mapping.getOcsf()); + existingDoc.getLogTypes().add(logType.getName()); + } + })); + return fieldMappingMap.values().stream().collect(Collectors.toList()); + } + + public String logTypeIndexMapping() { + try (InputStream is = getClass().getClassLoader().getResourceAsStream(LOG_TYPE_INDEX_MAPPING_FILE)) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + Streams.copy(is, out); + return out.toString(StandardCharsets.UTF_8.name()); + } catch (Exception e) { + logger.error( + () -> new ParameterizedMessage("failed to load log-type-index mapping file [{}]", LOG_TYPE_INDEX_MAPPING_FILE), + e + ); + throw new IllegalStateException("failed to load log-type-index mapping file [" + LOG_TYPE_INDEX_MAPPING_FILE + "]", e); + } - public List getAllLogTypes() { - return BuiltinLogTypeLoader.getAllLogTypes(); } - public LogType getLogTypeByName(String logType) { - return BuiltinLogTypeLoader.getLogTypeByName(logType); + private Settings logTypeIndexSettings() { + return Settings.builder().put(IndexMetadata.INDEX_HIDDEN_SETTING.getKey(), "true").build(); + } + + private int getConfigIndexMappingVersion(IndexMetadata metadata) { + MappingMetadata mappingMetadata = metadata.mapping(); + if (mappingMetadata == null) { + return 0; + } + @SuppressWarnings("unchecked") + Map meta = (Map) mappingMetadata.sourceAsMap().get("_meta"); + if (meta == null || meta.containsKey(LOG_TYPE_MAPPING_VERSION_META_FIELD) == false) { + return 1; // The mapping was created before meta field was introduced + } + return (int) meta.get(LOG_TYPE_MAPPING_VERSION_META_FIELD); + } + + public List getAllBuiltinLogTypes() { + return builtinLogTypeLoader.getAllLogTypes(); + } + + public void getRuleFieldMappings(ActionListener>> listener) { + ensureConfigIndexIsInitialized(ActionListener.wrap(() -> + getAllFieldMappings(ActionListener.delegateFailure( + listener, + (delegatedListener, fieldMappingDocs) -> { + Map> mappings = new HashMap<>(); + for (FieldMappingDoc fieldMappingDoc: fieldMappingDocs) { + Set logTypes = fieldMappingDoc.getLogTypes(); + if (logTypes != null) { + for (String logType: logTypes) { + Map mappingsByLogTypes = mappings.containsKey(logType)? mappings.get(logType): new HashMap<>(); + mappingsByLogTypes.put(fieldMappingDoc.getRawField(), fieldMappingDoc.getSchemaFields().get(defaultSchemaField)); + mappings.put(logType, mappingsByLogTypes); + } + } + } + delegatedListener.onResponse(mappings); + } + )) + )); } /** - * Returns sigmaRule rawField to ECS field mapping + * Returns sigmaRule rawField to default_schema_field(ECS) mapping * * @param logType Log type - * @return Map of rawField to ecs field + * Returns Map of rawField to ecs field via listener + */ + public void getRuleFieldMappings(String logType, ActionListener> listener) { + + if (builtinLogTypeLoader.logTypeExists(logType)) { + LogType lt = builtinLogTypeLoader.getLogTypeByName(logType); + if (lt.getMappings() == null) { + listener.onResponse(Map.of()); + } else { + listener.onResponse( + lt.getMappings() + .stream() + .collect(Collectors.toMap(LogType.Mapping::getRawField, LogType.Mapping::getEcs)) + ); + } + return; + } + + getFieldMappingsByLogType( + logType, + ActionListener.delegateFailure( + listener, + (delegatedListener, fieldMappingDocs) -> { + Map ruleFieldMappings = new HashMap<>(fieldMappingDocs.size()); + fieldMappingDocs.forEach( e -> { + ruleFieldMappings.put(e.getRawField(), e.getSchemaFields().get(defaultSchemaField)); + }); + delegatedListener.onResponse(ruleFieldMappings); + } + ) + ); + return; + } + + public void getRuleFieldMappingsAllSchemas(String logType, ActionListener> listener) { + + if (builtinLogTypeLoader.logTypeExists(logType)) { + LogType lt = builtinLogTypeLoader.getLogTypeByName(logType); + if (lt.getMappings() == null) { + listener.onResponse(List.of()); + } else { + listener.onResponse(lt.getMappings()); + } + return; + } + + getFieldMappingsByLogType( + logType, + ActionListener.delegateFailure( + listener, + (delegatedListener, fieldMappingDocs) -> { + List ruleFieldMappings = new ArrayList<>(); + fieldMappingDocs.forEach( e -> { + ruleFieldMappings.add(new LogType.Mapping(e.getRawField(), e.getSchemaFields().get("ecs"), e.getSchemaFields().get("ocsf"))); + }); + delegatedListener.onResponse(ruleFieldMappings); + } + ) + ); + return; + } + /** + * Provides required fields for a log type in order for all rules to work + * */ + public void getRequiredFields(String logType, ActionListener> listener) { + + getFieldMappingsByLogType( + logType, + ActionListener.delegateFailure( + listener, + (delegatedListener, fieldMappingDocs) -> { + List requiredFields = new ArrayList<>(); + fieldMappingDocs.forEach( e -> { + LogType.Mapping requiredField = new LogType.Mapping( + e.getRawField(), + e.getSchemaFields().get(defaultSchemaField), + e.getSchemaFields().get("ocsf") + ); + requiredFields.add(requiredField); + }); + delegatedListener.onResponse(requiredFields); + } + ) + ); + } + + /** + * Provides required fields for all log types in a form of map + * */ + public void getRequiredFieldsForAllLogTypes(ActionListener>> listener) { + ensureConfigIndexIsInitialized(ActionListener.wrap(() -> + getAllFieldMappings( + ActionListener.delegateFailure( + listener, + (delegatedListener, fieldMappingDocs) -> { + Map> requiredFieldsMap = new HashMap<>(); + fieldMappingDocs.forEach( e -> { + // Init sets if first time seeing this logType + e.getLogTypes().forEach(logType -> { + if (!requiredFieldsMap.containsKey(logType)) { + requiredFieldsMap.put(logType, new HashSet<>()); + } + }); + String requiredField = e.getSchemaFields().get(defaultSchemaField); + if (requiredField == null) { + requiredField = e.getRawField(); // Always fallback to rawField if defaultSchema one is missing + } + final String _requiredField = requiredField; + e.getLogTypes().forEach(logType -> { + requiredFieldsMap.get(logType).add(_requiredField); + }); + + }); + delegatedListener.onResponse(requiredFieldsMap); + } + ) + ) + )); + } + + /** + * Returns sigmaRule rawField to default_schema_field(ECS) mapping, but works with builtin types only! + * + * @param builtinLogType Built-in (prepackaged) Log type + * @return Map of rawField to ecs field via listener */ - public Map getRuleFieldMappings(String logType) { - LogType lt = getLogTypeByName(logType); + public Map getRuleFieldMappingsForBuiltinLogType(String builtinLogType) { - if (lt == null) { - throw SecurityAnalyticsException.wrap(new IllegalArgumentException("Can't get rule field mappings for invalid logType: [" + logType + "]")); + if (!builtinLogTypeLoader.logTypeExists(builtinLogType)) { + return null; } + + LogType lt = builtinLogTypeLoader.getLogTypeByName(builtinLogType); if (lt.getMappings() == null) { return Map.of(); } else { return lt.getMappings() - .stream() - .collect(Collectors.toMap(LogType.Mapping::getRawField, LogType.Mapping::getEcs)); + .stream() + .collect(Collectors.toMap(LogType.Mapping::getRawField, LogType.Mapping::getEcs)); + } } + + + public String getDefaultSchemaField() { + return defaultSchemaField; + } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/logtype/MappingSchema.java b/src/main/java/org/opensearch/securityanalytics/logtype/MappingSchema.java new file mode 100644 index 000000000..7e16c0078 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/logtype/MappingSchema.java @@ -0,0 +1,15 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.logtype; + +public enum MappingSchema { + ECS("ecs"), + OCSF("ocsf"); + + + MappingSchema(String name) { + + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java index e50fa2d62..b21f34f64 100644 --- a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java +++ b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java @@ -32,10 +32,15 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.Strings; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.rest.RestStatus; import org.opensearch.securityanalytics.action.GetIndexMappingsResponse; import org.opensearch.securityanalytics.action.GetMappingsViewResponse; +import org.opensearch.securityanalytics.logtype.LogTypeService; import org.opensearch.securityanalytics.model.CreateMappingResult; +import org.opensearch.securityanalytics.model.LogType; import org.opensearch.securityanalytics.util.IndexUtils; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; @@ -51,21 +56,23 @@ public class MapperService { private IndicesAdminClient indicesClient; private IndexNameExpressionResolver indexNameExpressionResolver; private IndexTemplateManager indexTemplateManager; + private LogTypeService logTypeService; public MapperService() {} - public MapperService(Client client, ClusterService clusterService, IndexNameExpressionResolver indexNameExpressionResolver, IndexTemplateManager indexTemplateManager) { + public MapperService(Client client, ClusterService clusterService, IndexNameExpressionResolver indexNameExpressionResolver, IndexTemplateManager indexTemplateManager, LogTypeService logTypeService) { this.indicesClient = client.admin().indices(); this.clusterService = clusterService; this.indexNameExpressionResolver = indexNameExpressionResolver; this.indexTemplateManager = indexTemplateManager; + this.logTypeService = logTypeService; } - public void createMappingAction(String indexName, String ruleTopic, boolean partial, ActionListener actionListener) { - this.createMappingAction(indexName, ruleTopic, null, partial, actionListener); + public void createMappingAction(String indexName, String logType, boolean partial, ActionListener actionListener) { + this.createMappingAction(indexName, logType, null, partial, actionListener); } - public void createMappingAction(String indexName, String ruleTopic, String aliasMappings, boolean partial, ActionListener actionListener) { + public void createMappingAction(String indexName, String logType, String aliasMappings, boolean partial, ActionListener actionListener) { // If indexName is Datastream it is enough to apply mappings to writeIndex only // since you can't update documents in non-write indices @@ -82,7 +89,7 @@ public void createMappingAction(String indexName, String ruleTopic, String alias indicesClient.getMappings(getMappingsRequest, new ActionListener<>() { @Override public void onResponse(GetMappingsResponse getMappingsResponse) { - applyAliasMappings(getMappingsResponse.getMappings(), ruleTopic, aliasMappings, partial, new ActionListener<>() { + applyAliasMappings(getMappingsResponse.getMappings(), logType, aliasMappings, partial, new ActionListener<>() { @Override public void onResponse(Collection createMappingResponse) { // We will return ack==false if one of the requests returned that @@ -115,10 +122,11 @@ public void onFailure(Exception e) { }); } - private void applyAliasMappings(Map indexMappings, String ruleTopic, String aliasMappings, boolean partial, ActionListener> actionListener) { + private void applyAliasMappings(Map indexMappings, String logType, String aliasMappings, boolean partial, ActionListener> actionListener) { int numOfIndices = indexMappings.size(); - GroupedActionListener doCreateMappingActionsListener = new GroupedActionListener(new ActionListener>() { @Override + GroupedActionListener doCreateMappingActionsListener = new GroupedActionListener(new ActionListener>() { + @Override public void onResponse(Collection response) { actionListener.onResponse(response); } @@ -126,9 +134,9 @@ public void onResponse(Collection response) { @Override public void onFailure(Exception e) { actionListener.onFailure( - new SecurityAnalyticsException( - "Failed applying mappings to index", RestStatus.INTERNAL_SERVER_ERROR, e - ) + new SecurityAnalyticsException( + "Failed applying mappings to index", RestStatus.INTERNAL_SERVER_ERROR, e + ) ); } }, numOfIndices); @@ -137,7 +145,7 @@ public void onFailure(Exception e) { String indexName = k; MappingMetadata mappingMetadata = v; // Try to apply mapping to index - doCreateMapping(indexName, mappingMetadata, ruleTopic, aliasMappings, partial, doCreateMappingActionsListener); + doCreateMapping(indexName, mappingMetadata, logType, aliasMappings, partial, doCreateMappingActionsListener); }); } @@ -145,7 +153,7 @@ public void onFailure(Exception e) { * Applies alias mappings to index. * @param indexName Index name * @param mappingMetadata Index mappings - * @param ruleTopic Rule topic spcifying specific alias templates + * @param logType Rule topic spcifying specific alias templates * @param aliasMappings User-supplied alias mappings * @param partial Partial flag indicating if we should apply mappings partially, in case source index doesn't have all paths specified in alias mappings * @param actionListener actionListener used to return response/error @@ -153,71 +161,140 @@ public void onFailure(Exception e) { private void doCreateMapping( String indexName, MappingMetadata mappingMetadata, - String ruleTopic, + String logType, String aliasMappings, boolean partial, ActionListener actionListener ) { try { - - String aliasMappingsJSON; - // aliasMappings parameter has higher priority then ruleTopic if (aliasMappings != null) { - aliasMappingsJSON = aliasMappings; - } else { - aliasMappingsJSON = MapperTopicStore.aliasMappings(ruleTopic); - } + Pair, List> validationResult = MapperUtils.validateIndexMappings(indexName, mappingMetadata, aliasMappings); + List missingPathsInIndex = validationResult.getLeft(); + List presentPathsInIndex = validationResult.getRight(); + + if (missingPathsInIndex.size() > 0) { + // If user didn't allow partial apply, we should error out here + if (!partial) { + actionListener.onFailure( + new IllegalArgumentException("Not all paths were found in index mappings: " + + missingPathsInIndex.stream() + .collect(Collectors.joining(", ", "[", "]"))) + ); + } + } - Pair, List> validationResult = MapperUtils.validateIndexMappings(indexName, mappingMetadata, aliasMappingsJSON); - List missingPathsInIndex = validationResult.getLeft(); - List presentPathsInIndex = validationResult.getRight(); + // Filter out mappings of sourceIndex fields to which we're applying alias mappings + Map presentPathsMappings = MapperUtils.getFieldMappingsFlat(mappingMetadata, presentPathsInIndex); + // Filtered alias mappings -- contains only aliases which are applicable to index: + // 1. fields in path params exists in index + // 2. alias isn't named as one of existing fields in index + Map filteredAliasMappings = filterNonApplicableAliases( + mappingMetadata, + missingPathsInIndex, + aliasMappings + ); + Map allMappings = new HashMap<>(presentPathsMappings); + allMappings.putAll((Map) filteredAliasMappings.get(PROPERTIES)); + + Map mappingsRoot = new HashMap<>(); + mappingsRoot.put(PROPERTIES, allMappings); + // Apply mappings to sourceIndex + PutMappingRequest request = new PutMappingRequest(indexName).source(filteredAliasMappings); + indicesClient.putMapping(request, new ActionListener<>() { + @Override + public void onResponse(AcknowledgedResponse acknowledgedResponse) { + CreateMappingResult result = new CreateMappingResult( + acknowledgedResponse, + indexName, + mappingsRoot + ); + actionListener.onResponse(result); + } - if(missingPathsInIndex.size() > 0) { - // If user didn't allow partial apply, we should error out here - if (!partial) { - actionListener.onFailure( - new IllegalArgumentException("Not all paths were found in index mappings: " + - missingPathsInIndex.stream() - .collect(Collectors.joining(", ", "[", "]"))) - ); - } - } + @Override + public void onFailure(Exception e) { + actionListener.onFailure(e); + } + }); + } else { + logTypeService.getRuleFieldMappingsAllSchemas(logType, new ActionListener<>() { + @Override + public void onResponse(List mappings) { + try { + List indexFields = MapperUtils.extractAllFieldsFlat(mappingMetadata); + Map> aliasMappingFields = new HashMap<>(); + XContentBuilder aliasMappingsObj = XContentFactory.jsonBuilder().startObject(); + for (LogType.Mapping mapping: mappings) { + if (indexFields.contains(mapping.getRawField())) { + aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getRawField())); + } else if (indexFields.contains(mapping.getOcsf())) { + aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf())); + } + } + aliasMappingsObj.field("properties", aliasMappingFields); + String aliasMappings = Strings.toString(aliasMappingsObj.endObject()); + + Pair, List> validationResult = MapperUtils.validateIndexMappings(indexName, mappingMetadata, aliasMappings); + List missingPathsInIndex = validationResult.getLeft(); + List presentPathsInIndex = validationResult.getRight(); + + if (missingPathsInIndex.size() > 0) { + // If user didn't allow partial apply, we should error out here + if (!partial) { + actionListener.onFailure( + new IllegalArgumentException("Not all paths were found in index mappings: " + + missingPathsInIndex.stream() + .collect(Collectors.joining(", ", "[", "]"))) + ); + } + } - // Filter out mappings of sourceIndex fields to which we're applying alias mappings - Map presentPathsMappings = MapperUtils.getFieldMappingsFlat(mappingMetadata, presentPathsInIndex); - // Filtered alias mappings -- contains only aliases which are applicable to index: - // 1. fields in path params exists in index - // 2. alias isn't named as one of existing fields in index - Map filteredAliasMappings = filterNonApplicableAliases( - mappingMetadata, - missingPathsInIndex, - aliasMappingsJSON - ); - Map allMappings = new HashMap<>(presentPathsMappings); - allMappings.putAll((Map) filteredAliasMappings.get(PROPERTIES)); - - Map mappingsRoot = new HashMap<>(); - mappingsRoot.put(PROPERTIES, allMappings); - // Apply mappings to sourceIndex - PutMappingRequest request = new PutMappingRequest(indexName).source(filteredAliasMappings); - indicesClient.putMapping(request, new ActionListener<>() { - @Override - public void onResponse(AcknowledgedResponse acknowledgedResponse) { - CreateMappingResult result = new CreateMappingResult( - acknowledgedResponse, - indexName, - mappingsRoot - ); - actionListener.onResponse(result); - } + // Filter out mappings of sourceIndex fields to which we're applying alias mappings + Map presentPathsMappings = MapperUtils.getFieldMappingsFlat(mappingMetadata, presentPathsInIndex); + // Filtered alias mappings -- contains only aliases which are applicable to index: + // 1. fields in path params exists in index + // 2. alias isn't named as one of existing fields in index + Map filteredAliasMappings = filterNonApplicableAliases( + mappingMetadata, + missingPathsInIndex, + aliasMappings + ); + Map allMappings = new HashMap<>(presentPathsMappings); + allMappings.putAll((Map) filteredAliasMappings.get(PROPERTIES)); + + Map mappingsRoot = new HashMap<>(); + mappingsRoot.put(PROPERTIES, allMappings); + // Apply mappings to sourceIndex + PutMappingRequest request = new PutMappingRequest(indexName).source(filteredAliasMappings); + indicesClient.putMapping(request, new ActionListener<>() { + @Override + public void onResponse(AcknowledgedResponse acknowledgedResponse) { + CreateMappingResult result = new CreateMappingResult( + acknowledgedResponse, + indexName, + mappingsRoot + ); + actionListener.onResponse(result); + } - @Override - public void onFailure(Exception e) { - actionListener.onFailure(e); - } - }); - } catch (IOException | IllegalArgumentException e) { + @Override + public void onFailure(Exception e) { + actionListener.onFailure(e); + } + }); + } catch (IOException ex) { + actionListener.onFailure(ex); + } + } + + @Override + public void onFailure(Exception e) { + actionListener.onFailure(e); + } + }); + } + } catch(IOException | IllegalArgumentException e){ actionListener.onFailure(e); } } @@ -302,57 +379,52 @@ public void doGetMappingAction(String indexName, String concreteIndexName, Actio indicesClient.getMappings(getMappingsRequest, new ActionListener<>() { @Override public void onResponse(GetMappingsResponse getMappingsResponse) { - try { - // Extract MappingMetadata - MappingMetadata mappingMetadata = getMappingsResponse.mappings().entrySet().iterator().next().getValue(); - // List of all found applied aliases on index - Set appliedAliases = new HashSet<>(); - // Get list of alias -> path pairs from index mappings - List> indexAliasPathPairs = MapperUtils.getAllAliasPathPairs(mappingMetadata); - - Map aliasMappingsMap = MapperTopicStore.getAliasMappingsMap(); - for (String mapperTopic : aliasMappingsMap.keySet()) { - // Get stored Alias Mappings as JSON string - String aliasMappingsJson = MapperTopicStore.aliasMappings(mapperTopic); - // Get list of alias -> path pairs from stored alias mappings - List> aliasPathPairs = MapperUtils.getAllAliasPathPairs(aliasMappingsJson); - // Try to find any alias mappings in index mappings which are present in stored alias mappings - for (Pair p1 : indexAliasPathPairs) { - for (Pair p2 : aliasPathPairs) { - // Match by alias only here since user can match alias to some other path - if (p1.getKey().equals(p2.getKey())) { - // Maintain list of found alias mappings + logTypeService.getRequiredFieldsForAllLogTypes(ActionListener.wrap(requiredFieldMap -> { + try { + // Extract MappingMetadata + MappingMetadata mappingMetadata = getMappingsResponse.mappings().entrySet().iterator().next().getValue(); + // List of all found applied aliases on index + Set appliedAliases = new HashSet<>(); + // Get list of alias -> path pairs from index mappings + List> indexAliasPathPairs = MapperUtils.getAllAliasPathPairs(mappingMetadata); + + for (String logType : requiredFieldMap.keySet()) { + // Get stored Alias Mappings as JSON string + Set requiredFields = requiredFieldMap.get(logType); + // Try to find any alias mappings in index mappings which are present in requiredFields set + for (Pair p1 : indexAliasPathPairs) { + if (requiredFields.contains(p1.getKey())) { appliedAliases.add(p1.getKey()); } } } - } - if (appliedAliases.size() == 0) { - actionListener.onFailure(SecurityAnalyticsException.wrap( - new OpenSearchStatusException("No applied aliases found", RestStatus.NOT_FOUND)) - ); - return; - } + if (appliedAliases.size() == 0) { + actionListener.onFailure(SecurityAnalyticsException.wrap( + new OpenSearchStatusException("No applied aliases found", RestStatus.NOT_FOUND)) + ); + return; + } - // Traverse mappings and do copy with excluded type=alias properties - MappingsTraverser mappingsTraverser = new MappingsTraverser(mappingMetadata); - // Resulting mapping after filtering - Map filteredMapping = mappingsTraverser.traverseAndCopyWithFilter(appliedAliases); + // Traverse mappings and do copy with excluded type=alias properties + MappingsTraverser mappingsTraverser = new MappingsTraverser(mappingMetadata); + // Resulting mapping after filtering + Map filteredMapping = mappingsTraverser.traverseAndCopyWithFilter(appliedAliases); - // Construct filtered mappings and return them as result - Map outIndexMappings = new HashMap<>(); - Map root = Map.of(org.opensearch.index.mapper.MapperService.SINGLE_MAPPING_NAME, filteredMapping); - MappingMetadata outMappingMetadata = new MappingMetadata(org.opensearch.index.mapper.MapperService.SINGLE_MAPPING_NAME, root); - outIndexMappings.put(indexName, outMappingMetadata); + // Construct filtered mappings and return them as result + Map outIndexMappings = new HashMap<>(); + Map root = Map.of(org.opensearch.index.mapper.MapperService.SINGLE_MAPPING_NAME, filteredMapping); + MappingMetadata outMappingMetadata = new MappingMetadata(org.opensearch.index.mapper.MapperService.SINGLE_MAPPING_NAME, root); + outIndexMappings.put(indexName, outMappingMetadata); - actionListener.onResponse(new GetIndexMappingsResponse(outIndexMappings)); - } catch (IOException e) { - actionListener.onFailure(e); - } + actionListener.onResponse(new GetIndexMappingsResponse(outIndexMappings)); + } catch (IOException e) { + actionListener.onFailure(e); + } + }, actionListener::onFailure)); } - @Override + @Override public void onFailure(Exception e) { actionListener.onFailure(e); } @@ -361,7 +433,7 @@ public void onFailure(Exception e) { public void getMappingsViewAction( String indexName, - String mapperTopic, + String logType, ActionListener actionListener ) { try { @@ -369,7 +441,7 @@ public void getMappingsViewAction( resolveConcreteIndex(indexName, new ActionListener<>() { @Override public void onResponse(String concreteIndex) { - doGetMappingsView(mapperTopic, actionListener, concreteIndex); + doGetMappingsView(logType, actionListener, concreteIndex); } @Override @@ -386,58 +458,72 @@ public void onFailure(Exception e) { /** * Constructs Mappings View of index - * @param mapperTopic Mapper Topic describing set of alias mappings + * @param logType Log Type * @param actionListener Action Listener * @param concreteIndex Concrete Index name for which we're computing Mappings View */ - private void doGetMappingsView(String mapperTopic, ActionListener actionListener, String concreteIndex) { + private void doGetMappingsView(String logType, ActionListener actionListener, String concreteIndex) { GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(concreteIndex); indicesClient.getMappings(getMappingsRequest, new ActionListener<>() { @Override public void onResponse(GetMappingsResponse getMappingsResponse) { - try { - // Extract MappingMetadata from GET _mapping response - MappingMetadata mappingMetadata = getMappingsResponse.mappings().entrySet().iterator().next().getValue(); - // Get list of all non-alias fields in index - List allFieldsFromIndex = MapperUtils.getAllNonAliasFieldsFromIndex(mappingMetadata); - // Get stored Alias Mappings as JSON string - String aliasMappingsJson = MapperTopicStore.aliasMappings(mapperTopic); - // Get list of alias -> path pairs from stored alias mappings - List> aliasPathPairs = MapperUtils.getAllAliasPathPairs(aliasMappingsJson); - // List of all found applied aliases on index - List applyableAliases = new ArrayList<>(); - // List of paths of found - List pathsOfApplyableAliases = new ArrayList<>(); - // List of unapplayable aliases - List unmappedFieldAliases = new ArrayList<>(); - - for (Pair p : aliasPathPairs) { - String alias = p.getKey(); - String path = p.getValue(); - if (allFieldsFromIndex.contains(path)) { - // Maintain list of found paths in index - applyableAliases.add(alias); - pathsOfApplyableAliases.add(path); - } else if (allFieldsFromIndex.contains(alias) == false) { - // we don't want to send back aliases which have same name as existing field in index - unmappedFieldAliases.add(alias); + logTypeService.getRequiredFields(logType, ActionListener.wrap(requiredFields -> { + try { + // Extract MappingMetadata from GET _mapping response + MappingMetadata mappingMetadata = getMappingsResponse.mappings().entrySet().iterator().next().getValue(); + // Get list of all non-alias fields in index + List allFieldsFromIndex = MapperUtils.getAllNonAliasFieldsFromIndex(mappingMetadata); + // List of all found applied aliases on index + List applyableAliases = new ArrayList<>(); + // List of paths of found + List pathsOfApplyableAliases = new ArrayList<>(); + // List of unapplayable aliases + List unmappedFieldAliases = new ArrayList<>(); + + for (LogType.Mapping requiredField: requiredFields) { + String alias = requiredField.getEcs(); + String rawPath = requiredField.getRawField(); + String ocsfPath = requiredField.getOcsf(); + if (allFieldsFromIndex.contains(rawPath)) { + // Maintain list of found paths in index + applyableAliases.add(alias); + pathsOfApplyableAliases.add(rawPath); + } else if (allFieldsFromIndex.contains(ocsfPath)) { + applyableAliases.add(alias); + pathsOfApplyableAliases.add(ocsfPath); + } else if (allFieldsFromIndex.contains(alias) == false) { + // we don't want to send back aliases which have same name as existing field in index + unmappedFieldAliases.add(alias); + } + } + + Map> aliasMappingFields = new HashMap<>(); + XContentBuilder aliasMappingsObj = XContentFactory.jsonBuilder().startObject(); + for (LogType.Mapping mapping: requiredFields) { + if (allFieldsFromIndex.contains(mapping.getOcsf())) { + aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf())); + } else if (mapping.getEcs() != null) { + aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getRawField())); + } } + aliasMappingsObj.field("properties", aliasMappingFields); + String aliasMappingsJson = Strings.toString(aliasMappingsObj.endObject()); + // Gather all applyable alias mappings + Map aliasMappings = + MapperUtils.getAliasMappingsWithFilter(aliasMappingsJson, applyableAliases); + // Unmapped fields from index for which we don't have alias to apply to + List unmappedIndexFields = allFieldsFromIndex + .stream() + .filter(e -> pathsOfApplyableAliases.contains(e) == false) + .collect(Collectors.toList()); + + actionListener.onResponse( + new GetMappingsViewResponse(aliasMappings, unmappedIndexFields, unmappedFieldAliases) + ); + } catch (Exception e) { + actionListener.onFailure(e); } - // Gather all applyable alias mappings - Map aliasMappings = - MapperUtils.getAliasMappingsWithFilter(aliasMappingsJson, applyableAliases); - // Unmapped fields from index for which we don't have alias to apply to - List unmappedIndexFields = allFieldsFromIndex - .stream() - .filter(e -> pathsOfApplyableAliases.contains(e) == false) - .collect(Collectors.toList()); - - actionListener.onResponse( - new GetMappingsViewResponse(aliasMappings, unmappedIndexFields, unmappedFieldAliases) - ); - } catch (Exception e) { - actionListener.onFailure(e); - } + }, actionListener::onFailure)); } @Override public void onFailure(Exception e) { @@ -473,7 +559,7 @@ public void onResponse(GetIndexResponse getIndexResponse) { actionListener.onResponse(writeIndex); } else { actionListener.onResponse( - IndexUtils.getNewestIndexByCreationDate(indices, MapperService.this.clusterService.state()) + IndexUtils.getNewestIndexByCreationDate(indices, MapperService.this.clusterService.state()) ); } } diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MapperTopicStore.java b/src/main/java/org/opensearch/securityanalytics/mapper/MapperTopicStore.java deleted file mode 100644 index d2f399917..000000000 --- a/src/main/java/org/opensearch/securityanalytics/mapper/MapperTopicStore.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.mapper; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.OpenSearchParseException; -import org.opensearch.common.settings.SettingsException; -import org.opensearch.common.xcontent.XContentHelper; -import org.opensearch.common.xcontent.json.JsonXContent; - -public class MapperTopicStore { - - private static final String MAPPER_CONFIG_FILE = "OSMapping/mapper_topics.json"; - - private static final Logger log = LogManager.getLogger(MapperTopicStore.class); - - private Map mapperMap; - private static MapperTopicStore INSTANCE = new MapperTopicStore(); - private MapperTopicStore() { - - String mapperTopicsJson; - try ( - InputStream is = MapperTopicStore.class.getClassLoader().getResourceAsStream(MAPPER_CONFIG_FILE) - ) { - mapperMap = new HashMap<>(); - mapperTopicsJson = new String(Objects.requireNonNull(is).readAllBytes(), StandardCharsets.UTF_8); - - if (mapperTopicsJson != null) { - Map configMap = - XContentHelper.convertToMap(JsonXContent.jsonXContent, mapperTopicsJson, false); - - mapperMap = configMap.entrySet() - .stream() - .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toString())); - - log.info("Loaded {} mapper topics", mapperMap.size()); - } - } catch (OpenSearchParseException e) { - throw e; - } catch (Exception e) { - throw new SettingsException("Failed to load settings from [" + MAPPER_CONFIG_FILE + "]", e); - } - } - - public static String aliasMappings(String mapperTopic) throws IOException { - if (INSTANCE.mapperMap.containsKey(mapperTopic.toLowerCase(Locale.ROOT))) { - return new String(Objects.requireNonNull( - - INSTANCE.getClass().getClassLoader().getResourceAsStream(INSTANCE. - mapperMap.get(mapperTopic.toLowerCase(Locale.ROOT)))).readAllBytes(), - StandardCharsets.UTF_8); - } - throw new IllegalArgumentException("Mapper not found: [" + mapperTopic + "]"); - } - - public static void putAliasMappings(String mapperTopic, String mappingFilePath) { - INSTANCE.mapperMap.put(mapperTopic, mappingFilePath); - } - - public static Map getAliasMappingsMap() { - return INSTANCE.mapperMap; - } -} diff --git a/src/main/java/org/opensearch/securityanalytics/model/Detector.java b/src/main/java/org/opensearch/securityanalytics/model/Detector.java index adbbe6776..fa9f0fa81 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/Detector.java +++ b/src/main/java/org/opensearch/securityanalytics/model/Detector.java @@ -39,6 +39,7 @@ public class Detector implements Writeable, ToXContentObject { private static final String DETECTOR_TYPE = "detector"; private static final String TYPE_FIELD = "type"; public static final String DETECTOR_TYPE_FIELD = "detector_type"; + private static final String LOG_TYPE_FIELD = "log_type"; public static final String NAME_FIELD = "name"; private static final String USER_FIELD = "user"; public static final String ENABLED_FIELD = "enabled"; @@ -86,7 +87,7 @@ public class Detector implements Writeable, ToXContentObject { private Instant enabledTime; - private DetectorType detectorType; + private String logType; private User user; @@ -113,7 +114,7 @@ public class Detector implements Writeable, ToXContentObject { private final String type; public Detector(String id, Long version, String name, Boolean enabled, Schedule schedule, - Instant lastUpdateTime, Instant enabledTime, DetectorType detectorType, + Instant lastUpdateTime, Instant enabledTime, String logType, User user, List inputs, List triggers, List monitorIds, String ruleIndex, String alertsIndex, String alertsHistoryIndex, String alertsHistoryIndexPattern, String findingsIndex, String findingsIndexPattern, Map rulePerMonitor) { @@ -126,7 +127,6 @@ public Detector(String id, Long version, String name, Boolean enabled, Schedule this.schedule = schedule; this.lastUpdateTime = lastUpdateTime; this.enabledTime = enabledTime; - this.detectorType = detectorType; this.user = user; this.inputs = inputs; this.triggers = triggers; @@ -138,6 +138,7 @@ public Detector(String id, Long version, String name, Boolean enabled, Schedule this.findingsIndex = findingsIndex; this.findingsIndexPattern = findingsIndexPattern; this.ruleIdMonitorIdMap = rulePerMonitor; + this.logType = logType; if (enabled) { Objects.requireNonNull(enabledTime); @@ -153,7 +154,7 @@ public Detector(StreamInput sin) throws IOException { Schedule.readFrom(sin), sin.readInstant(), sin.readOptionalInstant(), - sin.readEnum(DetectorType.class), + sin.readString(), sin.readBoolean() ? new User(sin) : null, sin.readList(DetectorInput::readFrom), sin.readList(DetectorTrigger::readFrom), @@ -182,7 +183,7 @@ public void writeTo(StreamOutput out) throws IOException { schedule.writeTo(out); out.writeInstant(lastUpdateTime); out.writeOptionalInstant(enabledTime); - out.writeEnum(detectorType); + out.writeString(logType); out.writeBoolean(user != null); if (user != null) { user.writeTo(out); @@ -231,7 +232,8 @@ public enum DetectorType { OKTA("okta", 19), AZURE("azure", 20), S3("s3", 21), - TEST_WINDOWS("test_windows", 22); + TEST_WINDOWS("test_windows", 22), + VPCFLOW("vpcflow", 23); private String type; private int dim; @@ -257,7 +259,7 @@ private XContentBuilder createXContentBuilder(XContentBuilder builder, ToXConten } builder.field(TYPE_FIELD, type) .field(NAME_FIELD, name) - .field(DETECTOR_TYPE_FIELD, detectorType.getDetectorType()); + .field(DETECTOR_TYPE_FIELD, logType); if (!secure) { if (user == null) { @@ -329,6 +331,7 @@ public static Detector parse(XContentParser xcp, String id, Long version) throws String name = null; String detectorType = null; + String logType = null; User user = null; Schedule schedule = null; Instant lastUpdateTime = null; @@ -356,12 +359,7 @@ public static Detector parse(XContentParser xcp, String id, Long version) throws name = xcp.text(); break; case DETECTOR_TYPE_FIELD: - detectorType = xcp.text(); - List allowedTypes = Arrays.stream(DetectorType.values()).map(DetectorType::getDetectorType).collect(Collectors.toList()); - - if (!allowedTypes.contains(detectorType.toLowerCase(Locale.ROOT))) { - throw new IllegalArgumentException(String.format(Locale.getDefault(), "Detector type should be one of %s", allowedTypes)); - } + logType = xcp.text(); break; case USER_FIELD: if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { @@ -449,6 +447,10 @@ public static Detector parse(XContentParser xcp, String id, Long version) throws enabledTime = null; } + if (logType == null) { + logType = detectorType; + } + return new Detector( id, version, @@ -457,7 +459,7 @@ public static Detector parse(XContentParser xcp, String id, Long version) throws Objects.requireNonNull(schedule, "Detector schedule is null"), lastUpdateTime != null ? lastUpdateTime : Instant.now(), enabledTime, - DetectorType.valueOf(detectorType.toUpperCase(Locale.ROOT)), + logType, user, inputs, triggers, @@ -468,7 +470,8 @@ public static Detector parse(XContentParser xcp, String id, Long version) throws alertsHistoryIndexPattern, findingsIndex, findingsIndexPattern, - rulePerMonitor); + rulePerMonitor + ); } public static Detector readFrom(StreamInput sin) throws IOException { @@ -504,7 +507,7 @@ public Instant getEnabledTime() { } public String getDetectorType() { - return detectorType.getDetectorType(); + return logType.toLowerCase(Locale.ROOT); } public User getUser() { @@ -613,11 +616,11 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Detector detector = (Detector) o; - return Objects.equals(id, detector.id) && Objects.equals(version, detector.version) && Objects.equals(name, detector.name) && Objects.equals(enabled, detector.enabled) && Objects.equals(schedule, detector.schedule) && Objects.equals(lastUpdateTime, detector.lastUpdateTime) && Objects.equals(enabledTime, detector.enabledTime) && detectorType == detector.detectorType && ((user == null && detector.user == null) || Objects.equals(user, detector.user)) && Objects.equals(inputs, detector.inputs) && Objects.equals(triggers, detector.triggers) && Objects.equals(type, detector.type) && Objects.equals(monitorIds, detector.monitorIds) && Objects.equals(ruleIndex, detector.ruleIndex); + return Objects.equals(id, detector.id) && Objects.equals(version, detector.version) && Objects.equals(name, detector.name) && Objects.equals(enabled, detector.enabled) && Objects.equals(schedule, detector.schedule) && Objects.equals(lastUpdateTime, detector.lastUpdateTime) && Objects.equals(enabledTime, detector.enabledTime) && Objects.equals(logType, detector.logType) && ((user == null && detector.user == null) || Objects.equals(user, detector.user)) && Objects.equals(inputs, detector.inputs) && Objects.equals(triggers, detector.triggers) && Objects.equals(type, detector.type) && Objects.equals(monitorIds, detector.monitorIds) && Objects.equals(ruleIndex, detector.ruleIndex); } @Override public int hashCode() { - return Objects.hash(id, version, name, enabled, schedule, lastUpdateTime, enabledTime, detectorType, user, inputs, triggers, type, monitorIds, ruleIndex); + return Objects.hash(id, version, name, enabled, schedule, lastUpdateTime, enabledTime, logType, user, inputs, triggers, type, monitorIds, ruleIndex); } } diff --git a/src/main/java/org/opensearch/securityanalytics/model/FieldMappingDoc.java b/src/main/java/org/opensearch/securityanalytics/model/FieldMappingDoc.java new file mode 100644 index 000000000..b2bd7763d --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/model/FieldMappingDoc.java @@ -0,0 +1,150 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.model; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.common.io.stream.Writeable; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.search.SearchHit; + +public class FieldMappingDoc implements ToXContent, Writeable { + + public static final String RAW_FIELD = "raw_field"; + public static final String LOG_TYPES = "log_types"; + + private String id; + private String rawField; + private String defaultSchemaFieldValue; + private Map schemaFields; + private Set logTypes; + + private boolean isDirty; + + public FieldMappingDoc(String id, String rawField, Map schemaFields, Set logTypes) { + this(rawField, schemaFields, logTypes); + this.id = id; + } + + public FieldMappingDoc(String rawField, Map schemaFields, Set logTypes) { + Objects.requireNonNull(schemaFields); + Objects.requireNonNull(logTypes); + this.rawField = rawField; + this.schemaFields = schemaFields; + this.logTypes = logTypes; + } + + public FieldMappingDoc(String rawField, Set logTypes) { + this.rawField = rawField; + this.schemaFields = new HashMap<>(); + this.logTypes = logTypes; + } + + public FieldMappingDoc(StreamInput sin) throws IOException { + this.rawField = sin.readString(); + this.schemaFields = sin.readMap(StreamInput::readString, StreamInput::readString); + Collections.addAll(this.logTypes, sin.readStringArray()); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(RAW_FIELD, rawField); + builder.mapContents(schemaFields); + builder.array(LOG_TYPES, logTypes.toArray(new String[0])); + return builder.endObject(); + } + + public static FieldMappingDoc parse(SearchHit hit, NamedXContentRegistry xContentRegistry) throws IOException { + XContentParser xcp = XContentHelper.createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, + hit.getSourceRef(), + XContentType.JSON + ); + return parse(xcp, hit.getId()); + } + + public static FieldMappingDoc parse(XContentParser xcp, String id) throws IOException { + String rawField = null; + Map schemaFields = new HashMap<>(); + Set logTypes = new HashSet<>(); + if (xcp.currentToken() == null) { + xcp.nextToken(); + } + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case RAW_FIELD: + rawField = xcp.text(); + break; + case LOG_TYPES: + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + logTypes.add(xcp.text()); + } + break; + default: + if (xcp.textOrNull() != null) { + schemaFields.put(fieldName, xcp.text()); + } + } + } + return new FieldMappingDoc(id, rawField, schemaFields, logTypes); + } + + + public String getRawField() { + return rawField; + } + + public Map getSchemaFields() { + return schemaFields; + } + + public Set getLogTypes() { + return logTypes; + } + + public String getId() { + return id; + } + + public boolean isDirty() { + return isDirty; + } + + public void setIsDirty(boolean isDirty) { + this.isDirty = isDirty; + } + + public Object get(String field) { + return schemaFields.get(field); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(this.rawField); + out.writeMap(schemaFields, StreamOutput::writeString, StreamOutput::writeString); + out.writeStringArray(logTypes.toArray(new String[0])); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/model/LogType.java b/src/main/java/org/opensearch/securityanalytics/model/LogType.java index 80fa003d4..0d221d1d7 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/LogType.java +++ b/src/main/java/org/opensearch/securityanalytics/model/LogType.java @@ -23,8 +23,8 @@ public class LogType implements Writeable { private static final String IS_BUILTIN = "is_builtin"; private static final String MAPPINGS = "mappings"; private static final String RAW_FIELD = "raw_field"; - private static final String ECS = "ecs"; - private static final String OCSF = "ocsf"; + public static final String ECS = "ecs"; + public static final String OCSF = "ocsf"; private String id; private String name; diff --git a/src/main/java/org/opensearch/securityanalytics/model/RuleCategory.java b/src/main/java/org/opensearch/securityanalytics/model/RuleCategory.java index d189465b7..a2abfa3ba 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/RuleCategory.java +++ b/src/main/java/org/opensearch/securityanalytics/model/RuleCategory.java @@ -57,38 +57,5 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public String getName() { return name; } - - - private static final String RULE_CATEGORIES_CONFIG_FILE = "rules/rule_categories.json"; - - // Rule category is the same as detector type - public static final List ALL_RULE_CATEGORIES; - - static { - List ruleCategories = new ArrayList<>(); - String ruleCategoriesJson; - try ( - InputStream is = RuleCategory.class.getClassLoader().getResourceAsStream(RULE_CATEGORIES_CONFIG_FILE) - ) { - ruleCategoriesJson = new String(Objects.requireNonNull(is).readAllBytes(), StandardCharsets.UTF_8); - - if (ruleCategoriesJson != null) { - Map configMap = - XContentHelper.convertToMap(JsonXContent.jsonXContent, ruleCategoriesJson, false); - List> categories = (List>) configMap.get("rule_categories"); - for (Map c : categories) { - ruleCategories.add(new RuleCategory( - (String) c.get(KEY), - (String) c.get(DISPLAY_NAME) - )); - } - } - } catch (OpenSearchParseException e) { - throw e; - } catch (Exception e) { - throw new SettingsException("Failed to load settings from [" + RULE_CATEGORIES_CONFIG_FILE + "]", e); - } - ALL_RULE_CATEGORIES = Collections.unmodifiableList(ruleCategories); - } } diff --git a/src/main/java/org/opensearch/securityanalytics/resthandler/RestGetAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/resthandler/RestGetAlertsAction.java index 69a558e8a..0d6bcb52d 100644 --- a/src/main/java/org/opensearch/securityanalytics/resthandler/RestGetAlertsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/resthandler/RestGetAlertsAction.java @@ -56,7 +56,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli GetAlertsRequest req = new GetAlertsRequest( detectorId, - detectorType != null ? Detector.DetectorType.valueOf(detectorType.toUpperCase(Locale.ROOT)) : null, + detectorType, table, severityLevel, alertState diff --git a/src/main/java/org/opensearch/securityanalytics/resthandler/RestGetFindingsAction.java b/src/main/java/org/opensearch/securityanalytics/resthandler/RestGetFindingsAction.java index 5ff727376..efc04e1e5 100644 --- a/src/main/java/org/opensearch/securityanalytics/resthandler/RestGetFindingsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/resthandler/RestGetFindingsAction.java @@ -52,7 +52,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli GetFindingsRequest req = new GetFindingsRequest( detectorId, - detectorType != null ? Detector.DetectorType.valueOf(detectorType.toUpperCase(Locale.ROOT)) : null, + detectorType, table ); diff --git a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java index a3836673a..9a0aebc3c 100644 --- a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java +++ b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java @@ -7,6 +7,7 @@ import java.util.concurrent.TimeUnit; import org.opensearch.common.settings.Setting; import org.opensearch.common.unit.TimeValue; +import org.opensearch.securityanalytics.model.FieldMappingDoc; public class SecurityAnalyticsSettings { public static final String CORRELATION_INDEX = "index.correlation"; @@ -104,4 +105,10 @@ public class SecurityAnalyticsSettings { new TimeValue(5, TimeUnit.MINUTES), Setting.Property.NodeScope, Setting.Property.Dynamic ); + + public static final Setting DEFAULT_MAPPING_SCHEMA = Setting.simpleString( + "plugins.security_analytics.mappings.default_schema", + "ecs", + Setting.Property.NodeScope, Setting.Property.Dynamic + ); } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportGetAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportGetAlertsAction.java index bfc73ac97..8af3d0fd8 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportGetAlertsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportGetAlertsAction.java @@ -84,7 +84,7 @@ protected void doExecute(Task task, GetAlertsRequest request, ActionListener { private final ThreadPool threadPool; + private final LogTypeService logTypeService; @Inject public TransportGetAllRuleCategoriesAction( TransportService transportService, ActionFilters actionFilters, GetAllRuleCategoriesAction getAllRuleCategoriesAction, - ClusterService clusterService, + LogTypeService logTypeService, ThreadPool threadPool ) { super(getAllRuleCategoriesAction.NAME, transportService, actionFilters, GetAllRuleCategoriesRequest::new); this.threadPool = threadPool; + this.logTypeService = logTypeService; } @Override protected void doExecute(Task task, GetAllRuleCategoriesRequest request, ActionListener actionListener) { this.threadPool.getThreadContext().stashContext(); - actionListener.onResponse(new GetAllRuleCategoriesResponse(RuleCategory.ALL_RULE_CATEGORIES)); + logTypeService.getAllLogTypes(ActionListener.wrap(logTypes -> { + actionListener.onResponse( + new GetAllRuleCategoriesResponse( + logTypes.stream().map(logType -> new RuleCategory(logType, logType)).collect(Collectors.toList()) + ) + ); + }, e -> actionListener.onFailure(SecurityAnalyticsException.wrap(e)))); + } } diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportGetFindingsAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportGetFindingsAction.java index 4ec9adeb5..5af8eb608 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportGetFindingsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportGetFindingsAction.java @@ -31,6 +31,7 @@ import org.opensearch.securityanalytics.action.GetFindingsResponse; import org.opensearch.securityanalytics.action.SearchDetectorRequest; import org.opensearch.securityanalytics.findings.FindingsService; +import org.opensearch.securityanalytics.logtype.LogTypeService; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.util.DetectorIndices; @@ -59,19 +60,32 @@ public class TransportGetFindingsAction extends HandledTransportAction> rul if (!docLevelRules.isEmpty()) { monitorRequests.add(createDocLevelMonitorRequest(docLevelRules, detector, refreshPolicy, Monitor.NO_ID, Method.POST)); } + if (!bucketLevelRules.isEmpty()) { - monitorRequests.addAll(buildBucketLevelMonitorRequests(bucketLevelRules, detector, refreshPolicy, Monitor.NO_ID, Method.POST)); - } - // Do nothing if detector doesn't have any monitor - if (monitorRequests.isEmpty()){ - listener.onResponse(Collections.emptyList()); - return; - } + StepListener> bucketLevelMonitorRequests = new StepListener<>(); + buildBucketLevelMonitorRequests(bucketLevelRules, detector, refreshPolicy, Monitor.NO_ID, Method.POST, bucketLevelMonitorRequests); + bucketLevelMonitorRequests.whenComplete(indexMonitorRequests -> { + monitorRequests.addAll(indexMonitorRequests); + // Do nothing if detector doesn't have any monitor + if (monitorRequests.isEmpty()) { + listener.onResponse(Collections.emptyList()); + return; + } - List monitorResponses = new ArrayList<>(); - StepListener addFirstMonitorStep = new StepListener(); - - // Indexing monitors in two steps in order to prevent all shards failed error from alerting - // https://github.com/opensearch-project/alerting/issues/646 - AlertingPluginInterface.INSTANCE.indexMonitor((NodeClient) client, monitorRequests.get(0), namedWriteableRegistry, addFirstMonitorStep); - addFirstMonitorStep.whenComplete(addedFirstMonitorResponse -> { - monitorResponses.add(addedFirstMonitorResponse); - int numberOfUnprocessedResponses = monitorRequests.size() - 1; - if (numberOfUnprocessedResponses == 0){ - listener.onResponse(monitorResponses); - } else { - GroupedActionListener monitorResponseListener = new GroupedActionListener( - new ActionListener>() { - @Override - public void onResponse(Collection indexMonitorResponse) { - monitorResponses.addAll(indexMonitorResponse.stream().collect(Collectors.toList())); + List monitorResponses = new ArrayList<>(); + StepListener addFirstMonitorStep = new StepListener(); + + // Indexing monitors in two steps in order to prevent all shards failed error from alerting + // https://github.com/opensearch-project/alerting/issues/646 + AlertingPluginInterface.INSTANCE.indexMonitor((NodeClient) client, monitorRequests.get(0), namedWriteableRegistry, addFirstMonitorStep); + addFirstMonitorStep.whenComplete(addedFirstMonitorResponse -> { + monitorResponses.add(addedFirstMonitorResponse); + int numberOfUnprocessedResponses = monitorRequests.size() - 1; + if (numberOfUnprocessedResponses == 0) { listener.onResponse(monitorResponses); + } else { + GroupedActionListener monitorResponseListener = new GroupedActionListener( + new ActionListener>() { + @Override + public void onResponse(Collection indexMonitorResponse) { + monitorResponses.addAll(indexMonitorResponse.stream().collect(Collectors.toList())); + listener.onResponse(monitorResponses); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }, numberOfUnprocessedResponses); + + for (int i = 1; i < monitorRequests.size(); i++) { + AlertingPluginInterface.INSTANCE.indexMonitor((NodeClient) client, monitorRequests.get(i), namedWriteableRegistry, monitorResponseListener); + } } - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }, numberOfUnprocessedResponses); + }, + listener::onFailure + ); + }, listener::onFailure); + } else { + // Do nothing if detector doesn't have any monitor + if (monitorRequests.isEmpty()) { + listener.onResponse(Collections.emptyList()); + return; + } - for(int i = 1; i < monitorRequests.size(); i++){ - AlertingPluginInterface.INSTANCE.indexMonitor((NodeClient) client, monitorRequests.get(i), namedWriteableRegistry, monitorResponseListener); - } - } - }, - listener::onFailure - ); + List monitorResponses = new ArrayList<>(); + StepListener addFirstMonitorStep = new StepListener(); + + // Indexing monitors in two steps in order to prevent all shards failed error from alerting + // https://github.com/opensearch-project/alerting/issues/646 + AlertingPluginInterface.INSTANCE.indexMonitor((NodeClient) client, monitorRequests.get(0), namedWriteableRegistry, addFirstMonitorStep); + addFirstMonitorStep.whenComplete(addedFirstMonitorResponse -> { + monitorResponses.add(addedFirstMonitorResponse); + int numberOfUnprocessedResponses = monitorRequests.size() - 1; + if (numberOfUnprocessedResponses == 0) { + listener.onResponse(monitorResponses); + } else { + GroupedActionListener monitorResponseListener = new GroupedActionListener( + new ActionListener>() { + @Override + public void onResponse(Collection indexMonitorResponse) { + monitorResponses.addAll(indexMonitorResponse.stream().collect(Collectors.toList())); + listener.onResponse(monitorResponses); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }, numberOfUnprocessedResponses); + + for (int i = 1; i < monitorRequests.size(); i++) { + AlertingPluginInterface.INSTANCE.indexMonitor((NodeClient) client, monitorRequests.get(i), namedWriteableRegistry, monitorResponseListener); + } + } + }, + listener::onFailure + ); + } } private void updateMonitorFromQueries(String index, List> rulesById, Detector detector, ActionListener> listener, WriteRequest.RefreshPolicy refreshPolicy) throws SigmaError, IOException { @@ -291,58 +338,90 @@ private void updateMonitorFromQueries(String index, List> rul List monitorsToBeAdded = new ArrayList<>(); // Process bucket level monitors if (!bucketLevelRules.isEmpty()) { - List ruleCategories = bucketLevelRules.stream().map(Pair::getRight).map(Rule::getCategory).distinct().collect( - Collectors.toList()); - Map queryBackendMap = new HashMap<>(); - for(String category: ruleCategories) { - Map fieldMappings = logTypeService.getRuleFieldMappings(category); - queryBackendMap.put(category, new OSQueryBackend(fieldMappings, true, true)); - } + logTypeService.getRuleFieldMappings(new ActionListener<>() { + @Override + public void onResponse(Map> ruleFieldMappings) { + try { + List ruleCategories = bucketLevelRules.stream().map(Pair::getRight).map(Rule::getCategory).distinct().collect( + Collectors.toList()); + Map queryBackendMap = new HashMap<>(); + for (String category : ruleCategories) { + Map fieldMappings = ruleFieldMappings.get(category); + queryBackendMap.put(category, new OSQueryBackend(fieldMappings, true, true)); + } + + // Pair of RuleId - MonitorId for existing monitors of the detector + Map monitorPerRule = detector.getRuleIdMonitorIdMap(); + + for (Pair query : bucketLevelRules) { + Rule rule = query.getRight(); + if (rule.getAggregationQueries() != null) { + // Detect if the monitor should be added or updated + if (monitorPerRule.containsKey(rule.getId())) { + String monitorId = monitorPerRule.get(rule.getId()); + monitorsToBeUpdated.add(createBucketLevelMonitorRequest(query.getRight(), + detector, + refreshPolicy, + monitorId, + Method.PUT, + queryBackendMap.get(rule.getCategory()))); + } else { + monitorsToBeAdded.add(createBucketLevelMonitorRequest(query.getRight(), + detector, + refreshPolicy, + Monitor.NO_ID, + Method.POST, + queryBackendMap.get(rule.getCategory()))); + } + } + } + + List> docLevelRules = rulesById.stream().filter(it -> !it.getRight().isAggregationRule()).collect( + Collectors.toList()); - // Pair of RuleId - MonitorId for existing monitors of the detector - Map monitorPerRule = detector.getRuleIdMonitorIdMap(); - - for (Pair query: bucketLevelRules) { - Rule rule = query.getRight(); - if (rule.getAggregationQueries() != null){ - // Detect if the monitor should be added or updated - if (monitorPerRule.containsKey(rule.getId())) { - String monitorId = monitorPerRule.get(rule.getId()); - monitorsToBeUpdated.add(createBucketLevelMonitorRequest(query.getRight(), - detector, - refreshPolicy, - monitorId, - Method.PUT, - queryBackendMap.get(rule.getCategory()))); - } else { - monitorsToBeAdded.add(createBucketLevelMonitorRequest(query.getRight(), - detector, - refreshPolicy, - Monitor.NO_ID, - Method.POST, - queryBackendMap.get(rule.getCategory()))); + // Process doc level monitors + if (!docLevelRules.isEmpty()) { + if (detector.getDocLevelMonitorId() == null) { + monitorsToBeAdded.add(createDocLevelMonitorRequest(docLevelRules, detector, refreshPolicy, Monitor.NO_ID, Method.POST)); + } else { + monitorsToBeUpdated.add(createDocLevelMonitorRequest(docLevelRules, detector, refreshPolicy, detector.getDocLevelMonitorId(), Method.PUT)); + } + } + + List monitorIdsToBeDeleted = detector.getRuleIdMonitorIdMap().values().stream().collect(Collectors.toList()); + monitorIdsToBeDeleted.removeAll(monitorsToBeUpdated.stream().map(IndexMonitorRequest::getMonitorId).collect( + Collectors.toList())); + + updateAlertingMonitors(monitorsToBeAdded, monitorsToBeUpdated, monitorIdsToBeDeleted, refreshPolicy, listener); + } catch (IOException | SigmaError ex) { + listener.onFailure(ex); } } - } - } - List> docLevelRules = rulesById.stream().filter(it -> !it.getRight().isAggregationRule()).collect( - Collectors.toList()); + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } else { + List> docLevelRules = rulesById.stream().filter(it -> !it.getRight().isAggregationRule()).collect( + Collectors.toList()); - // Process doc level monitors - if (!docLevelRules.isEmpty()) { - if (detector.getDocLevelMonitorId() == null) { - monitorsToBeAdded.add(createDocLevelMonitorRequest(docLevelRules, detector, refreshPolicy, Monitor.NO_ID, Method.POST)); - } else { - monitorsToBeUpdated.add(createDocLevelMonitorRequest(docLevelRules, detector, refreshPolicy, detector.getDocLevelMonitorId(), Method.PUT)); + // Process doc level monitors + if (!docLevelRules.isEmpty()) { + if (detector.getDocLevelMonitorId() == null) { + monitorsToBeAdded.add(createDocLevelMonitorRequest(docLevelRules, detector, refreshPolicy, Monitor.NO_ID, Method.POST)); + } else { + monitorsToBeUpdated.add(createDocLevelMonitorRequest(docLevelRules, detector, refreshPolicy, detector.getDocLevelMonitorId(), Method.PUT)); + } } - } - List monitorIdsToBeDeleted = detector.getRuleIdMonitorIdMap().values().stream().collect(Collectors.toList()); - monitorIdsToBeDeleted.removeAll(monitorsToBeUpdated.stream().map(IndexMonitorRequest::getMonitorId).collect( - Collectors.toList())); + List monitorIdsToBeDeleted = detector.getRuleIdMonitorIdMap().values().stream().collect(Collectors.toList()); + monitorIdsToBeDeleted.removeAll(monitorsToBeUpdated.stream().map(IndexMonitorRequest::getMonitorId).collect( + Collectors.toList())); - updateAlertingMonitors(monitorsToBeAdded, monitorsToBeUpdated, monitorIdsToBeDeleted, refreshPolicy, listener); + updateAlertingMonitors(monitorsToBeAdded, monitorsToBeUpdated, monitorIdsToBeDeleted, refreshPolicy, listener); + } } /** @@ -436,55 +515,69 @@ private IndexMonitorRequest createDocLevelMonitorRequest(List } Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, detector.getName(), detector.getEnabled(), detector.getSchedule(), detector.getLastUpdateTime(), detector.getEnabledTime(), - Monitor.MonitorType.DOC_LEVEL_MONITOR, detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(), - new DataSources(detector.getRuleIndex(), - detector.getFindingsIndex(), - detector.getFindingsIndexPattern(), - detector.getAlertsIndex(), - detector.getAlertsHistoryIndex(), - detector.getAlertsHistoryIndexPattern(), - DetectorMonitorConfig.getRuleIndexMappingsByType(detector.getDetectorType()), - true), PLUGIN_OWNER_FIELD); + Monitor.MonitorType.DOC_LEVEL_MONITOR, detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(), + new DataSources(detector.getRuleIndex(), + detector.getFindingsIndex(), + detector.getFindingsIndexPattern(), + detector.getAlertsIndex(), + detector.getAlertsHistoryIndex(), + detector.getAlertsHistoryIndexPattern(), + DetectorMonitorConfig.getRuleIndexMappingsByType(), + true), PLUGIN_OWNER_FIELD); return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); } - private List buildBucketLevelMonitorRequests(List> queries, Detector detector, WriteRequest.RefreshPolicy refreshPolicy, String monitorId, RestRequest.Method restMethod) throws IOException, SigmaError { - List ruleCategories = queries.stream().map(Pair::getRight).map(Rule::getCategory).distinct().collect( - Collectors.toList()); - Map queryBackendMap = new HashMap<>(); + private void buildBucketLevelMonitorRequests(List> queries, Detector detector, WriteRequest.RefreshPolicy refreshPolicy, String monitorId, RestRequest.Method restMethod, ActionListener> listener) throws IOException, SigmaError { - for(String category: ruleCategories) { - Map fieldMappings = logTypeService.getRuleFieldMappings(category); - queryBackendMap.put(category, new OSQueryBackend(fieldMappings, true, true)); - } + logTypeService.getRuleFieldMappings(new ActionListener<>() { + @Override + public void onResponse(Map> ruleFieldMappings) { + try { + List ruleCategories = queries.stream().map(Pair::getRight).map(Rule::getCategory).distinct().collect( + Collectors.toList()); + Map queryBackendMap = new HashMap<>(); + for(String category: ruleCategories) { + Map fieldMappings = ruleFieldMappings.get(category); + queryBackendMap.put(category, new OSQueryBackend(fieldMappings, true, true)); + } - List monitorRequests = new ArrayList<>(); + List monitorRequests = new ArrayList<>(); - for (Pair query: queries) { - Rule rule = query.getRight(); + for (Pair query: queries) { + Rule rule = query.getRight(); - // Creating bucket level monitor per each aggregation rule - if (rule.getAggregationQueries() != null){ - monitorRequests.add(createBucketLevelMonitorRequest( - query.getRight(), - detector, - refreshPolicy, - Monitor.NO_ID, - Method.POST, - queryBackendMap.get(rule.getCategory()))); + // Creating bucket level monitor per each aggregation rule + if (rule.getAggregationQueries() != null){ + monitorRequests.add(createBucketLevelMonitorRequest( + query.getRight(), + detector, + refreshPolicy, + Monitor.NO_ID, + Method.POST, + queryBackendMap.get(rule.getCategory()))); + } + } + listener.onResponse(monitorRequests); + } catch (IOException | SigmaError ex) { + listener.onFailure(ex); + } } - } - return monitorRequests; + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); } private IndexMonitorRequest createBucketLevelMonitorRequest( - Rule rule, - Detector detector, - WriteRequest.RefreshPolicy refreshPolicy, - String monitorId, - RestRequest.Method restMethod, - QueryBackend queryBackend + Rule rule, + Detector detector, + WriteRequest.RefreshPolicy refreshPolicy, + String monitorId, + RestRequest.Method restMethod, + QueryBackend queryBackend ) throws SigmaError { List indices = detector.getInputs().get(0).getIndices(); @@ -561,7 +654,7 @@ private IndexMonitorRequest createBucketLevelMonitorRequest( detector.getAlertsIndex(), detector.getAlertsHistoryIndex(), detector.getAlertsHistoryIndexPattern(), - DetectorMonitorConfig.getRuleIndexMappingsByType(detector.getDetectorType()), + DetectorMonitorConfig.getRuleIndexMappingsByType(), true), PLUGIN_OWNER_FIELD); return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexRuleAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexRuleAction.java index d846ce4fc..180616636 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexRuleAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexRuleAction.java @@ -44,6 +44,7 @@ import org.opensearch.securityanalytics.action.IndexRuleResponse; import org.opensearch.securityanalytics.logtype.LogTypeService; import org.opensearch.securityanalytics.model.Detector; +import org.opensearch.securityanalytics.model.FieldMappingDoc; import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.rules.backend.OSQueryBackend; import org.opensearch.securityanalytics.rules.backend.QueryBackend; @@ -179,31 +180,42 @@ public void onFailure(Exception e) { void prepareRuleIndexing() { String rule = request.getRule(); String category = request.getLogType().toLowerCase(Locale.ROOT); + logTypeService.getRuleFieldMappings( + category, + new ActionListener<>() { + @Override + public void onResponse(Map fieldMappings) { + try { + SigmaRule parsedRule = SigmaRule.fromYaml(rule, true); + if (parsedRule.getErrors() != null && parsedRule.getErrors().size() > 0) { + onFailures(parsedRule.getErrors().toArray(new SigmaError[]{})); + return; + } + QueryBackend backend = new OSQueryBackend(fieldMappings, true, true); + + List queries = backend.convertRule(parsedRule); + Set queryFieldNames = backend.getQueryFields().keySet(); + Rule ruleDoc = new Rule( + NO_ID, NO_VERSION, parsedRule, category, + queries, + new ArrayList<>(queryFieldNames), + rule + ); + indexRule(ruleDoc, fieldMappings); + } catch (IOException | SigmaError e) { + onFailures(e); + } + } - try { - SigmaRule parsedRule = SigmaRule.fromYaml(rule, true); - if (parsedRule.getErrors() != null && parsedRule.getErrors().size() > 0) { - onFailures(parsedRule.getErrors().toArray(new SigmaError[]{})); - return; + @Override + public void onFailure(Exception e) { + onFailures(e); + } } - - Map fieldMappings = logTypeService.getRuleFieldMappings(category); - final QueryBackend backend = new OSQueryBackend(fieldMappings, true, true); - List queries = backend.convertRule(parsedRule); - Set queryFieldNames = backend.getQueryFields().keySet(); - Rule ruleDoc = new Rule( - NO_ID, NO_VERSION, parsedRule, category, - queries, - new ArrayList<>(queryFieldNames), - rule - ); - indexRule(ruleDoc); - } catch (IOException | SigmaError e) { - onFailures(e); - } + ); } - void indexRule(Rule rule) throws IOException { + void indexRule(Rule rule, Map ruleFieldMappings) throws IOException { if (request.getMethod() == RestRequest.Method.PUT) { if (detectorIndices.detectorIndexExists()) { searchDetectors(request.getRuleId(), new ActionListener<>() { @@ -232,13 +244,13 @@ public void onResponse(SearchResponse response) { detectors.add(detector); } - updateRule(rule, detectors); + updateRule(rule, ruleFieldMappings, detectors); } catch (IOException ex) { onFailures(ex); } } else { try { - updateRule(rule, List.of()); + updateRule(rule, ruleFieldMappings, List.of()); } catch (IOException ex) { onFailures(ex); } @@ -251,7 +263,7 @@ public void onFailure(Exception e) { } }); } else { - updateRule(rule, List.of()); + updateRule(rule, ruleFieldMappings, List.of()); } } else { IndexRequest indexRequest = new IndexRequest(Rule.CUSTOM_RULES_INDEX) @@ -263,7 +275,11 @@ public void onFailure(Exception e) { @Override public void onResponse(IndexResponse response) { rule.setId(response.getId()); - onOperation(response, rule); + updateFieldMappings( + rule, + ruleFieldMappings, + ActionListener.wrap(() -> onOperation(response, rule) ) + ); } @Override @@ -312,7 +328,7 @@ public void onFailure(Exception e) { } } - private void updateRule(Rule rule, List detectors) throws IOException { + private void updateRule(Rule rule, Map ruleFieldMappings, List detectors) throws IOException { IndexRequest indexRequest = new IndexRequest(Rule.CUSTOM_RULES_INDEX) .setRefreshPolicy(request.getRefreshPolicy()) .source(rule.toXContent(XContentFactory.jsonBuilder(), new ToXContent.MapParams(Map.of("with_type", "true")))) @@ -324,11 +340,13 @@ private void updateRule(Rule rule, List detectors) throws IOException public void onResponse(IndexResponse response) { rule.setId(response.getId()); - if (detectors.size() > 0) { - updateDetectors(response, rule, detectors); - } else { - onOperation(response, rule); - } + updateFieldMappings(rule, ruleFieldMappings, ActionListener.wrap(() -> { + if (detectors.size() > 0) { + updateDetectors(response, rule, detectors); + } else { + onOperation(response, rule); + } + })); } @Override @@ -338,6 +356,21 @@ public void onFailure(Exception e) { }); } + private void updateFieldMappings(Rule rule, Map ruleFieldMappings, ActionListener listener) { + List fieldMappingDocs = new ArrayList<>(); + rule.getQueryFieldNames().forEach(field -> { + FieldMappingDoc mappingDoc = new FieldMappingDoc(field.getValue(), Set.of(rule.getCategory())); + if (ruleFieldMappings.containsKey(field.getValue())) { + mappingDoc.getSchemaFields().put(logTypeService.getDefaultSchemaField(), ruleFieldMappings.get(field.getValue())); + } + fieldMappingDocs.add(mappingDoc); + }); + logTypeService.indexFieldMappings( + fieldMappingDocs, + ActionListener.wrap(listener::onResponse, this::onFailures) + ); + } + private void onComplete(IndexResponse response, Rule rule, int target) { if (checker.incrementAndGet() == target) { onOperation(response, rule); diff --git a/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java b/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java index 590a14488..ea0e86806 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java @@ -262,7 +262,7 @@ private void ingestQueries(Map> logIndexToRules, WriteReque List queries = new ArrayList<>(); for (Map.Entry> logIndexToRule: logIndexToRules.entrySet()) { - Map fieldMappings = logTypeService.getRuleFieldMappings(logIndexToRule.getKey()); + Map fieldMappings = logTypeService.getRuleFieldMappingsForBuiltinLogType(logIndexToRule.getKey()); final QueryBackend backend = new OSQueryBackend(fieldMappings, true, true); queries.addAll(getQueries(backend, logIndexToRule.getKey(), logIndexToRule.getValue())); } @@ -284,7 +284,7 @@ private List getQueries(QueryBackend backend, String category, List(queryFieldNames), ruleStr ); diff --git a/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java b/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java index b866bf2a6..69639d4e2 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java @@ -9,6 +9,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.ActionListener; @@ -22,6 +23,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; +import org.opensearch.securityanalytics.logtype.LogTypeService; public class RuleTopicIndices { private static final Logger log = LogManager.getLogger(DetectorIndices.class); @@ -29,10 +31,12 @@ public class RuleTopicIndices { private final Client client; private final ClusterService clusterService; + private final LogTypeService logTypeService; - public RuleTopicIndices(Client client, ClusterService clusterService) { + public RuleTopicIndices(Client client, ClusterService clusterService, LogTypeService logTypeService) { this.client = client; this.clusterService = clusterService; + this.logTypeService = logTypeService; } public static String ruleTopicIndexSettings() throws IOException { @@ -41,33 +45,35 @@ public static String ruleTopicIndexSettings() throws IOException { public void initRuleTopicIndexTemplate(ActionListener actionListener) throws IOException { if (!ruleTopicIndexTemplateExists()) { - // Compose list of all patterns to cover all query indices - List indexPatterns = new ArrayList<>(); - for(String ruleIndex : DetectorMonitorConfig.getAllRuleIndices()) { - indexPatterns.add(ruleIndex + "*"); - } + getAllRuleIndices(ActionListener.wrap(allRuleIndices -> { + // Compose list of all patterns to cover all query indices + List indexPatterns = new ArrayList<>(); + for(String ruleIndex : allRuleIndices) { + indexPatterns.add(ruleIndex + "*"); + } - ComposableIndexTemplate template = new ComposableIndexTemplate( - indexPatterns, - new Template( - Settings.builder().loadFromSource(ruleTopicIndexSettings(), XContentType.JSON).build(), - null, - null - ), - null, - 500L, - null, - null - ); + ComposableIndexTemplate template = new ComposableIndexTemplate( + indexPatterns, + new Template( + Settings.builder().loadFromSource(ruleTopicIndexSettings(), XContentType.JSON).build(), + null, + null + ), + null, + 500L, + null, + null + ); - client.execute( - PutComposableIndexTemplateAction.INSTANCE, - new PutComposableIndexTemplateAction.Request(DetectorMonitorConfig.OPENSEARCH_SAP_RULE_INDEX_TEMPLATE) - .indexTemplate(template) - .create(true), - actionListener - ); + client.execute( + PutComposableIndexTemplateAction.INSTANCE, + new PutComposableIndexTemplateAction.Request(DetectorMonitorConfig.OPENSEARCH_SAP_RULE_INDEX_TEMPLATE) + .indexTemplate(template) + .create(true), + actionListener + ); + }, actionListener::onFailure)); } else { actionListener.onResponse(new AcknowledgedResponse(true)); } @@ -78,4 +84,16 @@ public boolean ruleTopicIndexTemplateExists() { return clusterState.metadata().templatesV2() .get(DetectorMonitorConfig.OPENSEARCH_SAP_RULE_INDEX_TEMPLATE) != null; } + + private void getAllRuleIndices(ActionListener> listener) { + + logTypeService.getAllLogTypes(ActionListener.wrap(logTypes -> { + listener.onResponse( + logTypes + .stream() + .map(logType -> DetectorMonitorConfig.getRuleIndex(logType)) + .collect(Collectors.toList()) + ); + }, listener::onFailure)); + } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/NetFlowMapping.json b/src/main/resources/OSMapping/NetFlowMapping.json deleted file mode 100644 index efd159e60..000000000 --- a/src/main/resources/OSMapping/NetFlowMapping.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "properties": { - "srcaddr": { - "type": "alias", - "path": "netflow.event_data.SourceAddress" - }, - "dstport": { - "type": "alias", - "path": "netflow.event_data.DestinationPort" - }, - "dstaddr": { - "type": "alias", - "path": "netflow.event_data.DestAddress" - }, - "srcport": { - "type": "alias", - "path": "netflow.event_data.SourcePort" - }, - "user": { - "type": "nested", - "properties": { - "user_alias": { - "type": "alias", - "path": "user.first" - } - } - } - } -} \ No newline at end of file diff --git a/src/main/resources/OSMapping/ad_ldap/mappings.json b/src/main/resources/OSMapping/ad_ldap/mappings.json deleted file mode 100644 index 3bd2ae2b5..000000000 --- a/src/main/resources/OSMapping/ad_ldap/mappings.json +++ /dev/null @@ -1,84 +0,0 @@ -{ - "properties": { - "azure-signinlogs-properties-user_id": { - "path": "azure.signinlogs.props.user_id", - "type": "alias" - }, - "azure-activitylogs-category": { - "path": "azure.activitylogs.category", - "type": "alias" - }, - "azure-platformlogs-operation_name": { - "path": "azure.platformlogs.operation_name", - "type": "alias" - }, - "modified_properties-new_value": { - "path": "modified_properties.new_value", - "type": "alias" - }, - "azure-resource-provider": { - "path": "azure.resource.provider", - "type": "alias" - }, - "azure-signinlogs-properties-conditional_access_status": { - "path": "azure.signinlogs.props.conditional_access_status", - "type": "alias" - }, - "SearchFilter": { - "path": "SearchFilter", - "type": "alias" - }, - "azure-platformlogs-result_type": { - "path": "azure.platformlogs.result_type", - "type": "alias" - }, - "azure-signinlogs-properties-device_detail-is_compliant": { - "path": "azure.signinlogs.props.device_detail.is_compliant", - "type": "alias" - }, - "ResourceDisplayName": { - "path": "ResourceDisplayName", - "type": "alias" - }, - "azure-signinlogs-properties-authentication_requirement": { - "path": "azure.signinlogs.props.authentication_requirement", - "type": "alias" - }, - "TargetResources": { - "path": "TargetResources", - "type": "alias" - }, - "Workload": { - "path": "Workload", - "type": "alias" - }, - "azure-signinlogs-properties-device_detail-device_id": { - "path": "azure.signinlogs.props.device_detail.device_id", - "type": "alias" - }, - "azure-signinlogs-properties-resource_id": { - "path": "azure.signinlogs.props.resource_id", - "type": "alias" - }, - "azure-signinlogs-result_description": { - "path": "azure.signinlogs.result_description", - "type": "alias" - }, - "EventID": { - "path": "EventID", - "type": "alias" - }, - "azure-signinlogs-properties-network_location_details": { - "path": "azure.signinlogs.props.network_location_details", - "type": "alias" - }, - "azure-auditlogs-properties-activity_display_name": { - "path": "azure.auditlogs.props.activity_display_name", - "type": "alias" - }, - "timestamp": { - "path": "@timestamp", - "type": "alias" - } - } -} \ No newline at end of file diff --git a/src/main/resources/OSMapping/apache_access/fieldmappings.yml b/src/main/resources/OSMapping/apache_access/fieldmappings.yml deleted file mode 100644 index 9f649aeee..000000000 --- a/src/main/resources/OSMapping/apache_access/fieldmappings.yml +++ /dev/null @@ -1,8 +0,0 @@ -# this file provides pre-defined mappings for Sigma fields defined for all Sigma rules under windows log group to their corresponding ECS Fields. -fieldmappings: - EventID: event_uid - HiveName: unmapped.HiveName - fieldB: mappedB - fieldA1: mappedA - CommandLine: windows-event_data-CommandLine - creationTime: timestamp diff --git a/src/main/resources/OSMapping/apache_access/mappings.json b/src/main/resources/OSMapping/apache_access/mappings.json deleted file mode 100644 index dc7fc31b5..000000000 --- a/src/main/resources/OSMapping/apache_access/mappings.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "properties": { - "windows-event_data-CommandLine": { - "type": "alias", - "path": "CommandLine" - }, - "event_uid": { - "type": "alias", - "path": "EventID" - }, - "timestamp": { - "path": "creationTime", - "type": "alias" - } - } -} \ No newline at end of file diff --git a/src/main/resources/OSMapping/azure/fieldmappings.yml b/src/main/resources/OSMapping/azure/fieldmappings.yml deleted file mode 100644 index 91a4c4407..000000000 --- a/src/main/resources/OSMapping/azure/fieldmappings.yml +++ /dev/null @@ -1,31 +0,0 @@ -fieldmappings: - Resultdescription: azure-signinlogs-result_description - eventSource: eventSource - eventName: eventName - Status: azure-platformlogs-status - LoggedByService: azure-auditlogs-properties-logged_by_service - properties_message: properties_message - status: azure-platformlogs-status - TargetUserName: azure-signinlogs-properties-user_id - creationTime: timestamp - Category: azure-activitylogs-category - OperationName: azure-platformlogs-operation_name - ModifiedProperties_NewValue: modified_properties-new_value - ResourceProviderValue: azure-resource-provider - conditionalAccessStatus: azure-signinlogs-properties-conditional_access_status - SearchFilter: search_filter - Operation: azure-platformlogs-operation_name - ResultType: azure-platformlogs-result_type - DeviceDetail_isCompliant: azure-signinlogs-properties-device_detail-is_compliant - ResourceDisplayName: resource_display_name - AuthenticationRequirement: azure-signinlogs-properties-authentication_requirement - TargetResources: target_resources - Workload: Workload - DeviceDetail_deviceId: azure-signinlogs-properties-device_detail-device_id - OperationNameValue: azure-platformlogs-operation_name - ResourceId: azure-signinlogs-properties-resource_id - ResultDescription: azure-signinlogs-result-description - EventID: EventID - NetworkLocationDetails: azure-signinlogs-properties-network_location_details - CategoryValue: azure-activitylogs-category - ActivityDisplayName: azure-auditlogs-properties-activity_display_name diff --git a/src/main/resources/OSMapping/azure/mappings.json b/src/main/resources/OSMapping/azure/mappings.json deleted file mode 100644 index 003f29d73..000000000 --- a/src/main/resources/OSMapping/azure/mappings.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "properties": { - "azure-signinlogs-properties-user_id": { - "path": "azure.signinlogs.props.user_id", - "type": "alias" - }, - "azure-activitylogs-category": { - "path": "azure.activitylogs.category", - "type": "alias" - }, - "azure-platformlogs-operation_name": { - "path": "azure.platformlogs.operation_name", - "type": "alias" - }, - "modified_properties-new_value": { - "path": "modified_properties.new_value", - "type": "alias" - }, - "azure-resource-provider": { - "path": "azure.resource.provider", - "type": "alias" - }, - "azure-signinlogs-properties-conditional_access_status": { - "path": "azure.signinlogs.props.conditional_access_status", - "type": "alias" - }, - "SearchFilter": { - "path": "SearchFilter", - "type": "alias" - }, - "azure-platformlogs-result_type": { - "path": "azure.platformlogs.result_type", - "type": "alias" - }, - "azure-signinlogs-properties-device_detail-is_compliant": { - "path": "azure.signinlogs.props.device_detail.is_compliant", - "type": "alias" - }, - "ResourceDisplayName": { - "path": "ResourceDisplayName", - "type": "alias" - }, - "azure-signinlogs-properties-authentication_requirement": { - "path": "azure.signinlogs.props.authentication_requirement", - "type": "alias" - }, - "TargetResources": { - "path": "TargetResources", - "type": "alias" - }, - "Workload": { - "path": "Workload", - "type": "alias" - }, - "azure-signinlogs-properties-device_detail-device_id": { - "path": "azure.signinlogs.props.device_detail.device_id", - "type": "alias" - }, - "azure-signinlogs-properties-resource_id": { - "path": "azure.signinlogs.props.resource_id", - "type": "alias" - }, - "EventID": { - "path": "EventID", - "type": "alias" - }, - "azure-signinlogs-properties-network_location_details": { - "path": "azure.signinlogs.props.network_location_details", - "type": "alias" - }, - "azure-auditlogs-properties-activity_display_name": { - "path": "azure.auditlogs.props.activity_display_name", - "type": "alias" - }, - "azure-signinlogs-result-description": { - "path": "azure.signinlogs.result-description", - "type": "alias" - }, - "eventSource": { - "path": "eventSource", - "type": "alias" - }, - "eventName": { - "path": "eventName", - "type": "alias" - }, - "azure-platformlogs-status": { - "path": "azure.platformlogs.status", - "type": "alias" - }, - "azure-auditlogs-properties-logged_by_service": { - "path": "azure.auditlogs.props.logged_by_service", - "type": "alias" - }, - "properties_message": { - "path": "properties_message", - "type": "alias" - }, - "timestamp": { - "path": "@timestamp", - "type": "alias" - } - } -} \ No newline at end of file diff --git a/src/main/resources/OSMapping/cloudtrail/fieldmappings.yml b/src/main/resources/OSMapping/cloudtrail/fieldmappings.yml deleted file mode 100644 index 68f459e50..000000000 --- a/src/main/resources/OSMapping/cloudtrail/fieldmappings.yml +++ /dev/null @@ -1,17 +0,0 @@ -fieldmappings: - eventName: aws-cloudtrail-event_name - eventSource: aws-cloudtrail-event_source - eventType: aws-cloudtrail-event_type - errorMessage: aws-cloudtrail-error_message - errorCode: aws-cloudtrail-error_code - responseElements: aws-cloudtrail-response_elements-text - responseElements.pendingModifiedValues.masterUserPassword: aws-cloudtrail-response_elements-pending_modified_values-master_user_password - responseElements.publiclyAccessible: aws-cloudtrail-response_elements-publicly_accessible - requestParameters.arn: aws-cloudtrail-request_parameters-arn - requestParameters.attribute: aws-cloudtrail-request_parameters-attribute - requestParameters.userName: aws-cloudtrail-request_parameters-username - requestParameters.containerDefinitions.command: aws-cloudtrail-request_parameters-container_definitions-command - userIdentity.type: aws-cloudtrail-user_identity-type - userIdentity.arn: aws-cloudtrail-user_identity-arn - userIdentity.sessionContext.sessionIssuer.type: aws-cloudtrail-user_identity-session_context-session_issuer-type - eventTime: timestamp \ No newline at end of file diff --git a/src/main/resources/OSMapping/cloudtrail/mappings.json b/src/main/resources/OSMapping/cloudtrail/mappings.json deleted file mode 100644 index ec730def0..000000000 --- a/src/main/resources/OSMapping/cloudtrail/mappings.json +++ /dev/null @@ -1,120 +0,0 @@ -{ - "properties": { - "cloud.account.id": { - "type": "alias", - "path": "aws.cloudtrail.recipient_account_id" - }, - "cloud.region": { - "type": "alias", - "path": "region_id" - }, - "source.geo.country_iso_code": { - "type": "alias", - "path": "src_country_iso_code" - }, - "source.geo.country_name": { - "type": "alias", - "path": "src_geo_country_name" - }, - "source.as.organization.name": { - "type": "alias", - "path": "src_as_org_name" - }, - "source.ip": { - "type": "alias", - "path": "src_ip" - }, - "userIdentity.arn": { - "type": "alias", - "path": "aws.cloudtrail.user_identity.arn" - }, - "eventName": { - "type": "alias", - "path": "aws.cloudtrail.eventName" - }, - "eventType": { - "type": "alias", - "path": "aws.cloudtrail.eventType" - }, - "errorCode": { - "type": "alias", - "path": "aws.cloudtrail.error_code" - }, - "eventSource": { - "type": "alias", - "path": "aws.cloudtrail.eventType" - }, - "tlsDetails.tlsVersion": { - "type": "alias", - "path": "tls_version" - }, - "user_agent.name": { - "type": "alias", - "path": "user_agent" - }, - "threat.matched.providers": { - "type": "alias", - "path": "thread_matched_providers" - }, - "aws-cloudtrail-event_name": { - "type": "alias", - "path": "aws.cloudtrail.event_name" - }, - "aws-cloudtrail-event_source": { - "type": "alias", - "path": "aws.cloudtrail.event_source" - }, - "aws-cloudtrail-event_type": { - "type": "alias", - "path": "aws.cloudtrail.event_type" - }, - "aws-cloudtrail-error_message": { - "type": "alias", - "path": "aws.cloudtrail.error_message" - }, - "aws-cloudtrail-error_code": { - "type": "alias", - "path": "aws.cloudtrail.error_code" - }, - "aws-cloudtrail-response_elements-text": { - "type": "alias", - "path": "aws.cloudtrail.response_elements.text" - }, - "aws-cloudtrail-response_elements-pending_modified_values-master_user_password": { - "type": "alias", - "path": "aws.cloudtrail.response_elements.pending_modified_values.master_user_password" - }, - "aws-cloudtrail-response_elements-publicly_accessible": { - "type": "alias", - "path": "aws.cloudtrail.response_elements.publicly_accessible" - }, - "aws-cloudtrail-request_parameters-arn": { - "type": "alias", - "path": "aws.cloudtrail.request_parameters.arn" - }, - "aws-cloudtrail-request_parameters-attribute": { - "type": "alias", - "path": "aws.cloudtrail.request_parameters.attribute" - }, - "aws-cloudtrail-request_parameters-username": { - "type": "alias", - "path": "aws.cloudtrail.request_parameters.username" - }, - "aws-cloudtrail-request_parameters-container_definitions-command": { - "type": "alias", - "path": "aws.cloudtrail.request_parameters.container_definitions.command" - }, - "aws-cloudtrail-user_identity-session_context-session_issuer-type": { - "type": "alias", - "path": "aws.cloudtrail.user_identity.session_context.session_issuer.type" - }, - "aws-cloudtrail-user_identity-arn": { - "type": "alias", - "path": "aws-cloudtrail-user_identity-arn" - }, - "timestamp": { - "path": "@timestamp", - "type": "alias" - } - } -} \ No newline at end of file diff --git a/src/main/resources/OSMapping/cloudtrail_logtype.json b/src/main/resources/OSMapping/cloudtrail_logtype.json index a6be253a6..f326a7146 100644 --- a/src/main/resources/OSMapping/cloudtrail_logtype.json +++ b/src/main/resources/OSMapping/cloudtrail_logtype.json @@ -5,67 +5,218 @@ "mappings": [ { "raw_field":"eventName", - "ecs":"aws.cloudtrail.event_name" + "ecs":"aws.cloudtrail.event_name", + "ocsf": "api.operation" }, { "raw_field":"eventSource", - "ecs":"aws.cloudtrail.event_source" + "ecs":"aws.cloudtrail.event_source", + "ocsf": "api.service.name" + }, + { + "raw_field":"eventVersion", + "ecs":"aws.cloudtrail.event_version", + "ocsf": "metadata.product.version" + }, + { + "raw_field":"eventID", + "ecs":"aws.cloudtrail.event_id", + "ocsf": "metadata.uid" }, { "raw_field":"eventType", - "ecs":"aws.cloudtrail.event_type" + "ecs":"aws.cloudtrail.event_type", + "ocsf": "unmapped.eventType" + }, + { + "raw_field":"eventCategory", + "ecs":"aws.cloudtrail.event_category", + "ocsf": "metadata.product.feature.name" }, { "raw_field":"errorMessage", - "ecs":"aws.cloudtrail.error_message" + "ecs":"aws.cloudtrail.error_message", + "ocsf": "api.response.message" }, { "raw_field":"errorCode", - "ecs":"aws.cloudtrail.error_code" + "ecs":"aws.cloudtrail.error_code", + "ocsf": "api.response.error" + }, + { + "raw_field":"apiVersion", + "ecs":"aws.cloudtrail.api_version", + "ocsf": "api.version" + }, + { + "raw_field":"awsRegion", + "ecs":"aws.cloudtrail.aws_region", + "ocsf": "cloud.region" + }, + { + "raw_field":"additionalEventData.LoginTo", + "ecs":"aws.cloudtrail.additional_event_data.loginTo", + "ocsf": "dst_endpoint.svc_name" + }, + { + "raw_field":"additionalEventData.MFAUsed", + "ecs":"aws.cloudtrail.additional_event_data.mfaUsed", + "ocsf": "mfa" }, { "raw_field":"responseElements", - "ecs":"aws.cloudtrail.response_elements.text" + "ecs":"aws.cloudtrail.response_elements.text", + "ocsf": "unmapped.responseElements" + }, + { + "raw_field":"requestID", + "ecs":"aws.cloudtrail.request_id", + "ocsf": "api.request.uid" + }, + { + "raw_field":"sourceIPAddress", + "ecs":"aws.cloudtrail.source_ip_address", + "ocsf": "src_endpoint.ip" + }, + { + "raw_field":"userAgent", + "ecs":"aws.cloudtrail.user_agent", + "ocsf": "http_request.user_agent" + }, + { + "raw_field":"vpcEndpointId", + "ecs":"aws.cloudtrail.vpc_endpoint_id", + "ocsf": "src_endpoint.uid" }, { "raw_field":"responseElements.pendingModifiedValues.masterUserPassword", - "ecs":"aws.cloudtrail.response_elements.pending_modified_values.master_user_password" + "ecs":"aws.cloudtrail.response_elements.pending_modified_values.master_user_password", + "ocsf": "unmapped.responseElements.pendingModifiedValues.masterUserPassword" }, { "raw_field":"responseElements.publiclyAccessible", - "ecs":"aws.cloudtrail.response_elements.publicly_accessible" + "ecs":"aws.cloudtrail.response_elements.publicly_accessible", + "ocsf": "unmapped.responseElements.publiclyAccessible" + }, + { + "raw_field":"responseElements.ConsoleLogin", + "ecs":"aws.cloudtrail.response_elements.publicly_accessible", + "ocsf": "status_id" }, { "raw_field":"requestParameters.arn", - "ecs":"aws.cloudtrail.request_parameters.arn" + "ecs":"aws.cloudtrail.request_parameters.arn", + "ocsf": "unmapped.requestParameters.arn" }, { "raw_field":"requestParameters.attribute", - "ecs":"aws.cloudtrail.request_parameters.attribute" + "ecs":"aws.cloudtrail.request_parameters.attribute", + "ocsf": "unmapped.requestParameters.attribute" }, { "raw_field":"requestParameters.userName", - "ecs":"aws.cloudtrail.request_parameters.username" + "ecs":"aws.cloudtrail.request_parameters.username", + "ocsf": "unmapped.requestParameters.userName" + }, + { + "raw_field":"requestParameters.roleArn", + "ecs":"aws.cloudtrail.request_parameters.roleArn", + "ocsf": "user.uuid" + }, + { + "raw_field":"requestParameters.roleSessionName", + "ecs":"aws.cloudtrail.request_parameters.roleSessionName", + "ocsf": "user.name" }, { "raw_field":"requestParameters.containerDefinitions.command", - "ecs":"aws.cloudtrail.request_parameters.container_definitions.command" + "ecs":"aws.cloudtrail.request_parameters.container_definitions.command", + "ocsf": "unmapped.requestParameters.containerDefinitions.command" }, { "raw_field":"userIdentity.type", - "ecs":"aws.cloudtrail.user_identity.type" + "ecs":"aws.cloudtrail.user_identity.type", + "ocsf": "actor.user.type" + }, + { + "raw_field":"userIdentity.principalId", + "ecs":"aws.cloudtrail.user_identity.principalId", + "ocsf": "actor.user.uid" }, { "raw_field":"userIdentity.arn", - "ecs":"aws.cloudtrail.user_identity.arn" + "ecs":"aws.cloudtrail.user_identity.arn", + "ocsf": "actor.user.uuid" + }, + { + "raw_field":"userIdentity.accountId", + "ecs":"aws.cloudtrail.user_identity.accountId", + "ocsf": "actor.user.account_uid" + }, + { + "raw_field":"userIdentity.accessKeyId", + "ecs":"aws.cloudtrail.user_identity.accessKeyId", + "ocsf": "actor.user.credential_uid" + }, + { + "raw_field":"userIdentity.identityProvider", + "ecs":"aws.cloudtrail.user_identity.identityProvider", + "ocsf": "actor.idp.name" + }, + { + "raw_field":"userIdentity.userName", + "ecs":"aws.cloudtrail.user_identity.userName", + "ocsf": "actor.user.name" + }, + { + "raw_field":"userIdentity.invokedBy", + "ecs":"aws.cloudtrail.user_identity.invokedBy", + "ocsf": "actor.invoked_by" }, { "raw_field":"userIdentity.sessionContext.sessionIssuer.type", - "ecs":"aws.cloudtrail.user_identity.session_context.session_issuer.type" + "ecs":"aws.cloudtrail.user_identity.session_context.session_issuer.type", + "ocsf": "unmapped.userIdentity.sessionContext.sessionIssuer.type" + }, + { + "raw_field":"userIdentity.sessionContext.sessionIssuer.arn", + "ecs":"aws.cloudtrail.user_identity.session_context.session_issuer.arn", + "ocsf": "actor.session.issuer" + }, + { + "raw_field":"userIdentity.sessionContext.attributes.creationDate", + "ecs":"aws.cloudtrail.user_identity.session_context.attributes.creationDate", + "ocsf": "actor.session.created_time" + }, + { + "raw_field":"userIdentity.sessionContext.attributes.mfaAuthenticated", + "ecs":"aws.cloudtrail.user_identity.session_context.attributes.mfaAuthenticated", + "ocsf": "actor.session.mfa" + }, + { + "raw_field":"userIdentity.webIdFederationData.federatedProvider", + "ecs":"aws.cloudtrail.user_identity.web_id_federation_data.federatedProvider", + "ocsf": "actor.idp.name" + }, + { + "raw_field":"resources[].ARN", + "ecs":"aws.cloudtrail.resources.ARN", + "ocsf": "resources[].uid" + }, + { + "raw_field":"resources[].accountId", + "ecs":"aws.cloudtrail.resources.account_uid", + "ocsf": "resources[].account_uid" + }, + { + "raw_field":"resources[].type", + "ecs":"aws.cloudtrail.resources.type", + "ocsf": "resources[].type" }, { "raw_field":"eventTime", - "ecs":"timestamp" + "ecs":"timestamp", + "ocsf": "time" } ] } \ No newline at end of file diff --git a/src/main/resources/OSMapping/dns/fieldmappings.yml b/src/main/resources/OSMapping/dns/fieldmappings.yml deleted file mode 100644 index 7672ee77a..000000000 --- a/src/main/resources/OSMapping/dns/fieldmappings.yml +++ /dev/null @@ -1,5 +0,0 @@ -fieldmappings: - record_type: dns-answers-type - query: dns-question-name - parent_domain: dns-question-registered_domain - creationTime: timestamp \ No newline at end of file diff --git a/src/main/resources/OSMapping/dns/mappings.json b/src/main/resources/OSMapping/dns/mappings.json deleted file mode 100644 index 0905e2c9b..000000000 --- a/src/main/resources/OSMapping/dns/mappings.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "properties": { - "dns-answers-type": { - "type": "alias", - "path": "dns.answers.type" - }, - "dns-question-name": { - "type": "alias", - "path": "dns.question.name" - }, - "dns-question-registered_domain": { - "type": "alias", - "path": "dns.question.registered_domain" - }, - "timestamp": { - "path": "creationTime", - "type": "alias" - } - } -} diff --git a/src/main/resources/OSMapping/dns_logtype.json b/src/main/resources/OSMapping/dns_logtype.json index fa2294507..b286eb342 100644 --- a/src/main/resources/OSMapping/dns_logtype.json +++ b/src/main/resources/OSMapping/dns_logtype.json @@ -5,19 +5,113 @@ "mappings": [ { "raw_field":"record_type", - "ecs":"dns.answers.type" + "ecs":"dns.answers.type", + "ocsf": "unmapped.record_type" + }, + { + "raw_field":"answers[].Type", + "ecs":"aws.route53.answers.Type", + "ocsf": "answers[].type" + }, + { + "raw_field":"answers[].Rdata", + "ecs":"aws.route53.answers.Rdata", + "ocsf": "answers[].rdata" + }, + { + "raw_field":"answers[].Class", + "ecs":"aws.route53.answers.Class", + "ocsf": "answers[].class" }, { "raw_field":"query", - "ecs":"dns.question.name" + "ecs":"dns.question.name", + "ocsf": "unmapped.query" + }, + { + "raw_field":"query_name", + "ecs":"aws.route53.query_name", + "ocsf": "query.hostname" }, { "raw_field":"parent_domain", - "ecs":"dns.question.registered_domain" + "ecs":"dns.question.registered_domain", + "ocsf": "unmapped.parent_domain" + }, + { + "raw_field":"version", + "ecs":"aws.route53.version", + "ocsf": "metadata.product.version" + }, + { + "raw_field":"account_id", + "ecs":"aws.route53.account_id", + "ocsf": "cloud.account_uid" + }, + { + "raw_field":"region", + "ecs":"aws.route53.region", + "ocsf": "cloud.region" + }, + { + "raw_field":"vpc_id", + "ecs":"aws.route53.vpc_id", + "ocsf": "src_endpoint.vpc_uid" + }, + { + "raw_field":"query_timestamp", + "ecs":"aws.route53.query_timestamp", + "ocsf": "time" + }, + { + "raw_field":"query_class", + "ecs":"aws.route53.query_class", + "ocsf": "query.class" + }, + { + "raw_field":"query_type", + "ecs":"aws.route53.query_type", + "ocsf": "query.type" + }, + { + "raw_field":"srcaddr", + "ecs":"aws.route53.srcaddr", + "ocsf": "src_endpoint.ip" + }, + { + "raw_field":"srcport", + "ecs":"aws.route53.srcport", + "ocsf": "src_endpoint.port" + }, + { + "raw_field":"transport", + "ecs":"aws.route53.transport", + "ocsf": "connection_info.protocol_name" + }, + { + "raw_field":"srcids.instance", + "ecs":"aws.route53.srcids.instance", + "ocsf": "src_endpoint.instance_uid" + }, + { + "raw_field":"srcids.resolver_endpoint", + "ecs":"aws.route53.srcids.resolver_endpoint", + "ocsf": "dst_endpoint.instance_uid" + }, + { + "raw_field":"srcids.resolver_network_interface", + "ecs":"aws.route53.srcids.resolver_network_interface", + "ocsf": "dst_endpoint.interface_uid" + }, + { + "raw_field":"firewall_rule_action", + "ecs":"aws.route53.srcids.firewall_rule_action", + "ocsf": "disposition_id" }, { "raw_field":"creationTime", - "ecs":"timestamp" + "ecs":"timestamp", + "ocsf": "unmapped.creationTime" } ] } \ No newline at end of file diff --git a/src/main/resources/OSMapping/github/fieldmappings.yml b/src/main/resources/OSMapping/github/fieldmappings.yml deleted file mode 100644 index 8b0c86406..000000000 --- a/src/main/resources/OSMapping/github/fieldmappings.yml +++ /dev/null @@ -1,2 +0,0 @@ -fieldmappings: - action: github-action \ No newline at end of file diff --git a/src/main/resources/OSMapping/github/mappings.json b/src/main/resources/OSMapping/github/mappings.json deleted file mode 100644 index 6041dcbcb..000000000 --- a/src/main/resources/OSMapping/github/mappings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "properties": { - "github-action": { - "type": "alias", - "path": "github.action" - } - } -} \ No newline at end of file diff --git a/src/main/resources/OSMapping/gworkspace/fieldmappings.yml b/src/main/resources/OSMapping/gworkspace/fieldmappings.yml deleted file mode 100644 index ef8e00a72..000000000 --- a/src/main/resources/OSMapping/gworkspace/fieldmappings.yml +++ /dev/null @@ -1,4 +0,0 @@ -fieldmappings: - eventSource: google_workspace-admin-service-name - eventName: google_workspace-event-name # not sure if beat is missing this one... I see only google_workspace.event.type which IS NOT eventName - new_value: google_workspace-admin-new_value diff --git a/src/main/resources/OSMapping/gworkspace/mappings.json b/src/main/resources/OSMapping/gworkspace/mappings.json deleted file mode 100644 index 15ff468f2..000000000 --- a/src/main/resources/OSMapping/gworkspace/mappings.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "properties": { - "google_workspace-admin-service-name": { - "type": "alias", - "path": "google_workspace.admin.service.name" - }, - "google_workspace-event-name": { - "type": "alias", - "path": "google_workspace.event.name" - }, - "google_workspace-admin-new_value": { - "type": "alias", - "path": "google_workspace.admin.new_value" - } - } -} \ No newline at end of file diff --git a/src/main/resources/OSMapping/linux/fieldmappings.yml b/src/main/resources/OSMapping/linux/fieldmappings.yml deleted file mode 100644 index d58617df6..000000000 --- a/src/main/resources/OSMapping/linux/fieldmappings.yml +++ /dev/null @@ -1,15 +0,0 @@ -fieldmappings: - name: user-filesystem-name - a0: auditd-log-a0 - comm: auditd-log-comm - exe: auditd-log-exe - uid: auditd-log-uid - USER: system-auth-user - User: system-auth-user - Image: process-exe - DestinationHostname: rsa-web-remote_domain - CommandLine: process-command_line - ParentImage: process-parent-executable - CurrentDirectory: process-working_directory - LogonId: process-real_user-id - creationTime: timestamp diff --git a/src/main/resources/OSMapping/linux/mappings.json b/src/main/resources/OSMapping/linux/mappings.json deleted file mode 100644 index 5c69c0c48..000000000 --- a/src/main/resources/OSMapping/linux/mappings.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "properties": { - "user-filesystem-name": { - "path": "user.filesystem.name", - "type": "alias" - }, - "auditd-log-a0": { - "path": "auditd.log.a0", - "type": "alias" - }, - "auditd-log-comm": { - "path": "auditd.log.comm", - "type": "alias" - }, - "auditd-log-exe": { - "path": "auditd.log.exe", - "type": "alias" - }, - "auditd-log-uid": { - "path": "auditd.log.uid", - "type": "alias" - }, - "system-auth-user": { - "path": "system.auth.user", - "type": "alias" - }, - "process-exe": { - "path": "process.exe", - "type": "alias" - }, - "rsa-web-remote_domain": { - "path": "rsa.web.remote_domain", - "type": "alias" - }, - "process-command_line": { - "path": "process.command_line", - "type": "alias" - }, - "process-parent-executable": { - "path": "process.parent.executable", - "type": "alias" - }, - "process-working_directory": { - "path": "process.working_directory", - "type": "alias" - }, - "process-real_user-id": { - "path": "process.real_user.id", - "type": "alias" - }, - "timestamp": { - "path": "creationTime", - "type": "alias" - } - } -} \ No newline at end of file diff --git a/src/main/resources/OSMapping/m365/fieldmappings.yml b/src/main/resources/OSMapping/m365/fieldmappings.yml deleted file mode 100644 index 6f641a816..000000000 --- a/src/main/resources/OSMapping/m365/fieldmappings.yml +++ /dev/null @@ -1,5 +0,0 @@ -fieldmappings: - eventSource: rsa-misc-event_source - eventName: rsa-misc-event_desc - status: rsa-misc-status - Payload: rsa-misc-payload_dst \ No newline at end of file diff --git a/src/main/resources/OSMapping/m365/mappings.json b/src/main/resources/OSMapping/m365/mappings.json deleted file mode 100644 index 98c586796..000000000 --- a/src/main/resources/OSMapping/m365/mappings.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "properties": { - "rsa-misc-event_desc": { - "type": "alias", - "path": "rsa.misc.event_desc" - }, - "rsa-misc-event_source": { - "type": "alias", - "path": "rsa.misc.event_source" - }, - "rsa-misc-status": { - "type": "alias", - "path": "rsa.misc.status" - }, - "rsa-misc-payload_dst": { - "type": "alias", - "path": "rsa.misc.payload_dst" - } - } -} \ No newline at end of file diff --git a/src/main/resources/OSMapping/mapper_topics.json b/src/main/resources/OSMapping/mapper_topics.json deleted file mode 100644 index 3dc5090ba..000000000 --- a/src/main/resources/OSMapping/mapper_topics.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "netflow": "OSMapping/network/NetFlowMapping.json", - "ad_ldap": "OSMapping/ad_ldap/mappings.json", - "apache_access": "OSMapping/apache_access/mappings.json", - "cloudtrail": "OSMapping/cloudtrail/mappings.json", - "dns": "OSMapping/dns/mappings.json", - "network": "OSMapping/network/mappings.json", - "others_application": "OSMapping/others_application/mappings.json", - "others_apt": "OSMapping/others_apt/mappings.json", - "others_cloud": "OSMapping/others_cloud/mappings.json", - "others_compliance": "OSMapping/others_compliance/mappings.json", - "others_macos": "OSMapping/others_macos/mappings.json", - "others_proxy": "OSMapping/others_proxy/mappings.json", - "others_web": "OSMapping/others_web/mappings.json", - "s3": "OSMapping/s3/mappings.json", - "okta": "OSMapping/okta/mappings.json", - "m365": "OSMapping/m365/mappings.json", - "gworkspace": "OSMapping/gworkspace/mappings.json", - "github": "OSMapping/github/mappings.json", - "azure": "OSMapping/azure/mappings.json", - "windows": "OSMapping/windows/mappings.json", - "test_windows": "OSMapping/test_windows/mappings.json", - "linux": "OSMapping/linux/mappings.json" -} \ No newline at end of file diff --git a/src/main/resources/OSMapping/netflow_logtype.json b/src/main/resources/OSMapping/netflow_logtype.json index f63bbc7c9..2b8259f4c 100644 --- a/src/main/resources/OSMapping/netflow_logtype.json +++ b/src/main/resources/OSMapping/netflow_logtype.json @@ -4,19 +4,19 @@ "is_builtin": true, "mappings": [ { - "raw_field":"source.ip", + "raw_field":"netflow.source_ipv4_address", "ecs":"source.ip" }, { - "raw_field":"source.port", + "raw_field":"netflow.source_transport_port", "ecs":"source.port" }, { - "raw_field":"destination.ip", + "raw_field":"netflow.destination_ipv4_address", "ecs":"destination.ip" }, { - "raw_field":"destination.port", + "raw_field":"netflow.destination_transport_port", "ecs":"destination.port" }, { diff --git a/src/main/resources/OSMapping/network/NetFlowMapping.json b/src/main/resources/OSMapping/network/NetFlowMapping.json deleted file mode 100644 index 9e404a67e..000000000 --- a/src/main/resources/OSMapping/network/NetFlowMapping.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "properties": { - "source.ip": { - "type": "alias", - "path": "netflow.source_ipv4_address" - }, - "source.port": { - "type": "alias", - "path": "netflow.source_transport_port" - }, - "destination.ip": { - "type": "alias", - "path": "netflow.destination_ipv4_address" - }, - "destination.port": { - "type": "alias", - "path": "netflow.destination_transport_port" - }, - "http.request.method": { - "type": "alias", - "path": "netflow.http_request_method" - }, - "http.response.status_code": { - "type": "alias", - "path": "netflow.http_status_code" - }, - "timestamp": { - "path": "creationTime", - "type": "alias" - } - } -} \ No newline at end of file diff --git a/src/main/resources/OSMapping/network/fieldmappings.yml b/src/main/resources/OSMapping/network/fieldmappings.yml deleted file mode 100644 index 2a10f24f4..000000000 --- a/src/main/resources/OSMapping/network/fieldmappings.yml +++ /dev/null @@ -1,18 +0,0 @@ -fieldmappings: - action: netflow-firewall_event - certificate.serial: zeek-x509-certificate-serial - name: zeek-smb_files-name - path: zeek-smb_files-path - dst_port: netflow-tcp_destination_port - netflow-destination_transport_port: netflow-destination_transport_port - qtype_name: zeek-dns-qtype_name - operation: zeek-dce_rpc-operation - endpoint: zeek-dce_rpc-endpoint - zeek-dce_rpc-endpoint: zeek-dce_rpc-endpoint - answers: zeek-dns-answers - query: zeek-dns-query - client_header_names: zeek-http-client_header_names - resp_mime_types: zeek-http-resp_mime_types - cipher: zeek-kerberos-cipher - request_type: zeek-kerberos-request_type - creationTime: timestamp \ No newline at end of file diff --git a/src/main/resources/OSMapping/network/mappings.json b/src/main/resources/OSMapping/network/mappings.json deleted file mode 100644 index a3743673e..000000000 --- a/src/main/resources/OSMapping/network/mappings.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "properties": { - "cloud.account.id": { - "path": "account_id", - "type": "alias" - }, - "cloud.region": { - "path": "region", - "type": "alias" - }, - "network.packets": { - "path": "network_packets", - "type": "alias" - }, - "source.packets": { - "path": "source_packets", - "type": "alias" - }, - "source.ip": { - "path": "source_ip", - "type": "alias" - }, - "source.geo.country_iso_code": { - "path": "source_geo_country_iso_code", - "type": "alias" - }, - "zeek-smb_files-name": { - "path": "zeek.smb_files.name", - "type": "alias" - }, - "zeek-x509-certificate-serial": { - "path": "zeek.x509-certificate.serial", - "type": "alias" - }, - "netflow-tcp_destination_port": { - "path": "netflow.tcp_destination_port", - "type": "alias" - }, - "netflow-destination_transport_port": { - "path": "netflow-destination_transport_port", - "type": "alias" - }, - "netflow-firewall_event": { - "path": "netflow.firewall_event", - "type": "alias" - }, - "zeek-smb_files-path": { - "path": "zeek.smb_files.path", - "type": "alias" - }, - "zeek-dns-qtype_name": { - "path": "zeek.dns.qtype_name", - "type": "alias" - }, - "zeek-dce_rpc-endpoint": { - "path": "zeek.dce_rpc.endpoint", - "type": "alias" - }, - "zeek-dce_rpc-operation": { - "path": "zeek.dce_rpc.operation", - "type": "alias" - }, - "zeek-dns-answers": { - "path": "zeek.dns.answers", - "type": "alias" - }, - "zeek-dns-query": { - "path": "zeek.dns.query", - "type": "alias" - }, - "zeek-http-client_header_names": { - "path": "zeek.http.client_header_names", - "type": "alias" - }, - "zeek-http-resp_mime_types": { - "path": "zeek.http.resp_mime_types", - "type": "alias" - }, - "zeek-kerberos-cipher": { - "path": "zeek.kerberos.cipher", - "type": "alias" - }, - "zeek-kerberos-request_type": { - "path": "zeek.kerberos.request_type", - "type": "alias" - }, - "timestamp": { - "path": "creationTime", - "type": "alias" - } - } -} diff --git a/src/main/resources/OSMapping/okta/fieldmappings.yml b/src/main/resources/OSMapping/okta/fieldmappings.yml deleted file mode 100644 index 939428d44..000000000 --- a/src/main/resources/OSMapping/okta/fieldmappings.yml +++ /dev/null @@ -1,3 +0,0 @@ -fieldmappings: - eventtype: okta-event_type - displaymessage: okta-display_message \ No newline at end of file diff --git a/src/main/resources/OSMapping/okta/mappings.json b/src/main/resources/OSMapping/okta/mappings.json deleted file mode 100644 index 7a4f300ba..000000000 --- a/src/main/resources/OSMapping/okta/mappings.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "properties": { - "okta-event_type": { - "type": "alias", - "path": "okta.event_type" - }, - "okta-display_message": { - "type": "alias", - "path": "okta.display_message" - } - } -} \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_application/fieldmappings.yml b/src/main/resources/OSMapping/others_application/fieldmappings.yml deleted file mode 100644 index 853eb1cba..000000000 --- a/src/main/resources/OSMapping/others_application/fieldmappings.yml +++ /dev/null @@ -1,4 +0,0 @@ -# this file provides pre-defined mappings for Sigma fields defined for all Sigma rules under application log group to their corresponding ECS Fields. -fieldmappings: - Signature: abusech-malware-signature - Filename: file-name \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_application/mappings.json b/src/main/resources/OSMapping/others_application/mappings.json deleted file mode 100644 index 33a27986b..000000000 --- a/src/main/resources/OSMapping/others_application/mappings.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "properties": { - "abusech-malware-signature": { - "type": "alias", - "path": "abusech.malware.signature" - }, - "file-name": { - "type": "alias", - "path": "file.name" - } - } -} \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_apt/fieldmappings.yml b/src/main/resources/OSMapping/others_apt/fieldmappings.yml deleted file mode 100644 index 26234ae87..000000000 --- a/src/main/resources/OSMapping/others_apt/fieldmappings.yml +++ /dev/null @@ -1,4 +0,0 @@ -# this file provides pre-defined mappings for Sigma fields defined for all Sigma rules under apt log group to their corresponding ECS Fields. -fieldmappings: - Image: process-exe - CommandLine: process-command_line diff --git a/src/main/resources/OSMapping/others_apt/mappings.json b/src/main/resources/OSMapping/others_apt/mappings.json deleted file mode 100644 index 2f12a6177..000000000 --- a/src/main/resources/OSMapping/others_apt/mappings.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "properties": { - "process-exe": { - "type": "alias", - "path": "process.exe" - }, - "process-command_line": { - "type": "alias", - "path": "process.command_line" - } - } -} \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_cloud/fieldmappings.yml b/src/main/resources/OSMapping/others_cloud/fieldmappings.yml deleted file mode 100644 index ea0592520..000000000 --- a/src/main/resources/OSMapping/others_cloud/fieldmappings.yml +++ /dev/null @@ -1,24 +0,0 @@ -# this file provides pre-defined mappings for Sigma fields defined for all Sigma rules under cloud log group to their corresponding ECS Fields. -fieldmappings: - eventSource: winlog-provider_name - status: azure-platformlogs-status - eventService: winlog-event_data-ServiceName - ResultType: azure-platformlogs-result_type - ResultDescription: azure-signinlogs-result_description - Operation: azure-activitylogs-operation_name - Resultdescription: azure-signinlogs-result_description - AuthenticationRequirement: azure-signinlogs-properties-authentication_requirement - Status: azure-platformlogs-status - OperationName: azure-auditlogs-operation_name - ResourceId: azure-resource-id - OperationNameValue: azure-auditlogs-operation_name - TargetResources: azure-auditlogs-properties-target_resources - NetworkLocationDetails: azure-signinlogs-properties-network_location_details - DeviceDetail.deviceId: azure-signinlogs-properties-device_detail-device_id - ResourceDisplayName: azure-signinlogs-properties-resource_display_name - conditionalAccessStatus: azure-signinlogs-properties-conditional_access_status - LoggedByService: azure-auditlogs-properties-logged_by_service - DeviceDetail.isCompliant: azure-signinlogs-properties-device_detail-is_compliant - ActivityDisplayName: azure-auditlogs-properties-activity_display_name - gcp.audit.method_name: gcp-audit-method_name - diff --git a/src/main/resources/OSMapping/others_cloud/mappings.json b/src/main/resources/OSMapping/others_cloud/mappings.json deleted file mode 100644 index dfd43cc08..000000000 --- a/src/main/resources/OSMapping/others_cloud/mappings.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "properties": { - "winlog-provider_name": { - "path": "winlog.provider_name", - "type": "alias" - }, - "azure-platformlogs-status": { - "path": "azure.platformlogs.status", - "type": "alias" - }, - "winlog-event_data-ServiceName": { - "path": "winlog.event_data.ServiceName", - "type": "alias" - }, - "azure-platformlogs-result_type": { - "path": "azure.platformlogs.result_type", - "type": "alias" - }, - "azure-signinlogs-result_description": { - "path": "azure.signinlogs.result_description", - "type": "alias" - }, - "azure-activitylogs-operation_name": { - "path": "azure.activitylogs.operation_name", - "type": "alias" - }, - "azure-signinlogs-properties-authentication_requirement": { - "path": "azure.signinlogs.props.authentication_requirement", - "type": "alias" - }, - "azure-auditlogs-operation_name": { - "path": "azure.auditlogs.operation_name", - "type": "alias" - }, - "azure-resource-id": { - "path": "azure.resource.id", - "type": "alias" - }, - "azure-auditlogs-properties-target_resources": { - "path": "azure.auditlogs.props.target_resources", - "type": "alias" - }, - "azure-signinlogs-properties-network_location_details": { - "path": "azure.signinlogs.props.network_location_details", - "type": "alias" - }, - "azure-signinlogs-properties-device_detail-device_id": { - "path": "azure.signinlogs.props.device_detail.device_id", - "type": "alias" - }, - "azure-signinlogs-properties-resource_display_name": { - "path": "azure.signinlogs.props.resource_display_name", - "type": "alias" - }, - "azure-signinlogs-properties-conditional_access_status": { - "path": "azure.signinlogs.props.conditional_access_status", - "type": "alias" - }, - "azure-auditlogs-properties-logged_by_service": { - "path": "azure.auditlogs.props.logged_by_service", - "type": "alias" - }, - "azure-signinlogs-properties-device_detail-is_compliant": { - "path": "azure.signinlogs.props.device_detail.is_compliant", - "type": "alias" - }, - "azure-auditlogs-properties-activity_display_name": { - "path": "azure.auditlogs.props.activity_display_name", - "type": "alias" - }, - "gcp-audit-method_name": { - "path": "gcp.audit.method_name", - "type": "alias" - } - } -} diff --git a/src/main/resources/OSMapping/others_compliance/fieldmappings.yml b/src/main/resources/OSMapping/others_compliance/fieldmappings.yml deleted file mode 100644 index 743cef93d..000000000 --- a/src/main/resources/OSMapping/others_compliance/fieldmappings.yml +++ /dev/null @@ -1,7 +0,0 @@ -# this file provides pre-defined mappings for Sigma fields defined for all Sigma rules under compliance log group to their corresponding ECS Fields. -fieldmappings: - host.scan.vuln: vulnerability-id - EventID: winlog-event_id - host.scan.vuln_name: vulnerability-enumeration - destination.port: netflow-tcp_destination_port - creationTime: timestamp diff --git a/src/main/resources/OSMapping/others_compliance/mappings.json b/src/main/resources/OSMapping/others_compliance/mappings.json deleted file mode 100644 index 31c3db63a..000000000 --- a/src/main/resources/OSMapping/others_compliance/mappings.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "properties": { - "vulnerability-id": { - "path": "vulnerability.id", - "type": "alias" - }, - "winlog-event_id": { - "path": "winlog.event_id", - "type": "alias" - }, - "vulnerability-enumeration": { - "path": "vulnerability.enumeration", - "type": "alias" - }, - "netflow-tcp_destination_port": { - "path": "netflow.tcp_destination_port", - "type": "alias" - }, - "timestamp": { - "path": "creationTime", - "type": "alias" - } - } -} diff --git a/src/main/resources/OSMapping/others_macos/fieldmappings.yml b/src/main/resources/OSMapping/others_macos/fieldmappings.yml deleted file mode 100644 index 1754fcddc..000000000 --- a/src/main/resources/OSMapping/others_macos/fieldmappings.yml +++ /dev/null @@ -1,7 +0,0 @@ -# this file provides pre-defined mappings for Sigma fields defined for all Sigma rules under macos log group to their corresponding ECS Fields. -fieldmappings: - EventID: event_uid - HiveName: unmapped.HiveName - fieldB: mappedB - fieldA1: mappedA - creationTime: timestamp \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_macos/mappings.json b/src/main/resources/OSMapping/others_macos/mappings.json deleted file mode 100644 index 5da91e3d9..000000000 --- a/src/main/resources/OSMapping/others_macos/mappings.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "properties": { - "macos.event_data.CommandLine": { - "type": "alias", - "path": "CommandLine" - }, - "timestamp": { - "path": "creationTime", - "type": "alias" - } - } -} \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_proxy/fieldmappings.yml b/src/main/resources/OSMapping/others_proxy/fieldmappings.yml deleted file mode 100644 index 08b377f19..000000000 --- a/src/main/resources/OSMapping/others_proxy/fieldmappings.yml +++ /dev/null @@ -1,7 +0,0 @@ -# this file provides pre-defined mappings for Sigma fields defined for all Sigma rules under proxy log group to their corresponding ECS Fields. -fieldmappings: - EventID: event_uid - HiveName: unmapped.HiveName - fieldB: mappedB - fieldA1: mappedA - creationTime: timestamp diff --git a/src/main/resources/OSMapping/others_proxy/mappings.json b/src/main/resources/OSMapping/others_proxy/mappings.json deleted file mode 100644 index a3ccdca77..000000000 --- a/src/main/resources/OSMapping/others_proxy/mappings.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "properties": { - "windows-event_data-CommandLine": { - "type": "alias", - "path": "CommandLine" - }, - "event_uid": { - "type": "alias", - "path": "EventID" - }, - "windows-hostname": { - "type": "alias", - "path": "HostName" - }, - "windows-message": { - "type": "alias", - "path": "Message" - }, - "windows-provider-name": { - "type": "alias", - "path": "Provider_Name" - }, - "windows-servicename": { - "type": "alias", - "path": "ServiceName" - }, - "timestamp": { - "path": "creationTime", - "type": "alias" - } - } -} \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_web/fieldmappings.yml b/src/main/resources/OSMapping/others_web/fieldmappings.yml deleted file mode 100644 index 17eb29dda..000000000 --- a/src/main/resources/OSMapping/others_web/fieldmappings.yml +++ /dev/null @@ -1,9 +0,0 @@ -# this file provides pre-defined mappings for Sigma fields defined for all Sigma rules under web log group to their corresponding ECS Fields. -fieldmappings: - c-uri: rsa-web-p_url - sc-status: rsa-misc-status - c-useragent: rsa-web-p_user_agent - cs-method: rsa-web-p_web_method - cs-uri-query: rsa-web-web_ref_query - cs-username: rsa-misc-username_fld - creationTime: timestamp diff --git a/src/main/resources/OSMapping/others_web/mappings.json b/src/main/resources/OSMapping/others_web/mappings.json deleted file mode 100644 index ea2dd3910..000000000 --- a/src/main/resources/OSMapping/others_web/mappings.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "properties": { - "rsa-web-p_url": { - "path": "rsa.web.p_url", - "type": "alias" - }, - "rsa-misc-status": { - "path": "rsa.misc.status", - "type": "alias" - }, - "rsa-web-p_user_agent": { - "path": "rsa.web.p_user_agent", - "type": "alias" - }, - "rsa-web-p_web_method": { - "path": "rsa.web.p_web_method", - "type": "alias" - }, - "rsa-web-web_ref_query": { - "path": "rsa.web.web_ref_query", - "type": "alias" - }, - "rsa-misc-username_fld": { - "path": "rsa.misc.username_fld", - "type": "alias" - }, - "timestamp": { - "path": "creationTime", - "type": "alias" - } - } -} diff --git a/src/main/resources/OSMapping/s3/fieldmappings.yml b/src/main/resources/OSMapping/s3/fieldmappings.yml deleted file mode 100644 index 7f56ba15d..000000000 --- a/src/main/resources/OSMapping/s3/fieldmappings.yml +++ /dev/null @@ -1,4 +0,0 @@ -fieldmappings: - eventName: aws-cloudtrail-event_name - eventSource: aws-cloudtrail-event_source - eventTime: timestamp diff --git a/src/main/resources/OSMapping/s3/mappings.json b/src/main/resources/OSMapping/s3/mappings.json deleted file mode 100644 index f08127f93..000000000 --- a/src/main/resources/OSMapping/s3/mappings.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "properties": { - "cloud.account.id": { - "type": "alias", - "path": "aws.s3access.requester" - }, - "cloud.region": { - "type": "alias", - "path": "region_id" - }, - "source.geo.country_iso_code": { - "type": "alias", - "path": "src_country_iso_code" - }, - "source.ip": { - "type": "alias", - "path": "aws.s3access.remote_ip" - }, - "Bucket": { - "type": "alias", - "path": "aws.s3access.bucket" - }, - "ErrorCode": { - "type": "alias", - "path": "aws.s3access.error_code" - }, - "HTTPstatus": { - "type": "alias", - "path": "aws.s3access.http_status" - }, - "Operation": { - "type": "alias", - "path": "aws.s3access.operation" - }, - "RequestURI_key": { - "type": "alias", - "path": "aws.s3access.key" - }, - "Requester": { - "type": "alias", - "path": "aws.s3access.requester" - }, - "aws-cloudtrail-event_source": { - "type": "alias", - "path": "aws.cloudtrail.event_source" - }, - "aws-cloudtrail-event_name": { - "type": "alias", - "path": "aws.cloudtrail.event_name" - }, - "timestamp": { - "path": "@timestamp", - "type": "alias" - } - } -} diff --git a/src/main/resources/OSMapping/test_windows/fieldmappings.yml b/src/main/resources/OSMapping/test_windows/fieldmappings.yml deleted file mode 100644 index 04fff5ead..000000000 --- a/src/main/resources/OSMapping/test_windows/fieldmappings.yml +++ /dev/null @@ -1,12 +0,0 @@ -# this file provides pre-defined mappings for Sigma fields defined for all Sigma rules under windows log group to their corresponding ECS Fields. -fieldmappings: - EventID: event_uid - HiveName: unmapped.HiveName - fieldB: mappedB - fieldA1: mappedA - CommandLine: windows-event_data-CommandLine - HostName: windows-hostname - Message: windows-message - Provider_Name: windows-provider-name - ServiceName: windows-servicename - creationTime: timestamp diff --git a/src/main/resources/OSMapping/test_windows/mappings.json b/src/main/resources/OSMapping/test_windows/mappings.json deleted file mode 100644 index a3ccdca77..000000000 --- a/src/main/resources/OSMapping/test_windows/mappings.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "properties": { - "windows-event_data-CommandLine": { - "type": "alias", - "path": "CommandLine" - }, - "event_uid": { - "type": "alias", - "path": "EventID" - }, - "windows-hostname": { - "type": "alias", - "path": "HostName" - }, - "windows-message": { - "type": "alias", - "path": "Message" - }, - "windows-provider-name": { - "type": "alias", - "path": "Provider_Name" - }, - "windows-servicename": { - "type": "alias", - "path": "ServiceName" - }, - "timestamp": { - "path": "creationTime", - "type": "alias" - } - } -} \ No newline at end of file diff --git a/src/main/resources/OSMapping/vpcflow_logtype.json b/src/main/resources/OSMapping/vpcflow_logtype.json new file mode 100644 index 000000000..9c8fc38b0 --- /dev/null +++ b/src/main/resources/OSMapping/vpcflow_logtype.json @@ -0,0 +1,132 @@ +{ + "name": "vpcflow", + "description": "VPC Flow Log Type", + "is_builtin": true, + "mappings": [ + { + "raw_field":"version", + "ecs":"netflow.version", + "ocsf": "metadata.product.version" + }, + { + "raw_field":"account_id", + "ecs":"netflow.account_id", + "ocsf": "cloud.account_uid" + }, + { + "raw_field":"region", + "ecs":"netflow.region", + "ocsf": "cloud.region" + }, + { + "raw_field":"az_id", + "ecs":"netflow.az_id", + "ocsf": "cloud.zone" + }, + { + "raw_field":"srcport", + "ecs":"netflow.srcport", + "ocsf": "src_endpoint.port" + }, + { + "raw_field":"dstport", + "ecs":"netflow.dstport", + "ocsf": "dst_endpoint.port" + }, + { + "raw_field":"protocol", + "ecs":"netflow.protocol", + "ocsf": "connection_info.protocol_num" + }, + { + "raw_field":"packets", + "ecs":"netflow.packets", + "ocsf": "traffic.packets" + }, + { + "raw_field":"bytes", + "ecs":"netflow.bytes", + "ocsf": "traffic.bytes" + }, + { + "raw_field":"end", + "ecs":"netflow.end", + "ocsf": "end_time" + }, + { + "raw_field":"tcp_flags", + "ecs":"netflow.tcp_flags", + "ocsf": "connection_info.tcp_flags" + }, + { + "raw_field":"protocol_ver", + "ecs":"netflow.protocol_ver", + "ocsf": "connection_info.protocol_ver" + }, + { + "raw_field":"pkt_src_aws_service", + "ecs":"netflow.pkt_src_aws_service", + "ocsf": "src_endpoint.svc_name" + }, + { + "raw_field":"pkt_dst_aws_service", + "ecs":"netflow.pkt_dst_aws_service", + "ocsf": "dst_endpoint.svc_name" + }, + { + "raw_field":"log_status", + "ecs":"netflow.log_status", + "ocsf": "status_code" + }, + { + "raw_field":"action", + "ecs":"netflow.action", + "ocsf": "disposition_id" + }, + { + "raw_field":"traffic_path", + "ecs":"netflow.traffic_path", + "ocsf": "boundary_id" + }, + { + "raw_field":"flow_direction", + "ecs":"netflow.flow_direction", + "ocsf": "connection_info.direction_id" + }, + { + "raw_field":"dstaddr", + "ecs":"netflow.dstaddr", + "ocsf": "dst_endpoint.ip" + }, + { + "raw_field":"srcaddr", + "ecs":"netflow.srcaddr", + "ocsf": "src_endpoint.ip" + }, + { + "raw_field":"interface_id", + "ecs":"netflow.interface_id", + "ocsf": "dst_endpoint.interface_uid" + }, + { + "raw_field":"vpc_id", + "ecs":"netflow.vpc_id", + "ocsf": "dst_endpoint.vpc_uid" + }, + { + "raw_field":"instance_id", + "ecs":"netflow.instance_id", + "ocsf": "dst_endpoint.instance_uid" + }, + { + "raw_field":"subnet_id", + "ecs":"netflow.subnet_id", + "ocsf": "dst_endpoint.subnet_uid" + }, + { + "raw_field":"start", + "ecs":"timestamp", + "ocsf": "time" + } + ] +} \ No newline at end of file diff --git a/src/main/resources/OSMapping/windows/fieldmappings.yml b/src/main/resources/OSMapping/windows/fieldmappings.yml deleted file mode 100644 index 8e905ad52..000000000 --- a/src/main/resources/OSMapping/windows/fieldmappings.yml +++ /dev/null @@ -1,65 +0,0 @@ -# this file provides pre-defined mappings for Sigma fields defined for all Sigma rules under windows log group to their corresponding ECS Fields. -fieldmappings: - AccountName: winlog-computerObject-name - AuthenticationPackageName: winlog-event_data-AuthenticationPackageName - Channel: winlog-channel - Company: winlog-event_data-Company - ComputerName: winlog-computer_name - Description: winlog-event_data-Description - Details: winlog-event_data-Detail - Device: winlog-event_data-DeviceName - DeviceName: winlog-event_data-DeviceName - FileName: winlog-event_data-OriginalFileName - FileVersion: winlog-event_data-FileVersion - IntegrityLevel: winlog-event_data-IntegrityLevel - IpAddress: winlog-event_data-IpAddress - KeyLength: winlog-event_data-KeyLength - Keywords: winlog-keywords - LogonId: winlog-event_data-LogonId - LogonProcessName: winlog-event_data-LogonProcessName - LogonType: winlog-event_data-LogonType - OriginalFileName: winlog-event_data-OriginalFileName - OriginalFilename: winlog-event_data-OriginalFileName - Path: winlog-event_data-Path - PrivilegeList: winlog-event_data-PrivilegeList - ProcessId: winlog-event_data-ProcessId - Product: winlog-event_data-Product - Provider: winlog-provider_name - ProviderName: winlog-provider_name - ScriptBlockText: winlog-event_data-ScriptBlockText - ServerName: winlog-event_data-TargetServerName - Service: winlog-event_data-ServiceName - Signed: winlog-event_data-Signed - State: winlog-event_data-State - Status: winlog-event_data-Status - SubjectDomainName: winlog-event_data-SubjectDomainName - SubjectLogonId: winlog-event_data-SubjectLogonId - SubjectUserName: winlog-event_data-SubjectUserName - SubjectUserSid: winlog-event_data-SubjectUserSid - TargetLogonId: winlog-event_data-TargetLogonId - TargetName: winlog-event_data-TargetUserName - TargetServerName: winlog-event_data-TargetServerName - TargetUserName: winlog-event_data-TargetUserName - TargetUserSid: winlog-event_data-TargetUserSid - TaskName: winlog-task - Type: winlog-user-type - User: winlog-user-name - UserName: winlog-user-name - Workstation: winlog-event_data-Workstation - WorkstationName: winlog-event_data-Workstation - event_uid: winlog-event_id - CommandLine: server-user-hash - hostname: host-hostname - message: windows-message - Provider_Name: winlog-provider_name - EventId: winlog-event_id - processPath: winlog-event_data-ProcessPath - ProcessName: winlog-event_data-ProcessName - ObjectName: winlog-computerObject-name - param1: winlog-event_data-param1 - param2: winlog-event_data-param2 - windows-hostname: winlog-computer_name - windows-provider-name: winlog-provider_name - windows-servicename: winlog-event_data-ServiceName - creationTime: timestamp - diff --git a/src/main/resources/OSMapping/windows/mappings.json b/src/main/resources/OSMapping/windows/mappings.json deleted file mode 100644 index f2eef2a51..000000000 --- a/src/main/resources/OSMapping/windows/mappings.json +++ /dev/null @@ -1,196 +0,0 @@ -{ - "properties": { - "winlog-computerObject-name": { - "path": "winlog.computerObject.name", - "type": "alias" - }, - "winlog-event_data-AuthenticationPackageName": { - "path": "winlog.event_data.AuthenticationPackageName", - "type": "alias" - }, - "winlog-channel": { - "path": "winlog.channel", - "type": "alias" - }, - "winlog-event_data-Company": { - "path": "winlog.event_data.Company", - "type": "alias" - }, - "winlog-computer_name": { - "path": "winlog.computer_name", - "type": "alias" - }, - "winlog-event_data-Description": { - "path": "winlog.event_data.Description", - "type": "alias" - }, - "winlog-event_data-Detail": { - "path": "winlog.event_data.Detail", - "type": "alias" - }, - "winlog-event_data-DeviceName": { - "path": "winlog.event_data.DeviceName", - "type": "alias" - }, - "winlog-event_data-OriginalFileName": { - "path": "winlog.event_data.OriginalFileName", - "type": "alias" - }, - "winlog-event_data-FileVersion": { - "path": "winlog.event_data.FileVersion", - "type": "alias" - }, - "winlog-event_data-IntegrityLevel": { - "path": "winlog.event_data.IntegrityLevel", - "type": "alias" - }, - "winlog-event_data-IpAddress": { - "path": "winlog.event_data.IpAddress", - "type": "alias" - }, - "winlog-event_data-KeyLength": { - "path": "winlog.event_data.KeyLength", - "type": "alias" - }, - "winlog-keywords": { - "path": "winlog.keywords", - "type": "alias" - }, - "winlog-event_data-LogonId": { - "path": "winlog.event_data.LogonId", - "type": "alias" - }, - "winlog-event_data-LogonProcessName": { - "path": "winlog.event_data.LogonProcessName", - "type": "alias" - }, - "winlog-event_data-LogonType": { - "path": "winlog.event_data.LogonType", - "type": "alias" - }, - "winlog-event_data-Path": { - "path": "winlog.event_data.Path", - "type": "alias" - }, - "winlog-event_data-PrivilegeList": { - "path": "winlog.event_data.PrivilegeList", - "type": "alias" - }, - "winlog-event_data-ProcessId": { - "path": "winlog.event_data.ProcessId", - "type": "alias" - }, - "winlog-event_data-Product": { - "path": "winlog.event_data.Product", - "type": "alias" - }, - "winlog-provider_name": { - "path": "winlog.provider_name", - "type": "alias" - }, - "winlog-event_data-ScriptBlockText": { - "path": "winlog.event_data.ScriptBlockText", - "type": "alias" - }, - "winlog-event_data-TargetServerName": { - "path": "winlog.event_data.TargetServerName", - "type": "alias" - }, - "winlog-event_data-ServiceName": { - "path": "winlog.event_data.ServiceName", - "type": "alias" - }, - "winlog-event_data-Signed": { - "path": "winlog.event_data.Signed", - "type": "alias" - }, - "winlog-event_data-State": { - "path": "winlog.event_data.State", - "type": "alias" - }, - "winlog-event_data-Status": { - "path": "winlog.event_data.Status", - "type": "alias" - }, - "winlog-event_data-SubjectDomainName": { - "path": "winlog.event_data.SubjectDomainName", - "type": "alias" - }, - "winlog-event_data-SubjectLogonId": { - "path": "winlog.event_data.SubjectLogonId", - "type": "alias" - }, - "winlog-event_data-SubjectUserName": { - "path": "winlog.event_data.SubjectUserName", - "type": "alias" - }, - "winlog-event_data-SubjectUserSid": { - "path": "winlog.event_data.SubjectUserSid", - "type": "alias" - }, - "winlog-event_data-TargetLogonId": { - "path": "winlog.event_data.TargetLogonId", - "type": "alias" - }, - "winlog-event_data-TargetUserName": { - "path": "winlog.event_data.TargetUserName", - "type": "alias" - }, - "winlog-event_data-TargetUserSid": { - "path": "winlog.event_data.TargetUserSid", - "type": "alias" - }, - "winlog-task": { - "path": "winlog.task", - "type": "alias" - }, - "winlog-user-type": { - "path": "winlog.user.type", - "type": "alias" - }, - "winlog-user-name": { - "path": "winlog.user.name", - "type": "alias" - }, - "winlog-event_data-Workstation": { - "path": "winlog.event_data.Workstation", - "type": "alias" - }, - "winlog-event_id": { - "path": "winlog.event_id", - "type": "alias" - }, - "server-user-hash": { - "path": "server.user.hash", - "type": "alias" - }, - "host-hostname": { - "path": "host.hostname", - "type": "alias" - }, - "windows-message": { - "path": "windows.message", - "type": "alias" - }, - "winlog-event_data-ProcessPath": { - "path": "winlog.event_data.ProcessPath", - "type": "alias" - }, - "winlog-event_data-ProcessName": { - "path": "winlog.event_data.ProcessName", - "type": "alias" - }, - "winlog-event_data-param1": { - "path": "winlog.event_data.param1", - "type": "alias" - }, - "winlog-event_data-param2": { - "path": "winlog.event_data.param2", - "type": "alias" - }, - "timestamp": { - "path": "creationTime", - "type": "alias" - } - } -} \ No newline at end of file diff --git a/src/main/resources/mappings/log_type_config_mapping.json b/src/main/resources/mappings/log_type_config_mapping.json new file mode 100644 index 000000000..c150cc466 --- /dev/null +++ b/src/main/resources/mappings/log_type_config_mapping.json @@ -0,0 +1,35 @@ +{ + "_meta" : { + "schema_version": 1 + }, + "dynamic_templates": [ + { + "strings": { + "match_mapping_type": "string", + "mapping": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + ], + "properties": { + "raw_field": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "log_types": { + "type": "keyword" + } + } +} diff --git a/src/main/resources/rules/cloudtrail/aws_create_load_balancer_layer.yml b/src/main/resources/rules/cloudtrail/aws_create_load_balancer_layer.yml new file mode 100644 index 000000000..5689743c9 --- /dev/null +++ b/src/main/resources/rules/cloudtrail/aws_create_load_balancer_layer.yml @@ -0,0 +1,22 @@ +title: AWS Create Load Balancer +id: 97fbabf8-8e1b-47a2-b8d5-a418d2b95b3d +description: AWS Create Load Balancer +author: Austin Songer +status: experimental +date: 2021/09/23 +references: + - https://docs.aws.amazon.com/lambda/latest/dg/API_UpdateFunctionConfiguration.html +logsource: + product: aws + service: cloudtrail +detection: + selection: + eventSource: elasticloadbalancing.amazonaws.com + eventName|startswith: CreateLoadBalancer + condition: selection +level: medium +tags: + - attack.privilege_escalation +falsepositives: + - Lambda Layer being attached may be performed by a system administrator. Verify whether the user identity, user agent, and/or hostname should be making changes in your environment. + - Lambda Layer being attached from unfamiliar users should be investigated. If known behavior is causing false positives, it can be exempted from the rule. \ No newline at end of file diff --git a/src/main/resources/rules/rule_categories.json b/src/main/resources/rules/rule_categories.json index bf8520c78..25ddee7d8 100644 --- a/src/main/resources/rules/rule_categories.json +++ b/src/main/resources/rules/rule_categories.json @@ -51,6 +51,10 @@ { "key": "linux", "display_name": "System logs" + }, + { + "key": "vpcflow", + "display_name": "VpcFlow logs" } ] } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/LogTypeServiceTests.java b/src/test/java/org/opensearch/securityanalytics/LogTypeServiceTests.java new file mode 100644 index 000000000..8eb717e60 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/LogTypeServiceTests.java @@ -0,0 +1,151 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import org.junit.Before; +import org.opensearch.action.admin.indices.refresh.RefreshRequest; +import org.opensearch.action.support.PlainActionFuture; +import org.opensearch.common.settings.Setting; +import org.opensearch.plugins.Plugin; +import org.opensearch.securityanalytics.logtype.BuiltinLogTypeLoader; +import org.opensearch.securityanalytics.logtype.LogTypeService; +import org.opensearch.securityanalytics.model.FieldMappingDoc; +import org.opensearch.securityanalytics.model.LogType; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.transport.MockTransportService; + +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.securityanalytics.logtype.LogTypeService.LOG_TYPE_INDEX; + +public class LogTypeServiceTests extends OpenSearchIntegTestCase { + + private LogTypeService logTypeService; + + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(TestPlugin.class, MockTransportService.TestPlugin.class); + } + + @Before + protected void beforeTest() throws Exception { + if (logTypeService == null) { + BuiltinLogTypeLoader builtinLogTypeLoader = mock(BuiltinLogTypeLoader.class); + doNothing().when(builtinLogTypeLoader).ensureLogTypesLoaded(); + + List dummyLogTypes = List.of( + new LogType(null, "test_logtype", "", true, + List.of( + new LogType.Mapping("rawFld1", "ecsFld1", "ocsfFld1"), + new LogType.Mapping("rawFld2", "ecsFld2", "ocsfFld2"), + new LogType.Mapping("rawFld3", "ecsFld3", "ocsfFld3") + ) + ) + ); + when(builtinLogTypeLoader.getAllLogTypes()).thenReturn(dummyLogTypes); + logTypeService = new LogTypeService(client(), clusterService(), xContentRegistry(), builtinLogTypeLoader); + } + } + + public void testIndexMappings() throws ExecutionException, InterruptedException { + ensureGreen(); + + List fieldMappingDocs = List.of( + new FieldMappingDoc("fld1", Map.of("ecs", "ecs_fld1", "ocsf", "ocsf_fld1"), Set.of("windows")), + new FieldMappingDoc("fld2", Map.of("ecs", "ecs_fld2", "ocsf", "ocsf_fld2"), Set.of("windows")), + new FieldMappingDoc("fld3", Map.of("ecs", "ecs_winlog.fld3", "ocsf", "ocsf_fld3"), Set.of("windows")) + ); + + indexFieldMappings(fieldMappingDocs); + + client().admin().indices().refresh(new RefreshRequest(LOG_TYPE_INDEX)).get(); + + fieldMappingDocs = List.of( + new FieldMappingDoc("fld1", Map.of("ecs", "ecs_fld1", "ocsf", "ocsf_fld111"), Set.of("linux")), + new FieldMappingDoc("fld2", Map.of("ecs", "ecs_fld2", "ocsf", "ocsf_fld222"), Set.of("linux")), + new FieldMappingDoc("fld3", Map.of("ecs", "network.something", "ocsf", "ocsf_fld333"), Set.of("network")) + ); + + indexFieldMappings(fieldMappingDocs); + + client().admin().indices().refresh(new RefreshRequest(LOG_TYPE_INDEX)).get(); + + PlainActionFuture> getAllFieldMappingsFuture = new PlainActionFuture<>(); + logTypeService.getAllFieldMappings(getAllFieldMappingsFuture); + try { + List allFieldMappings = getAllFieldMappingsFuture.get(); + // 3 initial ones from test_logtype, fld1 and fld2 are inserted and then updated/merged and fld3 is inserted twice since ecs field is different + assertEquals(7, allFieldMappings.size()); + } catch (Exception e) { + fail(e.getMessage()); + } + + + List allLogTypes = getAllLogTypes(); + assertEquals(4, allLogTypes.size()); + assertTrue(allLogTypes.contains("windows")); + assertTrue(allLogTypes.contains("linux")); + assertTrue(allLogTypes.contains("test_logtype")); + assertTrue(allLogTypes.contains("network")); + + List fieldMappings = getFieldMappingsByLogTypes(List.of("linux")); + assertEquals(2, fieldMappings.size()); + fieldMappings = getFieldMappingsByLogTypes(List.of("windows")); + assertEquals(3, fieldMappings.size()); + + } + + + + private void indexFieldMappings(List fieldMappingDocs) { + PlainActionFuture fut = new PlainActionFuture<>(); + + logTypeService.indexFieldMappings(fieldMappingDocs, fut); + try { + fut.get(); + } catch (Exception e) { + fail(e.getMessage()); + } + } + + private List getFieldMappingsByLogTypes(List logTypes) { + PlainActionFuture> future = new PlainActionFuture<>(); + logTypeService.getFieldMappingsByLogTypes(logTypes, future); + try { + return future.get(); + } catch (Exception e) { + fail(e.getMessage()); + } + return null; + } + + private List getAllLogTypes() { + PlainActionFuture> getAllLogTypesFuture = new PlainActionFuture<>(); + logTypeService.getAllLogTypes(getAllLogTypesFuture); + try { + return getAllLogTypesFuture.get(); + } catch (Exception e) { + fail(e.getMessage()); + } + return null; + } + + public static class TestPlugin extends Plugin { + @Override + public List> getSettings() { + return Arrays.asList(SecurityAnalyticsSettings.DEFAULT_MAPPING_SCHEMA); + } + } + +} diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index c02276d1f..0246ccf3e 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -337,7 +337,7 @@ protected List executeSearch(String index, String request, Boolean re refreshIndex(index); } - Response response = makeRequest(client(), "GET", String.format(Locale.getDefault(), "%s/_search", index), Collections.emptyMap(), new StringEntity(request), new BasicHeader("Content-Type", "application/json")); + Response response = makeRequest(client(), "GET", String.format(Locale.getDefault(), "%s/_search", index), Map.of("preference", "_primary"), new StringEntity(request), new BasicHeader("Content-Type", "application/json")); Assert.assertEquals("Search failed", RestStatus.OK, restStatus(response)); SearchResponse searchResponse = SearchResponse.fromXContent(createParser(JsonXContent.jsonXContent, response.getEntity().getContent())); diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 54aab0846..df731b7ee 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -133,7 +133,7 @@ public static Detector randomDetector(String name, DetectorTrigger trigger = new DetectorTrigger(null, "windows-trigger", "1", List.of(randomDetectorType()), List.of("QuarksPwDump Clearing Access History"), List.of("high"), List.of("T0008"), List.of()); triggers.add(trigger); } - return new Detector(null, null, name, enabled, schedule, lastUpdateTime, enabledTime, detectorType, user, inputs, triggers, Collections.singletonList(""), "", "", "", "", "", "", Collections.emptyMap()); + return new Detector(null, null, name, enabled, schedule, lastUpdateTime, enabledTime, detectorType.getDetectorType(), user, inputs, triggers, Collections.singletonList(""), "", "", "", "", "", "", Collections.emptyMap()); } public static Detector randomDetectorWithNoUser() { @@ -145,7 +145,7 @@ public static Detector randomDetectorWithNoUser() { Instant enabledTime = enabled ? Instant.now().truncatedTo(ChronoUnit.MILLIS) : null; Instant lastUpdateTime = Instant.now().truncatedTo(ChronoUnit.MILLIS); - return new Detector(null, null, name, enabled, schedule, lastUpdateTime, enabledTime, detectorType, null, inputs, Collections.emptyList(),Collections.singletonList(""), "", "", "", "", "", "", Collections.emptyMap()); + return new Detector(null, null, name, enabled, schedule, lastUpdateTime, enabledTime, detectorType.getDetectorType(), null, inputs, Collections.emptyList(),Collections.singletonList(""), "", "", "", "", "", "", Collections.emptyMap()); } public static CorrelationRule randomCorrelationRule(String name) { diff --git a/src/test/java/org/opensearch/securityanalytics/action/IndexDetectorRequestTests.java b/src/test/java/org/opensearch/securityanalytics/action/IndexDetectorRequestTests.java index 9144f56ee..d1ed34947 100644 --- a/src/test/java/org/opensearch/securityanalytics/action/IndexDetectorRequestTests.java +++ b/src/test/java/org/opensearch/securityanalytics/action/IndexDetectorRequestTests.java @@ -50,7 +50,7 @@ public void testIndexDetectorPostRequest_2() throws IOException { DetectorInput input2 = new DetectorInput("windows detector for security analytics", List.of("windows-2"), Collections.emptyList(), rules.stream().map(DetectorRule::new).collect(Collectors.toList())); - Detector detector = randomDetectorWithInputs(List.of(input1, input2)); + Detector detector = randomDetectorWithInputs(List.of(input1)); IndexDetectorRequest request = new IndexDetectorRequest(detectorId, WriteRequest.RefreshPolicy.IMMEDIATE, RestRequest.Method.POST, detector); Assert.assertNotNull(request); diff --git a/src/test/java/org/opensearch/securityanalytics/action/IndexDetectorResponseTests.java b/src/test/java/org/opensearch/securityanalytics/action/IndexDetectorResponseTests.java index ad6a110e2..b60fd6738 100644 --- a/src/test/java/org/opensearch/securityanalytics/action/IndexDetectorResponseTests.java +++ b/src/test/java/org/opensearch/securityanalytics/action/IndexDetectorResponseTests.java @@ -39,7 +39,7 @@ public void testIndexDetectorPostResponse() throws IOException { cronSchedule, Instant.now(), Instant.now(), - detectorType, + detectorType.getDetectorType(), randomUser(), List.of(), List.of(), diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java index fc7c963ae..f3d234974 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java @@ -53,7 +53,7 @@ public void testGetAlerts_success() { new CronSchedule("31 * * * *", ZoneId.of("Asia/Kolkata"), Instant.ofEpochSecond(1538164858L)), Instant.now(), Instant.now(), - Detector.DetectorType.OTHERS_APPLICATION, + Detector.DetectorType.OTHERS_APPLICATION.getDetectorType(), null, List.of(), List.of(), @@ -229,7 +229,7 @@ public void testGetFindings_getFindingsByMonitorIdFailures() { new CronSchedule("31 * * * *", ZoneId.of("Asia/Kolkata"), Instant.ofEpochSecond(1538164858L)), Instant.now(), Instant.now(), - Detector.DetectorType.OTHERS_APPLICATION, + Detector.DetectorType.OTHERS_APPLICATION.getDetectorType(), null, List.of(), List.of(), diff --git a/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java index 0f79b5c15..e5111804a 100644 --- a/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java @@ -110,7 +110,6 @@ public void testListCorrelationsWorkflow() throws IOException, InterruptedExcept Assert.assertEquals(5, noOfSigmaRuleMatches); Thread.sleep(5000); - indexDoc(indices.vpcFlowsIndex, "1", randomVpcFlowDoc()); executeResponse = executeAlertingMonitor(vpcFlowMonitorId, Collections.emptyMap()); executeResults = entityAsMap(executeResponse); diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingServiceTests.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingServiceTests.java index e4651d501..1b23270c5 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingServiceTests.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingServiceTests.java @@ -53,7 +53,7 @@ public void testGetFindings_success() { new CronSchedule("31 * * * *", ZoneId.of("Asia/Kolkata"), Instant.ofEpochSecond(1538164858L)), Instant.now(), Instant.now(), - Detector.DetectorType.OTHERS_APPLICATION, + Detector.DetectorType.OTHERS_APPLICATION.getDetectorType(), null, List.of(), List.of(), @@ -173,7 +173,7 @@ public void testGetFindings_getFindingsByMonitorIdFailure() { new CronSchedule("31 * * * *", ZoneId.of("Asia/Kolkata"), Instant.ofEpochSecond(1538164858L)), Instant.now(), Instant.now(), - Detector.DetectorType.OTHERS_APPLICATION, + Detector.DetectorType.OTHERS_APPLICATION.getDetectorType(), null, List.of(), List.of(), diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java index aea00bac1..bcc3e56d9 100644 --- a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java @@ -25,7 +25,6 @@ import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; -import org.opensearch.core.common.Strings; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.json.JsonXContent; @@ -82,7 +81,7 @@ public void testGetMappingSuccess_1() throws IOException { indexDoc(testIndexName1, "1", sampleDoc); // puts mappings with timestamp alias - String createMappingsRequest = "{\"index_name\":\"my_index*\",\"rule_topic\":\"windows\",\"partial\":true,\"alias_mappings\":{\"properties\":{\"timestamp\":{\"type\":\"alias\",\"path\":\"lvl1field\"},\"winlog-computer_name\":{\"type\":\"alias\",\"path\":\"source1.port\"},\"winlog-event_data-AuthenticationPackageName\":{\"type\":\"alias\",\"path\":\"source1.ip\"},\"winlog-event_data-Company\":{\"type\":\"alias\",\"path\":\"some.very.long.field.name\"}}}}"; + String createMappingsRequest = "{\"index_name\":\"my_index*\",\"rule_topic\":\"windows\",\"partial\":true,\"alias_mappings\":{\"properties\":{\"timestamp\":{\"type\":\"alias\",\"path\":\"lvl1field\"},\"winlog.computer_name\":{\"type\":\"alias\",\"path\":\"source1.port\"},\"winlog.event_data.AuthenticationPackageName\":{\"type\":\"alias\",\"path\":\"source1.ip\"},\"winlog.event_data.Company\":{\"type\":\"alias\",\"path\":\"some.very.long.field.name\"}}}}"; Request request = new Request("POST", MAPPER_BASE_URI); // both req params and req body are supported @@ -95,8 +94,7 @@ public void testGetMappingSuccess_1() throws IOException { assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); Map respMap = (Map) responseAsMap(response); Map props = (Map)((Map) respMap.get(testIndexPattern)).get("mappings"); - props = (Map) props.get("properties"); - assertEquals(4, props.size()); + assertEquals(4, recurProps(props)); } public void testCreateMappingSuccess() throws IOException { @@ -307,7 +305,7 @@ public void testCreateIndexMappingsIndexMappingsEmpty() throws IOException { try { client().performRequest(request); } catch (ResponseException e) { - assertTrue(e.getMessage().contains("Mappings for index [my_index_alias_fail_1] are empty")); + assertTrue(e.getMessage().contains("Failed applying mappings to index")); } } @@ -1349,19 +1347,9 @@ private void deleteDatastream(String datastreamName) throws IOException { } private final String DNS_SAMPLE = "dns-sample.json"; - private final String CLOUDTRAIL_SAMPLE = "cloudtrail-sample.json"; - private final String CLOUDTRAIL_SAMPLE_S3 = "s3-sample.json"; - - private final String DNS_MAPPINGS = "OSMapping/dns/mappings.json"; - private final String CLOUDTRAIL_MAPPINGS = "OSMapping/cloudtrail/mappings.json"; - private final String S3_MAPPINGS = "OSMapping/s3/mappings.json"; - private final String NETWORK_MAPPINGS = "OSMapping/network/mappings.json"; - private final String LINUX_MAPPINGS = "OSMapping/linux/mappings.json"; - private final String WINDOWS_MAPPINGS = "OSMapping/windows/mappings.json"; - private final String APACHE_ACCESS_MAPPINGS = "OSMapping/apache_access/mappings.json"; - private final String AD_LDAP_MAPPINGS = "OSMapping/ad_ldap/mappings.json"; + private final String DNS_MAPPINGS = "OSMapping/dns_logtype.json"; private String readResource(String name) throws IOException { try (InputStream inputStream = SecurityAnalyticsPlugin.class.getClassLoader().getResourceAsStream(name)) { @@ -1375,8 +1363,8 @@ private String readResource(String name) throws IOException { } public void testReadResource() throws IOException { - String content = readResource(DNS_MAPPINGS); - assertTrue(content.contains("properties")); + String content = readResource(DNS_SAMPLE); + assertTrue(content.contains("query_type")); } public void testCreateDNSMapping() throws IOException{ @@ -1398,29 +1386,29 @@ public void testCreateDNSMapping() throws IOException{ NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, indexMapping); - Map mappings = (Map) parser.map().get("properties"); + List> mappings = (List>) parser.map().get("mappings"); GetMappingsResponse getMappingsResponse = SecurityAnalyticsClientUtils.executeGetMappingsRequest(INDEX_NAME); MappingsTraverser mappingsTraverser = new MappingsTraverser(getMappingsResponse.getMappings().entrySet().iterator().next().getValue()); List flatProperties = mappingsTraverser.extractFlatNonAliasFields(); - assertTrue(flatProperties.contains("dns.answers.type")); - assertTrue(flatProperties.contains("dns.question.name")); - assertTrue(flatProperties.contains("dns.question.registered_domain")); + assertTrue(flatProperties.contains("query_type")); //Loop over the mappings and run update request for each one specifying the index to be updated - mappings.entrySet().forEach(entry -> { - String key = entry.getKey(); + mappings.forEach(entry -> { + String key = entry.get("ecs").toString(); if("timestamp".equals(key)) return; - String path = ((Map) entry.getValue()).get("path").toString(); + String path = entry.get("raw_field").toString(); try { - Request updateRequest = new Request("PUT", SecurityAnalyticsPlugin.MAPPER_BASE_URI); - updateRequest.setJsonEntity(org.opensearch.common.Strings.toString(XContentFactory.jsonBuilder().map(Map.of( - "index_name", INDEX_NAME, - "field", path, - "alias", key)))); - Response apiResponse = client().performRequest(updateRequest); - assertEquals(HttpStatus.SC_OK, apiResponse.getStatusLine().getStatusCode()); + if (flatProperties.contains(path)) { + Request updateRequest = new Request("PUT", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + updateRequest.setJsonEntity(org.opensearch.common.Strings.toString(XContentFactory.jsonBuilder().map(Map.of( + "index_name", INDEX_NAME, + "field", path, + "alias", key)))); + Response apiResponse = client().performRequest(updateRequest); + assertEquals(HttpStatus.SC_OK, apiResponse.getStatusLine().getStatusCode()); + } } catch (IOException e) { throw new RuntimeException(e); @@ -1607,7 +1595,7 @@ public void testCloudtrailMappings() throws IOException { " }\n" + "}"; List hits = executeSearch(".opensearch-sap-cloudtrail-detectors-queries-000001", request); - Assert.assertEquals(31, hits.size()); + Assert.assertEquals(32, hits.size()); } public void testS3Mappings() throws IOException { @@ -1624,7 +1612,6 @@ public void testS3Mappings() throws IOException { //Expect only "timestamp" alias to be applied Map mappings = getIndexMappingsSAFlat(indexName); assertTrue(mappings.containsKey("timestamp")); - assertTrue(mappings.containsKey("Requester")); // Verify that all rules are working DetectorInput input = new DetectorInput("windows detector for security analytics", List.of(indexName), List.of(), @@ -1641,4 +1628,22 @@ public void testS3Mappings() throws IOException { List hits = executeSearch(".opensearch-sap-s3-detectors-queries-000001", request); Assert.assertEquals(1, hits.size()); } + + @SuppressWarnings("unchecked") + private int recurProps(Map props) { + int totalProps = 0; + if (props.containsKey("properties")) { + totalProps += recurProps((Map) props.get("properties")); + } else { + for (Map.Entry prop: props.entrySet()) { + Map propValue = (Map) prop.getValue(); + if (propValue.containsKey("properties")) { + totalProps += recurProps((Map) propValue.get("properties")); + } else { + ++totalProps; + } + } + } + return totalProps; + } } diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperServiceTests.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperServiceTests.java index bfb3b6b50..d04b74d06 100644 --- a/src/test/java/org/opensearch/securityanalytics/mapper/MapperServiceTests.java +++ b/src/test/java/org/opensearch/securityanalytics/mapper/MapperServiceTests.java @@ -106,7 +106,7 @@ public class MapperServiceTests extends OpenSearchTestCase { // }); // } - public void testGetMappingsView_successAliasesOnlyReturned() { +/* public void testGetMappingsView_successAliasesOnlyReturned() { // We expect JSON parser to throw "duplicate fields" error // Setup @@ -303,6 +303,6 @@ public void onFailure(Exception e) { assertTrue(e.getMessage().contains("Mapper not found: [incorrectTopicName]")); } }); - } + }*/ } diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperUtilsTests.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperUtilsTests.java index 632b149f2..1c2e6a641 100644 --- a/src/test/java/org/opensearch/securityanalytics/mapper/MapperUtilsTests.java +++ b/src/test/java/org/opensearch/securityanalytics/mapper/MapperUtilsTests.java @@ -13,10 +13,9 @@ import java.io.IOException; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; -public class MapperUtilsTests extends OpenSearchTestCase { +/*public class MapperUtilsTests extends OpenSearchTestCase { public void testValidateIndexMappingsMissingSome() throws IOException { @@ -94,4 +93,4 @@ public void testGetAllPathsFromAliasMappingsThrow() throws IOException { assertThrows(IllegalArgumentException.class, () -> MapperUtils.getAllPathsFromAliasMappings(MapperTopicStore.aliasMappings("test1"))); assertThrows(JsonParseException.class, () -> MapperUtils.getAllPathsFromAliasMappings(MapperTopicStore.aliasMappings("test2"))); } -} +}*/ diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java index 96a8d01bd..23215edf4 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java @@ -418,7 +418,7 @@ public void testSearchingDetectors() throws IOException { List> hits = ((List>) ((Map) searchResponseBody.get("hits")).get("hits")); Map hit = hits.get(0); String detectorTypeInResponse = (String) ((Map) hit.get("_source")).get("detector_type"); - Assert.assertEquals("Detector type incorrect", detectorTypeInResponse, randomDetectorType().toLowerCase(Locale.ROOT)); + Assert.assertEquals("Detector type incorrect", detectorTypeInResponse.toLowerCase(Locale.ROOT), randomDetectorType().toLowerCase(Locale.ROOT)); } @SuppressWarnings("unchecked") @@ -476,7 +476,7 @@ public void testCreatingADetectorWithCustomRules() throws IOException { SearchHit hit = hits.get(0); String detectorType = (String) ((Map) hit.getSourceAsMap().get("detector")).get("detector_type"); - Assert.assertEquals("Detector type incorrect", detectorType, randomDetectorType().toLowerCase(Locale.ROOT)); + Assert.assertEquals("Detector type incorrect", detectorType.toLowerCase(Locale.ROOT), randomDetectorType().toLowerCase(Locale.ROOT)); String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java new file mode 100644 index 000000000..676f3f4ca --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java @@ -0,0 +1,2767 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.resthandler; + +import org.apache.http.HttpStatus; +import org.apache.http.entity.StringEntity; +import org.apache.http.message.BasicHeader; +import org.junit.Assert; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.rest.RestStatus; +import org.opensearch.search.SearchHit; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.model.Detector; +import org.opensearch.securityanalytics.model.DetectorInput; +import org.opensearch.securityanalytics.model.DetectorRule; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.opensearch.securityanalytics.TestHelpers.*; + +public class OCSFDetectorRestApiIT extends SecurityAnalyticsRestTestCase { + + @SuppressWarnings("unchecked") + public void testCloudTrailAPIActivityOCSFDetector() throws IOException { + String index = createTestIndex("cloudtrail", ocsfCloudtrailMappings()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"cloudtrail\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithInputs(List.of(new DetectorInput("ocsf logs based cloudtrail detector for security analytics", List.of(index), List.of(), + getPrePackagedRules(Detector.DetectorType.CLOUDTRAIL.getDetectorType()).stream().map(DetectorRule::new).collect(Collectors.toList()))), Detector.DetectorType.CLOUDTRAIL); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String createdId = responseBody.get("_id").toString(); + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertNotEquals("response is missing Id", Detector.NO_ID, createdId); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, createdId), createResponse.getHeader("Location")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("rule_topic_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", "cloudtrail", detectorTypeInResponse); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", ocsfCloudtrailDoc()); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(1, noOfSigmaRuleMatches); + } + + @SuppressWarnings("unchecked") + public void testCloudTrailAPIActivityRawDetector() throws IOException { + String index = createTestIndex("cloudtrail", rawCloudtrailMappings()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"cloudtrail\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithInputs(List.of(new DetectorInput("raw logs based cloudtrail detector for security analytics", List.of(index), List.of(), + getPrePackagedRules(Detector.DetectorType.CLOUDTRAIL.getDetectorType()).stream().map(DetectorRule::new).collect(Collectors.toList()))), Detector.DetectorType.CLOUDTRAIL); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String createdId = responseBody.get("_id").toString(); + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertNotEquals("response is missing Id", Detector.NO_ID, createdId); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, createdId), createResponse.getHeader("Location")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("rule_topic_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", "cloudtrail", detectorTypeInResponse); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", rawCloudtrailDoc()); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(1, noOfSigmaRuleMatches); + } + + @SuppressWarnings("unchecked") + public void testRoute53OCSFDetector() throws IOException { + String rule = customRoute53Rule(); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "dns"), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String createdId = responseBody.get("_id").toString(); + + String index = createTestIndex("route53", ocsfRoute53Mappings()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"dns\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithInputs(List.of(new DetectorInput("raw logs based route53 detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + getPrePackagedRules(Detector.DetectorType.DNS.getDetectorType()).stream().map(DetectorRule::new).collect(Collectors.toList()))), Detector.DetectorType.DNS); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + createdId = responseBody.get("_id").toString(); + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertNotEquals("response is missing Id", Detector.NO_ID, createdId); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, createdId), createResponse.getHeader("Location")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("rule_topic_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", "dns", detectorTypeInResponse); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", ocsfRoute53Doc()); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(1, noOfSigmaRuleMatches); + } + + @SuppressWarnings("unchecked") + public void testRoute53RawDetector() throws IOException { + String rule = customRoute53Rule(); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "dns"), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String createdId = responseBody.get("_id").toString(); + + String index = createTestIndex("route53", rawRoute53DnsMappings()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"dns\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithInputs(List.of(new DetectorInput("raw logs based route53 detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + getPrePackagedRules(Detector.DetectorType.DNS.getDetectorType()).stream().map(DetectorRule::new).collect(Collectors.toList()))), Detector.DetectorType.DNS); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + createdId = responseBody.get("_id").toString(); + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertNotEquals("response is missing Id", Detector.NO_ID, createdId); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, createdId), createResponse.getHeader("Location")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("rule_topic_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", "dns", detectorTypeInResponse); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", rawRoute53Doc()); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(1, noOfSigmaRuleMatches); + } + + @SuppressWarnings("unchecked") + public void testVpcflowOcsfDetector() throws IOException { + String rule = customVpcFlowRule(); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "vpcflow"), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String createdId = responseBody.get("_id").toString(); + + String index = createTestIndex("vpcflow", ocsfVpcflowMappings()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"vpcflow\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithInputs(List.of(new DetectorInput("raw logs based vpcflow detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + getPrePackagedRules(Detector.DetectorType.DNS.getDetectorType()).stream().map(DetectorRule::new).collect(Collectors.toList()))), Detector.DetectorType.DNS); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + createdId = responseBody.get("_id").toString(); + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertNotEquals("response is missing Id", Detector.NO_ID, createdId); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, createdId), createResponse.getHeader("Location")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("rule_topic_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", "dns", detectorTypeInResponse); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", ocsfVpcflowDoc()); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(1, noOfSigmaRuleMatches); + } + + @SuppressWarnings("unchecked") + public void testVpcflowRawDetector() throws IOException { + String rule = customVpcFlowRule(); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "vpcflow"), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String createdId = responseBody.get("_id").toString(); + + String index = createTestIndex("vpcflow", rawVpcFlowMappings()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"vpcflow\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithInputs(List.of(new DetectorInput("raw logs based vpcflow detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + getPrePackagedRules(Detector.DetectorType.DNS.getDetectorType()).stream().map(DetectorRule::new).collect(Collectors.toList()))), Detector.DetectorType.DNS); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + createdId = responseBody.get("_id").toString(); + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertNotEquals("response is missing Id", Detector.NO_ID, createdId); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, createdId), createResponse.getHeader("Location")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("rule_topic_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", "dns", detectorTypeInResponse); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", rawVpcFlowDoc()); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(1, noOfSigmaRuleMatches); + } + + @SuppressWarnings("unchecked") + public void testOCSFCloudtrailGetMappingsViewApi() throws IOException { + String index = createTestIndex("cloudtrail", ocsfCloudtrailMappings()); + + Request request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI); + // both req params and req body are supported + request.addParameter("index_name", index); + request.addParameter("rule_topic", "cloudtrail"); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + Map respMap = responseAsMap(response); + // Verify alias mappings + Map props = (Map) respMap.get("properties"); + Assert.assertEquals(18, props.size()); + // Verify unmapped index fields + List unmappedIndexFields = (List) respMap.get("unmapped_index_fields"); + assertEquals(20, unmappedIndexFields.size()); + // Verify unmapped field aliases + List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); + assertEquals(25, unmappedFieldAliases.size()); + } + + @SuppressWarnings("unchecked") + public void testOCSFVpcflowGetMappingsViewApi() throws IOException { + String index = createTestIndex("vpcflow", ocsfVpcflowMappings()); + + Request request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI); + // both req params and req body are supported + request.addParameter("index_name", index); + request.addParameter("rule_topic", "vpcflow"); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + Map respMap = responseAsMap(response); + // Verify alias mappings + Map props = (Map) respMap.get("properties"); + Assert.assertEquals(20, props.size()); + // Verify unmapped index fields + List unmappedIndexFields = (List) respMap.get("unmapped_index_fields"); + assertEquals(26, unmappedIndexFields.size()); + // Verify unmapped field aliases + List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); + assertEquals(5, unmappedFieldAliases.size()); + } + + @SuppressWarnings("unchecked") + public void testOCSFRoute53GetMappingsViewApi() throws IOException { + String index = createTestIndex("route53", ocsfRoute53Mappings()); + + Request request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI); + // both req params and req body are supported + request.addParameter("index_name", index); + request.addParameter("rule_topic", "dns"); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + Map respMap = responseAsMap(response); + // Verify alias mappings + Map props = (Map) respMap.get("properties"); + Assert.assertEquals(11, props.size()); + // Verify unmapped index fields + List unmappedIndexFields = (List) respMap.get("unmapped_index_fields"); + assertEquals(28, unmappedIndexFields.size()); + // Verify unmapped field aliases + List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); + assertEquals(11, unmappedFieldAliases.size()); + } + + @SuppressWarnings("unchecked") + public void testRawCloudtrailGetMappingsViewApi() throws IOException { + String index = createTestIndex("cloudtrail", rawCloudtrailMappings()); + + Request request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI); + // both req params and req body are supported + request.addParameter("index_name", index); + request.addParameter("rule_topic", "cloudtrail"); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + Map respMap = responseAsMap(response); + // Verify alias mappings + Map props = (Map) respMap.get("properties"); + Assert.assertEquals(17, props.size()); + // Verify unmapped index fields + List unmappedIndexFields = (List) respMap.get("unmapped_index_fields"); + assertEquals(17, unmappedIndexFields.size()); + // Verify unmapped field aliases + List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); + assertEquals(26, unmappedFieldAliases.size()); + } + + @SuppressWarnings("unchecked") + public void testRawVpcflowGetMappingsViewApi() throws IOException { + String index = createTestIndex("vpcflow", rawVpcFlowMappings()); + + Request request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI); + // both req params and req body are supported + request.addParameter("index_name", index); + request.addParameter("rule_topic", "vpcflow"); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + Map respMap = responseAsMap(response); + // Verify alias mappings + Map props = (Map) respMap.get("properties"); + Assert.assertEquals(24, props.size()); + // Verify unmapped index fields + List unmappedIndexFields = (List) respMap.get("unmapped_index_fields"); + assertEquals(5, unmappedIndexFields.size()); + // Verify unmapped field aliases + List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); + assertEquals(1, unmappedFieldAliases.size()); + } + + @SuppressWarnings("unchecked") + public void testRawRoute53GetMappingsViewApi() throws IOException { + String index = createTestIndex("route53", rawRoute53DnsMappings()); + + Request request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI); + // both req params and req body are supported + request.addParameter("index_name", index); + request.addParameter("rule_topic", "dns"); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + Map respMap = responseAsMap(response); + // Verify alias mappings + Map props = (Map) respMap.get("properties"); + Assert.assertEquals(14, props.size()); + // Verify unmapped index fields + List unmappedIndexFields = (List) respMap.get("unmapped_index_fields"); + assertEquals(6, unmappedIndexFields.size()); + // Verify unmapped field aliases + List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); + assertEquals(8, unmappedFieldAliases.size()); + } + + private String rawCloudtrailDoc() { + return "{\n" + + " \"eventVersion\": \"1.03\",\n" + + " \"userIdentity\": {\n" + + " \"type\": \"IAMUser\",\n" + + " \"principalId\": \"123456789012\",\n" + + " \"arn\": \"arn:aws:iam::123456789012:user/Alice\",\n" + + " \"accountId\": \"123456789012\",\n" + + " \"accessKeyId\": \"AKIAIOSFODNN7EXAMPLE\",\n" + + " \"userName\": \"Alice\"\n" + + " },\n" + + " \"eventTime\": \"2016-04-01T15:31:48Z\",\n" + + " \"eventSource\": \"elasticloadbalancing.amazonaws.com\",\n" + + " \"eventName\": \"CreateLoadBalancer\",\n" + + " \"awsRegion\": \"us-west-2\",\n" + + " \"sourceIPAddress\": \"198.51.100.1\",\n" + + " \"userAgent\": \"aws-cli/1.10.10 Python/2.7.9 Windows/7 botocore/1.4.1\",\n" + + " \"requestParameters\": {\n" + + " \"subnets\": [\n" + + " \"subnet-8360a9e7\",\n" + + " \"subnet-b7d581c0\"\n" + + " ],\n" + + " \"securityGroups\": [\n" + + " \"sg-5943793c\"\n" + + " ],\n" + + " \"name\": \"my-load-balancer\",\n" + + " \"scheme\": \"internet-facing\"\n" + + " },\n" + + " \"responseElements\": {\n" + + " \"loadBalancers\": [\n" + + " {\n" + + " \"type\": \"application\",\n" + + " \"loadBalancerName\": \"my-load-balancer\",\n" + + " \"vpcId\": \"vpc-3ac0fb5f\",\n" + + " \"securityGroups\": [\n" + + " \"sg-5943793c\"\n" + + " ],\n" + + " \"state\": {\n" + + " \"code\": \"provisioning\"\n" + + " },\n" + + " \"availabilityZones\": [\n" + + " {\n" + + " \"subnetId\": \"subnet-8360a9e7\",\n" + + " \"zoneName\": \"us-west-2a\"\n" + + " },\n" + + " {\n" + + " \"subnetId\": \"subnet-b7d581c0\",\n" + + " \"zoneName\": \"us-west-2b\"\n" + + " }\n" + + " ],\n" + + " \"dNSName\": \"my-load-balancer-1836718677.us-west-2.elb.amazonaws.com\",\n" + + " \"canonicalHostedZoneId\": \"Z2P70J7HTTTPLU\",\n" + + " \"createdTime\": \"Apr 11, 2016 5:23:50 PM\",\n" + + " \"loadBalancerArn\": \"arn:aws:elasticloadbalancing:us-west-2:123456789012:loadbalancer/app/my-load-balancer/ffcddace1759e1d0\",\n" + + " \"scheme\": \"internet-facing\"\n" + + " }\n" + + " ]\n" + + " },\n" + + " \"requestID\": \"b9960276-b9b2-11e3-8a13-f1ef1EXAMPLE\",\n" + + " \"eventID\": \"6f4ab5bd-2daa-4d00-be14-d92efEXAMPLE\",\n" + + " \"eventType\": \"AwsApiCall\",\n" + + " \"apiVersion\": \"2015-12-01\",\n" + + " \"recipientAccountId\": \"123456789012\"\n" + + " }"; + } + + private String rawRoute53Doc() { + return "{\n" + + " \"version\": \"1.100000\",\n" + + " \"account_id\": \"123456789012\",\n" + + " \"region\": \"us-east-1\",\n" + + " \"vpc_id\": \"vpc-00000000000000000\",\n" + + " \"query_timestamp\": \"2022-10-13T21:02:36Z\",\n" + + " \"query_name\": \"ip-127-0-0-62.alert.firewall.canary.\",\n" + + " \"query_type\": \"A\",\n" + + " \"query_class\": \"IN\",\n" + + " \"rcode\": \"NOERROR\",\n" + + " \"answers\": [{\n" + + " \"Rdata\": \"127.0.0.62\",\n" + + " \"Type\": \"A\",\n" + + " \"Class\": \"IN\"\n" + + " }],\n" + + " \"srcaddr\": \"10.200.21.100\",\n" + + " \"srcport\": \"15083\",\n" + + " \"transport\": \"UDP\",\n" + + " \"srcids\": {\n" + + " \"resolver_endpoint\": \"rslvr-in-0000000000000000\",\n" + + " \"resolver_network_interface\": \"rni-0000000000000000\"\n" + + " },\n" + + " \"firewall_rule_action\": \"ALERT\",\n" + + " \"firewall_rule_group_id\": \"rslvr-frg-000000000000000\",\n" + + " \"firewall_domain_list_id\": \"rslvr-fdl-0000000000000\"\n" + + "}"; + } + + private String rawVpcFlowDoc() { + return "{\n" + + " \"account_id\": \"123456789012\",\n" + + " \"action\": \"REJECT\",\n" + + " \"az_id\": \"use1-az1\",\n" + + " \"bytes\": 40,\n" + + " \"dstaddr\": \"172.31.2.52\",\n" + + " \"dstport\": 39938,\n" + + " \"end\": 1649721788,\n" + + " \"flow_direction\": \"ingress\",\n" + + " \"instance_id\": \"i-000000000000000000\",\n" + + " \"interface_id\": \"eni-000000000000000000\",\n" + + " \"log_status\": \"OK\",\n" + + " \"packets\": 1,\n" + + " \"pkt_dst_aws_service\": \"-\",\n" + + " \"pkt_dstaddr\": \"172.31.2.52\",\n" + + " \"pkt_src_aws_service\": \"-\",\n" + + " \"pkt_srcaddr\": \"1.2.3.4\",\n" + + " \"protocol\": 6,\n" + + " \"region\": \"us-east-1\",\n" + + " \"srcaddr\": \"1.2.3.4\",\n" + + " \"srcport\": 56858,\n" + + " \"start\": 1649721732,\n" + + " \"sublocation_type\": \"-\",\n" + + " \"sublocation_id\": \"-\",\n" + + " \"subnet_id\": \"subnet-000000000000000000\",\n" + + " \"tcp_flags\": 2,\n" + + " \"traffic_path\": \"-\",\n" + + " \"type\": \"IPv4\",\n" + + " \"version\": 5,\n" + + " \"vpc_id\": \"vpc-00000000\"\n" + + "}"; + } + + private String customRoute53Rule() { + return "title: High DNS Requests Rate\n" + + "id: b4163085-4001-46a3-a79a-55d8bbbc7a3a\n" + + "status: experimental\n" + + "description: High DNS requests amount from host per short period of time\n" + + "author: Daniil Yugoslavskiy, oscd.community\n" + + "date: 2019/10/24\n" + + "modified: 2021/09/21\n" + + "tags:\n" + + " - attack.exfiltration\n" + + " - attack.t1048.003\n" + + " - attack.command_and_control\n" + + " - attack.t1071.004\n" + + "logsource:\n" + + " category: dns\n" + + "detection:\n" + + " selection:\n" + + " srcids.resolver_endpoint: 'rslvr-in-0000000000000000'\n" + + " timeframe: 1m\n" + + " condition: selection\n" + + "falsepositives:\n" + + " - Legitimate high DNS requests rate to domain name which should be added to whitelist\n" + + "level: medium"; + } + + private String customVpcFlowRule() { + return "title: High DNS Requests Rate\n" + + "id: b4163085-4001-46a3-a79a-55d8bbbc7a3a\n" + + "status: experimental\n" + + "description: High DNS requests amount from host per short period of time\n" + + "author: Daniil Yugoslavskiy, oscd.community\n" + + "date: 2019/10/24\n" + + "modified: 2021/09/21\n" + + "tags:\n" + + " - attack.exfiltration\n" + + " - attack.t1048.003\n" + + " - attack.command_and_control\n" + + " - attack.t1071.004\n" + + "logsource:\n" + + " category: dns\n" + + "detection:\n" + + " selection:\n" + + " srcaddr: '1.2.3.4'\n" + + " timeframe: 1m\n" + + " condition: selection\n" + + "falsepositives:\n" + + " - Legitimate high DNS requests rate to domain name which should be added to whitelist\n" + + "level: medium"; + } + + private String ocsfRoute53Doc() { + return "{\n" + + " \"metadata\": {\n" + + " \"product\": {\n" + + " \"version\": \"1.100000\",\n" + + " \"name\": \"Route 53\",\n" + + " \"feature\": {\n" + + " \"name\": \"Resolver Query Logs\"\n" + + " },\n" + + " \"vendor_name\": \"AWS\"\n" + + " },\n" + + " \"profiles\": [\n" + + " \"cloud\",\n" + + " \"security_control\"\n" + + " ],\n" + + " \"version\": \"1.0.0-rc.2\"\n" + + " },\n" + + " \"cloud\": {\n" + + " \"account_uid\": \"123456789012\",\n" + + " \"region\": \"us-east-1\",\n" + + " \"provider\": \"AWS\"\n" + + " },\n" + + " \"src_endpoint\": {\n" + + " \"vpc_uid\": \"vpc-00000000000000000\",\n" + + " \"ip\": \"10.200.21.100\",\n" + + " \"port\": 15083,\n" + + " \"instance_uid\": null\n" + + " },\n" + + " \"time\": 1665694956000,\n" + + " \"queries\": {\n" + + " \"hostname\": \"ip-127-0-0-62.alert.firewall.canary.\",\n" + + " \"type\": \"A\",\n" + + " \"class\": \"IN\"\n" + + " },\n" + + " \"answers\": [\n" + + " {\n" + + " \"type\": \"A\",\n" + + " \"rdata\": \"127.0.0.62\",\n" + + " \"class\": \"IN\"\n" + + " }\n" + + " ],\n" + + " \"connection_info\": {\n" + + " \"protocol_name\": \"UDP\",\n" + + " \"direction\": \"Unknown\",\n" + + " \"direction_id\": 0\n" + + " },\n" + + " \"dst_endpoint\": {\n" + + " \"instance_uid\": \"rslvr-in-0000000000000000\",\n" + + " \"interface_uid\": \"rni-0000000000000000\"\n" + + " },\n" + + " \"severity_id\": 1,\n" + + " \"severity\": \"Informational\",\n" + + " \"class_name\": \"DNS Activity\",\n" + + " \"class_uid\": 4003,\n" + + " \"category_name\": \"Network Activity\",\n" + + " \"category_uid\": 4,\n" + + " \"rcode_id\": 0,\n" + + " \"rcode\": \"NoError\",\n" + + " \"activity_id\": 2,\n" + + " \"activity_name\": \"Response\",\n" + + " \"type_name\": \"DNS Activity: Response\",\n" + + " \"type_uid\": 400302,\n" + + " \"disposition\": \"No Action\",\n" + + " \"disposition_id\": 16,\n" + + " \"unmapped\": [\n" + + " [\n" + + " \"firewall_rule_group_id\",\n" + + " \"rslvr-frg-000000000000000\"\n" + + " ],\n" + + " [\n" + + " \"firewall_domain_list_id\",\n" + + " \"rslvr-fdl-0000000000000\"\n" + + " ]\n" + + " ]\n" + + "}"; + } + + private String ocsfVpcflowDoc() { + return "{\n" + + " \"metadata\": {\n" + + " \"product\": {\n" + + " \"version\": \"5\",\n" + + " \"name\": \"Amazon VPC\",\n" + + " \"feature\": {\n" + + " \"name\": \"Flowlogs\"\n" + + " },\n" + + " \"vendor_name\": \"AWS\"\n" + + " },\n" + + " \"profiles\": [\n" + + " \"cloud\",\n" + + " \"security_control\"\n" + + " ],\n" + + " \"version\": \"1.0.0-rc.2\"\n" + + " },\n" + + " \"cloud\": {\n" + + " \"account_uid\": \"123456789012\",\n" + + " \"region\": \"us-east-2\",\n" + + " \"zone\": \"use1-az1\",\n" + + " \"provider\": \"AWS\"\n" + + " },\n" + + " \"src_endpoint\": {\n" + + " \"port\": 56858,\n" + + " \"svc_name\": \"-\",\n" + + " \"ip\": \"1.2.3.4\",\n" + + " \"intermediate_ips\": null,\n" + + " \"interface_uid\": \"eni-000000000000000000\",\n" + + " \"vpc_uid\": \"vpc-00000000\",\n" + + " \"instance_uid\": \"i-000000000000000000\",\n" + + " \"subnet_uid\": \"subnet-000000000000000000\"\n" + + " },\n" + + " \"dst_endpoint\": {\n" + + " \"port\": 39938,\n" + + " \"svc_name\": \"-\",\n" + + " \"ip\": \"172.31.2.52\",\n" + + " \"intermediate_ips\": null,\n" + + " \"interface_uid\": null,\n" + + " \"vpc_uid\": null,\n" + + " \"instance_uid\": null,\n" + + " \"subnet_uid\": null\n" + + " },\n" + + " \"connection_info\": {\n" + + " \"protocol_num\": 6,\n" + + " \"tcp_flags\": 2,\n" + + " \"protocol_ver\": \"IPv4\",\n" + + " \"boundary_id\": 99,\n" + + " \"boundary\": \"-\",\n" + + " \"direction_id\": 2,\n" + + " \"direction\": \"Outbound\"\n" + + " },\n" + + " \"traffic\": {\n" + + " \"packets\": 1,\n" + + " \"bytes\": 50\n" + + " },\n" + + " \"time\": 1649721732000,\n" + + " \"start_time\": 1649721732000,\n" + + " \"end_time\": 1649721788000,\n" + + " \"status_code\": \"OK\",\n" + + " \"severity_id\": 1,\n" + + " \"severity\": \"Informational\",\n" + + " \"class_name\": \"Network Activity\",\n" + + " \"class_uid\": 4001,\n" + + " \"category_name\": \"Network Activity\",\n" + + " \"category_uid\": 4,\n" + + " \"activity_name\": \"Traffic\",\n" + + " \"activity_id\": 6,\n" + + " \"disposition\": \"Allowed\",\n" + + " \"disposition_id\": 1,\n" + + " \"type_uid\": 400106,\n" + + " \"type_name\": \"Network Activity: Traffic\",\n" + + " \"unmapped\": [\n" + + " [\n" + + " \"sublocation_id\",\n" + + " \"-\"\n" + + " ],\n" + + " [\n" + + " \"sublocation_type\",\n" + + " \"-\"\n" + + " ]\n" + + " ]\n" + + "}"; + } + + private String ocsfCloudtrailDoc() { + return "{\n" + + " \"metadata\": {\n" + + " \"product\": {\n" + + " \"version\": \"1.03\",\n" + + " \"name\": \"CloudTrail\",\n" + + " \"vendor_name\": \"AWS\",\n" + + " \"feature\": {\n" + + " \"name\": \"Management, Data, and Insights\"\n" + + " }\n" + + " },\n" + + " \"uid\": \"6f4ab5bd-2daa-4d00-be14-d92efEXAMPLE\",\n" + + " \"profiles\": [\n" + + " \"cloud\"\n" + + " ],\n" + + " \"version\": \"1.0.0-rc.2\"\n" + + " },\n" + + " \"time\": 1459524708000,\n" + + " \"cloud\": {\n" + + " \"region\": \"us-west-2\",\n" + + " \"provider\": \"AWS\"\n" + + " },\n" + + " \"api\": {\n" + + " \"response\": {\n" + + " \"error\": null,\n" + + " \"message\": null\n" + + " },\n" + + " \"operation\": \"CreateLoadBalancer\",\n" + + " \"version\": \"2015-12-01\",\n" + + " \"service\": {\n" + + " \"name\": \"elasticloadbalancing.amazonaws.com\"\n" + + " },\n" + + " \"request\": {\n" + + " \"uid\": \"b9960276-b9b2-11e3-8a13-f1ef1EXAMPLE\"\n" + + " }\n" + + " },\n" + + " \"resources\": null,\n" + + " \"actor\": {\n" + + " \"user\": {\n" + + " \"type\": \"IAMUser\",\n" + + " \"name\": \"Alice\",\n" + + " \"uid\": \"123456789012\",\n" + + " \"uuid\": \"arn:aws:iam::123456789012:user/Alice\",\n" + + " \"account_uid\": \"123456789012\",\n" + + " \"credential_uid\": \"AKIAIOSFODNN7EXAMPLE\"\n" + + " },\n" + + " \"session\": {\n" + + " \"created_time\": null,\n" + + " \"mfa\": null,\n" + + " \"issuer\": null\n" + + " },\n" + + " \"invoked_by\": null,\n" + + " \"idp\": {\n" + + " \"name\": null\n" + + " }\n" + + " },\n" + + " \"http_request\": {\n" + + " \"user_agent\": \"aws-cli/1.10.10 Python/2.7.9 Windows/7 botocore/1.4.1\"\n" + + " },\n" + + " \"src_endpoint\": {\n" + + " \"uid\": null,\n" + + " \"ip\": \"198.51.100.1\",\n" + + " \"domain\": null\n" + + " },\n" + + " \"class_name\": \"API Activity\",\n" + + " \"class_uid\": 3005,\n" + + " \"category_name\": \"Audit Activity\",\n" + + " \"category_uid\": 3,\n" + + " \"severity_id\": 1,\n" + + " \"severity\": \"Informational\",\n" + + " \"status\": \"Success\",\n" + + " \"status_id\": 1,\n" + + " \"activity_name\": \"Create\",\n" + + " \"activity_id\": 1,\n" + + " \"type_uid\": 300501,\n" + + " \"type_name\": \"API Activity: Create\",\n" + + " \"unmapped\": [\n" + + " [\n" + + " \"responseElements.loadBalancers[].state.code\",\n" + + " \"provisioning\"\n" + + " ],\n" + + " [\n" + + " \"responseElements.loadBalancers[].canonicalHostedZoneId\",\n" + + " \"Z2P70J7HTTTPLU\"\n" + + " ],\n" + + " [\n" + + " \"responseElements.loadBalancers[].availabilityZones[].subnetId\",\n" + + " \"subnet-8360a9e7,subnet-b7d581c0\"\n" + + " ],\n" + + " [\n" + + " \"responseElements.loadBalancers[].loadBalancerName\",\n" + + " \"my-load-balancer\"\n" + + " ],\n" + + " [\n" + + " \"responseElements.loadBalancers[].type\",\n" + + " \"application\"\n" + + " ],\n" + + " [\n" + + " \"responseElements.loadBalancers[].dNSName\",\n" + + " \"my-load-balancer-1836718677.us-west-2.elb.amazonaws.com\"\n" + + " ],\n" + + " [\n" + + " \"eventType\",\n" + + " \"AwsApiCall\"\n" + + " ],\n" + + " [\n" + + " \"responseElements.loadBalancers[].vpcId\",\n" + + " \"vpc-3ac0fb5f\"\n" + + " ],\n" + + " [\n" + + " \"responseElements.loadBalancers[].securityGroups[]\",\n" + + " \"sg-5943793c\"\n" + + " ],\n" + + " [\n" + + " \"requestParameters.subnets[]\",\n" + + " \"subnet-8360a9e7,subnet-b7d581c0\"\n" + + " ],\n" + + " [\n" + + " \"requestParameters.securityGroups[]\",\n" + + " \"sg-5943793c\"\n" + + " ],\n" + + " [\n" + + " \"responseElements.loadBalancers[].scheme\",\n" + + " \"internet-facing\"\n" + + " ],\n" + + " [\n" + + " \"responseElements.loadBalancers[].availabilityZones[].zoneName\",\n" + + " \"us-west-2a,us-west-2b\"\n" + + " ],\n" + + " [\n" + + " \"recipientAccountId\",\n" + + " \"123456789012\"\n" + + " ],\n" + + " [\n" + + " \"responseElements.loadBalancers[].createdTime\",\n" + + " \"Apr 11, 2016 5:23:50 PM\"\n" + + " ],\n" + + " [\n" + + " \"responseElements.loadBalancers[].loadBalancerArn\",\n" + + " \"arn:aws:elasticloadbalancing:us-west-2:123456789012:loadbalancer/app/my-load-balancer/ffcddace1759e1d0\"\n" + + " ],\n" + + " [\n" + + " \"requestParameters.scheme\",\n" + + " \"internet-facing\"\n" + + " ],\n" + + " [\n" + + " \"requestParameters.name\",\n" + + " \"my-load-balancer\"\n" + + " ]\n" + + " ]\n" + + "}"; + } + + private String rawRoute53DnsMappings() { + return "\"properties\": {\n" + + " \"account_id\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"answers\": {\n" + + " \"properties\": {\n" + + " \"Class\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"Rdata\": {\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"Type\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"firewall_domain_list_id\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"firewall_rule_action\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"firewall_rule_group_id\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"query_class\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"query_name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"query_timestamp\": {\n" + + " \"type\": \"date\"\n" + + " },\n" + + " \"query_type\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"rcode\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"region\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"srcaddr\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"srcids\": {\n" + + " \"properties\": {\n" + + " \"resolver_endpoint\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"resolver_network_interface\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"srcport\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"transport\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"version\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"vpc_id\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }"; + } + + private String rawCloudtrailMappings() { + return "\"properties\": {\n" + + " \"apiVersion\": {\n" + + " \"type\": \"date\"\n" + + " },\n" + + " \"awsRegion\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventID\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventName\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventSource\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventTime\": {\n" + + " \"type\": \"date\"\n" + + " },\n" + + " \"eventType\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventVersion\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"recipientAccountId\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"requestID\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"requestParameters\": {\n" + + " \"properties\": {\n" + + " \"name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"scheme\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"securityGroups\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"subnets\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"responseElements\": {\n" + + " \"properties\": {\n" + + " \"loadBalancers\": {\n" + + " \"properties\": {\n" + + " \"availabilityZones\": {\n" + + " \"properties\": {\n" + + " \"subnetId\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"zoneName\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"canonicalHostedZoneId\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"createdTime\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"dNSName\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"loadBalancerArn\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"loadBalancerName\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"scheme\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"securityGroups\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"state\": {\n" + + " \"properties\": {\n" + + " \"code\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"type\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"vpcId\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"sourceIPAddress\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"userAgent\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"userIdentity\": {\n" + + " \"properties\": {\n" + + " \"accessKeyId\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"accountId\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"arn\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"principalId\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"type\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"userName\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }"; + } + + private String ocsfRoute53Mappings() { + return "\"properties\": {\n" + + " \"activity_id\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"activity_name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"answers\": {\n" + + " \"properties\": {\n" + + " \"class\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"rdata\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"type\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"category_name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"category_uid\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"class_name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"class_uid\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"cloud\": {\n" + + " \"properties\": {\n" + + " \"account_uid\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"provider\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"region\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"connection_info\": {\n" + + " \"properties\": {\n" + + " \"direction\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"direction_id\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"protocol_name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"disposition\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"disposition_id\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"dst_endpoint\": {\n" + + " \"properties\": {\n" + + " \"instance_uid\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"interface_uid\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"metadata\": {\n" + + " \"properties\": {\n" + + " \"product\": {\n" + + " \"properties\": {\n" + + " \"feature\": {\n" + + " \"properties\": {\n" + + " \"name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"vendor_name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"version\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"profiles\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"version\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"queries\": {\n" + + " \"properties\": {\n" + + " \"class\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"hostname\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"type\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"rcode\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"rcode_id\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"severity\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"severity_id\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"src_endpoint\": {\n" + + " \"properties\": {\n" + + " \"ip\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"port\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"vpc_uid\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"time\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"type_name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"type_uid\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"unmapped\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }"; + } + + private String ocsfVpcflowMappings() { + return "\"properties\": {\n" + + " \"activity_id\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"activity_name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"category_name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"category_uid\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"class_name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"class_uid\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"cloud\": {\n" + + " \"properties\": {\n" + + " \"account_uid\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"provider\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"region\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"zone\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"connection_info\": {\n" + + " \"properties\": {\n" + + " \"boundary\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"boundary_id\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"direction\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"direction_id\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"protocol_num\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"protocol_ver\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"tcp_flags\": {\n" + + " \"type\": \"long\"\n" + + " }\n" + + " }\n" + + " },\n" + + " \"disposition\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"disposition_id\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"dst_endpoint\": {\n" + + " \"properties\": {\n" + + " \"ip\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"port\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"svc_name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"end_time\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"metadata\": {\n" + + " \"properties\": {\n" + + " \"product\": {\n" + + " \"properties\": {\n" + + " \"feature\": {\n" + + " \"properties\": {\n" + + " \"name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"vendor_name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"version\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"profiles\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"version\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"severity\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"severity_id\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"src_endpoint\": {\n" + + " \"properties\": {\n" + + " \"instance_uid\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"interface_uid\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"ip\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"port\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"subnet_uid\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"svc_name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"vpc_uid\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"start_time\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"status_code\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"time\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"traffic\": {\n" + + " \"properties\": {\n" + + " \"bytes\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"packets\": {\n" + + " \"type\": \"long\"\n" + + " }\n" + + " }\n" + + " },\n" + + " \"type_name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"type_uid\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"unmapped\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }"; + } + + private String ocsfCloudtrailMappings() { + return "\"properties\": {\n" + + " \"activity_id\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"activity_name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"actor\": {\n" + + " \"properties\": {\n" + + " \"idp\": {\n" + + " \"type\": \"object\"\n" + + " },\n" + + " \"session\": {\n" + + " \"type\": \"object\"\n" + + " },\n" + + " \"user\": {\n" + + " \"properties\": {\n" + + " \"account_uid\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"credential_uid\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"type\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"uid\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"uuid\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"api\": {\n" + + " \"properties\": {\n" + + " \"operation\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"request\": {\n" + + " \"properties\": {\n" + + " \"uid\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"response\": {\n" + + " \"type\": \"object\"\n" + + " },\n" + + " \"service\": {\n" + + " \"properties\": {\n" + + " \"name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"version\": {\n" + + " \"type\": \"date\"\n" + + " }\n" + + " }\n" + + " },\n" + + " \"category_name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"category_uid\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"class_name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"class_uid\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"cloud\": {\n" + + " \"properties\": {\n" + + " \"provider\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"region\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"http_request\": {\n" + + " \"properties\": {\n" + + " \"user_agent\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"metadata\": {\n" + + " \"properties\": {\n" + + " \"product\": {\n" + + " \"properties\": {\n" + + " \"feature\": {\n" + + " \"properties\": {\n" + + " \"name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"vendor_name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"version\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"profiles\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"uid\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"version\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"severity\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"severity_id\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"src_endpoint\": {\n" + + " \"properties\": {\n" + + " \"ip\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"status\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"status_id\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"time\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"type_name\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"type_uid\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"unmapped\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }"; + } + + private String rawVpcFlowMappings() { + return "\"properties\": {\n" + + " \"account_id\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"action\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"az_id\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"bytes\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"dstaddr\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"dstport\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"end\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"flow_direction\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"instance_id\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"interface_id\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"log_status\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"packets\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"pkt_dst_aws_service\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"pkt_dstaddr\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"pkt_src_aws_service\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"pkt_srcaddr\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"protocol\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"region\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"srcaddr\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"srcport\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"start\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"sublocation_id\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"sublocation_type\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"subnet_id\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"tcp_flags\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"traffic_path\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"type\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"version\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"vpc_id\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }"; + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java index c025dafab..a57897d85 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java @@ -8,6 +8,7 @@ import org.apache.http.entity.StringEntity; import org.apache.http.message.BasicHeader; import org.junit.Assert; +import org.junit.Ignore; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; @@ -103,13 +104,14 @@ public void testCreatingARule() throws IOException { Assert.assertEquals(0, hits.size()); } - public void testCreatingARule_incorrect_category() throws IOException { + @Ignore + public void testCreatingARule_custom_category() throws IOException { String rule = randomRule(); try { makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "unknown_category"), new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); - fail("expected exception due to invalid category"); +// fail("expected exception due to invalid category"); } catch (ResponseException e) { assertEquals(HttpStatus.SC_BAD_REQUEST, e.getResponse().getStatusLine().getStatusCode()); Assert.assertTrue( @@ -355,7 +357,8 @@ public void testUpdatingUnusedRule() throws IOException { Assert.assertEquals("Update rule failed", RestStatus.OK, restStatus(updateResponse)); } - public void testUpdatingARule_incorrect_category() throws IOException { + @Ignore + public void testUpdatingARule_custom_category() throws IOException { String index = createTestIndex(randomIndex(), windowsIndexMapping()); // Execute CreateMappingsAction to add alias mapping for index @@ -775,19 +778,18 @@ public void testCustomRuleValidation() throws IOException { public void testGetAllRuleCategories() throws IOException { Response response = makeRequest(client(), "GET", SecurityAnalyticsPlugin.RULE_BASE_URI + "/categories", Collections.emptyMap(), null); List categories = (List) asMap(response).get("rule_categories"); - assertEquals(13, categories.size()); - assertTrue(((Map)categories.get(0)).get("key").equals("ad_ldap")); - assertTrue(((Map)categories.get(1)).get("key").equals("dns")); - assertTrue(((Map)categories.get(2)).get("key").equals("network")); - assertTrue(((Map)categories.get(3)).get("key").equals("apache_access")); - assertTrue(((Map)categories.get(4)).get("key").equals("cloudtrail")); - assertTrue(((Map)categories.get(5)).get("key").equals("s3")); - assertTrue(((Map)categories.get(6)).get("key").equals("windows")); - assertTrue(((Map)categories.get(7)).get("key").equals("gworkspace")); - assertTrue(((Map)categories.get(8)).get("key").equals("github")); - assertTrue(((Map)categories.get(9)).get("key").equals("m365")); - assertTrue(((Map)categories.get(10)).get("key").equals("okta")); - assertTrue(((Map)categories.get(11)).get("key").equals("azure")); - assertTrue(((Map)categories.get(12)).get("key").equals("linux")); + assertEquals(22, categories.size()); + assertTrue(categories.stream().anyMatch(e -> ((Map)e).get("key").equals("ad_ldap"))); + assertTrue(categories.stream().anyMatch(e -> ((Map)e).get("key").equals("dns"))); + assertTrue(categories.stream().anyMatch(e -> ((Map)e).get("key").equals("network"))); + assertTrue(categories.stream().anyMatch(e -> ((Map)e).get("key").equals("cloudtrail"))); + assertTrue(categories.stream().anyMatch(e -> ((Map)e).get("key").equals("s3"))); + assertTrue(categories.stream().anyMatch(e -> ((Map)e).get("key").equals("windows"))); + assertTrue(categories.stream().anyMatch(e -> ((Map)e).get("key").equals("gworkspace"))); + assertTrue(categories.stream().anyMatch(e -> ((Map)e).get("key").equals("github"))); + assertTrue(categories.stream().anyMatch(e -> ((Map)e).get("key").equals("m365"))); + assertTrue(categories.stream().anyMatch(e -> ((Map)e).get("key").equals("okta"))); + assertTrue(categories.stream().anyMatch(e -> ((Map)e).get("key").equals("azure"))); + assertTrue(categories.stream().anyMatch(e -> ((Map)e).get("key").equals("linux"))); } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/Writable/LogTypeTests.java b/src/test/java/org/opensearch/securityanalytics/writable/LogTypeTests.java similarity index 98% rename from src/test/java/org/opensearch/securityanalytics/Writable/LogTypeTests.java rename to src/test/java/org/opensearch/securityanalytics/writable/LogTypeTests.java index e8a709ef3..2013fdc29 100644 --- a/src/test/java/org/opensearch/securityanalytics/Writable/LogTypeTests.java +++ b/src/test/java/org/opensearch/securityanalytics/writable/LogTypeTests.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.Writable; +package org.opensearch.securityanalytics.writable; import java.io.IOException; import java.util.List; @@ -63,4 +63,4 @@ public void testLogTypeAsStreamNoMappings() throws IOException { assertEquals(logType.getIsBuiltIn(), newLogType.getIsBuiltIn()); assertEquals(logType.getMappings().size(), newLogType.getMappings().size()); } -} +} \ No newline at end of file diff --git a/src/test/resources/ad_ldap-sample.json b/src/test/resources/ad_ldap-sample.json index 10c05dc02..862522e99 100644 --- a/src/test/resources/ad_ldap-sample.json +++ b/src/test/resources/ad_ldap-sample.json @@ -18,5 +18,5 @@ "EventID": 12345, "azure.signinlogs.properties.network_location_details": "1234", "azure.auditlogs.properties.activity_display_name": "1234", - "@timestamp": "2022-12-27T20:29:31.734Z" + "creationTime": "2022-12-27T20:29:31.734Z" } \ No newline at end of file diff --git a/src/test/resources/azure-sample.json b/src/test/resources/azure-sample.json index d42291fbc..ab69da0b4 100644 --- a/src/test/resources/azure-sample.json +++ b/src/test/resources/azure-sample.json @@ -23,5 +23,5 @@ "azure.platformlogs.status": "111", "azure.auditlogs.props.logged_by_service": "111", "properties_message": "111", - "@timestamp": "2022-12-27T20:29:31.734Z" + "creationTime": "2022-12-27T20:29:31.734Z" } \ No newline at end of file diff --git a/src/test/resources/cloudtrail-sample.json b/src/test/resources/cloudtrail-sample.json index 1732bff36..02e9ee2fb 100644 --- a/src/test/resources/cloudtrail-sample.json +++ b/src/test/resources/cloudtrail-sample.json @@ -27,5 +27,5 @@ "aws-cloudtrail-request_parameters-container_definitions-command": "123", "aws-cloudtrail-user_identity-session_context-session_issuer-type": "123", "aws-cloudtrail-user_identity-arn": "123", - "@timestamp": "2022-12-27T20:29:31.734Z" + "eventTime": "2022-12-27T20:29:31.734Z" } diff --git a/src/test/resources/dns-sample.json b/src/test/resources/dns-sample.json index 1c4df84ce..84fd0d88f 100644 --- a/src/test/resources/dns-sample.json +++ b/src/test/resources/dns-sample.json @@ -1,50 +1,26 @@ { - "dns": { - "additionals_count": 0, - "answers": [ - { - "class": "IN", - "data": "192.168.73.66", - "name": "chat.testdomain.loc", - "ttl": "52", - "type": "A" - } - ], - "answers_count": 1, - "authorities_count": 0, - "flags": { - "authentic_data": false, - "authoritative": false, - "checking_disabled": false, - "recursion_available": true, - "recursion_desired": true, - "truncated_response": false - }, - "header_flags": [ - "RD", - "RA" - ], - "id": 59295, - "op_code": "QUERY", - "opt": { - "do": false, - "ext_rcode": "NOERROR", - "udp_size": 512, - "version": "0" - }, - "question": { - "class": "IN", - "etld_plus_one": "testdomain.loc", - "name": "chat.testdomain.loc", - "registered_domain": "testdomain.loc", - "subdomain": "chat", - "top_level_domain": "loc", - "type": "A" - }, - "resolved_ip": [ - "192.168.73.66" - ], - "response_code": "NOERROR", - "type": "answer" - } + "version": "1.100000", + "account_id": "123456789012", + "region": "us-east-1", + "vpc_id": "vpc-00000000000000000", + "query_timestamp": "2022-10-13T21:02:36Z", + "query_name": "ip-127-0-0-62.alert.firewall.canary.", + "query_type": "A", + "query_class": "IN", + "rcode": "NOERROR", + "answers": [{ + "Rdata": "127.0.0.62", + "Type": "A", + "Class": "IN" + }], + "srcaddr": "10.200.21.100", + "srcport": "15083", + "transport": "UDP", + "srcids": { + "resolver_endpoint": "rslvr-in-0000000000000000", + "resolver_network_interface": "rni-0000000000000000" + }, + "firewall_rule_action": "ALERT", + "firewall_rule_group_id": "rslvr-frg-000000000000000", + "firewall_domain_list_id": "rslvr-fdl-0000000000000" } \ No newline at end of file diff --git a/src/test/resources/s3-sample.json b/src/test/resources/s3-sample.json index 58253cd99..49669477b 100644 --- a/src/test/resources/s3-sample.json +++ b/src/test/resources/s3-sample.json @@ -10,5 +10,5 @@ "aws.s3access.requester": "1234", "aws-cloudtrail-event_source": "123", "aws-cloudtrail-event_name": "123", - "@timestamp": "123" + "eventTime": "123" } \ No newline at end of file