Skip to content

Commit

Permalink
adds ioc fields list in log type config files and ioc fields object i…
Browse files Browse the repository at this point in the history 
…n LogType POJO

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>
  • Loading branch information
@eirsep
eirsep committed Oct 16, 2023
1 parent d53085b commit 98bbd42
Show file tree
Hide file tree
Showing 8 changed files with 214 additions and 53 deletions.
2 changes: 1 addition & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ dependencies {
implementation group: 'org.apache.commons', name: 'commons-lang3', version: "${versions.commonslang}"
implementation "org.antlr:antlr4-runtime:4.10.1"
implementation "com.cronutils:cron-utils:9.1.6"
api "org.opensearch:common-utils:${common_utils_version}@jar"
api files("/Users/snistala/Documents/opensearch/common-utils/build/libs/common-utils-3.0.0.0-SNAPSHOT.jar")
api "org.opensearch.client:opensearch-rest-client:${opensearch_version}"
implementation "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"
implementation "org.opensearch:opensearch-job-scheduler-spi:${opensearch_build}"
Expand Down
8 changes: 8 additions & 0 deletions src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
Expand Down Expand Up @@ -660,6 +661,13 @@ public void getRuleFieldMappings(String logType, ActionListener<Map<String, Stri
return;
}

public List<LogType.IocFields> getIocFieldsList(String logType) {
LogType logTypeByName = builtinLogTypeLoader.getLogTypeByName(logType);
if(logTypeByName == null)
return Collections.emptyList();
return logTypeByName.getIocFieldsList();
}

public void getRuleFieldMappingsAllSchemas(String logType, ActionListener<List<LogType.Mapping>> listener) {

if (builtinLogTypeLoader.logTypeExists(logType)) {
Expand Down
66 changes: 63 additions & 3 deletions src/main/java/org/opensearch/securityanalytics/model/LogType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,13 @@

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;

public class LogType implements Writeable {

Expand All @@ -25,27 +24,33 @@ public class LogType implements Writeable {
private static final String RAW_FIELD = "raw_field";
public static final String ECS = "ecs";
public static final String OCSF = "ocsf";
public static final String IOC_FIELDS = "ioc_fields";
public static final String IOC = "ioc";
public static final String FIELDS = "fields";

private String id;
private String name;
private String description;
private Boolean isBuiltIn;
private List<Mapping> mappings;
private List<IocFields> iocFieldsList;

public LogType(StreamInput sin) throws IOException {
this.id = sin.readString();
this.isBuiltIn = sin.readOptionalBoolean();
this.name = sin.readString();
this.description = sin.readString();
this.mappings = sin.readList(Mapping::readFrom);
this.iocFieldsList = sin.readList(IocFields::readFrom);
}

public LogType(String id, String name, String description, boolean isBuiltIn, List<Mapping> mappings) {
public LogType(String id, String name, String description, boolean isBuiltIn, List<Mapping> mappings, List<IocFields> iocFieldsList) {
this.id = id;
this.name = name;
this.description = description;
this.isBuiltIn = isBuiltIn;
this.mappings = mappings == null ? List.of() : mappings;
this.iocFieldsList = iocFieldsList == null ? List.of() : iocFieldsList;
}

public LogType(Map<String, Object> logTypeAsMap) {
Expand All @@ -62,6 +67,17 @@ public LogType(Map<String, Object> logTypeAsMap) {
new Mapping(e.get(RAW_FIELD), e.get(ECS), e.get(OCSF))
).collect(Collectors.toList());
}
if(logTypeAsMap.containsKey(IOC_FIELDS)) {
List<Map<String, Object>> iocFieldsList = (List<Map<String, Object>>) logTypeAsMap.get(IOC_FIELDS);
if (iocFieldsList.size() > 0) {
this.iocFieldsList = new ArrayList<>(mappings.size());
this.iocFieldsList = iocFieldsList.stream().map(e ->
new IocFields(e.get(IOC).toString(), (List<String>) e.get(FIELDS))
).collect(Collectors.toList());
}
} else {
iocFieldsList = Collections.emptyList();
}
}

public String getName() {
Expand All @@ -74,6 +90,10 @@ public String getDescription() {

public boolean getIsBuiltIn() { return isBuiltIn; }

public List<IocFields> getIocFieldsList() {
return iocFieldsList;
}

public List<Mapping> getMappings() {
return mappings;
}
Expand All @@ -85,6 +105,7 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeString(name);
out.writeString(description);
out.writeCollection(mappings);
out.writeCollection(iocFieldsList);
}

@Override
Expand Down Expand Up @@ -134,4 +155,43 @@ public static Mapping readFrom(StreamInput sin) throws IOException {
}
}

/**
* stores information of list of field names that contain information for given IoC (Indicator of Compromise).
*/
public static class IocFields implements Writeable {
private final String ioc;

private final List<String> fields;

public IocFields(String ioc, List<String> fields) {
this.ioc = ioc;
this.fields = fields;
}

public IocFields(StreamInput sin) throws IOException {
this.ioc = sin.readString();
this.fields = sin.readStringList();
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(ioc);
out.writeStringCollection(fields);
}

public String getIoc() {
return ioc;
}

public List<String> getFields() {
return fields;
}


public static IocFields readFrom(StreamInput sin) throws IOException {
return new IocFields(sin);
}
}


}
76 changes: 54 additions & 22 deletions src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,19 @@
package org.opensearch.securityanalytics.threatIntel;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.client.Client;
import org.opensearch.common.settings.Settings;
import org.opensearch.commons.alerting.model.DocLevelQuery;
import org.opensearch.core.action.ActionListener;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.securityanalytics.SecurityAnalyticsPlugin;
import org.opensearch.securityanalytics.model.Detector;
import org.opensearch.securityanalytics.model.LogType;
import org.opensearch.securityanalytics.model.ThreatIntelFeedData;
import org.opensearch.securityanalytics.util.SecurityAnalyticsException;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
Expand All @@ -19,33 +23,56 @@

public class DetectorThreatIntelService {

private static final Logger log = LogManager.getLogger(DetectorThreatIntelService.class);

private final ThreatIntelFeedDataService threatIntelFeedDataService;

public DetectorThreatIntelService(ThreatIntelFeedDataService threatIntelFeedDataService) {
this.threatIntelFeedDataService = threatIntelFeedDataService;
}


/**
* Convert the feed data IOCs into query string query format to create doc level queries.
*/
public DocLevelQuery createDocLevelQueryFromThreatIntelList(
List<ThreatIntelFeedData> tifdList, String docLevelQueryId
public List<DocLevelQuery> createDocLevelQueriesFromThreatIntelList(
List<LogType.IocFields> iocFieldList, List<ThreatIntelFeedData> tifdList, Detector detector
) {
List<DocLevelQuery> queries = new ArrayList<>();
Set<String> iocs = tifdList.stream().map(ThreatIntelFeedData::getIocValue).collect(Collectors.toSet());
String query = buildQueryStringQueryWithIocList(iocs);
return new DocLevelQuery(
docLevelQueryId, tifdList.get(0).getFeedId(),
Collections.singletonList("*"),
query,
Collections.singletonList("threat_intel")
);
//ioc types supported by log type
List<String> logTypeIocs = iocFieldList.stream().map(LogType.IocFields::getIoc).collect(Collectors.toList());
// filter out ioc types not supported for given log types
Map<String, Set<String>> iocTypeToValues = tifdList.stream().filter(t -> logTypeIocs.contains(t.getIocType()))
.collect(Collectors.groupingBy(
ThreatIntelFeedData::getIocType,
Collectors.mapping(ThreatIntelFeedData::getIocValue, Collectors.toSet())
));

for (Map.Entry<String, Set<String>> entry : iocTypeToValues.entrySet()) {
String query = buildQueryStringQueryWithIocList(iocs);
List<String> fields = iocFieldList.stream().filter(t -> entry.getKey().matches(t.getIoc())).findFirst().get().getFields();

// create doc
for (String field : fields) { //todo increase max clause count from 1024
queries.add(new DocLevelQuery(
constructId(detector, entry.getKey()), tifdList.get(0).getFeedId(),
Collections.emptyList(),
String.format(query, field),
List.of("threat_intel", entry.getKey() /*ioc_type*/)
));
}
}
return queries;
}

private String buildQueryStringQueryWithIocList(Set<String> iocs) {
StringBuilder sb = new StringBuilder();
sb.append("%s");
sb.append(":");
sb.append("(");
for (String ioc : iocs) {
if (sb.length() > 2) {
if (sb.length() > 4) {
sb.append(" OR ");
}
sb.append(ioc);
Expand All @@ -55,42 +82,47 @@ private String buildQueryStringQueryWithIocList(Set<String> iocs) {
return sb.toString();
}

public void createDocLevelQueryFromThreatIntel(Detector detector, ActionListener<DocLevelQuery> listener) {
public void createDocLevelQueryFromThreatIntel(List<LogType.IocFields> iocFieldList, Detector detector, ActionListener<List<DocLevelQuery>> listener) {
try {
if (detector.getThreatIntelEnabled() == false) {
listener.onResponse(null);
if (false == detector.getThreatIntelEnabled() || iocFieldList.isEmpty()) {
listener.onResponse(Collections.emptyList());
return;

}

CountDownLatch latch = new CountDownLatch(1);
threatIntelFeedDataService.getThreatIntelFeedData(new ActionListener<>() {
@Override
public void onResponse(List<ThreatIntelFeedData> threatIntelFeedData) {
if (threatIntelFeedData.isEmpty()) {
listener.onResponse(null);
listener.onResponse(Collections.emptyList());
} else {
listener.onResponse(createDocLevelQueryFromThreatIntelList(
threatIntelFeedData,
detector.getName() + "_threat_intel" + UUID.randomUUID()
));
listener.onResponse(
createDocLevelQueriesFromThreatIntelList(iocFieldList, threatIntelFeedData, detector)
);
}
latch.countDown();
}

@Override
public void onFailure(Exception e) {
log.error("Failed to get threat intel feeds for doc level query creation", e);
listener.onFailure(e);
latch.countDown();
}
});

latch.await(30, TimeUnit.SECONDS);
} catch (InterruptedException e) {
log.error("Failed to create doc level queries from threat intel feeds", e);
listener.onFailure(e);
}

}

private static String constructId(Detector detector, String iocType) {
return detector.getName() + "_threat_intel_" + iocType + "_" + UUID.randomUUID();
}

public void updateDetectorsWithLatestThreatIntelRules() {

}
Expand Down
Loading

0 comments on commit 98bbd42

Please sign in to comment.