Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Threat Intel Feeds Updated Job Scheduler Framework #649

Closed
wants to merge 14 commits into from
Closed

Threat Intel Feeds Updated Job Scheduler Framework #649

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
cc98020
add mapping for indices storing threat intel feed data
eirsep Oct 2, 2023
f192d98
fix feed indices mapping
eirsep Oct 2, 2023
c93c492
add threat intel feed data dao
eirsep Oct 3, 2023
88f417d
add threatIntelEnabled field in detector.
eirsep Oct 3, 2023
2c0df71
add threat intel feed service and searching feeds
eirsep Oct 3, 2023
f9d5477
ti feed data to doc level query convertor logic added
eirsep Oct 3, 2023
c58d233
plug threat intel feed into detector creation
eirsep Oct 4, 2023
18f4498
Preliminary framework for jobscheduler and datasource (#626)
jowg-amazon Oct 5, 2023
c7507aa
with listener and processor
jowg-amazon Oct 5, 2023
0628964
removed actions
jowg-amazon Oct 6, 2023
93c2b3a
clean up
jowg-amazon Oct 7, 2023
d9f5fb7
added parser
jowg-amazon Oct 9, 2023
b894f11
add unit tests
jowg-amazon Oct 10, 2023
0b46131
Merge branch 'opensearch-project:main' into threatintelfeeds
jowg-amazon Oct 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,8 @@ dependencies {
api "org.opensearch:common-utils:${common_utils_version}@jar"
api "org.opensearch.client:opensearch-rest-client:${opensearch_version}"
implementation "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"
compileOnly "org.opensearch:opensearch-job-scheduler-spi:${opensearch_build}"
implementation "org.apache.commons:commons-csv:1.10.0"

// Needed for integ tests
zipArchive group: 'org.opensearch.plugin', name:'alerting', version: "${opensearch_build}"
Expand Down
74 changes: 53 additions & 21 deletions src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,13 @@
*/
package org.opensearch.securityanalytics;

import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.*;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

plz revert wildcard imports

import java.util.function.Supplier;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.cluster.routing.Preference;
import org.opensearch.core.action.ActionListener;
import org.opensearch.action.ActionRequest;
import org.opensearch.core.action.ActionResponse;
import org.opensearch.action.search.SearchRequest;
import org.opensearch.action.search.SearchResponse;
import org.opensearch.client.Client;
import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
import org.opensearch.cluster.node.DiscoveryNode;
Expand All @@ -38,18 +31,12 @@
import org.opensearch.index.codec.CodecServiceFactory;
import org.opensearch.index.engine.EngineFactory;
import org.opensearch.index.mapper.Mapper;
import org.opensearch.index.query.QueryBuilders;
import org.opensearch.plugins.ActionPlugin;
import org.opensearch.plugins.ClusterPlugin;
import org.opensearch.plugins.EnginePlugin;
import org.opensearch.plugins.MapperPlugin;
import org.opensearch.plugins.Plugin;
import org.opensearch.plugins.SearchPlugin;
import org.opensearch.indices.SystemIndexDescriptor;
import org.opensearch.plugins.*;
import org.opensearch.repositories.RepositoriesService;
import org.opensearch.rest.RestController;
import org.opensearch.rest.RestHandler;
import org.opensearch.script.ScriptService;
import org.opensearch.search.builder.SearchSourceBuilder;
import org.opensearch.securityanalytics.action.*;
import org.opensearch.securityanalytics.correlation.index.codec.CorrelationCodecService;
import org.opensearch.securityanalytics.correlation.index.mapper.CorrelationVectorFieldMapper;
Expand All @@ -60,7 +47,15 @@
import org.opensearch.securityanalytics.mapper.IndexTemplateManager;
import org.opensearch.securityanalytics.mapper.MapperService;
import org.opensearch.securityanalytics.model.CustomLogType;
import org.opensearch.securityanalytics.model.ThreatIntelFeedData;
import org.opensearch.securityanalytics.resthandler.*;
import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService;
import org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataService;
import org.opensearch.securityanalytics.threatIntel.common.ThreatIntelExecutor;
import org.opensearch.securityanalytics.threatIntel.common.ThreatIntelLockService;
import org.opensearch.securityanalytics.threatIntel.dao.DatasourceDao;
import org.opensearch.securityanalytics.threatIntel.jobscheduler.DatasourceRunner;
import org.opensearch.securityanalytics.threatIntel.jobscheduler.DatasourceUpdateService;
import org.opensearch.securityanalytics.transport.*;
import org.opensearch.securityanalytics.model.Rule;
import org.opensearch.securityanalytics.model.Detector;
Expand All @@ -72,10 +67,13 @@
import org.opensearch.securityanalytics.util.DetectorIndices;
import org.opensearch.securityanalytics.util.RuleIndices;
import org.opensearch.securityanalytics.util.RuleTopicIndices;
import org.opensearch.threadpool.ExecutorBuilder;
import org.opensearch.threadpool.ThreadPool;
import org.opensearch.watcher.ResourceWatcherService;

public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin {
import static org.opensearch.securityanalytics.threatIntel.jobscheduler.Datasource.THREAT_INTEL_DATA_INDEX_NAME_PREFIX;

public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin, SystemIndexPlugin {

private static final Logger log = LogManager.getLogger(SecurityAnalyticsPlugin.class);

Expand Down Expand Up @@ -116,6 +114,22 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map

private Client client;

// private DatasourceDao datasourceDao;

// private ThreatIntelFeedDataService threatIntelFeedDataService;

@Override
public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings){
return List.of(new SystemIndexDescriptor(THREAT_INTEL_DATA_INDEX_NAME_PREFIX, "System index used for threat intel data"));
}

@Override
public List<ExecutorBuilder<?>> getExecutorBuilders(Settings settings) {
List<ExecutorBuilder<?>> executorBuilders = new ArrayList<>();
executorBuilders.add(ThreatIntelExecutor.executorBuilder(settings));
return executorBuilders;
}

@Override
public Collection<Object> createComponents(Client client,
ClusterService clusterService,
Expand All @@ -128,6 +142,7 @@ public Collection<Object> createComponents(Client client,
NamedWriteableRegistry namedWriteableRegistry,
IndexNameExpressionResolver indexNameExpressionResolver,
Supplier<RepositoriesService> repositoriesServiceSupplier) {

builtinLogTypeLoader = new BuiltinLogTypeLoader();
logTypeService = new LogTypeService(client, clusterService, xContentRegistry, builtinLogTypeLoader);
detectorIndices = new DetectorIndices(client.admin(), clusterService, threadPool);
Expand All @@ -138,11 +153,22 @@ public Collection<Object> createComponents(Client client,
mapperService = new MapperService(client, clusterService, indexNameExpressionResolver, indexTemplateManager, logTypeService);
ruleIndices = new RuleIndices(logTypeService, client, clusterService, threadPool);
correlationRuleIndices = new CorrelationRuleIndices(client, clusterService);
ThreatIntelFeedDataService threatIntelFeedDataService = new ThreatIntelFeedDataService(clusterService.state(), clusterService, client, indexNameExpressionResolver, xContentRegistry);
DetectorThreatIntelService detectorThreatIntelService = new DetectorThreatIntelService(threatIntelFeedDataService);
DatasourceDao datasourceDao = new DatasourceDao(client, clusterService);

this.client = client;

DatasourceUpdateService datasourceUpdateService = new DatasourceUpdateService(clusterService, datasourceDao, threatIntelFeedDataService);
ThreatIntelExecutor threatIntelExecutor = new ThreatIntelExecutor(threadPool);
ThreatIntelLockService threatIntelLockService = new ThreatIntelLockService(clusterService, client);

DatasourceRunner.getJobRunnerInstance().initialize(clusterService,datasourceUpdateService, datasourceDao, threatIntelExecutor, threatIntelLockService);

return List.of(
detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices,
mapperService, indexTemplateManager, builtinLogTypeLoader
mapperService, indexTemplateManager, builtinLogTypeLoader, threatIntelFeedDataService, detectorThreatIntelService,
datasourceUpdateService, datasourceDao, threatIntelExecutor, threatIntelLockService
);
}

Expand Down Expand Up @@ -193,7 +219,8 @@ public List<NamedXContentRegistry.Entry> getNamedXContent() {
Detector.XCONTENT_REGISTRY,
DetectorInput.XCONTENT_REGISTRY,
Rule.XCONTENT_REGISTRY,
CustomLogType.XCONTENT_REGISTRY
CustomLogType.XCONTENT_REGISTRY,
ThreatIntelFeedData.XCONTENT_REGISTRY
);
}

Expand Down Expand Up @@ -243,7 +270,12 @@ public List<Setting<?>> getSettings() {
SecurityAnalyticsSettings.IS_CORRELATION_INDEX_SETTING,
SecurityAnalyticsSettings.CORRELATION_TIME_WINDOW,
SecurityAnalyticsSettings.DEFAULT_MAPPING_SCHEMA,
SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE
SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE,
SecurityAnalyticsSettings.DATASOURCE_ENDPOINT,
SecurityAnalyticsSettings.DATASOURCE_UPDATE_INTERVAL,
SecurityAnalyticsSettings.BATCH_SIZE,
SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT,
SecurityAnalyticsSettings.CACHE_SIZE
);
}

Expand Down Expand Up @@ -292,5 +324,5 @@ public void onFailure(Exception e) {
log.warn("Failed to initialize LogType config index and builtin log types");
}
});
}
}
}
1 change: 1 addition & 0 deletions src/main/java/org/opensearch/securityanalytics/action/GetDetectorResponse.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
.field(Detector.INPUTS_FIELD, detector.getInputs())
.field(Detector.LAST_UPDATE_TIME_FIELD, detector.getLastUpdateTime())
.field(Detector.ENABLED_TIME_FIELD, detector.getEnabledTime())
.field(Detector.THREAT_INTEL_ENABLED_FIELD, detector.getThreatIntelEnabled())
.endObject();
return builder.endObject();
}
Expand Down
1 change: 1 addition & 0 deletions src/main/java/org/opensearch/securityanalytics/action/IndexDetectorResponse.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
.field(Detector.TRIGGERS_FIELD, detector.getTriggers())
.field(Detector.LAST_UPDATE_TIME_FIELD, detector.getLastUpdateTime())
.field(Detector.ENABLED_TIME_FIELD, detector.getEnabledTime())
.field(Detector.THREAT_INTEL_ENABLED_FIELD, detector.getThreatIntelEnabled())
.endObject();
return builder.endObject();
}
Expand Down
3 changes: 3 additions & 0 deletions src/main/java/org/opensearch/securityanalytics/config/monitors/opensearch_security.policy
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
grant {
permission java.lang.management.ManagementPermission "reputation.alienvault.com:443" "connect,resolve";
};
27 changes: 20 additions & 7 deletions src/main/java/org/opensearch/securityanalytics/model/Detector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,11 @@
import java.io.IOException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Objects;

import java.util.stream.Collectors;

public class Detector implements Writeable, ToXContentObject {

private static final Logger log = LogManager.getLogger(Detector.class);
Expand All @@ -51,6 +48,7 @@ public class Detector implements Writeable, ToXContentObject {
public static final String TRIGGERS_FIELD = "triggers";
public static final String LAST_UPDATE_TIME_FIELD = "last_update_time";
public static final String ENABLED_TIME_FIELD = "enabled_time";
public static final String THREAT_INTEL_ENABLED_FIELD = "threat_intel_enabled";
public static final String ALERTING_MONITOR_ID = "monitor_id";

public static final String ALERTING_WORKFLOW_ID = "workflow_ids";
Expand Down Expand Up @@ -118,11 +116,14 @@ public class Detector implements Writeable, ToXContentObject {

private final String type;

private final Boolean threatIntelEnabled;

public Detector(String id, Long version, String name, Boolean enabled, Schedule schedule,
Instant lastUpdateTime, Instant enabledTime, String logType,
User user, List<DetectorInput> inputs, List<DetectorTrigger> triggers, List<String> monitorIds,
String ruleIndex, String alertsIndex, String alertsHistoryIndex, String alertsHistoryIndexPattern,
String findingsIndex, String findingsIndexPattern, Map<String, String> rulePerMonitor, List<String> workflowIds) {
String findingsIndex, String findingsIndexPattern, Map<String, String> rulePerMonitor,
List<String> workflowIds, Boolean threatIntelEnabled) {
this.type = DETECTOR_TYPE;

this.id = id != null ? id : NO_ID;
Expand All @@ -145,6 +146,7 @@ public Detector(String id, Long version, String name, Boolean enabled, Schedule
this.ruleIdMonitorIdMap = rulePerMonitor;
this.logType = logType;
this.workflowIds = workflowIds != null ? workflowIds : null;
this.threatIntelEnabled = threatIntelEnabled != null && threatIntelEnabled;

if (enabled) {
Objects.requireNonNull(enabledTime);
Expand Down Expand Up @@ -172,7 +174,8 @@ public Detector(StreamInput sin) throws IOException {
sin.readString(),
sin.readString(),
sin.readMap(StreamInput::readString, StreamInput::readString),
sin.readStringList()
sin.readStringList(),
sin.readOptionalBoolean()
);
}

Expand Down Expand Up @@ -211,6 +214,7 @@ public void writeTo(StreamOutput out) throws IOException {
if (workflowIds != null) {
out.writeStringCollection(workflowIds);
}
out.writeOptionalBoolean(threatIntelEnabled);
}

public XContentBuilder toXContentWithUser(XContentBuilder builder, Params params) throws IOException {
Expand Down Expand Up @@ -239,6 +243,7 @@ private XContentBuilder createXContentBuilder(XContentBuilder builder, ToXConten
}
}

builder.field(THREAT_INTEL_ENABLED_FIELD, threatIntelEnabled);
builder.field(ENABLED_FIELD, enabled);

if (enabledTime == null) {
Expand Down Expand Up @@ -280,7 +285,6 @@ private XContentBuilder createXContentBuilder(XContentBuilder builder, ToXConten
builder.field(FINDINGS_INDEX, findingsIndex);
builder.field(FINDINGS_INDEX_PATTERN, findingsIndexPattern);


if (params.paramAsBoolean("with_type", false)) {
builder.endObject();
}
Expand Down Expand Up @@ -327,6 +331,7 @@ public static Detector parse(XContentParser xcp, String id, Long version) throws
String alertsHistoryIndexPattern = null;
String findingsIndex = null;
String findingsIndexPattern = null;
Boolean enableThreatIntel = false;

XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp);
while (xcp.nextToken() != XContentParser.Token.END_OBJECT) {
Expand All @@ -350,6 +355,9 @@ public static Detector parse(XContentParser xcp, String id, Long version) throws
case ENABLED_FIELD:
enabled = xcp.booleanValue();
break;
case THREAT_INTEL_ENABLED_FIELD:
enableThreatIntel = xcp.booleanValue();
break;
case SCHEDULE_FIELD:
schedule = Schedule.parse(xcp);
break;
Expand Down Expand Up @@ -459,7 +467,8 @@ public static Detector parse(XContentParser xcp, String id, Long version) throws
findingsIndex,
findingsIndexPattern,
rulePerMonitor,
workflowIds
workflowIds,
enableThreatIntel
);
}

Expand Down Expand Up @@ -612,6 +621,10 @@ public boolean isWorkflowSupported() {
return workflowIds != null && !workflowIds.isEmpty();
}

public Boolean getThreatIntelEnabled() {
return threatIntelEnabled;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
Expand Down
Loading
Loading