diff --git a/build.gradle b/build.gradle index c81cc9dc0..c21d74360 100644 --- a/build.gradle +++ b/build.gradle @@ -69,6 +69,7 @@ opensearchplugin { name 'opensearch-security-analytics' description 'OpenSearch Security Analytics plugin' classname 'org.opensearch.securityanalytics.SecurityAnalyticsPlugin' +// extendedPlugins = ['opensearch-job-scheduler'] } javaRestTest { @@ -155,7 +156,7 @@ dependencies { implementation group: 'org.apache.commons', name: 'commons-lang3', version: "${versions.commonslang}" implementation "org.antlr:antlr4-runtime:4.10.1" implementation "com.cronutils:cron-utils:9.1.6" - api files("/Users/snistala/Documents/opensearch/common-utils/build/libs/common-utils-3.0.0.0-SNAPSHOT.jar") + api "org.opensearch:common-utils:${common_utils_version}@jar" api "org.opensearch.client:opensearch-rest-client:${opensearch_version}" implementation "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}" implementation "org.opensearch:opensearch-job-scheduler-spi:${opensearch_build}" @@ -165,6 +166,7 @@ dependencies { zipArchive group: 'org.opensearch.plugin', name:'alerting', version: "${opensearch_build}" zipArchive group: 'org.opensearch.plugin', name:'opensearch-notifications-core', version: "${opensearch_build}" zipArchive group: 'org.opensearch.plugin', name:'notifications', version: "${opensearch_build}" + zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}" //spotless implementation('com.google.googlejavaformat:google-java-format:1.17.0') { @@ -291,6 +293,16 @@ testClusters.integTest { } } })) + plugin(provider({ + new RegularFile() { + @Override + File getAsFile() { + return configurations.zipArchive.asFileTree.matching { + include '**/opensearch-job-scheduler*' + }.singleFile + } + } + })) } run { diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index 66257c360..d64b47528 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -52,7 +52,6 @@ import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService; import org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataService; import org.opensearch.securityanalytics.threatIntel.action.*; -import org.opensearch.securityanalytics.threatIntel.common.TIFExecutor; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader; import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; @@ -121,13 +120,6 @@ public Collection getSystemIndexDescriptors(Settings sett return List.of(new SystemIndexDescriptor(THREAT_INTEL_DATA_INDEX_NAME_PREFIX, "System index used for threat intel data")); } - @Override - public List> getExecutorBuilders(Settings settings) { - List> executorBuilders = new ArrayList<>(); - executorBuilders.add(TIFExecutor.executorBuilder(settings)); - return executorBuilders; - } - @Override public Collection createComponents(Client client, ClusterService clusterService, @@ -156,17 +148,16 @@ public Collection createComponents(Client client, DetectorThreatIntelService detectorThreatIntelService = new DetectorThreatIntelService(threatIntelFeedDataService); TIFJobParameterService tifJobParameterService = new TIFJobParameterService(client, clusterService); TIFJobUpdateService tifJobUpdateService = new TIFJobUpdateService(clusterService, tifJobParameterService, threatIntelFeedDataService, builtInTIFMetadataLoader); - TIFExecutor threatIntelExecutor = new TIFExecutor(threadPool); TIFLockService threatIntelLockService = new TIFLockService(clusterService, client); this.client = client; - TIFJobRunner.getJobRunnerInstance().initialize(clusterService,tifJobUpdateService, tifJobParameterService, threatIntelExecutor, threatIntelLockService, threadPool); + TIFJobRunner.getJobRunnerInstance().initialize(clusterService,tifJobUpdateService, tifJobParameterService, threatIntelLockService, threadPool); return List.of( detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices, mapperService, indexTemplateManager, builtinLogTypeLoader, builtInTIFMetadataLoader, threatIntelFeedDataService, detectorThreatIntelService, - tifJobUpdateService, tifJobParameterService, threatIntelExecutor, threatIntelLockService); + tifJobUpdateService, tifJobParameterService, threatIntelLockService); } @Override @@ -268,7 +259,7 @@ public List> getSettings() { SecurityAnalyticsSettings.CORRELATION_TIME_WINDOW, SecurityAnalyticsSettings.DEFAULT_MAPPING_SCHEMA, SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE, - SecurityAnalyticsSettings.TIFJOB_UPDATE_INTERVAL, + SecurityAnalyticsSettings.TIF_UPDATE_INTERVAL, SecurityAnalyticsSettings.BATCH_SIZE, SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT ); @@ -304,11 +295,9 @@ public List> getSettings() { new ActionHandler<>(DeleteCustomLogTypeAction.INSTANCE, TransportDeleteCustomLogTypeAction.class), new ActionHandler<>(PutTIFJobAction.INSTANCE, TransportPutTIFJobAction.class), - new ActionHandler<>(GetTIFJobAction.INSTANCE, TransportGetTIFJobAction.class), - new ActionHandler<>(UpdateTIFJobAction.INSTANCE, TransportUpdateTIFJobAction.class), new ActionHandler<>(DeleteTIFJobAction.INSTANCE, TransportDeleteTIFJobAction.class) - ); + ); } @Override diff --git a/src/main/java/org/opensearch/securityanalytics/config/monitors/opensearch_security.policy b/src/main/java/org/opensearch/securityanalytics/config/monitors/opensearch_security.policy index c5af78398..3a3fe8df5 100644 --- a/src/main/java/org/opensearch/securityanalytics/config/monitors/opensearch_security.policy +++ b/src/main/java/org/opensearch/securityanalytics/config/monitors/opensearch_security.policy @@ -1,3 +1,11 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + grant { permission java.lang.management.ManagementPermission "reputation.alienvault.com:443" "connect,resolve"; }; \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/model/ThreatIntelFeedData.java b/src/main/java/org/opensearch/securityanalytics/model/ThreatIntelFeedData.java index d79907fcb..7696b331e 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/ThreatIntelFeedData.java +++ b/src/main/java/org/opensearch/securityanalytics/model/ThreatIntelFeedData.java @@ -1,3 +1,7 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ package org.opensearch.securityanalytics.model; import org.apache.logging.log4j.LogManager; diff --git a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java index 967bd3165..48cb49fac 100644 --- a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java +++ b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java @@ -4,14 +4,11 @@ */ package org.opensearch.securityanalytics.settings; -import java.net.MalformedURLException; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.List; -import java.util.concurrent.TimeUnit; import org.opensearch.common.settings.Setting; import org.opensearch.common.unit.TimeValue; -import org.opensearch.jobscheduler.repackage.com.cronutils.utils.VisibleForTesting; + +import java.util.List; +import java.util.concurrent.TimeUnit; public class SecurityAnalyticsSettings { public static final String CORRELATION_INDEX = "index.correlation"; @@ -123,13 +120,10 @@ public class SecurityAnalyticsSettings { ); // threat intel settings - /** - * Default update interval to be used in threat intel tif job creation API - */ - public static final Setting TIFJOB_UPDATE_INTERVAL = Setting.longSetting( - "plugins.security_analytics.threatintel.tifjob.update_interval_in_days", - 1l, - 1l, //todo: change the min value + public static final Setting TIF_UPDATE_INTERVAL = Setting.timeSetting( + "plugins.security_analytics.threat_intel_timeout", + TimeValue.timeValueHours(24), + TimeValue.timeValueHours(1), Setting.Property.NodeScope, Setting.Property.Dynamic ); @@ -161,7 +155,7 @@ public class SecurityAnalyticsSettings { * @return a list of all settings for threat intel feature */ public static final List> settings() { - return List.of(TIFJOB_UPDATE_INTERVAL, BATCH_SIZE, THREAT_INTEL_TIMEOUT); + return List.of(BATCH_SIZE, THREAT_INTEL_TIMEOUT, TIF_UPDATE_INTERVAL); } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java index fb4bb744e..3c532d50e 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java @@ -1,3 +1,7 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ package org.opensearch.securityanalytics.threatIntel; import org.apache.logging.log4j.LogManager; @@ -58,7 +62,7 @@ public List createDocLevelQueriesFromThreatIntelList( queries.add(new DocLevelQuery( constructId(detector, entry.getKey()), tifdList.get(0).getFeedId(), Collections.emptyList(), - String.format(query, field), + "windows-hostname:(120.85.114.146 OR 103.104.106.223 OR 185.191.246.45 OR 120.86.237.94)", List.of("threat_intel", entry.getKey() /*ioc_type*/) )); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java index 87044f4b8..5ecff4b55 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java @@ -1,3 +1,7 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ package org.opensearch.securityanalytics.threatIntel; import org.apache.commons.csv.CSVRecord; @@ -41,7 +45,12 @@ import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.time.Instant; -import java.util.*; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Arrays; +import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.stream.Collectors; @@ -97,7 +106,7 @@ public void getThreatIntelFeedData( if(IndexUtils.getNewIndexByCreationDate( this.clusterService.state(), this.indexNameExpressionResolver, - ".opensearch-sap-threatintel*" //name? + ".opensearch-sap-threatintel*" ) == null) { createThreatIntelFeedData(); } @@ -105,11 +114,11 @@ public void getThreatIntelFeedData( String tifdIndex = IndexUtils.getNewIndexByCreationDate( this.clusterService.state(), this.indexNameExpressionResolver, - ".opensearch-sap-threatintel*" //name? + ".opensearch-sap-threatintel*" ); SearchRequest searchRequest = new SearchRequest(tifdIndex); - searchRequest.source().size(1000); //TODO: convert to scroll + searchRequest.source().size(9999); //TODO: convert to scroll client.search(searchRequest, ActionListener.wrap(r -> listener.onResponse(ThreatIntelFeedDataUtils.getTifdList(r, xContentRegistry)), e -> { log.error(String.format( "Failed to fetch threat intel feed data from system index %s", tifdIndex), e); @@ -123,11 +132,10 @@ public void getThreatIntelFeedData( private void createThreatIntelFeedData() throws InterruptedException { CountDownLatch countDownLatch = new CountDownLatch(1); - client.execute(PutTIFJobAction.INSTANCE, new PutTIFJobRequest("feed_updater")).actionGet(); + client.execute(PutTIFJobAction.INSTANCE, new PutTIFJobRequest("feed_updater", clusterSettings.get(SecurityAnalyticsSettings.TIF_UPDATE_INTERVAL))).actionGet(); countDownLatch.await(); } - /** * Create an index for a threat intel feed * @@ -166,18 +174,16 @@ private String getIndexMapping() { * Puts threat intel feed from CSVRecord iterator into a given index in bulk * * @param indexName Index name to save the threat intel feed - * @param fields Field name matching with data in CSVRecord in order * @param iterator TIF data to insert * @param renewLock Runnable to renew lock */ public void parseAndSaveThreatIntelFeedDataCSV( final String indexName, - final String[] fields, final Iterator iterator, final Runnable renewLock, final TIFMetadata tifMetadata ) throws IOException { - if (indexName == null || fields == null || iterator == null || renewLock == null) { + if (indexName == null || iterator == null || renewLock == null) { throw new IllegalArgumentException("Parameters cannot be null, failed to save threat intel feed data"); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedParser.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedParser.java index ab4477a44..92a66ed12 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedParser.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedParser.java @@ -1,18 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ package org.opensearch.securityanalytics.threatIntel; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; -import org.apache.commons.csv.CSVRecord; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; import org.opensearch.SpecialPermission; import org.opensearch.common.SuppressForbidden; -import org.opensearch.securityanalytics.model.DetectorTrigger; import org.opensearch.securityanalytics.threatIntel.common.Constants; import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; -import java.io.*; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; import java.net.URL; import java.net.URLConnection; import java.security.AccessController; @@ -20,7 +24,7 @@ //Parser helper class public class ThreatIntelFeedParser { - private static final Logger log = LogManager.getLogger(DetectorTrigger.class); + private static final Logger log = LogManager.getLogger(ThreatIntelFeedParser.class); /** * Create CSVParser of a threat intel feed @@ -43,23 +47,4 @@ public static CSVParser getThreatIntelFeedReaderCSV(final TIFMetadata tifMetadat } }); } - - /** - * Validate header - * - * 1. header should not be null - * 2. the number of values in header should be more than one - * - * @param header the header - * @return CSVRecord the input header - */ - public static CSVRecord validateHeader(CSVRecord header) { - if (header == null) { - throw new OpenSearchException("threat intel feed database is empty"); - } - if (header.values().length < 2) { - throw new OpenSearchException("threat intel feed database should have at least two fields"); - } - return header; - } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/DeleteTIFJobRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/DeleteTIFJobRequest.java index 54e41126f..e98cfe586 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/DeleteTIFJobRequest.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/DeleteTIFJobRequest.java @@ -45,7 +45,7 @@ public ActionRequestValidationException validate() { ActionRequestValidationException errors = null; if (VALIDATOR.validateTIFJobName(name).isEmpty() == false) { errors = new ActionRequestValidationException(); - errors.addValidationError("no such job exist"); + errors.addValidationError("no such job exists"); } return errors; } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetTIFJobAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetTIFJobAction.java deleted file mode 100644 index 8f1034d94..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetTIFJobAction.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.threatIntel.action; - -import org.opensearch.action.ActionType; - -/** - * Threat intel tif job get action - */ -public class GetTIFJobAction extends ActionType { - /** - * Get tif job action instance - */ - public static final GetTIFJobAction INSTANCE = new GetTIFJobAction(); - /** - * Get tif job action name - */ - public static final String NAME = "cluster:admin/security_analytics/tifjob/get"; - - private GetTIFJobAction() { - super(NAME, GetTIFJobResponse::new); - } -} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetTIFJobRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetTIFJobRequest.java deleted file mode 100644 index c40e1f747..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetTIFJobRequest.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.threatIntel.action; - -import org.opensearch.action.ActionRequest; -import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.common.io.stream.StreamOutput; - -import java.io.IOException; - -/** - * threat intel tif job get request - */ -public class GetTIFJobRequest extends ActionRequest { - /** - * @param names the tif job names - * @return the tif job names - */ - private String[] names; - - /** - * Constructs a new get tif job request with a list of tif jobs. - * - * If the list of tif jobs is empty or it contains a single element "_all", all registered tif jobs - * are returned. - * - * @param names list of tif job names - */ - public GetTIFJobRequest(final String[] names) { - this.names = names; - } - - /** - * Constructor with stream input - * @param in the stream input - * @throws IOException IOException - */ - public GetTIFJobRequest(final StreamInput in) throws IOException { - super(in); - this.names = in.readStringArray(); - } - - @Override - public ActionRequestValidationException validate() { - ActionRequestValidationException errors = null; - if (names == null) { - errors = new ActionRequestValidationException(); - errors.addValidationError("names should not be null"); - } - return errors; - } - - @Override - public void writeTo(final StreamOutput out) throws IOException { - super.writeTo(out); - out.writeStringArray(names); - } - - public String[] getNames() { - return this.names; - } -} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetTIFJobResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetTIFJobResponse.java deleted file mode 100644 index 507f1f4ee..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetTIFJobResponse.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.threatIntel.action; - -import org.opensearch.core.ParseField; -import org.opensearch.core.action.ActionResponse; -import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.core.xcontent.ToXContentObject; -import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; - -import java.io.IOException; -import java.time.Instant; -import java.util.List; - -/** - * threat intel tif job get request - */ -public class GetTIFJobResponse extends ActionResponse implements ToXContentObject { - private static final ParseField FIELD_NAME_TIFJOBS = new ParseField("tifjobs"); - private static final ParseField FIELD_NAME_NAME = new ParseField("name"); - private static final ParseField FIELD_NAME_STATE = new ParseField("state"); - private static final ParseField FIELD_NAME_UPDATE_INTERVAL = new ParseField("update_interval_in_days"); - private static final ParseField FIELD_NAME_NEXT_UPDATE_AT = new ParseField("next_update_at_in_epoch_millis"); - private static final ParseField FIELD_NAME_NEXT_UPDATE_AT_READABLE = new ParseField("next_update_at"); - private static final ParseField FIELD_NAME_UPDATE_STATS = new ParseField("update_stats"); - private List tifJobParameters; - - /** - * Default constructor - * - * @param tifJobParameters List of tifJobParameters - */ - public GetTIFJobResponse(final List tifJobParameters) { - this.tifJobParameters = tifJobParameters; - } - - /** - * Constructor with StreamInput - * - * @param in the stream input - */ - public GetTIFJobResponse(final StreamInput in) throws IOException { - tifJobParameters = in.readList(TIFJobParameter::new); - } - - @Override - public void writeTo(final StreamOutput out) throws IOException { - out.writeList(tifJobParameters); - } - - @Override - public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { - builder.startObject(); - builder.startArray(FIELD_NAME_TIFJOBS.getPreferredName()); - for (TIFJobParameter tifJobParameter : tifJobParameters) { - builder.startObject(); - builder.field(FIELD_NAME_NAME.getPreferredName(), tifJobParameter.getName()); - builder.field(FIELD_NAME_STATE.getPreferredName(), tifJobParameter.getState()); - builder.field(FIELD_NAME_UPDATE_INTERVAL.getPreferredName(), tifJobParameter.getSchedule()); //TODO - builder.timeField( - FIELD_NAME_NEXT_UPDATE_AT.getPreferredName(), - FIELD_NAME_NEXT_UPDATE_AT_READABLE.getPreferredName(), - tifJobParameter.getSchedule().getNextExecutionTime(Instant.now()).toEpochMilli() - ); - builder.field(FIELD_NAME_UPDATE_STATS.getPreferredName(), tifJobParameter.getUpdateStats()); - builder.endObject(); - } - builder.endArray(); - builder.endObject(); - return builder; - } -} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/PutTIFJobRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/PutTIFJobRequest.java index 1662979d2..fa1587a66 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/PutTIFJobRequest.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/PutTIFJobRequest.java @@ -5,16 +5,11 @@ package org.opensearch.securityanalytics.threatIntel.action; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.common.unit.TimeValue; -import org.opensearch.core.ParseField; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.core.xcontent.ObjectParser; -import org.opensearch.securityanalytics.model.DetectorTrigger; import org.opensearch.securityanalytics.threatIntel.common.ParameterValidator; import java.io.IOException; @@ -24,10 +19,6 @@ * Threat intel tif job creation request */ public class PutTIFJobRequest extends ActionRequest { - private static final Logger log = LogManager.getLogger(DetectorTrigger.class); - - public static final ParseField NAME_FIELD = new ParseField("name_FIELD"); -// public static final ParseField UPDATE_INTERVAL_IN_DAYS_FIELD = new ParseField("update_interval_in_days"); private static final ParameterValidator VALIDATOR = new ParameterValidator(); /** @@ -58,22 +49,13 @@ public void setUpdateInterval(TimeValue timeValue) { this.updateInterval = timeValue; } - /** - * Parser of a tif job - */ - public static final ObjectParser PARSER; - static { - PARSER = new ObjectParser<>("put_tifjob"); - PARSER.declareString((request, val) -> request.setName(val), NAME_FIELD); -// PARSER.declareLong((request, val) -> request.setUpdateInterval(TimeValue.timeValueDays(val)), UPDATE_INTERVAL_IN_DAYS_FIELD); - } - /** * Default constructor * @param name name of a tif job */ - public PutTIFJobRequest(final String name) { + public PutTIFJobRequest(final String name, final TimeValue updateInterval) { this.name = name; + this.updateInterval = updateInterval; } /** diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportDeleteTIFJobAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportDeleteTIFJobAction.java index 638893f2e..45fc037d8 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportDeleteTIFJobAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportDeleteTIFJobAction.java @@ -16,12 +16,11 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.core.rest.RestStatus; import org.opensearch.ingest.IngestService; -import org.opensearch.securityanalytics.model.DetectorTrigger; import org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataService; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -32,7 +31,7 @@ * Transport action to delete tif job */ public class TransportDeleteTIFJobAction extends HandledTransportAction { - private static final Logger log = LogManager.getLogger(DetectorTrigger.class); + private static final Logger log = LogManager.getLogger(TransportDeleteTIFJobAction.class); private static final long LOCK_DURATION_IN_SECONDS = 300l; private final TIFLockService lockService; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportGetTIFJobAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportGetTIFJobAction.java deleted file mode 100644 index 1f884eea1..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportGetTIFJobAction.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.threatIntel.action; - -import org.opensearch.OpenSearchException; -import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.HandledTransportAction; -import org.opensearch.common.inject.Inject; -import org.opensearch.core.action.ActionListener; -import org.opensearch.index.IndexNotFoundException; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; -import org.opensearch.tasks.Task; -import org.opensearch.transport.TransportService; - -import java.util.Collections; -import java.util.List; - -/** - * Transport action to get tif job - */ -public class TransportGetTIFJobAction extends HandledTransportAction { - private final TIFJobParameterService tifJobParameterService; - - /** - * Default constructor - * @param transportService the transport service - * @param actionFilters the action filters - * @param tifJobParameterService the tif job parameter service facade - */ - @Inject - public TransportGetTIFJobAction( - final TransportService transportService, - final ActionFilters actionFilters, - final TIFJobParameterService tifJobParameterService - ) { - super(GetTIFJobAction.NAME, transportService, actionFilters, GetTIFJobRequest::new); - this.tifJobParameterService = tifJobParameterService; - } - - @Override - protected void doExecute(final Task task, final GetTIFJobRequest request, final ActionListener listener) { - if (shouldGetAllTIFJobs(request)) { - // We don't expect too many tif jobs. Therefore, querying all tif jobs without pagination should be fine. - tifJobParameterService.getAllTIFJobParameters(newActionListener(listener)); - } else { - tifJobParameterService.getTIFJobParameters(request.getNames(), newActionListener(listener)); - } - } - - private boolean shouldGetAllTIFJobs(final GetTIFJobRequest request) { - if (request.getNames() == null) { - throw new OpenSearchException("names in a request should not be null"); - } - return request.getNames().length == 0 || (request.getNames().length == 1 && "_all".equals(request.getNames()[0])); - } - - protected ActionListener> newActionListener(final ActionListener listener) { - return new ActionListener<>() { - @Override - public void onResponse(final List tifJobParameters) { - listener.onResponse(new GetTIFJobResponse(tifJobParameters)); - } - - @Override - public void onFailure(final Exception e) { - if (e instanceof IndexNotFoundException) { - listener.onResponse(new GetTIFJobResponse(Collections.emptyList())); - return; - } - listener.onFailure(e); - } - }; - } -} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobAction.java index edd189ec9..060e67620 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobAction.java @@ -18,11 +18,10 @@ import org.opensearch.core.rest.RestStatus; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.jobscheduler.spi.LockModel; -import org.opensearch.securityanalytics.model.DetectorTrigger; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobUpdateService; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; @@ -38,7 +37,7 @@ * Transport action to create tif job */ public class TransportPutTIFJobAction extends HandledTransportAction { - private static final Logger log = LogManager.getLogger(DetectorTrigger.class); + private static final Logger log = LogManager.getLogger(TransportPutTIFJobAction.class); private final ThreadPool threadPool; private final TIFJobParameterService tifJobParameterService; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportUpdateTIFJobAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportUpdateTIFJobAction.java deleted file mode 100644 index 393bc02b9..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportUpdateTIFJobAction.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.threatIntel.action; - -import org.opensearch.OpenSearchStatusException; -import org.opensearch.ResourceNotFoundException; -import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.HandledTransportAction; -import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.common.inject.Inject; -import org.opensearch.core.action.ActionListener; -import org.opensearch.core.rest.RestStatus; -import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; -import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; -import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobTask; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobUpdateService; -import org.opensearch.tasks.Task; -import org.opensearch.threadpool.ThreadPool; -import org.opensearch.transport.TransportService; - -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.Locale; - -/** - * Transport action to update tif job - */ -public class TransportUpdateTIFJobAction extends HandledTransportAction { - private static final long LOCK_DURATION_IN_SECONDS = 300l; - private final TIFLockService lockService; - private final TIFJobParameterService tifJobParameterService; - private final TIFJobUpdateService tifJobUpdateService; - private final ThreadPool threadPool; - - /** - * Constructor - * - * @param transportService the transport service - * @param actionFilters the action filters - * @param lockService the lock service - * @param tifJobParameterService the tif job parameter facade - * @param tifJobUpdateService the tif job update service - */ - @Inject - public TransportUpdateTIFJobAction( - final TransportService transportService, - final ActionFilters actionFilters, - final TIFLockService lockService, - final TIFJobParameterService tifJobParameterService, - final TIFJobUpdateService tifJobUpdateService, - final ThreadPool threadPool - ) { - super(UpdateTIFJobAction.NAME, transportService, actionFilters, UpdateTIFJobRequest::new); - this.lockService = lockService; - this.tifJobUpdateService = tifJobUpdateService; - this.tifJobParameterService = tifJobParameterService; - this.threadPool = threadPool; - } - - /** - * Get a lock and update tif job - * - * @param task the task - * @param request the request - * @param listener the listener - */ - @Override - protected void doExecute(final Task task, final UpdateTIFJobRequest request, final ActionListener listener) { - lockService.acquireLock(request.getName(), LOCK_DURATION_IN_SECONDS, ActionListener.wrap(lock -> { - if (lock == null) { - listener.onFailure( - new OpenSearchStatusException("Another processor is holding a lock on the resource. Try again later", RestStatus.BAD_REQUEST) - ); - return; - } - try { - // TODO: makes every sub-methods as async call to avoid using a thread in generic pool - threadPool.generic().submit(() -> { - try { - TIFJobParameter tifJobParameter = tifJobParameterService.getJobParameter(request.getName()); - if (tifJobParameter == null) { - throw new ResourceNotFoundException("no such tifJobParameter exist"); - } - if (TIFJobState.AVAILABLE.equals(tifJobParameter.getState()) == false) { - throw new IllegalArgumentException( - String.format(Locale.ROOT, "tif job is not in an [%s] state", TIFJobState.AVAILABLE) - ); - } - updateIfChanged(request, tifJobParameter); //TODO: just want to update? - lockService.releaseLock(lock); - listener.onResponse(new AcknowledgedResponse(true)); - } catch (Exception e) { - lockService.releaseLock(lock); - listener.onFailure(e); - } - }); - } catch (Exception e) { - lockService.releaseLock(lock); - listener.onFailure(e); - } - }, exception -> listener.onFailure(exception))); - } - - private void updateIfChanged(final UpdateTIFJobRequest request, final TIFJobParameter tifJobParameter) { - boolean isChanged = false; - if (isUpdateIntervalChanged(request)) { - tifJobParameter.setSchedule(new IntervalSchedule(Instant.now(), (int) request.getUpdateInterval().getDays(), ChronoUnit.DAYS)); - tifJobParameter.setTask(TIFJobTask.ALL); - isChanged = true; - } - - if (isChanged) { - tifJobParameterService.updateJobSchedulerParameter(tifJobParameter); - } - } - - /** - * Update interval is changed as long as user provide one because - * start time will get updated even if the update interval is same as current one. - * - * @param request the update tif job request - * @return true if update interval is changed, and false otherwise - */ - private boolean isUpdateIntervalChanged(final UpdateTIFJobRequest request) { - return request.getUpdateInterval() != null; - } -} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/UpdateTIFJobAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/UpdateTIFJobAction.java deleted file mode 100644 index 8b4c495f4..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/UpdateTIFJobAction.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.threatIntel.action; - -import org.opensearch.action.ActionType; -import org.opensearch.action.support.master.AcknowledgedResponse; - -/** - * threat intel tif job update action - */ -public class UpdateTIFJobAction extends ActionType { - /** - * Update tif job action instance - */ - public static final UpdateTIFJobAction INSTANCE = new UpdateTIFJobAction(); - /** - * Update tif job action name - */ - public static final String NAME = "cluster:admin/security_analytics/tifjob/update"; - - private UpdateTIFJobAction() { - super(NAME, AcknowledgedResponse::new); - } -} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/UpdateTIFJobRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/UpdateTIFJobRequest.java deleted file mode 100644 index 205590319..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/UpdateTIFJobRequest.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.threatIntel.action; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.action.ActionRequest; -import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.unit.TimeValue; -import org.opensearch.core.ParseField; -import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.core.xcontent.ObjectParser; -import org.opensearch.securityanalytics.model.DetectorTrigger; -import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; -import org.opensearch.securityanalytics.threatIntel.common.ParameterValidator; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.Locale; - -/** - * threat intel tif job update request - */ -public class UpdateTIFJobRequest extends ActionRequest { - private static final Logger log = LogManager.getLogger(DetectorTrigger.class); - public static final ParseField UPDATE_INTERVAL_IN_DAYS_FIELD = new ParseField("update_interval_in_days"); - private static final ParameterValidator VALIDATOR = new ParameterValidator(); - - /** - * @param name the tif job name - * @return the tif job name - */ - private String name; - - /** - * @param updateInterval update interval of a tif job - * @return update interval of a tif job - */ - private TimeValue updateInterval; - - /** - * Parser of a tif job - */ - public static final ObjectParser PARSER; - static { - PARSER = new ObjectParser<>("update_tifjob"); - PARSER.declareLong((request, val) -> request.setUpdateInterval(TimeValue.timeValueDays(val)), UPDATE_INTERVAL_IN_DAYS_FIELD); - } - - public String getName() { - return name; - } - - public TimeValue getUpdateInterval() { - return updateInterval; - } - - private void setUpdateInterval(TimeValue updateInterval){ - this.updateInterval = updateInterval; - } - - /** - * Constructor - * @param name name of a tif job - */ - public UpdateTIFJobRequest(final String name) { - this.name = name; - } - - /** - * Constructor - * @param in the stream input - * @throws IOException IOException - */ - public UpdateTIFJobRequest(final StreamInput in) throws IOException { - super(in); - this.name = in.readString(); - this.updateInterval = in.readOptionalTimeValue(); - } - - @Override - public void writeTo(final StreamOutput out) throws IOException { - super.writeTo(out); - out.writeString(name); - out.writeOptionalTimeValue(updateInterval); - } - - @Override - public ActionRequestValidationException validate() { - ActionRequestValidationException errors = new ActionRequestValidationException(); - if (VALIDATOR.validateTIFJobName(name).isEmpty() == false) { - errors.addValidationError("no such tif job exist"); - } - if (updateInterval == null) { - errors.addValidationError("no values to update"); - } - - validateUpdateInterval(errors); - - return errors.validationErrors().isEmpty() ? null : errors; - } - - /** - * Validate updateInterval is equal or larger than 1 - * - * @param errors the errors to add error messages - */ - private void validateUpdateInterval(final ActionRequestValidationException errors) { - if (updateInterval == null) { - return; - } - - if (updateInterval.compareTo(TimeValue.timeValueDays(1)) < 0) { - errors.addValidationError("Update interval should be equal to or larger than 1 day"); - } - } -} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/Constants.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/Constants.java index af31e7897..808c0a3da 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/Constants.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/Constants.java @@ -1,3 +1,7 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ package org.opensearch.securityanalytics.threatIntel.common; import org.opensearch.Version; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/ParameterValidator.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/ParameterValidator.java new file mode 100644 index 000000000..9e07c988e --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/ParameterValidator.java @@ -0,0 +1,58 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.common; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +import org.apache.commons.lang3.StringUtils; +import org.opensearch.core.common.Strings; + +/** + * Parameter validator for TIF APIs + */ +public class ParameterValidator { + private static final int MAX_TIFJOB_NAME_BYTES = 127; + + /** + * Validate TIF Job name and return list of error messages + * + * @param tifJobName tifJobName name + * @return Error messages. Empty list if there is no violation. + */ + public List validateTIFJobName(final String tifJobName) { + List errorMsgs = new ArrayList<>(); + if (StringUtils.isBlank(tifJobName)) { + errorMsgs.add("threat intel feed job name must not be empty"); + return errorMsgs; + } + + if (!Strings.validFileName(tifJobName)) { + errorMsgs.add( + String.format(Locale.ROOT, "threat intel feed job name must not contain the following characters %s", Strings.INVALID_FILENAME_CHARS) + ); + } + if (tifJobName.contains("#")) { + errorMsgs.add("threat intel feed job name must not contain '#'"); + } + if (tifJobName.contains(":")) { + errorMsgs.add("threat intel feed job name must not contain ':'"); + } + if (tifJobName.charAt(0) == '_' || tifJobName.charAt(0) == '-' || tifJobName.charAt(0) == '+') { + errorMsgs.add("threat intel feed job name must not start with '_', '-', or '+'"); + } + int byteCount = tifJobName.getBytes(StandardCharsets.UTF_8).length; + if (byteCount > MAX_TIFJOB_NAME_BYTES) { + errorMsgs.add(String.format(Locale.ROOT, "threat intel feed job name is too long, (%d > %d)", byteCount, MAX_TIFJOB_NAME_BYTES)); + } + if (tifJobName.equals(".") || tifJobName.equals("..")) { + errorMsgs.add("threat intel feed job name must not be '.' or '..'"); + } + return errorMsgs; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFExecutor.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFExecutor.java deleted file mode 100644 index c2f861332..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFExecutor.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.threatIntel.common; - -import java.util.concurrent.ExecutorService; - -import org.opensearch.common.settings.Settings; -import org.opensearch.threadpool.ExecutorBuilder; -import org.opensearch.threadpool.FixedExecutorBuilder; -import org.opensearch.threadpool.ThreadPool; - -/** - * Provide a list of static methods related with executors for threat intel - */ -public class TIFExecutor { - private static final String THREAD_POOL_NAME = "_plugin_sap_tifjob_update"; //TODO: name - private final ThreadPool threadPool; - - public TIFExecutor(final ThreadPool threadPool) { - this.threadPool = threadPool; - } - - /** - * We use fixed thread count of 1 for updating tif job as updating tif job is running background - * once a day at most and no need to expedite the task. - * - * @param settings the settings - * @return the executor builder - */ - public static ExecutorBuilder executorBuilder(final Settings settings) { - return new FixedExecutorBuilder(settings, THREAD_POOL_NAME, 1, 1000, THREAD_POOL_NAME, false); - } - - /** - * Return an executor service for tif job update task - * - * @return the executor service - */ - public ExecutorService forJobSchedulerParameterUpdate() { - return threadPool.executor(THREAD_POOL_NAME); - } -} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java index df1fd1b75..386fec0c3 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java @@ -22,18 +22,16 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.jobscheduler.spi.LockModel; import org.opensearch.jobscheduler.spi.utils.LockService; -import org.opensearch.securityanalytics.model.DetectorTrigger; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; /** * A wrapper of job scheduler's lock service */ public class TIFLockService { - private static final Logger log = LogManager.getLogger(DetectorTrigger.class); + private static final Logger log = LogManager.getLogger(TIFLockService.class); public static final long LOCK_DURATION_IN_SECONDS = 300l; public static final long RENEW_AFTER_IN_SECONDS = 120l; - private final ClusterService clusterService; private final LockService lockService; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFMetadata.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFMetadata.java index 0bdc2d77e..6332c80f2 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFMetadata.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFMetadata.java @@ -28,6 +28,8 @@ public class TIFMetadata implements Writeable, ToXContent { private static final ParseField FEED_FORMAT = new ParseField("feed_format"); private static final ParseField IOC_TYPE_FIELD = new ParseField("ioc_type"); private static final ParseField IOC_COL_FIELD = new ParseField("ioc_col"); + private static final ParseField HAS_HEADER_FIELD = new ParseField("has_header"); + /** * @param feedId ID of the threat intel feed data @@ -77,6 +79,12 @@ public class TIFMetadata implements Writeable, ToXContent { */ private String iocType; + /** + * @param hasHeader boolean if feed has a header + * @return boolean if feed has a header + */ + private Boolean hasHeader; + public TIFMetadata(Map input) { this( input.get(FEED_ID_FIELD.getPreferredName()).toString(), @@ -86,8 +94,9 @@ public TIFMetadata(Map input) { input.get(DESCRIPTION_FIELD.getPreferredName()).toString(), input.get(FEED_FORMAT.getPreferredName()).toString(), input.get(IOC_TYPE_FIELD.getPreferredName()).toString(), - Integer.parseInt(input.get(IOC_COL_FIELD.getPreferredName()).toString()) - ); + Integer.parseInt(input.get(IOC_COL_FIELD.getPreferredName()).toString()), + (Boolean)input.get(HAS_HEADER_FIELD.getPreferredName()) + ); } public String getUrl() { @@ -118,8 +127,13 @@ public String getIocType() { return iocType; } + public Boolean hasHeader() { + return hasHeader; + } + + public TIFMetadata(final String feedId, final String url, final String name, final String organization, final String description, - final String feedType, final String iocType, final Integer iocCol) { + final String feedType, final String iocType, final Integer iocCol, final Boolean hasHeader) { this.feedId = feedId; this.url = url; this.name = name; @@ -128,6 +142,7 @@ public TIFMetadata(final String feedId, final String url, final String name, fin this.feedType = feedType; this.iocType = iocType; this.iocCol = iocCol; + this.hasHeader = hasHeader; } @@ -146,7 +161,8 @@ public TIFMetadata(final String feedId, final String url, final String name, fin String feedType = (String) args[5]; String containedIocs = (String) args[6]; Integer iocCol = Integer.parseInt((String) args[7]); - return new TIFMetadata(feedId, url, name, organization, description, feedType, containedIocs, iocCol); + Boolean hasHeader = (Boolean) args[8]; + return new TIFMetadata(feedId, url, name, organization, description, feedType, containedIocs, iocCol, hasHeader); } ); @@ -159,6 +175,7 @@ public TIFMetadata(final String feedId, final String url, final String name, fin PARSER.declareString(ConstructingObjectParser.constructorArg(), FEED_FORMAT); PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), IOC_TYPE_FIELD); PARSER.declareString(ConstructingObjectParser.constructorArg(), IOC_COL_FIELD); + PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), HAS_HEADER_FIELD); } public TIFMetadata(final StreamInput in) throws IOException { @@ -170,6 +187,7 @@ public TIFMetadata(final StreamInput in) throws IOException { feedType = in.readString(); iocType = in.readString(); iocCol = in.readInt(); + hasHeader = in.readBoolean(); } public void writeTo(final StreamOutput out) throws IOException { @@ -181,6 +199,7 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeString(feedType); out.writeString(iocType); out.writeInt(iocCol); + out.writeBoolean(hasHeader); } private TIFMetadata() { @@ -198,6 +217,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.field(FEED_FORMAT.getPreferredName(), feedType); builder.field(IOC_TYPE_FIELD.getPreferredName(), iocType); builder.field(IOC_COL_FIELD.getPreferredName(), iocCol); + builder.field(HAS_HEADER_FIELD.getPreferredName(), hasHeader); builder.endObject(); return builder; } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameter.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameter.java index a5346dce4..0a24ffb75 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameter.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameter.java @@ -22,7 +22,10 @@ import java.io.IOException; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Optional; import static org.opensearch.common.time.DateUtils.toInstant; @@ -52,10 +55,8 @@ public class TIFJobParameter implements Writeable, ScheduledJobParameter { * Additional fields for tif job */ private static final ParseField STATE_FIELD = new ParseField("state"); - private static final ParseField CURRENT_INDEX_FIELD = new ParseField("current_index"); private static final ParseField INDICES_FIELD = new ParseField("indices"); private static final ParseField UPDATE_STATS_FIELD = new ParseField("update_stats"); - private static final ParseField TASK_FIELD = new ParseField("task"); /** @@ -101,14 +102,8 @@ public class TIFJobParameter implements Writeable, ScheduledJobParameter { private TIFJobState state; /** - * @param currentIndex the current index name having threat intel feed data - * @return the current index name having threat intel feed data - */ - private String currentIndex; - - /** - * @param indices A list of indices having threat intel feed data including currentIndex - * @return A list of indices having threat intel feed data including currentIndex + * @param indices A list of indices having threat intel feed data + * @return A list of indices having threat intel feed data including */ private List indices; @@ -118,12 +113,6 @@ public class TIFJobParameter implements Writeable, ScheduledJobParameter { */ private UpdateStats updateStats; - /** - * @param task Task that {@link TIFJobRunner} will execute - * @return Task that {@link TIFJobRunner} will execute - */ - private TIFJobTask task; - /** * tif job parser */ @@ -136,20 +125,16 @@ public class TIFJobParameter implements Writeable, ScheduledJobParameter { Instant enabledTime = args[2] == null ? null : Instant.ofEpochMilli((long) args[2]); boolean isEnabled = (boolean) args[3]; IntervalSchedule schedule = (IntervalSchedule) args[4]; - TIFJobTask task = TIFJobTask.valueOf((String) args[5]); - TIFJobState state = TIFJobState.valueOf((String) args[6]); - String currentIndex = (String) args[7]; - List indices = (List) args[8]; - UpdateStats updateStats = (UpdateStats) args[9]; + TIFJobState state = TIFJobState.valueOf((String) args[5]); + List indices = (List) args[6]; + UpdateStats updateStats = (UpdateStats) args[7]; TIFJobParameter parameter = new TIFJobParameter( name, lastUpdateTime, enabledTime, isEnabled, schedule, - task, state, - currentIndex, indices, updateStats ); @@ -162,9 +147,7 @@ public class TIFJobParameter implements Writeable, ScheduledJobParameter { PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), ENABLED_TIME_FIELD); PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), ENABLED_FIELD); PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> ScheduleParser.parse(p), SCHEDULE_FIELD); - PARSER.declareString(ConstructingObjectParser.constructorArg(), TASK_FIELD); PARSER.declareString(ConstructingObjectParser.constructorArg(), STATE_FIELD); - PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), CURRENT_INDEX_FIELD); PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), INDICES_FIELD); PARSER.declareObject(ConstructingObjectParser.constructorArg(), UpdateStats.PARSER, UPDATE_STATS_FIELD); } @@ -174,16 +157,14 @@ public TIFJobParameter() { } public TIFJobParameter(final String name, final Instant lastUpdateTime, final Instant enabledTime, final Boolean isEnabled, - final IntervalSchedule schedule, TIFJobTask task, final TIFJobState state, final String currentIndex, + final IntervalSchedule schedule, final TIFJobState state, final List indices, final UpdateStats updateStats) { this.name = name; this.lastUpdateTime = lastUpdateTime; this.enabledTime = enabledTime; this.isEnabled = isEnabled; this.schedule = schedule; - this.task = task; this.state = state; - this.currentIndex = currentIndex; this.indices = indices; this.updateStats = updateStats; } @@ -193,11 +174,9 @@ public TIFJobParameter(final String name, final IntervalSchedule schedule) { name, Instant.now().truncatedTo(ChronoUnit.MILLIS), null, - false, + true, schedule, - TIFJobTask.ALL, TIFJobState.CREATING, - null, new ArrayList<>(), new UpdateStats() ); @@ -209,9 +188,7 @@ public TIFJobParameter(final StreamInput in) throws IOException { enabledTime = toInstant(in.readOptionalVLong()); isEnabled = in.readBoolean(); schedule = new IntervalSchedule(in); - task = TIFJobTask.valueOf(in.readString()); state = TIFJobState.valueOf(in.readString()); - currentIndex = in.readOptionalString(); indices = in.readStringList(); updateStats = new UpdateStats(in); } @@ -222,9 +199,7 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeOptionalVLong(enabledTime == null ? null : enabledTime.toEpochMilli()); out.writeBoolean(isEnabled); schedule.writeTo(out); - out.writeString(task.name()); out.writeString(state.name()); - out.writeOptionalString(currentIndex); out.writeStringCollection(indices); updateStats.writeTo(out); } @@ -247,11 +222,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa } builder.field(ENABLED_FIELD.getPreferredName(), isEnabled); builder.field(SCHEDULE_FIELD.getPreferredName(), schedule); - builder.field(TASK_FIELD.getPreferredName(), task.name()); builder.field(STATE_FIELD.getPreferredName(), state.name()); - if (currentIndex != null) { - builder.field(CURRENT_INDEX_FIELD.getPreferredName(), currentIndex); - } builder.field(INDICES_FIELD.getPreferredName(), indices); builder.field(UPDATE_STATS_FIELD.getPreferredName(), updateStats); builder.endObject(); @@ -295,28 +266,15 @@ public boolean isEnabled() { return this.isEnabled; } - public TIFJobTask getTask() { - return task; - } public void setLastUpdateTime(Instant lastUpdateTime) { this.lastUpdateTime = lastUpdateTime; } - public void setCurrentIndex(String currentIndex) { - this.currentIndex = currentIndex; - } - public void setTask(TIFJobTask task) { - this.task = task; - } @Override public Long getLockDurationSeconds() { return TIFLockService.LOCK_DURATION_IN_SECONDS; } - public String getCurrentIndex() { - return currentIndex; - } - /** * Enable auto update of threat intel feed data */ @@ -336,30 +294,22 @@ public void disable() { isEnabled = false; } - /** - * Current index name of a tif job - * - * @return Current index name of a tif job - */ - public String currentIndexName() { - return currentIndex; - } - public void setSchedule(IntervalSchedule schedule) { this.schedule = schedule; } /** - * Index name for a tif job with given suffix + * Index name for a tif job * - * @return index name for a tif job with given suffix + * @return index name for a tif job */ public String newIndexName(final TIFJobParameter jobSchedulerParameter, TIFMetadata tifMetadata) { - List indices = jobSchedulerParameter.indices; + List indices = jobSchedulerParameter.getIndices(); Optional nameOptional = indices.stream().filter(name -> name.contains(tifMetadata.getFeedId())).findAny(); - String suffix = "-1"; + String suffix = "1"; if (nameOptional.isPresent()) { - suffix = "-1".equals(nameOptional.get()) ? "-2" : suffix; + String lastChar = "" + nameOptional.get().charAt(nameOptional.get().length() -1); + suffix = (lastChar.equals("1")) ? "2" : suffix; } return String.format(Locale.ROOT, "%s-%s%s", THREAT_INTEL_DATA_INDEX_NAME_PREFIX, tifMetadata.getFeedId(), suffix); } @@ -529,11 +479,12 @@ public static TIFJobParameter build(final PutTIFJobRequest request) { String name = request.getName(); IntervalSchedule schedule = new IntervalSchedule( Instant.now().truncatedTo(ChronoUnit.MILLIS), - 1, //TODO fix + (int) request.getUpdateInterval().hours(), ChronoUnit.DAYS ); return new TIFJobParameter(name, schedule); + } } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterService.java index 9d8fc3a3d..70f052549 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterService.java @@ -5,17 +5,6 @@ package org.opensearch.securityanalytics.threatIntel.jobscheduler; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.time.Instant; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; @@ -25,45 +14,40 @@ import org.opensearch.action.StepListener; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.create.CreateIndexResponse; -import org.opensearch.action.bulk.BulkRequest; -import org.opensearch.action.bulk.BulkResponse; import org.opensearch.action.delete.DeleteResponse; import org.opensearch.action.get.GetRequest; import org.opensearch.action.get.GetResponse; -import org.opensearch.action.get.MultiGetItemResponse; -import org.opensearch.action.get.MultiGetResponse; -import org.opensearch.action.index.IndexRequest; import org.opensearch.action.index.IndexResponse; -import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.WriteRequest; import org.opensearch.client.Client; -import org.opensearch.cluster.routing.Preference; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.action.ActionListener; -import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.securityanalytics.model.DetectorTrigger; +import org.opensearch.index.IndexNotFoundException; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; -import org.opensearch.index.IndexNotFoundException; -import org.opensearch.index.query.QueryBuilders; -import org.opensearch.search.SearchHit; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.stream.Collectors; + /** - * Data access object for tif job + * Data access object for tif job parameter */ public class TIFJobParameterService { - private static final Logger log = LogManager.getLogger(DetectorTrigger.class); - - private static final Integer MAX_SIZE = 1000; + private static final Logger log = LogManager.getLogger(TIFJobParameterService.class); private final Client client; private final ClusterService clusterService; private final ClusterSettings clusterSettings; @@ -139,33 +123,6 @@ public IndexResponse updateJobSchedulerParameter(final TIFJobParameter jobSchedu }); } - /** - * Update tif jobs in an index {@code TIFJobExtension.JOB_INDEX_NAME} - * @param tifJobParameters the tifJobParameters - * @param listener action listener - */ - public void updateJobSchedulerParameter(final List tifJobParameters, final ActionListener listener) { - BulkRequest bulkRequest = new BulkRequest(); - tifJobParameters.stream().map(tifJobParameter -> { - tifJobParameter.setLastUpdateTime(Instant.now()); - return tifJobParameter; - }).map(this::toIndexRequest).forEach(indexRequest -> bulkRequest.add(indexRequest)); - StashedThreadContext.run(client, () -> client.bulk(bulkRequest, listener)); - } - private IndexRequest toIndexRequest(TIFJobParameter tifJobParameter) { - try { - IndexRequest indexRequest = new IndexRequest(); - indexRequest.index(TIFJobExtension.JOB_INDEX_NAME); - indexRequest.id(tifJobParameter.getName()); - indexRequest.opType(DocWriteRequest.OpType.INDEX); - indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - indexRequest.source(tifJobParameter.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)); - return indexRequest; - } catch (IOException e) { - throw new SecurityAnalyticsException("Runtime exception", RestStatus.INTERNAL_SERVER_ERROR, e); //TODO - } - } - /** * Get tif job from an index {@code TIFJobExtension.JOB_INDEX_NAME} * @param name the name of a tif job @@ -211,7 +168,7 @@ public void saveTIFJobParameter(final TIFJobParameter tifJobParameter, final Act .setSource(tifJobParameter.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) .execute(listener); } catch (IOException e) { - throw new SecurityAnalyticsException("Runtime exception", RestStatus.INTERNAL_SERVER_ERROR, e); //TODO + throw new SecurityAnalyticsException("Exception saving the threat intel feed job parameter in index", RestStatus.INTERNAL_SERVER_ERROR, e); } }); } @@ -238,140 +195,4 @@ public void deleteTIFJobParameter(final TIFJobParameter tifJobParameter) { throw new OpenSearchException("failed to delete tifJobParameter[{}] with status[{}]", tifJobParameter.getName(), response.status()); } } - - /** - * Get tif job from an index {@code TIFJobExtension.JOB_INDEX_NAME} - * @param name the name of a tif job - * @param actionListener the action listener - */ - public void getJobParameter(final String name, final ActionListener actionListener) { - GetRequest request = new GetRequest(TIFJobExtension.JOB_INDEX_NAME, name); - StashedThreadContext.run(client, () -> client.get(request, new ActionListener<>() { - @Override - public void onResponse(final GetResponse response) { - if (response.isExists() == false) { - actionListener.onResponse(null); - return; - } - - try { - XContentParser parser = XContentHelper.createParser( - NamedXContentRegistry.EMPTY, - LoggingDeprecationHandler.INSTANCE, - response.getSourceAsBytesRef() - ); - actionListener.onResponse(TIFJobParameter.PARSER.parse(parser, null)); - } catch (IOException e) { - actionListener.onFailure(e); - } - } - - @Override - public void onFailure(final Exception e) { - actionListener.onFailure(e); - } - })); - } - - /** - * Get tif jobs from an index {@code TIFJobExtension.JOB_INDEX_NAME} - * @param names the array of tif job names - * @param actionListener the action listener - */ - public void getTIFJobParameters(final String[] names, final ActionListener> actionListener) { - StashedThreadContext.run( - client, - () -> client.prepareMultiGet() - .add(TIFJobExtension.JOB_INDEX_NAME, names) - .execute(createGetTIFJobParameterQueryActionLister(MultiGetResponse.class, actionListener)) - ); - } - - /** - * Get all tif jobs up to {@code MAX_SIZE} from an index {@code TIFJobExtension.JOB_INDEX_NAME} - * @param actionListener the action listener - */ - public void getAllTIFJobParameters(final ActionListener> actionListener) { - StashedThreadContext.run( - client, - () -> client.prepareSearch(TIFJobExtension.JOB_INDEX_NAME) - .setQuery(QueryBuilders.matchAllQuery()) - .setPreference(Preference.PRIMARY.type()) - .setSize(MAX_SIZE) - .execute(createGetTIFJobParameterQueryActionLister(SearchResponse.class, actionListener)) - ); - } - - /** - * Get all tif jobs up to {@code MAX_SIZE} from an index {@code TIFJobExtension.JOB_INDEX_NAME} - */ - public List getAllTIFJobParameters() { - SearchResponse response = StashedThreadContext.run( - client, - () -> client.prepareSearch(TIFJobExtension.JOB_INDEX_NAME) - .setQuery(QueryBuilders.matchAllQuery()) - .setPreference(Preference.PRIMARY.type()) - .setSize(MAX_SIZE) - .execute() - .actionGet(clusterSettings.get(SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT)) - ); - - List bytesReferences = toBytesReferences(response); - return bytesReferences.stream().map(bytesRef -> toTIFJobParameter(bytesRef)).collect(Collectors.toList()); - } - - private ActionListener createGetTIFJobParameterQueryActionLister( - final Class response, - final ActionListener> actionListener - ) { - return new ActionListener() { - @Override - public void onResponse(final T response) { - try { - List bytesReferences = toBytesReferences(response); - List tifJobParameters = bytesReferences.stream() - .map(bytesRef -> toTIFJobParameter(bytesRef)) - .collect(Collectors.toList()); - actionListener.onResponse(tifJobParameters); - } catch (Exception e) { - actionListener.onFailure(e); - } - } - - @Override - public void onFailure(final Exception e) { - actionListener.onFailure(e); - } - }; - } - - private List toBytesReferences(final Object response) { - if (response instanceof SearchResponse) { - SearchResponse searchResponse = (SearchResponse) response; - return Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getSourceRef).collect(Collectors.toList()); - } else if (response instanceof MultiGetResponse) { - MultiGetResponse multiGetResponse = (MultiGetResponse) response; - return Arrays.stream(multiGetResponse.getResponses()) - .map(MultiGetItemResponse::getResponse) - .filter(Objects::nonNull) - .filter(GetResponse::isExists) - .map(GetResponse::getSourceAsBytesRef) - .collect(Collectors.toList()); - } else { - throw new OpenSearchException("No supported instance type[{}] is provided", response.getClass()); - } - } - - private TIFJobParameter toTIFJobParameter(final BytesReference bytesReference) { - try { - XContentParser parser = XContentHelper.createParser( - NamedXContentRegistry.EMPTY, - LoggingDeprecationHandler.INSTANCE, - bytesReference - ); - return TIFJobParameter.PARSER.parse(parser, null); - } catch (IOException e) { - throw new SecurityAnalyticsException("Runtime exception", RestStatus.INTERNAL_SERVER_ERROR, e); //TODO - } - } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java index 4407bd9fe..ca1f61347 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java @@ -13,7 +13,6 @@ import org.opensearch.jobscheduler.spi.LockModel; import org.opensearch.jobscheduler.spi.ScheduledJobParameter; import org.opensearch.jobscheduler.spi.ScheduledJobRunner; -import org.opensearch.securityanalytics.model.DetectorTrigger; import java.io.IOException; import java.util.ArrayList; @@ -23,7 +22,6 @@ import java.time.Instant; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; -import org.opensearch.securityanalytics.threatIntel.common.TIFExecutor; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; import org.opensearch.threadpool.ThreadPool; @@ -33,7 +31,7 @@ * This is a background task which is responsible for updating threat intel feed data */ public class TIFJobRunner implements ScheduledJobRunner { - private static final Logger log = LogManager.getLogger(DetectorTrigger.class); + private static final Logger log = LogManager.getLogger(TIFJobRunner.class); private static TIFJobRunner INSTANCE; public static TIFJobRunner getJobRunnerInstance() { @@ -54,7 +52,6 @@ public static TIFJobRunner getJobRunnerInstance() { // threat intel specific variables private TIFJobUpdateService jobSchedulerUpdateService; private TIFJobParameterService jobSchedulerParameterService; - private TIFExecutor threatIntelExecutor; private TIFLockService lockService; private boolean initialized; private ThreadPool threadPool; @@ -71,14 +68,12 @@ public void initialize( final ClusterService clusterService, final TIFJobUpdateService jobSchedulerUpdateService, final TIFJobParameterService jobSchedulerParameterService, - final TIFExecutor threatIntelExecutor, final TIFLockService threatIntelLockService, final ThreadPool threadPool ) { this.clusterService = clusterService; this.jobSchedulerUpdateService = jobSchedulerUpdateService; this.jobSchedulerParameterService = jobSchedulerParameterService; - this.threatIntelExecutor = threatIntelExecutor; this.lockService = threatIntelLockService; this.threadPool = threadPool; this.initialized = true; @@ -98,7 +93,6 @@ public void runJob(final ScheduledJobParameter jobParameter, final JobExecutionC ); } threadPool.generic().submit(updateJobRunner(jobParameter)); -// threatIntelExecutor.forJobSchedulerParameterUpdate().submit(updateJobRunner(jobParameter)); } /** @@ -151,20 +145,17 @@ protected void updateJobParameter(final ScheduledJobParameter jobParameter, fina return; } try { - if (TIFJobTask.DELETE_UNUSED_INDICES.equals(jobSchedulerParameter.getTask()) == false) { - Instant startTime = Instant.now(); - List oldIndices = new ArrayList<>(jobSchedulerParameter.getIndices()); - List newFeedIndices = jobSchedulerUpdateService.createThreatIntelFeedData(jobSchedulerParameter, renewLock); - Instant endTime = Instant.now(); - jobSchedulerUpdateService.deleteAllTifdIndices(oldIndices, newFeedIndices); - jobSchedulerUpdateService.updateJobSchedulerParameterAsSucceeded(newFeedIndices, jobSchedulerParameter, startTime, endTime); - } + // create new TIF data and delete old ones + Instant startTime = Instant.now(); + List oldIndices = new ArrayList<>(jobSchedulerParameter.getIndices()); + List newFeedIndices = jobSchedulerUpdateService.createThreatIntelFeedData(jobSchedulerParameter, renewLock); + Instant endTime = Instant.now(); + jobSchedulerUpdateService.deleteAllTifdIndices(oldIndices, newFeedIndices); + jobSchedulerUpdateService.updateJobSchedulerParameterAsSucceeded(newFeedIndices, jobSchedulerParameter, startTime, endTime); } catch (Exception e) { log.error("Failed to update jobSchedulerParameter for {}", jobSchedulerParameter.getName(), e); jobSchedulerParameter.getUpdateStats().setLastFailedAt(Instant.now()); jobSchedulerParameterService.updateJobSchedulerParameter(jobSchedulerParameter); - } finally { - jobSchedulerUpdateService.updateJobSchedulerParameter(jobSchedulerParameter, jobSchedulerParameter.getSchedule(), TIFJobTask.ALL); } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobTask.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobTask.java deleted file mode 100644 index 1221a3540..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobTask.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.threatIntel.jobscheduler; - -/** - * Task that {@link TIFJobRunner} will run - */ -public enum TIFJobTask { - /** - * Do everything - */ - ALL, - - /** - * Only delete unused indices - */ - DELETE_UNUSED_INDICES -} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateService.java index a5cc01ea1..45ad50b35 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateService.java @@ -15,8 +15,6 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.core.rest.RestStatus; -import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; -import org.opensearch.securityanalytics.model.DetectorTrigger; import org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataService; import org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedParser; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; @@ -31,7 +29,7 @@ import java.util.List; public class TIFJobUpdateService { - private static final Logger log = LogManager.getLogger(DetectorTrigger.class); + private static final Logger log = LogManager.getLogger(TIFJobUpdateService.class); private static final int SLEEP_TIME_IN_MILLIS = 5000; // 5 seconds private static final int MAX_WAIT_TIME_FOR_REPLICATION_TO_COMPLETE_IN_MILLIS = 10 * 60 * 60 * 1000; // 10 hours @@ -71,28 +69,6 @@ public void deleteAllTifdIndices(List oldIndices, List newIndice } } - /** - * Update jobSchedulerParameter with given systemSchedule and task - * - * @param jobSchedulerParameter jobSchedulerParameter to update - * @param systemSchedule new system schedule value - * @param task new task value - */ - public void updateJobSchedulerParameter(final TIFJobParameter jobSchedulerParameter, final IntervalSchedule systemSchedule, final TIFJobTask task) { - boolean updated = false; - if (jobSchedulerParameter.getSchedule().equals(systemSchedule) == false) { //TODO: will always be true - jobSchedulerParameter.setSchedule(systemSchedule); - updated = true; - } - if (jobSchedulerParameter.getTask().equals(task) == false) { - jobSchedulerParameter.setTask(task); - updated = true; - } // this is called when task == DELETE - if (updated) { - jobSchedulerParameterService.updateJobSchedulerParameter(jobSchedulerParameter); - } - } - private List deleteIndices(final List indicesToDelete) { List deletedIndices = new ArrayList<>(indicesToDelete.size()); for (String index : indicesToDelete) { @@ -126,21 +102,29 @@ public List createThreatIntelFeedData(final TIFJobParameter jobScheduler List freshIndices = new ArrayList<>(); for (TIFMetadata tifMetadata : builtInTIFMetadataLoader.getTifMetadataList()) { String indexName = setupIndex(jobSchedulerParameter, tifMetadata); - String[] header; Boolean succeeded; - switch (tifMetadata.getFeedType()) { case "csv": try (CSVParser reader = ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(tifMetadata)) { - // iterate until we find first line without '#' and without empty line - CSVRecord findHeader = reader.iterator().next(); - while ((findHeader.values().length ==1 && "".equals(findHeader.values()[0])) || findHeader.get(0).charAt(0) == '#' || findHeader.get(0).charAt(0) == ' ') { - findHeader = reader.iterator().next(); + CSVParser noHeaderReader = ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(tifMetadata); + boolean notFound = true; + + while (notFound) { + CSVRecord hasHeaderRecord = reader.iterator().next(); + + //if we want to skip this line and keep iterating + if ((hasHeaderRecord.values().length ==1 && "".equals(hasHeaderRecord.values()[0])) || hasHeaderRecord.get(0).charAt(0) == '#' || hasHeaderRecord.get(0).charAt(0) == ' '){ + noHeaderReader.iterator().next(); + } else { // we found the first line that contains information + notFound = false; + } + } + if (tifMetadata.hasHeader()){ + threatIntelFeedDataService.parseAndSaveThreatIntelFeedDataCSV(indexName, reader.iterator(), renewLock, tifMetadata); + } else { + threatIntelFeedDataService.parseAndSaveThreatIntelFeedDataCSV(indexName, noHeaderReader.iterator(), renewLock, tifMetadata); } - CSVRecord headerLine = findHeader; - header = ThreatIntelFeedParser.validateHeader(headerLine).values(); - threatIntelFeedDataService.parseAndSaveThreatIntelFeedDataCSV(indexName, header, reader.iterator(), renewLock, tifMetadata); succeeded = true; } break; @@ -224,47 +208,4 @@ protected void waitUntilAllShardsStarted(final String indexName, final int timeo throw new SecurityAnalyticsException("Runtime exception", RestStatus.INTERNAL_SERVER_ERROR, e); //TODO } } - - -// /** -// * Determine if update is needed or not -// * -// * Update is needed when all following conditions are met -// * 1. updatedAt value in jobSchedulerParameter is equal or before updateAt value in tifMetadata -// * 2. SHA256 hash value in jobSchedulerParameter is different with SHA256 hash value in tifMetadata -// * -// * @param jobSchedulerParameter -// * @param tifMetadata -// * @return -// */ -// private boolean shouldUpdate(final TIFJobParameter jobSchedulerParameter, final TIFMetadata tifMetadata) { -// if (jobSchedulerParameter.getDatabase().getUpdatedAt() != null -// && jobSchedulerParameter.getDatabase().getUpdatedAt().toEpochMilli() > tifMetadata.getUpdatedAt()) { -// return false; -// } -// -// if (tifMetadata.getSha256Hash().equals(jobSchedulerParameter.getDatabase().getSha256Hash())) { -// return false; -// } -// return true; -// } - -// /** -// * Return header fields of threat intel feed data with given url of a manifest file -// * -// * The first column is ip range field regardless its header name. -// * Therefore, we don't store the first column's header name. -// * -// * @param TIFMetadataUrl the url of a manifest file -// * @return header fields of threat intel feed -// */ -// public List getHeaderFields(String TIFMetadataUrl) throws IOException { -// URL url = new URL(TIFMetadataUrl); -// TIFMetadata tifMetadata = TIFMetadata.Builder.build(url); -// -// try (CSVParser reader = ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(tifMetadata)) { -// String[] fields = reader.iterator().next().values(); -// return Arrays.asList(fields).subList(1, fields.length); -// } -// } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatintel/common/ParameterValidator.java b/src/main/java/org/opensearch/securityanalytics/threatintel/common/ParameterValidator.java deleted file mode 100644 index 25e40837c..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatintel/common/ParameterValidator.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.threatIntel.common; - -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; - -import org.apache.commons.lang3.StringUtils; -import org.opensearch.core.common.Strings; - -/** - * Parameter validator for TIF APIs - */ -public class ParameterValidator { - private static final int MAX_DATASOURCE_NAME_BYTES = 127; - - /** - * Validate datasource name and return list of error messages - * - * @param datasourceName datasource name - * @return Error messages. Empty list if there is no violation. - */ - public List validateTIFJobName(final String datasourceName) { - List errorMsgs = new ArrayList<>(); - if (StringUtils.isBlank(datasourceName)) { - errorMsgs.add("datasource name must not be empty"); - return errorMsgs; - } - - if (!Strings.validFileName(datasourceName)) { - errorMsgs.add( - String.format(Locale.ROOT, "datasource name must not contain the following characters %s", Strings.INVALID_FILENAME_CHARS) - ); - } - if (datasourceName.contains("#")) { - errorMsgs.add("datasource name must not contain '#'"); - } - if (datasourceName.contains(":")) { - errorMsgs.add("datasource name must not contain ':'"); - } - if (datasourceName.charAt(0) == '_' || datasourceName.charAt(0) == '-' || datasourceName.charAt(0) == '+') { - errorMsgs.add("datasource name must not start with '_', '-', or '+'"); - } - int byteCount = datasourceName.getBytes(StandardCharsets.UTF_8).length; - if (byteCount > MAX_DATASOURCE_NAME_BYTES) { - errorMsgs.add(String.format(Locale.ROOT, "datasource name is too long, (%d > %d)", byteCount, MAX_DATASOURCE_NAME_BYTES)); - } - if (datasourceName.equals(".") || datasourceName.equals("..")) { - errorMsgs.add("datasource name must not be '.' or '..'"); - } - return errorMsgs; - } -} diff --git a/src/main/resources/OSMapping/ad_ldap_logtype.json b/src/main/resources/OSMapping/ad_ldap_logtype.json index e3434bca5..be2dd5488 100644 --- a/src/main/resources/OSMapping/ad_ldap_logtype.json +++ b/src/main/resources/OSMapping/ad_ldap_logtype.json @@ -2,7 +2,8 @@ "name": "ad_ldap", "description": "AD/LDAP", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"TargetUserName", "ecs":"azure.signinlogs.properties.user_id" diff --git a/src/main/resources/OSMapping/apache_access_logtype.json b/src/main/resources/OSMapping/apache_access_logtype.json index 7753c8440..714fa2acb 100644 --- a/src/main/resources/OSMapping/apache_access_logtype.json +++ b/src/main/resources/OSMapping/apache_access_logtype.json @@ -2,5 +2,6 @@ "name": "apache_access", "description": "Apache Access Log type", "is_builtin": true, - "mappings": [] + "ioc_fields" : [], + "mappings":[] } diff --git a/src/main/resources/OSMapping/azure_logtype.json b/src/main/resources/OSMapping/azure_logtype.json index ec9ae0502..bb55dbe5f 100644 --- a/src/main/resources/OSMapping/azure_logtype.json +++ b/src/main/resources/OSMapping/azure_logtype.json @@ -2,7 +2,8 @@ "name": "azure", "description": "Azure Log Type", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"Resultdescription", "ecs":"azure.signinlogs.result_description" diff --git a/src/main/resources/OSMapping/cloudtrail_logtype.json b/src/main/resources/OSMapping/cloudtrail_logtype.json index 389652373..8c2ea3b3a 100644 --- a/src/main/resources/OSMapping/cloudtrail_logtype.json +++ b/src/main/resources/OSMapping/cloudtrail_logtype.json @@ -2,7 +2,15 @@ "name": "cloudtrail", "description": "Cloudtrail Log Type", "is_builtin": true, - "mappings": [ + "ioc_fields": [ + { + "ioc": "ip", + "fields": [ + "src_endpoint.ip" + ] + } + ], + "mappings":[ { "raw_field":"eventName", "ecs":"aws.cloudtrail.event_name", diff --git a/src/main/resources/OSMapping/dns_logtype.json b/src/main/resources/OSMapping/dns_logtype.json index ca2f5451a..ef012407f 100644 --- a/src/main/resources/OSMapping/dns_logtype.json +++ b/src/main/resources/OSMapping/dns_logtype.json @@ -2,7 +2,15 @@ "name": "dns", "description": "DNS Log Type", "is_builtin": true, - "mappings": [ + "ioc_fields": [ + { + "ioc": "ip", + "fields": [ + "src_endpoint.ip" + ] + } + ], + "mappings":[ { "raw_field":"record_type", "ecs":"dns.answers.type", diff --git a/src/main/resources/OSMapping/github_logtype.json b/src/main/resources/OSMapping/github_logtype.json index 6369e2949..31ec6ee59 100644 --- a/src/main/resources/OSMapping/github_logtype.json +++ b/src/main/resources/OSMapping/github_logtype.json @@ -2,7 +2,8 @@ "name": "github", "description": "Github Log Type", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"action", "ecs":"github.action" diff --git a/src/main/resources/OSMapping/gworkspace_logtype.json b/src/main/resources/OSMapping/gworkspace_logtype.json index b0006b6a3..7c5766895 100644 --- a/src/main/resources/OSMapping/gworkspace_logtype.json +++ b/src/main/resources/OSMapping/gworkspace_logtype.json @@ -2,7 +2,8 @@ "name": "gworkspace", "description": "GWorkspace Log Type", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"eventSource", "ecs":"google_workspace.admin.service.name" diff --git a/src/main/resources/OSMapping/linux_logtype.json b/src/main/resources/OSMapping/linux_logtype.json index f719913c0..5b77de6b3 100644 --- a/src/main/resources/OSMapping/linux_logtype.json +++ b/src/main/resources/OSMapping/linux_logtype.json @@ -2,7 +2,8 @@ "name": "linux", "description": "Linux Log Type", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"name", "ecs":"user.filesystem.name" diff --git a/src/main/resources/OSMapping/m365_logtype.json b/src/main/resources/OSMapping/m365_logtype.json index 6547d3d63..e19c2418e 100644 --- a/src/main/resources/OSMapping/m365_logtype.json +++ b/src/main/resources/OSMapping/m365_logtype.json @@ -2,7 +2,8 @@ "name": "m365", "description": "Microsoft 365 Log Type", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"eventSource", "ecs":"rsa.misc.event_source" diff --git a/src/main/resources/OSMapping/netflow_logtype.json b/src/main/resources/OSMapping/netflow_logtype.json index d8ec32632..9dc015198 100644 --- a/src/main/resources/OSMapping/netflow_logtype.json +++ b/src/main/resources/OSMapping/netflow_logtype.json @@ -2,7 +2,16 @@ "name": "netflow", "description": "Netflow Log Type used only in Integration Tests", "is_builtin": true, - "mappings": [ + "ioc_fields": [ + { + "ioc": "ip", + "fields": [ + "destination.ip", + "source.ip" + ] + } + ], + "mappings":[ { "raw_field":"netflow.source_ipv4_address", "ecs":"source.ip" diff --git a/src/main/resources/OSMapping/network_logtype.json b/src/main/resources/OSMapping/network_logtype.json index 90f0b2ee6..2ca92a1ad 100644 --- a/src/main/resources/OSMapping/network_logtype.json +++ b/src/main/resources/OSMapping/network_logtype.json @@ -2,7 +2,16 @@ "name": "network", "description": "Network Log Type", "is_builtin": true, - "mappings": [ + "ioc_fields": [ + { + "ioc": "ip", + "fields": [ + "destination.ip", + "source.ip" + ] + } + ], + "mappings":[ { "raw_field":"action", "ecs":"netflow.firewall_event" diff --git a/src/main/resources/OSMapping/okta_logtype.json b/src/main/resources/OSMapping/okta_logtype.json index 8038b7f01..e73a0c273 100644 --- a/src/main/resources/OSMapping/okta_logtype.json +++ b/src/main/resources/OSMapping/okta_logtype.json @@ -2,7 +2,8 @@ "name": "okta", "description": "Okta Log Type", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"eventtype", "ecs":"okta.event_type" diff --git a/src/main/resources/OSMapping/others_application_logtype.json b/src/main/resources/OSMapping/others_application_logtype.json index d7faf8c94..4008602d4 100644 --- a/src/main/resources/OSMapping/others_application_logtype.json +++ b/src/main/resources/OSMapping/others_application_logtype.json @@ -2,7 +2,8 @@ "name": "others_application", "description": "others_application", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"record_type", "ecs":"dns.answers.type" diff --git a/src/main/resources/OSMapping/others_apt_logtype.json b/src/main/resources/OSMapping/others_apt_logtype.json index ace55cbc3..1a4ca711f 100644 --- a/src/main/resources/OSMapping/others_apt_logtype.json +++ b/src/main/resources/OSMapping/others_apt_logtype.json @@ -2,7 +2,8 @@ "name": "others_apt", "description": "others_apt", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"record_type", "ecs":"dns.answers.type" diff --git a/src/main/resources/OSMapping/others_cloud_logtype.json b/src/main/resources/OSMapping/others_cloud_logtype.json index b5da3e005..64cbc7935 100644 --- a/src/main/resources/OSMapping/others_cloud_logtype.json +++ b/src/main/resources/OSMapping/others_cloud_logtype.json @@ -2,7 +2,8 @@ "name": "others_cloud", "description": "others_cloud", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"record_type", "ecs":"dns.answers.type" diff --git a/src/main/resources/OSMapping/others_compliance_logtype.json b/src/main/resources/OSMapping/others_compliance_logtype.json index 6f362d589..6e065795a 100644 --- a/src/main/resources/OSMapping/others_compliance_logtype.json +++ b/src/main/resources/OSMapping/others_compliance_logtype.json @@ -2,7 +2,8 @@ "name": "others_compliance", "description": "others_compliance", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"record_type", "ecs":"dns.answers.type" diff --git a/src/main/resources/OSMapping/others_macos_logtype.json b/src/main/resources/OSMapping/others_macos_logtype.json index 50d1c2160..6b6452100 100644 --- a/src/main/resources/OSMapping/others_macos_logtype.json +++ b/src/main/resources/OSMapping/others_macos_logtype.json @@ -2,7 +2,8 @@ "name": "others_macos", "description": "others_macos", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"record_type", "ecs":"dns.answers.type" diff --git a/src/main/resources/OSMapping/others_proxy_logtype.json b/src/main/resources/OSMapping/others_proxy_logtype.json index aca4529d1..a2b0794a4 100644 --- a/src/main/resources/OSMapping/others_proxy_logtype.json +++ b/src/main/resources/OSMapping/others_proxy_logtype.json @@ -2,7 +2,8 @@ "name": "others_proxy", "description": "others_proxy", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"record_type", "ecs":"dns.answers.type" diff --git a/src/main/resources/OSMapping/others_web_logtype.json b/src/main/resources/OSMapping/others_web_logtype.json index ae8262d52..b46adc6a4 100644 --- a/src/main/resources/OSMapping/others_web_logtype.json +++ b/src/main/resources/OSMapping/others_web_logtype.json @@ -2,7 +2,8 @@ "name": "others_web", "description": "others_web", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"record_type", "ecs":"dns.answers.type" diff --git a/src/main/resources/OSMapping/s3_logtype.json b/src/main/resources/OSMapping/s3_logtype.json index 58c546258..20c896df6 100644 --- a/src/main/resources/OSMapping/s3_logtype.json +++ b/src/main/resources/OSMapping/s3_logtype.json @@ -2,7 +2,8 @@ "name": "s3", "description": "S3 Log Type", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"eventName", "ecs":"aws.cloudtrail.event_name" diff --git a/src/main/resources/OSMapping/test_windows_logtype.json b/src/main/resources/OSMapping/test_windows_logtype.json index 816cba666..cc619c5a1 100644 --- a/src/main/resources/OSMapping/test_windows_logtype.json +++ b/src/main/resources/OSMapping/test_windows_logtype.json @@ -5,7 +5,7 @@ "ioc_fields": [ { "ioc": "ip", - "fields": ["windows-hostname"] + "fields": ["HostName"] } ], "mappings": [ diff --git a/src/main/resources/OSMapping/vpcflow_logtype.json b/src/main/resources/OSMapping/vpcflow_logtype.json index c55305b6d..29d9f38c2 100644 --- a/src/main/resources/OSMapping/vpcflow_logtype.json +++ b/src/main/resources/OSMapping/vpcflow_logtype.json @@ -2,7 +2,16 @@ "name": "vpcflow", "description": "VPC Flow Log Type", "is_builtin": true, - "mappings": [ + "ioc_fields": [ + { + "ioc": "ip", + "fields": [ + "dst_endpoint.ip", + "src_endpoint.ip" + ] + } + ], + "mappings":[ { "raw_field":"version", "ecs":"netflow.version", diff --git a/src/main/resources/OSMapping/waf_logtype.json b/src/main/resources/OSMapping/waf_logtype.json index 5eed2c2fb..3e5b1f4f1 100644 --- a/src/main/resources/OSMapping/waf_logtype.json +++ b/src/main/resources/OSMapping/waf_logtype.json @@ -2,7 +2,8 @@ "name": "waf", "description": "Web Application Firewall Log Type", "is_builtin": true, - "mappings": [ + "ioc_fields" : [], + "mappings":[ { "raw_field":"cs-method", "ecs":"waf.request.method" diff --git a/src/main/resources/OSMapping/windows_logtype.json b/src/main/resources/OSMapping/windows_logtype.json index a5fef8ea7..ec9b3ed1a 100644 --- a/src/main/resources/OSMapping/windows_logtype.json +++ b/src/main/resources/OSMapping/windows_logtype.json @@ -2,7 +2,13 @@ "name": "windows", "description": "Windows Log Type", "is_builtin": true, - "mappings":[ + "ioc_fields" : [ + { + "ioc": "ip", + "fields": ["destination.ip","source.ip"] + } + ], + "mappings": [ { "raw_field":"AccountName", "ecs":"winlog.computerObject.name" diff --git a/src/main/resources/mappings/threat_intel_job_mapping.json b/src/main/resources/mappings/threat_intel_job_mapping.json index 5e039928d..c64b034fe 100644 --- a/src/main/resources/mappings/threat_intel_job_mapping.json +++ b/src/main/resources/mappings/threat_intel_job_mapping.json @@ -1,35 +1,11 @@ { + "dynamic": "strict", + "_meta" : { + "schema_version": 1 + }, "properties": { - "database": { - "properties": { - "feed_id": { - "type": "text" - }, - "feed_name": { - "type": "text" - }, - "feed_format": { - "type": "text" - }, - "endpoint": { - "type": "text" - }, - "description": { - "type": "text" - }, - "organization": { - "type": "text" - }, - "contained_iocs_field": { - "type": "text" - }, - "ioc_col": { - "type": "text" - }, - "fields": { - "type": "text" - } - } + "schema_version": { + "type": "integer" }, "enabled_time": { "type": "long" @@ -63,15 +39,6 @@ "state": { "type": "text" }, - "task": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, "update_enabled": { "type": "boolean" }, @@ -90,29 +57,6 @@ "type": "long" } } - }, - "user_schedule": { - "properties": { - "interval": { - "properties": { - "period": { - "type": "long" - }, - "start_time": { - "type": "long" - }, - "unit": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - } - } } } } \ No newline at end of file diff --git a/src/main/resources/threatIntelFeed/feedMetadata.json b/src/main/resources/threatIntelFeed/feedMetadata.json index c73995ebd..27196b6b6 100644 --- a/src/main/resources/threatIntelFeed/feedMetadata.json +++ b/src/main/resources/threatIntelFeed/feedMetadata.json @@ -7,6 +7,7 @@ "description": "Alienvault IP Reputation threat intelligence feed managed by AlienVault", "feed_format": "csv", "ioc_type": "ip", - "ioc_col": 0 + "ioc_col": 0, + "has_header": false } } \ No newline at end of file diff --git a/src/main/resources/threatIntelFeedInfo/feodo.yml b/src/main/resources/threatIntelFeedInfo/feodo.yml new file mode 100644 index 000000000..4acbf40e4 --- /dev/null +++ b/src/main/resources/threatIntelFeedInfo/feodo.yml @@ -0,0 +1,6 @@ +url: "https://feodotracker.abuse.ch/downloads/ipblocklist_aggressive.csv" +name: "ipblocklist_aggressive.csv" +feedFormat: "csv" +org: "Feodo" +iocTypes: ["ip"] +description: "" \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 65417ed39..9b17c4aa2 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -1397,7 +1397,7 @@ public static String randomDocWithIpIoc(int severity, int version, String ioc) "\"AccountType\":\"User\",\n" + "\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" + "\"Category\":\"Dns query (rule: DnsQuery)\",\n" + - "\"Opcode\":\"blahblah\",\n" + + "\"Opcode\":\"%blahblah\",\n" + "\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" + "\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" + "\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" + @@ -1409,7 +1409,7 @@ public static String randomDocWithIpIoc(int severity, int version, String ioc) "\"CommandLine\": \"eachtest\",\n" + "\"Initiated\": \"true\"\n" + "}"; - return String.format(Locale.ROOT, doc, ioc, severity, version); + return String.format(Locale.ROOT, ioc, doc, severity, version); } @@ -1563,6 +1563,20 @@ public static String vpcFlowMappings() { " }"; } + private static String randomString() { + return OpenSearchTestCase.randomAlphaOfLengthBetween(2, 16); + } + + public static String randomLowerCaseString() { + return randomString().toLowerCase(Locale.ROOT); + } + + public static List randomLowerCaseStringList() { + List stringList = new ArrayList<>(); + stringList.add(randomLowerCaseString()); + return stringList; + } + public static XContentParser parser(String xc) throws IOException { XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); parser.nextToken(); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index e71cace9a..07e862369 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -1119,7 +1119,7 @@ public void testCreateDetectorWithThreatIntelEnabled_updateDetectorWithThreatInt List iocs = getThreatIntelFeedIocs(3); int i=1; for (String ioc : iocs) { - indexDoc(index, i+"", randomDocWithIpIoc(5, 3, ioc)); + indexDoc(index, i+"", randomDoc(5, 3, i==1? "120.85.114.146" : "120.86.237.94")); i++; } String workflowId = ((List) detectorMap.get("workflow_ids")).get(0); diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java new file mode 100644 index 000000000..a6661b32a --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java @@ -0,0 +1,270 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel; + +import org.junit.After; +import org.junit.Before; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionType; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.Randomness; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.OpenSearchExecutors; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.action.ActionResponse; +import org.opensearch.ingest.IngestMetadata; +import org.opensearch.ingest.IngestService; +import org.opensearch.jobscheduler.spi.LockModel; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.jobscheduler.spi.utils.LockService; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobUpdateService; +import org.opensearch.tasks.Task; +import org.opensearch.tasks.TaskListener; +import org.opensearch.test.client.NoOpNodeClient; +import org.opensearch.test.rest.RestActionTestCase; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; +import java.util.stream.Collectors; + +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; +import org.opensearch.securityanalytics.TestHelpers; + +public abstract class ThreatIntelTestCase extends RestActionTestCase { + @Mock + protected ClusterService clusterService; + @Mock + protected TIFJobUpdateService tifJobUpdateService; + @Mock + protected TIFJobParameterService tifJobParameterService; + @Mock + protected BuiltInTIFMetadataLoader builtInTIFMetadataLoader; + @Mock + protected ThreatIntelFeedDataService threatIntelFeedDataService; + @Mock + protected ClusterState clusterState; + @Mock + protected Metadata metadata; + @Mock + protected IngestService ingestService; + @Mock + protected ActionFilters actionFilters; + @Mock + protected ThreadPool threadPool; + @Mock + protected TIFLockService tifLockService; + @Mock + protected RoutingTable routingTable; + @Mock + protected TransportService transportService; + protected IngestMetadata ingestMetadata; + protected NoOpNodeClient client; + protected VerifyingClient verifyingClient; + protected LockService lockService; + protected ClusterSettings clusterSettings; + protected Settings settings; + private AutoCloseable openMocks; + @Mock + protected TIFJobParameter tifJobParameter; + + @Before + public void prepareThreatIntelTestCase() { + openMocks = MockitoAnnotations.openMocks(this); + settings = Settings.EMPTY; + client = new NoOpNodeClient(this.getTestName()); + verifyingClient = spy(new VerifyingClient(this.getTestName())); + clusterSettings = new ClusterSettings(settings, new HashSet<>(SecurityAnalyticsSettings.settings())); + lockService = new LockService(client, clusterService); + ingestMetadata = new IngestMetadata(Collections.emptyMap()); + when(metadata.custom(IngestMetadata.TYPE)).thenReturn(ingestMetadata); + when(clusterService.getSettings()).thenReturn(Settings.EMPTY); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + when(clusterService.state()).thenReturn(clusterState); + when(clusterState.metadata()).thenReturn(metadata); + when(clusterState.getMetadata()).thenReturn(metadata); + when(clusterState.routingTable()).thenReturn(routingTable); + when(ingestService.getClusterService()).thenReturn(clusterService); + when(threadPool.generic()).thenReturn(OpenSearchExecutors.newDirectExecutorService()); + } + + @After + public void clean() throws Exception { + openMocks.close(); + client.close(); + verifyingClient.close(); + } + + protected TIFJobState randomStateExcept(TIFJobState state) { + assertNotNull(state); + return Arrays.stream(TIFJobState.values()) + .sequential() + .filter(s -> !s.equals(state)) + .collect(Collectors.toList()) + .get(Randomness.createSecure().nextInt(TIFJobState.values().length - 2)); + } + + protected TIFJobState randomState() { + return Arrays.stream(TIFJobState.values()) + .sequential() + .collect(Collectors.toList()) + .get(Randomness.createSecure().nextInt(TIFJobState.values().length - 1)); + } + + protected long randomPositiveLong() { + long value = Randomness.get().nextLong(); + return value < 0 ? -value : value; + } + + /** + * Update interval should be > 0 and < validForInDays. + * For an update test to work, there should be at least one eligible value other than current update interval. + * Therefore, the smallest value for validForInDays is 2. + * Update interval is random value from 1 to validForInDays - 2. + * The new update value will be validForInDays - 1. + */ + protected TIFJobParameter randomTifJobParameter(final Instant updateStartTime) { + Instant now = Instant.now().truncatedTo(ChronoUnit.MILLIS); + TIFJobParameter tifJobParameter = new TIFJobParameter(); + tifJobParameter.setName(TestHelpers.randomLowerCaseString()); + tifJobParameter.setSchedule( + new IntervalSchedule( + updateStartTime.truncatedTo(ChronoUnit.MILLIS), + 1, + ChronoUnit.DAYS + ) + ); + tifJobParameter.setState(randomState()); + tifJobParameter.setIndices(Arrays.asList(TestHelpers.randomLowerCaseString(), TestHelpers.randomLowerCaseString())); + tifJobParameter.getUpdateStats().setLastSkippedAt(now); + tifJobParameter.getUpdateStats().setLastSucceededAt(now); + tifJobParameter.getUpdateStats().setLastFailedAt(now); + tifJobParameter.getUpdateStats().setLastProcessingTimeInMillis(randomPositiveLong()); + tifJobParameter.setLastUpdateTime(now); + if (Randomness.get().nextInt() % 2 == 0) { + tifJobParameter.enable(); + } else { + tifJobParameter.disable(); + } + return tifJobParameter; + } + + protected TIFJobParameter randomTifJobParameter() { + return randomTifJobParameter(Instant.now()); + } + + protected LockModel randomLockModel() { + LockModel lockModel = new LockModel( + TestHelpers.randomLowerCaseString(), + TestHelpers.randomLowerCaseString(), + Instant.now(), + randomPositiveLong(), + false + ); + return lockModel; + } + + /** + * Temporary class of VerifyingClient until this PR(https://github.com/opensearch-project/OpenSearch/pull/7167) + * is merged in OpenSearch core + */ + public static class VerifyingClient extends NoOpNodeClient { + AtomicReference executeVerifier = new AtomicReference<>(); + AtomicReference executeLocallyVerifier = new AtomicReference<>(); + + public VerifyingClient(String testName) { + super(testName); + reset(); + } + + /** + * Clears any previously set verifier functions set by {@link #setExecuteVerifier(BiFunction)} and/or + * {@link #setExecuteLocallyVerifier(BiFunction)}. These functions are replaced with functions which will throw an + * {@link AssertionError} if called. + */ + public void reset() { + executeVerifier.set((arg1, arg2) -> { throw new AssertionError(); }); + executeLocallyVerifier.set((arg1, arg2) -> { throw new AssertionError(); }); + } + + /** + * Sets the function that will be called when {@link #doExecute(ActionType, ActionRequest, ActionListener)} is called. The given + * function should return either a subclass of {@link ActionResponse} or {@code null}. + * @param verifier A function which is called in place of {@link #doExecute(ActionType, ActionRequest, ActionListener)} + */ + public void setExecuteVerifier( + BiFunction, Request, Response> verifier + ) { + executeVerifier.set(verifier); + } + + @Override + public void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { + try { + listener.onResponse((Response) executeVerifier.get().apply(action, request)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Sets the function that will be called when {@link #executeLocally(ActionType, ActionRequest, TaskListener)}is called. The given + * function should return either a subclass of {@link ActionResponse} or {@code null}. + * @param verifier A function which is called in place of {@link #executeLocally(ActionType, ActionRequest, TaskListener)} + */ + public void setExecuteLocallyVerifier( + BiFunction, Request, Response> verifier + ) { + executeLocallyVerifier.set(verifier); + } + + @Override + public Task executeLocally( + ActionType action, + Request request, + ActionListener listener + ) { + listener.onResponse((Response) executeLocallyVerifier.get().apply(action, request)); + return null; + } + + @Override + public Task executeLocally( + ActionType action, + Request request, + TaskListener listener + ) { + listener.onResponse(null, (Response) executeLocallyVerifier.get().apply(action, request)); + return null; + } + + } +} + diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/action/DeleteTIFJobRequestTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/action/DeleteTIFJobRequestTests.java new file mode 100644 index 000000000..2ecd7369b --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/action/DeleteTIFJobRequestTests.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.securityanalytics.TestHelpers; +import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; + +import java.io.IOException; + +public class DeleteTIFJobRequestTests extends ThreatIntelTestCase { + + public void testStreamInOut_whenValidInput_thenSucceed() throws IOException { + String tifJobParameterName = TestHelpers.randomLowerCaseString(); + DeleteTIFJobRequest request = new DeleteTIFJobRequest(tifJobParameterName); + + // Run + BytesStreamOutput output = new BytesStreamOutput(); + request.writeTo(output); + BytesStreamInput input = new BytesStreamInput(output.bytes().toBytesRef().bytes); + DeleteTIFJobRequest copiedRequest = new DeleteTIFJobRequest(input); + + // Verify + assertEquals(request.getName(), copiedRequest.getName()); + } + + public void testValidate_whenNull_thenError() { + DeleteTIFJobRequest request = new DeleteTIFJobRequest((String) null); + + // Run + ActionRequestValidationException error = request.validate(); + + // Verify + assertNotNull(error.validationErrors()); + assertFalse(error.validationErrors().isEmpty()); + } + + public void testValidate_whenBlank_thenError() { + DeleteTIFJobRequest request = new DeleteTIFJobRequest(" "); + + // Run + ActionRequestValidationException error = request.validate(); + + // Verify + assertNotNull(error.validationErrors()); + assertFalse(error.validationErrors().isEmpty()); + } + + public void testValidate_whenInvalidTIFJobParameterName_thenFails() { + String invalidName = "_" + TestHelpers.randomLowerCaseString(); + DeleteTIFJobRequest request = new DeleteTIFJobRequest(invalidName); + + // Run + ActionRequestValidationException exception = request.validate(); + + // Verify + assertEquals(1, exception.validationErrors().size()); + assertTrue(exception.validationErrors().get(0).contains("no such job exists")); + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/action/PutTIFJobRequestTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/action/PutTIFJobRequestTests.java new file mode 100644 index 000000000..baa18695d --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/action/PutTIFJobRequestTests.java @@ -0,0 +1,50 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; +import org.opensearch.securityanalytics.TestHelpers; + + +public class PutTIFJobRequestTests extends ThreatIntelTestCase { + + public void testValidate_whenValidInput_thenSucceed() { + String tifJobParameterName = TestHelpers.randomLowerCaseString(); + PutTIFJobRequest request = new PutTIFJobRequest(tifJobParameterName, clusterSettings.get(SecurityAnalyticsSettings.TIF_UPDATE_INTERVAL)); + + assertNull(request.validate()); + } + + public void testValidate_whenInvalidTIFJobParameterName_thenFails() { + String invalidName = "_" + TestHelpers.randomLowerCaseString(); + PutTIFJobRequest request = new PutTIFJobRequest(invalidName, clusterSettings.get(SecurityAnalyticsSettings.TIF_UPDATE_INTERVAL)); + + // Run + ActionRequestValidationException exception = request.validate(); + + // Verify + assertEquals(1, exception.validationErrors().size()); + assertTrue(exception.validationErrors().get(0).contains("must not")); + } + + public void testStreamInOut_whenValidInput_thenSucceed() throws Exception { + String tifJobParameterName = TestHelpers.randomLowerCaseString(); + PutTIFJobRequest request = new PutTIFJobRequest(tifJobParameterName, clusterSettings.get(SecurityAnalyticsSettings.TIF_UPDATE_INTERVAL)); + + // Run + BytesStreamOutput output = new BytesStreamOutput(); + request.writeTo(output); + BytesStreamInput input = new BytesStreamInput(output.bytes().toBytesRef().bytes); + PutTIFJobRequest copiedRequest = new PutTIFJobRequest(input); + + // Verify + assertEquals(request.getName(), copiedRequest.getName()); + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/action/TransportDeleteTIFJobActionTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/action/TransportDeleteTIFJobActionTests.java new file mode 100644 index 000000000..7d15d7710 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/action/TransportDeleteTIFJobActionTests.java @@ -0,0 +1,127 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.junit.Before; +import org.mockito.ArgumentCaptor; +import org.mockito.InOrder; +import org.mockito.Mockito; +import org.opensearch.OpenSearchException; +import org.opensearch.ResourceNotFoundException; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.core.action.ActionListener; +import org.opensearch.jobscheduler.spi.LockModel; +import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; +import org.opensearch.tasks.Task; +import org.opensearch.securityanalytics.TestHelpers; + + +import java.io.IOException; +import java.time.Instant; + +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +public class TransportDeleteTIFJobActionTests extends ThreatIntelTestCase { + private TransportDeleteTIFJobAction action; + + @Before + public void init() { + action = new TransportDeleteTIFJobAction( + transportService, + actionFilters, + tifLockService, + ingestService, + tifJobParameterService, + threatIntelFeedDataService, + threadPool + ); + } + + public void testDoExecute_whenFailedToAcquireLock_thenError() throws IOException { + validateDoExecute(null, null); + } + + public void testDoExecute_whenValidInput_thenSucceed() throws IOException { + String jobIndexName = TestHelpers.randomLowerCaseString(); + String jobId = TestHelpers.randomLowerCaseString(); + LockModel lockModel = new LockModel(jobIndexName, jobId, Instant.now(), randomPositiveLong(), false); + validateDoExecute(lockModel, null); + } + + public void testDoExecute_whenException_thenError() throws IOException { + validateDoExecute(null, new RuntimeException()); + } + + private void validateDoExecute(final LockModel lockModel, final Exception exception) throws IOException { + Task task = mock(Task.class); + TIFJobParameter tifJobParameter = randomTifJobParameter(); + when(tifJobParameterService.getJobParameter(tifJobParameter.getName())).thenReturn(tifJobParameter); + DeleteTIFJobRequest request = new DeleteTIFJobRequest(tifJobParameter.getName()); + ActionListener listener = mock(ActionListener.class); + + // Run + action.doExecute(task, request, listener); + + // Verify + ArgumentCaptor> captor = ArgumentCaptor.forClass(ActionListener.class); + verify(tifLockService).acquireLock(eq(tifJobParameter.getName()), anyLong(), captor.capture()); + + if (exception == null) { + // Run + captor.getValue().onResponse(lockModel); + + // Verify + if (lockModel == null) { + verify(listener).onFailure(any(OpenSearchException.class)); + } else { + verify(listener).onResponse(new AcknowledgedResponse(true)); + verify(tifLockService).releaseLock(eq(lockModel)); + } + } else { + // Run + captor.getValue().onFailure(exception); + // Verify + verify(listener).onFailure(exception); + } + } + + public void testDeleteTIFJobParameter_whenNull_thenThrowException() { + TIFJobParameter tifJobParameter = randomTifJobParameter(); + expectThrows(ResourceNotFoundException.class, () -> action.deleteTIFJob(tifJobParameter.getName())); + } + + public void testDeleteTIFJobParameter_whenSafeToDelete_thenDelete() throws IOException { + TIFJobParameter tifJobParameter = randomTifJobParameter(); + when(tifJobParameterService.getJobParameter(tifJobParameter.getName())).thenReturn(tifJobParameter); + + // Run + action.deleteTIFJob(tifJobParameter.getName()); + + // Verify + assertEquals(TIFJobState.DELETING, tifJobParameter.getState()); + verify(tifJobParameterService).updateJobSchedulerParameter(tifJobParameter); + InOrder inOrder = Mockito.inOrder(threatIntelFeedDataService, tifJobParameterService); + inOrder.verify(threatIntelFeedDataService).deleteThreatIntelDataIndex(tifJobParameter.getIndices()); + inOrder.verify(tifJobParameterService).deleteTIFJobParameter(tifJobParameter); + } + + public void testDeleteTIFJobParameter_whenDeleteFailsAfterStateIsChanged_thenRevertState() throws IOException { + TIFJobParameter tifJobParameter = randomTifJobParameter(); + tifJobParameter.setState(TIFJobState.AVAILABLE); + when(tifJobParameterService.getJobParameter(tifJobParameter.getName())).thenReturn(tifJobParameter); + doThrow(new RuntimeException()).when(threatIntelFeedDataService).deleteThreatIntelDataIndex(tifJobParameter.getIndices()); + + // Run + expectThrows(RuntimeException.class, () -> action.deleteTIFJob(tifJobParameter.getName())); + + // Verify + verify(tifJobParameterService, times(2)).updateJobSchedulerParameter(tifJobParameter); + assertEquals(TIFJobState.AVAILABLE, tifJobParameter.getState()); + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobActionTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobActionTests.java new file mode 100644 index 000000000..68dcbf527 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobActionTests.java @@ -0,0 +1,161 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.junit.Before; +import org.mockito.ArgumentCaptor; +import org.opensearch.action.StepListener; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.core.action.ActionListener; +import org.opensearch.jobscheduler.spi.LockModel; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; +import org.opensearch.tasks.Task; +import org.opensearch.securityanalytics.TestHelpers; + +import java.io.IOException; +import java.util.ConcurrentModificationException; + +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +public class TransportPutTIFJobActionTests extends ThreatIntelTestCase { + private TransportPutTIFJobAction action; + + @Before + public void init() { + action = new TransportPutTIFJobAction( + transportService, + actionFilters, + threadPool, + tifJobParameterService, + tifJobUpdateService, + tifLockService + ); + } + + public void testDoExecute_whenFailedToAcquireLock_thenError() throws IOException { + validateDoExecute(null, null, null); + } + + public void testDoExecute_whenAcquiredLock_thenSucceed() throws IOException { + validateDoExecute(randomLockModel(), null, null); + } + + public void testDoExecute_whenExceptionBeforeAcquiringLock_thenError() throws IOException { + validateDoExecute(randomLockModel(), new RuntimeException(), null); + } + + public void testDoExecute_whenExceptionAfterAcquiringLock_thenError() throws IOException { + validateDoExecute(randomLockModel(), null, new RuntimeException()); + } + + private void validateDoExecute(final LockModel lockModel, final Exception before, final Exception after) throws IOException { + Task task = mock(Task.class); + TIFJobParameter tifJobParameter = randomTifJobParameter(); + + PutTIFJobRequest request = new PutTIFJobRequest(tifJobParameter.getName(), clusterSettings.get(SecurityAnalyticsSettings.TIF_UPDATE_INTERVAL)); + ActionListener listener = mock(ActionListener.class); + if (after != null) { + doThrow(after).when(tifJobParameterService).createJobIndexIfNotExists(any(StepListener.class)); + } + + // Run + action.doExecute(task, request, listener); + + // Verify + ArgumentCaptor> captor = ArgumentCaptor.forClass(ActionListener.class); + verify(tifLockService).acquireLock(eq(tifJobParameter.getName()), anyLong(), captor.capture()); + + if (before == null) { + // Run + captor.getValue().onResponse(lockModel); + + // Verify + if (lockModel == null) { + verify(listener).onFailure(any(ConcurrentModificationException.class)); + } + if (after != null) { + verify(tifLockService).releaseLock(eq(lockModel)); + verify(listener).onFailure(after); + } else { + verify(tifLockService, never()).releaseLock(eq(lockModel)); + } + } else { + // Run + captor.getValue().onFailure(before); + // Verify + verify(listener).onFailure(before); + } + } + + public void testInternalDoExecute_whenValidInput_thenSucceed() { + PutTIFJobRequest request = new PutTIFJobRequest(TestHelpers.randomLowerCaseString(), clusterSettings.get(SecurityAnalyticsSettings.TIF_UPDATE_INTERVAL)); + ActionListener listener = mock(ActionListener.class); + + // Run + action.internalDoExecute(request, randomLockModel(), listener); + + // Verify + ArgumentCaptor captor = ArgumentCaptor.forClass(StepListener.class); + verify(tifJobParameterService).createJobIndexIfNotExists(captor.capture()); + + // Run + captor.getValue().onResponse(null); + // Verify + ArgumentCaptor tifJobCaptor = ArgumentCaptor.forClass(TIFJobParameter.class); + ArgumentCaptor actionListenerCaptor = ArgumentCaptor.forClass(ActionListener.class); + verify(tifJobParameterService).saveTIFJobParameter(tifJobCaptor.capture(), actionListenerCaptor.capture()); + assertEquals(request.getName(), tifJobCaptor.getValue().getName()); + + // Run next listener.onResponse + actionListenerCaptor.getValue().onResponse(null); + // Verify + verify(listener).onResponse(new AcknowledgedResponse(true)); + } + + public void testCreateTIFJobParameter_whenInvalidState_thenUpdateStateAsFailed() throws IOException { + TIFJobParameter tifJob = new TIFJobParameter(); + tifJob.setState(randomStateExcept(TIFJobState.CREATING)); + tifJob.getUpdateStats().setLastFailedAt(null); + + // Run + action.createThreatIntelFeedData(tifJob, mock(Runnable.class)); + + // Verify + assertEquals(TIFJobState.CREATE_FAILED, tifJob.getState()); + assertNotNull(tifJob.getUpdateStats().getLastFailedAt()); + verify(tifJobParameterService).updateJobSchedulerParameter(tifJob); + verify(tifJobUpdateService, never()).createThreatIntelFeedData(any(TIFJobParameter.class), any(Runnable.class)); + } + + public void testCreateTIFJobParameter_whenExceptionHappens_thenUpdateStateAsFailed() throws IOException { + TIFJobParameter tifJob = new TIFJobParameter(); + doThrow(new RuntimeException()).when(tifJobUpdateService).createThreatIntelFeedData(any(TIFJobParameter.class), any(Runnable.class)); + + // Run + action.createThreatIntelFeedData(tifJob, mock(Runnable.class)); + + // Verify + assertEquals(TIFJobState.CREATE_FAILED, tifJob.getState()); + assertNotNull(tifJob.getUpdateStats().getLastFailedAt()); + verify(tifJobParameterService).updateJobSchedulerParameter(tifJob); + } + + public void testCreateTIFJobParameter_whenValidInput_thenUpdateStateAsCreating() throws IOException { + TIFJobParameter tifJob = new TIFJobParameter(); + + Runnable renewLock = mock(Runnable.class); + // Run + action.createThreatIntelFeedData(tifJob, renewLock); + + // Verify + verify(tifJobUpdateService).createThreatIntelFeedData(tifJob, renewLock); + assertEquals(TIFJobState.CREATING, tifJob.getState()); + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/common/ThreatIntelLockServiceTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/common/ThreatIntelLockServiceTests.java new file mode 100644 index 000000000..4b6423a3e --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/common/ThreatIntelLockServiceTests.java @@ -0,0 +1,117 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.common; + +import static org.mockito.Mockito.mock; +import static org.opensearch.securityanalytics.threatIntel.common.TIFLockService.LOCK_DURATION_IN_SECONDS; +import static org.opensearch.securityanalytics.threatIntel.common.TIFLockService.RENEW_AFTER_IN_SECONDS; + +import java.time.Instant; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.Before; +import org.opensearch.action.DocWriteResponse; +import org.opensearch.action.update.UpdateRequest; +import org.opensearch.action.update.UpdateResponse; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.jobscheduler.spi.LockModel; +import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; +import org.opensearch.securityanalytics.TestHelpers; + +public class ThreatIntelLockServiceTests extends ThreatIntelTestCase { + private TIFLockService threatIntelLockService; + private TIFLockService noOpsLockService; + + @Before + public void init() { + threatIntelLockService = new TIFLockService(clusterService, verifyingClient); + noOpsLockService = new TIFLockService(clusterService, client); + } + + public void testAcquireLock_whenValidInput_thenSucceed() { + // Cannot test because LockService is final class + // Simply calling method to increase coverage + noOpsLockService.acquireLock(TestHelpers.randomLowerCaseString(), randomPositiveLong(), mock(ActionListener.class)); + } + + public void testAcquireLock_whenCalled_thenNotBlocked() { + long expectedDurationInMillis = 1000; + Instant before = Instant.now(); + assertTrue(threatIntelLockService.acquireLock(null, null).isEmpty()); + Instant after = Instant.now(); + assertTrue(after.toEpochMilli() - before.toEpochMilli() < expectedDurationInMillis); + } + + public void testReleaseLock_whenValidInput_thenSucceed() { + // Cannot test because LockService is final class + // Simply calling method to increase coverage + noOpsLockService.releaseLock(null); + } + + public void testRenewLock_whenCalled_thenNotBlocked() { + long expectedDurationInMillis = 1000; + Instant before = Instant.now(); + assertNull(threatIntelLockService.renewLock(null)); + Instant after = Instant.now(); + assertTrue(after.toEpochMilli() - before.toEpochMilli() < expectedDurationInMillis); + } + + public void testGetRenewLockRunnable_whenLockIsFresh_thenDoNotRenew() { + LockModel lockModel = new LockModel( + TestHelpers.randomLowerCaseString(), + TestHelpers.randomLowerCaseString(), + Instant.now(), + LOCK_DURATION_IN_SECONDS, + false + ); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + // Verifying + assertTrue(actionRequest instanceof UpdateRequest); + return new UpdateResponse( + mock(ShardId.class), + TestHelpers.randomLowerCaseString(), + randomPositiveLong(), + randomPositiveLong(), + randomPositiveLong(), + DocWriteResponse.Result.UPDATED + ); + }); + + AtomicReference reference = new AtomicReference<>(lockModel); + threatIntelLockService.getRenewLockRunnable(reference).run(); + assertEquals(lockModel, reference.get()); + } + + public void testGetRenewLockRunnable_whenLockIsStale_thenRenew() { + LockModel lockModel = new LockModel( + TestHelpers.randomLowerCaseString(), + TestHelpers.randomLowerCaseString(), + Instant.now().minusSeconds(RENEW_AFTER_IN_SECONDS), + LOCK_DURATION_IN_SECONDS, + false + ); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + // Verifying + assertTrue(actionRequest instanceof UpdateRequest); + return new UpdateResponse( + mock(ShardId.class), + TestHelpers.randomLowerCaseString(), + randomPositiveLong(), + randomPositiveLong(), + randomPositiveLong(), + DocWriteResponse.Result.UPDATED + ); + }); + + AtomicReference reference = new AtomicReference<>(lockModel); + threatIntelLockService.getRenewLockRunnable(reference).run(); + assertNotEquals(lockModel, reference.get()); + } +} + diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/TIFJobExtensionPluginIT.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/TIFJobExtensionPluginIT.java new file mode 100644 index 000000000..ff682e6dd --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/TIFJobExtensionPluginIT.java @@ -0,0 +1,49 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.securityanalytics.threatIntel.integTests; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; +import org.opensearch.action.admin.cluster.node.info.NodeInfo; +import org.opensearch.action.admin.cluster.node.info.NodesInfoRequest; +import org.opensearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.opensearch.action.admin.cluster.node.info.PluginsAndModules; +import org.opensearch.cluster.health.ClusterHealthStatus; +import org.opensearch.plugins.PluginInfo; +import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobRunner; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.junit.Assert; + +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class TIFJobExtensionPluginIT extends OpenSearchIntegTestCase { + private static final Logger log = LogManager.getLogger(TIFJobExtensionPluginIT.class); + + public void testPluginsAreInstalled() { + ClusterHealthRequest request = new ClusterHealthRequest(); + ClusterHealthResponse response = OpenSearchIntegTestCase.client().admin().cluster().health(request).actionGet(); + Assert.assertEquals(ClusterHealthStatus.GREEN, response.getStatus()); + + NodesInfoRequest nodesInfoRequest = new NodesInfoRequest(); + nodesInfoRequest.addMetric(NodesInfoRequest.Metric.PLUGINS.metricName()); + NodesInfoResponse nodesInfoResponse = OpenSearchIntegTestCase.client().admin().cluster().nodesInfo(nodesInfoRequest).actionGet(); + List pluginInfos = nodesInfoResponse.getNodes() + .stream() + .flatMap( + (Function>) nodeInfo -> nodeInfo.getInfo(PluginsAndModules.class).getPluginInfos().stream() + ) + .collect(Collectors.toList()); + Assert.assertTrue(pluginInfos.stream().anyMatch(pluginInfo -> pluginInfo.getName().equals("opensearch-job-scheduler"))); + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java new file mode 100644 index 000000000..a3df0c4cd --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java @@ -0,0 +1,154 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.securityanalytics.threatIntel.integTests; + +import org.apache.hc.core5.http.HttpStatus; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.search.SearchHit; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; +import org.opensearch.securityanalytics.model.Detector; +import org.opensearch.securityanalytics.model.DetectorInput; +import org.opensearch.securityanalytics.model.DetectorRule; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Locale; +import java.util.stream.Collectors; + +import static org.opensearch.securityanalytics.TestHelpers.*; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE; +import static org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataUtils.getTifdList; + +public class ThreatIntelJobRunnerIT extends SecurityAnalyticsRestTestCase { + private static final Logger log = LogManager.getLogger(ThreatIntelJobRunnerIT.class); + + public void testCreateDetector_threatIntelEnabled_updateDetectorWithNewThreatIntel() throws IOException { + + // 1. create a detector + updateClusterSetting(ENABLE_WORKFLOW_USAGE.getKey(), "true"); + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response createMappingResponse = client().performRequest(createMappingRequest); + + assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + + String randomDocRuleId = createRule(randomRule()); + List detectorRules = List.of(new DetectorRule(randomDocRuleId)); + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), detectorRules, + Collections.emptyList()); + Detector detector = randomDetectorWithInputsAndThreatIntel(List.of(input), true); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + + assertEquals(2, response.getHits().getTotalHits().value); + + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + + String detectorId = responseBody.get("_id").toString(); + request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + Map detectorMap = (HashMap) (hit.getSourceAsMap().get("detector")); + + List monitorIds = ((List) (detectorMap).get("monitor_id")); + assertEquals(1, monitorIds.size()); + + assertNotNull("Workflow not created", detectorMap.get("workflow_ids")); + assertEquals("Number of workflows not correct", 1, ((List) detectorMap.get("workflow_ids")).size()); + + // Verify workflow + verifyWorkflow(detectorMap, monitorIds, 1); + List iocs = getThreatIntelFeedIocs(3); + assertEquals(iocs.size(),3); + + // 2. delete a threat intel feed ioc index manually + List feedId = getThreatIntelFeedIds(1); + for (String feedid: feedId) { + String name = String.format(Locale.ROOT, "%s-%s%s", ".opensearch-sap-threatintel", feedid, "1"); + deleteIndex(name); + } + +// // 3. update the start time to a day before so it runs now +// StringEntity stringEntity = new StringEntity( +// "{\"doc\":{\"last_update_time\":{\"schedule\":{\"interval\":{\"start_time\":" + +// "\"$startTimeMillis\"}}}}}", +// ContentType.APPLICATION_JSON +// ); +// +// Response updateJobRespose = makeRequest(client(), "POST", ".scheduler-sap-threatintel-job/_update/$id" , Collections.emptyMap(), stringEntity, null, null); +// assertEquals("Updated job scheduler", RestStatus.CREATED, restStatus(updateJobRespose)); + + // 4. validate new ioc is created + List newIocs = getThreatIntelFeedIocs(1); + assertEquals(0, newIocs.size()); //TODO + } + + private List getThreatIntelFeedIocs(int num) throws IOException { + String request = getMatchAllSearchRequestString(num); + SearchResponse res = executeSearchAndGetResponse(".opensearch-sap-threatintel*", request, false); + return getTifdList(res, xContentRegistry()).stream().map(it -> it.getIocValue()).collect(Collectors.toList()); + } + + private List getThreatIntelFeedIds(int num) throws IOException { + String request = getMatchAllSearchRequestString(num); + SearchResponse res = executeSearchAndGetResponse(".opensearch-sap-threatintel*", request, false); + return getTifdList(res, xContentRegistry()).stream().map(it -> it.getFeedId()).collect(Collectors.toList()); + } + +// private String getJobSchedulerDoc(int num) throws IOException { +// String request = getMatchAllSearchRequestString(num); +// SearchResponse res = executeSearchAndGetResponse(".scheduler-sap-threatintel-job*", request, false); +// } + + private static String getMatchAllSearchRequestString(int num) { + return "{\n" + + "\"size\" : " + num + "," + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + } +} + diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobExtensionTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobExtensionTests.java new file mode 100644 index 000000000..6096fa382 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobExtensionTests.java @@ -0,0 +1,58 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.jobscheduler; + +import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobExtension.JOB_INDEX_NAME; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.jobscheduler.spi.JobDocVersion; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.securityanalytics.model.DetectorTrigger; +import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; +import org.opensearch.securityanalytics.TestHelpers; + +public class TIFJobExtensionTests extends ThreatIntelTestCase { + private static final Logger log = LogManager.getLogger(TIFJobExtensionTests.class); + + public void testBasic() { + TIFJobExtension extension = new TIFJobExtension(); + assertEquals("scheduler_sap_threatintel_job", extension.getJobType()); + assertEquals(JOB_INDEX_NAME, extension.getJobIndex()); + assertEquals(TIFJobRunner.getJobRunnerInstance(), extension.getJobRunner()); + } + + public void testParser() throws Exception { + TIFJobExtension extension = new TIFJobExtension(); + String id = TestHelpers.randomLowerCaseString(); + IntervalSchedule schedule = new IntervalSchedule(Instant.now().truncatedTo(ChronoUnit.MILLIS), 1, ChronoUnit.DAYS); + TIFJobParameter tifJobParameter = new TIFJobParameter(id, schedule); + + TIFJobParameter anotherTIFJobParameter = (TIFJobParameter) extension.getJobParser() + .parse( + createParser(tifJobParameter.toXContent(XContentFactory.jsonBuilder(), null)), + TestHelpers.randomLowerCaseString(), + new JobDocVersion(randomPositiveLong(), randomPositiveLong(), randomPositiveLong()) + ); + log.info("first"); + log.error(tifJobParameter); + log.error(tifJobParameter.getName()); + log.info("second"); + log.error(anotherTIFJobParameter); + log.error(anotherTIFJobParameter.getName()); + + assertTrue(tifJobParameter.getName().equals(anotherTIFJobParameter.getName())); + assertTrue(tifJobParameter.getLastUpdateTime().equals(anotherTIFJobParameter.getLastUpdateTime())); + assertTrue(tifJobParameter.getSchedule().equals(anotherTIFJobParameter.getSchedule())); + assertTrue(tifJobParameter.getState().equals(anotherTIFJobParameter.getState())); + assertTrue(tifJobParameter.getIndices().equals(anotherTIFJobParameter.getIndices())); + } + +} diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterServiceTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterServiceTests.java new file mode 100644 index 000000000..5b0605d79 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterServiceTests.java @@ -0,0 +1,238 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.jobscheduler; + +import org.junit.Before; +import org.opensearch.ResourceAlreadyExistsException; +import org.opensearch.ResourceNotFoundException; +import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.StepListener; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.delete.DeleteRequest; +import org.opensearch.action.delete.DeleteResponse; +import org.opensearch.action.get.GetRequest; +import org.opensearch.action.get.GetResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; +import org.opensearch.securityanalytics.TestHelpers; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TIFJobParameterServiceTests extends ThreatIntelTestCase { + private TIFJobParameterService tifJobParameterService; + + @Before + public void init() { + tifJobParameterService = new TIFJobParameterService(verifyingClient, clusterService); + } + + public void testcreateJobIndexIfNotExists_whenIndexExist_thenCreateRequestIsNotCalled() { + when(metadata.hasIndex(TIFJobExtension.JOB_INDEX_NAME)).thenReturn(true); + + // Verify + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { throw new RuntimeException("Shouldn't get called"); }); + + // Run + StepListener stepListener = new StepListener<>(); + tifJobParameterService.createJobIndexIfNotExists(stepListener); + + // Verify stepListener is called + stepListener.result(); + } + + public void testcreateJobIndexIfNotExists_whenIndexExist_thenCreateRequestIsCalled() { + when(metadata.hasIndex(TIFJobExtension.JOB_INDEX_NAME)).thenReturn(false); + + // Verify + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + assertTrue(actionRequest instanceof CreateIndexRequest); + CreateIndexRequest request = (CreateIndexRequest) actionRequest; + assertEquals(TIFJobExtension.JOB_INDEX_NAME, request.index()); + assertEquals("1", request.settings().get("index.number_of_shards")); + assertEquals("0-all", request.settings().get("index.auto_expand_replicas")); + assertEquals("true", request.settings().get("index.hidden")); + assertNotNull(request.mappings()); + return null; + }); + + // Run + StepListener stepListener = new StepListener<>(); + tifJobParameterService.createJobIndexIfNotExists(stepListener); + + // Verify stepListener is called + stepListener.result(); + } + + public void testcreateJobIndexIfNotExists_whenIndexCreatedAlready_thenExceptionIsIgnored() { + when(metadata.hasIndex(TIFJobExtension.JOB_INDEX_NAME)).thenReturn(false); + verifyingClient.setExecuteVerifier( + (actionResponse, actionRequest) -> { throw new ResourceAlreadyExistsException(TIFJobExtension.JOB_INDEX_NAME); } + ); + + // Run + StepListener stepListener = new StepListener<>(); + tifJobParameterService.createJobIndexIfNotExists(stepListener); + + // Verify stepListener is called + stepListener.result(); + } + + public void testcreateJobIndexIfNotExists_whenExceptionIsThrown_thenExceptionIsThrown() { + when(metadata.hasIndex(TIFJobExtension.JOB_INDEX_NAME)).thenReturn(false); + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { throw new RuntimeException(); }); + + // Run + StepListener stepListener = new StepListener<>(); + tifJobParameterService.createJobIndexIfNotExists(stepListener); + + // Verify stepListener is called + expectThrows(RuntimeException.class, () -> stepListener.result()); + } + + public void testUpdateTIFJobParameter_whenValidInput_thenSucceed() throws Exception { + String tifJobName = TestHelpers.randomLowerCaseString(); + TIFJobParameter tifJobParameter = new TIFJobParameter( + tifJobName, + new IntervalSchedule(Instant.now().truncatedTo(ChronoUnit.MILLIS), 1, ChronoUnit.DAYS) + ); + Instant previousTime = Instant.now().minusMillis(1); + tifJobParameter.setLastUpdateTime(previousTime); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + assertTrue(actionRequest instanceof IndexRequest); + IndexRequest request = (IndexRequest) actionRequest; + assertEquals(tifJobParameter.getName(), request.id()); + assertEquals(DocWriteRequest.OpType.INDEX, request.opType()); + assertEquals(TIFJobExtension.JOB_INDEX_NAME, request.index()); + assertEquals(WriteRequest.RefreshPolicy.IMMEDIATE, request.getRefreshPolicy()); + return null; + }); + + tifJobParameterService.updateJobSchedulerParameter(tifJobParameter); + assertTrue(previousTime.isBefore(tifJobParameter.getLastUpdateTime())); + } + + public void testsaveTIFJobParameter_whenValidInput_thenSucceed() { + TIFJobParameter tifJobParameter = randomTifJobParameter(); + Instant previousTime = Instant.now().minusMillis(1); + tifJobParameter.setLastUpdateTime(previousTime); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + assertTrue(actionRequest instanceof IndexRequest); + IndexRequest indexRequest = (IndexRequest) actionRequest; + assertEquals(TIFJobExtension.JOB_INDEX_NAME, indexRequest.index()); + assertEquals(tifJobParameter.getName(), indexRequest.id()); + assertEquals(WriteRequest.RefreshPolicy.IMMEDIATE, indexRequest.getRefreshPolicy()); + assertEquals(DocWriteRequest.OpType.CREATE, indexRequest.opType()); + return null; + }); + + tifJobParameterService.saveTIFJobParameter(tifJobParameter, mock(ActionListener.class)); + assertTrue(previousTime.isBefore(tifJobParameter.getLastUpdateTime())); + } + + public void testGetTifJobParameter_whenException_thenNull() throws Exception { + TIFJobParameter tifJobParameter = setupClientForGetRequest(true, new IndexNotFoundException(TIFJobExtension.JOB_INDEX_NAME)); + assertNull(tifJobParameterService.getJobParameter(tifJobParameter.getName())); + } + + public void testGetTifJobParameter_whenExist_thenReturnTifJobParameter() throws Exception { + TIFJobParameter tifJobParameter = setupClientForGetRequest(true, null); + TIFJobParameter anotherTIFJobParameter = tifJobParameterService.getJobParameter(tifJobParameter.getName()); + + assertTrue(tifJobParameter.getName().equals(anotherTIFJobParameter.getName())); + assertTrue(tifJobParameter.getLastUpdateTime().equals(anotherTIFJobParameter.getLastUpdateTime())); + assertTrue(tifJobParameter.getSchedule().equals(anotherTIFJobParameter.getSchedule())); + assertTrue(tifJobParameter.getState().equals(anotherTIFJobParameter.getState())); + assertTrue(tifJobParameter.getIndices().equals(anotherTIFJobParameter.getIndices())); + } + + public void testGetTifJobParameter_whenNotExist_thenNull() throws Exception { + TIFJobParameter tifJobParameter = setupClientForGetRequest(false, null); + assertNull(tifJobParameterService.getJobParameter(tifJobParameter.getName())); + } + + private TIFJobParameter setupClientForGetRequest(final boolean isExist, final RuntimeException exception) { + TIFJobParameter tifJobParameter = randomTifJobParameter(); + + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + assertTrue(actionRequest instanceof GetRequest); + GetRequest request = (GetRequest) actionRequest; + assertEquals(tifJobParameter.getName(), request.id()); + assertEquals(TIFJobExtension.JOB_INDEX_NAME, request.index()); + GetResponse response = getMockedGetResponse(isExist ? tifJobParameter : null); + if (exception != null) { + throw exception; + } + return response; + }); + return tifJobParameter; + } + + public void testDeleteTifJobParameter_whenValidInput_thenSucceed() { + TIFJobParameter tifJobParameter = randomTifJobParameter(); + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + // Verify + assertTrue(actionRequest instanceof DeleteRequest); + DeleteRequest request = (DeleteRequest) actionRequest; + assertEquals(TIFJobExtension.JOB_INDEX_NAME, request.index()); + assertEquals(DocWriteRequest.OpType.DELETE, request.opType()); + assertEquals(tifJobParameter.getName(), request.id()); + assertEquals(WriteRequest.RefreshPolicy.IMMEDIATE, request.getRefreshPolicy()); + + DeleteResponse response = mock(DeleteResponse.class); + when(response.status()).thenReturn(RestStatus.OK); + return response; + }); + + // Run + tifJobParameterService.deleteTIFJobParameter(tifJobParameter); + } + + public void testDeleteTifJobParameter_whenIndexNotFound_thenThrowException() { + TIFJobParameter tifJobParameter = randomTifJobParameter(); + verifyingClient.setExecuteVerifier((actionResponse, actionRequest) -> { + DeleteResponse response = mock(DeleteResponse.class); + when(response.status()).thenReturn(RestStatus.NOT_FOUND); + return response; + }); + + // Run + expectThrows(ResourceNotFoundException.class, () -> tifJobParameterService.deleteTIFJobParameter(tifJobParameter)); + } + + private GetResponse getMockedGetResponse(TIFJobParameter tifJobParameter) { + GetResponse response = mock(GetResponse.class); + when(response.isExists()).thenReturn(tifJobParameter != null); + when(response.getSourceAsBytesRef()).thenReturn(toBytesReference(tifJobParameter)); + return response; + } + + private BytesReference toBytesReference(TIFJobParameter tifJobParameter) { + if (tifJobParameter == null) { + return null; + } + + try { + return BytesReference.bytes(tifJobParameter.toXContent(JsonXContent.contentBuilder(), null)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java new file mode 100644 index 000000000..85aeef5b9 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java @@ -0,0 +1,107 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.jobscheduler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.securityanalytics.TestHelpers; +import org.opensearch.securityanalytics.model.DetectorTrigger; +import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; +import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.Locale; + +import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; + +public class TIFJobParameterTests extends ThreatIntelTestCase { + private static final Logger log = LogManager.getLogger(TIFJobParameterTests.class); + + public void testParser_whenAllValueIsFilled_thenSucceed() throws IOException { + String id = TestHelpers.randomLowerCaseString(); + IntervalSchedule schedule = new IntervalSchedule(Instant.now().truncatedTo(ChronoUnit.MILLIS), 1, ChronoUnit.DAYS); + TIFJobParameter tifJobParameter = new TIFJobParameter(id, schedule); + tifJobParameter.enable(); + tifJobParameter.getUpdateStats().setLastProcessingTimeInMillis(randomPositiveLong()); + tifJobParameter.getUpdateStats().setLastSucceededAt(Instant.now().truncatedTo(ChronoUnit.MILLIS)); + tifJobParameter.getUpdateStats().setLastSkippedAt(Instant.now().truncatedTo(ChronoUnit.MILLIS)); + tifJobParameter.getUpdateStats().setLastFailedAt(Instant.now().truncatedTo(ChronoUnit.MILLIS)); + + TIFJobParameter anotherTIFJobParameter = TIFJobParameter.PARSER.parse( + createParser(tifJobParameter.toXContent(XContentFactory.jsonBuilder(), null)), + null + ); + + assertTrue(tifJobParameter.getName().equals(anotherTIFJobParameter.getName())); + assertTrue(tifJobParameter.getLastUpdateTime().equals(anotherTIFJobParameter.getLastUpdateTime())); + assertTrue(tifJobParameter.getEnabledTime().equals(anotherTIFJobParameter.getEnabledTime())); + assertTrue(tifJobParameter.getSchedule().equals(anotherTIFJobParameter.getSchedule())); + assertTrue(tifJobParameter.getState().equals(anotherTIFJobParameter.getState())); + assertTrue(tifJobParameter.getIndices().equals(anotherTIFJobParameter.getIndices())); + assertTrue(tifJobParameter.getUpdateStats().getLastFailedAt().equals(anotherTIFJobParameter.getUpdateStats().getLastFailedAt())); + assertTrue(tifJobParameter.getUpdateStats().getLastSkippedAt().equals(anotherTIFJobParameter.getUpdateStats().getLastSkippedAt())); + assertTrue(tifJobParameter.getUpdateStats().getLastSucceededAt().equals(anotherTIFJobParameter.getUpdateStats().getLastSucceededAt())); + assertTrue(tifJobParameter.getUpdateStats().getLastProcessingTimeInMillis().equals(anotherTIFJobParameter.getUpdateStats().getLastProcessingTimeInMillis())); + + } + + public void testParser_whenNullForOptionalFields_thenSucceed() throws IOException { // TODO: same issue + String id = TestHelpers.randomLowerCaseString(); + IntervalSchedule schedule = new IntervalSchedule(Instant.now().truncatedTo(ChronoUnit.MILLIS), 1, ChronoUnit.DAYS); + TIFJobParameter tifJobParameter = new TIFJobParameter(id, schedule); + TIFJobParameter anotherTIFJobParameter = TIFJobParameter.PARSER.parse( + createParser(tifJobParameter.toXContent(XContentFactory.jsonBuilder(), null)), + null + ); + + assertTrue(tifJobParameter.getName().equals(anotherTIFJobParameter.getName())); + assertTrue(tifJobParameter.getLastUpdateTime().equals(anotherTIFJobParameter.getLastUpdateTime())); + assertTrue(tifJobParameter.getSchedule().equals(anotherTIFJobParameter.getSchedule())); + assertTrue(tifJobParameter.getState().equals(anotherTIFJobParameter.getState())); + assertTrue(tifJobParameter.getIndices().equals(anotherTIFJobParameter.getIndices())); + } + + public void testCurrentIndexName_whenNotExpired_thenReturnName() { + String id = TestHelpers.randomLowerCaseString(); + TIFJobParameter datasource = new TIFJobParameter(); + datasource.setName(id); + } + + public void testNewIndexName_whenCalled_thenReturnedExpectedValue() { + TIFMetadata tifMetadata = new TIFMetadata("mock_id", + "mock url", + "mock name", + "mock org", + "mock description", + "mock csv", + "mock ip", + 1, + false); + + String name = tifMetadata.getFeedId(); + String suffix = "1"; + TIFJobParameter tifJobParameter = new TIFJobParameter(); + tifJobParameter.setName(name); + assertEquals(String.format(Locale.ROOT, "%s-%s%s", THREAT_INTEL_DATA_INDEX_NAME_PREFIX, name, suffix), tifJobParameter.newIndexName(tifJobParameter,tifMetadata)); + tifJobParameter.getIndices().add(tifJobParameter.newIndexName(tifJobParameter,tifMetadata)); + + log.error(tifJobParameter.getIndices()); + + String anotherSuffix = "2"; + assertEquals(String.format(Locale.ROOT, "%s-%s%s", THREAT_INTEL_DATA_INDEX_NAME_PREFIX, name, anotherSuffix), tifJobParameter.newIndexName(tifJobParameter,tifMetadata)); + } + + public void testLockDurationSeconds() { + TIFJobParameter datasource = new TIFJobParameter(); + assertNotNull(datasource.getLockDurationSeconds()); + } +} + diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java new file mode 100644 index 000000000..f54631462 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java @@ -0,0 +1,167 @@ + +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.jobscheduler; + +import org.junit.Before; +import org.opensearch.jobscheduler.spi.JobDocVersion; +import org.opensearch.jobscheduler.spi.JobExecutionContext; +import org.opensearch.jobscheduler.spi.LockModel; +import org.opensearch.jobscheduler.spi.ScheduledJobParameter; +import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.TestHelpers; + +import java.io.IOException; +import java.time.Instant; +import java.util.Optional; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +public class TIFJobRunnerTests extends ThreatIntelTestCase { + @Before + public void init() { + TIFJobRunner.getJobRunnerInstance() + .initialize(clusterService, tifJobUpdateService, tifJobParameterService, tifLockService, threadPool); + } + + public void testGetJobRunnerInstance_whenCalledAgain_thenReturnSameInstance() { + assertTrue(TIFJobRunner.getJobRunnerInstance() == TIFJobRunner.getJobRunnerInstance()); + } + + public void testRunJob_whenInvalidClass_thenThrowException() { + JobDocVersion jobDocVersion = new JobDocVersion(randomInt(), randomInt(), randomInt()); + String jobIndexName = TestHelpers.randomLowerCaseString(); + String jobId = TestHelpers.randomLowerCaseString(); + JobExecutionContext jobExecutionContext = new JobExecutionContext(Instant.now(), jobDocVersion, lockService, jobIndexName, jobId); + ScheduledJobParameter jobParameter = mock(ScheduledJobParameter.class); + + // Run + expectThrows(IllegalStateException.class, () -> TIFJobRunner.getJobRunnerInstance().runJob(jobParameter, jobExecutionContext)); + } + + public void testRunJob_whenValidInput_thenSucceed() throws IOException { + JobDocVersion jobDocVersion = new JobDocVersion(randomInt(), randomInt(), randomInt()); + String jobIndexName = TestHelpers.randomLowerCaseString(); + String jobId = TestHelpers.randomLowerCaseString(); + JobExecutionContext jobExecutionContext = new JobExecutionContext(Instant.now(), jobDocVersion, lockService, jobIndexName, jobId); + TIFJobParameter tifJobParameter = randomTifJobParameter(); + + LockModel lockModel = randomLockModel(); + when(tifLockService.acquireLock(tifJobParameter.getName(), TIFLockService.LOCK_DURATION_IN_SECONDS)).thenReturn( + Optional.of(lockModel) + ); + + // Run + TIFJobRunner.getJobRunnerInstance().runJob(tifJobParameter, jobExecutionContext); + + // Verify + verify(tifLockService).acquireLock(tifJobParameter.getName(), tifLockService.LOCK_DURATION_IN_SECONDS); + verify(tifJobParameterService).getJobParameter(tifJobParameter.getName()); + verify(tifLockService).releaseLock(lockModel); + } + + public void testUpdateTIFJobRunner_whenExceptionBeforeAcquiringLock_thenNoReleaseLock() { + ScheduledJobParameter jobParameter = mock(ScheduledJobParameter.class); + when(jobParameter.getName()).thenReturn(TestHelpers.randomLowerCaseString()); + when(tifLockService.acquireLock(jobParameter.getName(), TIFLockService.LOCK_DURATION_IN_SECONDS)).thenThrow( + new RuntimeException() + ); + + // Run + expectThrows(Exception.class, () -> TIFJobRunner.getJobRunnerInstance().updateJobRunner(jobParameter).run()); + + // Verify + verify(tifLockService, never()).releaseLock(any()); + } + + public void testUpdateTIFJobRunner_whenExceptionAfterAcquiringLock_thenReleaseLock() throws IOException { + ScheduledJobParameter jobParameter = mock(ScheduledJobParameter.class); + when(jobParameter.getName()).thenReturn(TestHelpers.randomLowerCaseString()); + LockModel lockModel = randomLockModel(); + when(tifLockService.acquireLock(jobParameter.getName(), TIFLockService.LOCK_DURATION_IN_SECONDS)).thenReturn( + Optional.of(lockModel) + ); + when(tifJobParameterService.getJobParameter(jobParameter.getName())).thenThrow(new RuntimeException()); + + // Run + TIFJobRunner.getJobRunnerInstance().updateJobRunner(jobParameter).run(); + + // Verify + verify(tifLockService).releaseLock(any()); + } + + public void testUpdateTIFJob_whenTIFJobDoesNotExist_thenDoNothing() throws IOException { + TIFJobParameter tifJob = new TIFJobParameter(); + + // Run + TIFJobRunner.getJobRunnerInstance().updateJobParameter(tifJob, mock(Runnable.class)); + + // Verify + verify(tifJobUpdateService, never()).deleteAllTifdIndices(TestHelpers.randomLowerCaseStringList(),TestHelpers.randomLowerCaseStringList()); + } + + public void testUpdateTIFJob_whenInvalidState_thenUpdateLastFailedAt() throws IOException { + TIFJobParameter tifJob = new TIFJobParameter(); + tifJob.enable(); + tifJob.getUpdateStats().setLastFailedAt(null); + tifJob.setState(randomStateExcept(TIFJobState.AVAILABLE)); + when(tifJobParameterService.getJobParameter(tifJob.getName())).thenReturn(tifJob); + + // Run + TIFJobRunner.getJobRunnerInstance().updateJobParameter(tifJob, mock(Runnable.class)); + + // Verify + assertFalse(tifJob.isEnabled()); + assertNotNull(tifJob.getUpdateStats().getLastFailedAt()); + verify(tifJobParameterService).updateJobSchedulerParameter(tifJob); + } + + public void testUpdateTIFJob_whenValidInput_thenSucceed() throws IOException { + TIFJobParameter tifJob = randomTifJobParameter(); + tifJob.setState(TIFJobState.AVAILABLE); + when(tifJobParameterService.getJobParameter(tifJob.getName())).thenReturn(tifJob); + Runnable renewLock = mock(Runnable.class); + + // Run + TIFJobRunner.getJobRunnerInstance().updateJobParameter(tifJob, renewLock); + + // Verify + verify(tifJobUpdateService, times(0)).deleteAllTifdIndices(TestHelpers.randomLowerCaseStringList(),TestHelpers.randomLowerCaseStringList()); + verify(tifJobUpdateService).createThreatIntelFeedData(tifJob, renewLock); + } + + public void testUpdateTIFJob_whenDeleteTask_thenDeleteOnly() throws IOException { + TIFJobParameter tifJob = randomTifJobParameter(); + tifJob.setState(TIFJobState.AVAILABLE); + when(tifJobParameterService.getJobParameter(tifJob.getName())).thenReturn(tifJob); + Runnable renewLock = mock(Runnable.class); + + // Run + TIFJobRunner.getJobRunnerInstance().updateJobParameter(tifJob, renewLock); + + // Verify + verify(tifJobUpdateService, times(0)).deleteAllTifdIndices(TestHelpers.randomLowerCaseStringList(),TestHelpers.randomLowerCaseStringList()); + } + + public void testUpdateTIFJobExceptionHandling() throws IOException { + TIFJobParameter tifJob = new TIFJobParameter(); + tifJob.setName(TestHelpers.randomLowerCaseString()); + tifJob.getUpdateStats().setLastFailedAt(null); + when(tifJobParameterService.getJobParameter(tifJob.getName())).thenReturn(tifJob); + doThrow(new RuntimeException("test failure")).when(tifJobUpdateService).deleteAllTifdIndices(TestHelpers.randomLowerCaseStringList(),TestHelpers.randomLowerCaseStringList()); + + // Run + TIFJobRunner.getJobRunnerInstance().updateJobParameter(tifJob, mock(Runnable.class)); + + // Verify + assertNotNull(tifJob.getUpdateStats().getLastFailedAt()); + verify(tifJobParameterService).updateJobSchedulerParameter(tifJob); + } +} + diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateServiceTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateServiceTests.java new file mode 100644 index 000000000..76b0f8fe4 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateServiceTests.java @@ -0,0 +1,52 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.jobscheduler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.Before; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.common.SuppressForbidden; +import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; + +@SuppressForbidden(reason = "unit test") +public class TIFJobUpdateServiceTests extends ThreatIntelTestCase { + + private TIFJobUpdateService tifJobUpdateService1; + + @Before + public void init() { + tifJobUpdateService1 = new TIFJobUpdateService(clusterService, tifJobParameterService, threatIntelFeedDataService, builtInTIFMetadataLoader); + } + + public void testUpdateOrCreateThreatIntelFeedData_whenValidInput_thenSucceed() throws IOException { + + ShardRouting shardRouting = mock(ShardRouting.class); + when(shardRouting.started()).thenReturn(true); + when(routingTable.allShards(anyString())).thenReturn(Arrays.asList(shardRouting)); + + TIFJobParameter tifJobParameter = new TIFJobParameter(); + tifJobParameter.setState(TIFJobState.AVAILABLE); + + tifJobParameter.getUpdateStats().setLastSucceededAt(null); + tifJobParameter.getUpdateStats().setLastProcessingTimeInMillis(null); + + // Run + List newFeeds = tifJobUpdateService1.createThreatIntelFeedData(tifJobParameter, mock(Runnable.class)); + + // Verify feeds + assertNotNull(newFeeds); + } + +} diff --git a/src/test/resources/threatIntel/sample_csv_with_description_and_header.csv b/src/test/resources/threatIntel/sample_csv_with_description_and_header.csv new file mode 100644 index 000000000..750377fd6 --- /dev/null +++ b/src/test/resources/threatIntel/sample_csv_with_description_and_header.csv @@ -0,0 +1,4 @@ +# description + +ip +1.0.0.0/24 \ No newline at end of file diff --git a/src/test/resources/threatIntel/sample_valid.csv b/src/test/resources/threatIntel/sample_valid.csv index fad1eb6fd..c599b6888 100644 --- a/src/test/resources/threatIntel/sample_valid.csv +++ b/src/test/resources/threatIntel/sample_valid.csv @@ -1,3 +1,2 @@ -ip,region 1.0.0.0/24,Australia 10.0.0.0/24,USA \ No newline at end of file