Merge pull request #77 from fireeye/elazar-changes

Elazar changes

README.md

@@ -6,7 +6,7 @@ HXTool is a web-based, standalone tool that can be used with FireEye Endpoint Se
 HXTool provides additional features not directly available in the product by leveraging FireEye Endpoint Security's rich API.
 
 ### Version
-4.7-pre
+4.7
 
 ## Installation
 To install HXTool:
@@ -33,13 +33,14 @@ Configuration for HXTool is held in the `conf.json` file, documentation is in [R
 ### Docker
 To build a Docker image from the HXTool source, execute the following: 
 ```bash
-docker build -t hxtool:latest .
+docker build --pull -t hxtool:latest .
 ```
 
 To run HXTool once the image build process is complete, execute the following:
 ```bash
-docker run -p 8080:8080 -d --name hxtool hxtool:latest
+docker run -p 8080:8080/tcp -d --cap-add=IPC_LOCK --name hxtool hxtool:latest
 ```
+IPC_LOCK is needed for the GNOME keyring daemon. See [README.DOCKER](README.DOCKER)
 
 ## Contribution
 

hxtool_api.py

@@ -1089,7 +1089,13 @@ def hxtool_api_streaming_indicators_export(hx_api_object):
 @valid_session_required
 def hxtool_api_streaming_indicators_import(hx_api_object):
 	files = request.files.getlist('ruleImport')
-
+
+	import_platform = request.form.get('platform', '')
+	if import_platform == "all":
+		import_platform = ['win', 'osx', 'linux']
+	else:
+		import_platform = [import_platform]
+
 	for file in files:
 		# we may have a zip file, or a list of one or more files
 		if (zipfile.is_zipfile(file)):
@@ -1100,18 +1106,41 @@ def hxtool_api_streaming_indicators_import(hx_api_object):
 				# for each zip file
 				for zfile in zfiles:
 					with myzip.open(zfile) as myfile:
-						iocs = json.loads(myfile.read())
+						file_content = myfile.read().decode(default_encoding)
+						try:
+							iocs = json.loads(file_content)	
+						except json.decoder.JSONDecodeError:
+							iocs = openioc_to_hxioc(file_content, import_platform)
+							if iocs is None:
+								app.logger.warn(format_activity_log(msg="rule action fail", reason="{} is not a valid Endpoint Security (HX) JSON or OpenIOC 1.1 indicator." .format(file.filename), action="import", user=session['ht_user'], controller=session['hx_ip']))
+								continue
+
 						hxtool_handle_streaming_indicator_import(hx_api_object=hx_api_object, iocs=iocs)
 		else:
 			# not a zip file
 			file.seek(0)  # rewind to the start of the file
-			iocs = json.loads(file.read().decode(default_encoding))
+			file_content = file.read().decode(default_encoding)
+			try:
+				iocs = json.loads(file_content)
+			except json.decoder.JSONDecodeError:
+				iocs = openioc_to_hxioc(file_content, import_platform)
+				if iocs is None:
+					app.logger.warn(format_activity_log(msg="rule action fail", reason="{} is not a valid Endpoint Security (HX) JSON or OpenIOC 1.1 indicator." .format(file.filename), action="import", user=session['ht_user'], controller=session['hx_ip']))
+					continue
+
 			hxtool_handle_streaming_indicator_import(hx_api_object=hx_api_object, iocs=iocs)
 
 	return(app.response_class(response=json.dumps("OK"), status=200, mimetype='application/json'))
 
 def hxtool_handle_streaming_indicator_import(hx_api_object, iocs):
 	for (_, ioc) in iocs.items():
+		# Add category for IOCs that don't have, i.e. OpenIOC 1.1 import
+		if not ioc.get('category', False):
+			ioc['category'] = 'Custom'
+		if not ioc.get('description', False):
+			ioc['description'] = ""
+		if not ioc.get('uri_name', False):
+			ioc['uri_name'] = _
 		# only import custom 
 		if ioc['category'].lower() == 'custom':
 				# create the new indicator.  If this is an update, the orginal will be removed below.
@@ -1123,6 +1152,15 @@ def hxtool_handle_streaming_indicator_import(hx_api_object, iocs):
 												description='{0}\n\nImported from {1}'.format(ioc['description'], ioc['uri_name']))
 				if ret:
 					new_ioc_id = response_data['id']
+
+					# Convert realtime format to streaming format
+					if not 'conditions' in ioc:
+						ioc['conditions'] = []
+					if 'presence' in ioc:
+						ioc['conditions'].extend(ioc['presence'])
+					if 'execution' in ioc:
+						ioc['conditions'].extend(ioc['execution'])
+
 					for condition in ioc['conditions']:
 						mytests = {"tests": []}
 						for test in condition:
@@ -1146,7 +1184,7 @@ def hxtool_handle_streaming_indicator_import(hx_api_object, iocs):
 						app.logger.info(format_activity_log(msg="streaming rule action", reason="imported indicator", action="import", name=ioc['name'], user=session['ht_user'], controller=session['hx_ip']))
 				else:
 					# failed to create IOC
-					app.logger.info(format_activity_log(msg="streaming rule action", reason="unable to import indicator", action="import", name=iocs['name'], user=session['ht_user'], controller=session['hx_ip']))
+					app.logger.info(format_activity_log(msg="streaming rule action", reason="unable to import indicator", action="import", name=ioc['name'], user=session['ht_user'], controller=session['hx_ip']))
 	return					
 
 
@@ -1316,16 +1354,17 @@ def hxtool_api_indicators_export(hx_api_object):
 def hxtool_api_indicators_import(hx_api_object):
 	files = request.files.getlist('ruleImport')
 
+	import_platform = request.form.get('platform', '')
+	if import_platform == "all":
+		import_platform = ['win', 'osx', 'linux']
+	else:
+		import_platform = [import_platform]
+
 	for file in files:
 		file_content = file.read().decode(default_encoding)
 		try: 
 			iocs = json.loads(file_content)
 		except json.decoder.JSONDecodeError:
-			import_platform = request.form.get('platform', '')
-			if import_platform == "all":
-				import_platform = ['win', 'osx', 'linux']
-			else:
-				import_platform = [import_platform]
 			iocs = openioc_to_hxioc(file_content, import_platform)
 			if iocs is None:
 				app.logger.warn(format_activity_log(msg="rule action fail", reason="{} is not a valid Endpoint Security (HX) JSON or OpenIOC 1.1 indicator." .format(file.filename), action="import", user=session['ht_user'], controller=session['hx_ip']))

hxtool_apicache.py

@@ -87,7 +87,7 @@ def apicache_processor(self, currOffset, objectType, records, myCache, refresh_i
 		hxtool_global.apicache['data'][objectType]['stats']['timeline'].append(myStats)
 
 		# Show log if we have updates or new records
-		if s_update is not 0 or s_add is not 0:
+		if s_update != 0 or s_add != 0:
 			self.logger.info("{}: [{}] {} records updated, {} records added in {} seconds".format(self.profile_id, objectType, s_update, s_add, (s_end - s_start).total_seconds()))
 
 		return currOffset

hxtool_mongodb.py

@@ -92,7 +92,7 @@ def auditQuery(self, query, sort = None, qlimit = 1000):
 			return self._db_audits.find(query).limit(qlimit)
 
 	def auditGetCollections(self):
-		pipeline = [{'$group': {'count': {'$sum': 1}, '_id': {'bulk_acquisition_id': '$bulk_acquisition_id'}}}, {'$limit': 1000}]
+		pipeline = [{'$group': {'count': {'$sum': 1}, '_id': {'bulk_acquisition_id': '$bulk_acquisition_id'}}}, {'$limit': 1000}, {'$sort': {'_id': 1}}]
 		return self._db_audits.aggregate(pipeline)
 
 	def profileCreate(self, hx_name, hx_host, hx_port):

hxtool_vars.py

@@ -5,7 +5,7 @@
 
 default_encoding = 'utf-8'
 HXTOOL_API_VERSION = 1
-__version__ = "4.7-pre"
+__version__ = "4.7"
 hxtool_schema_version = 40
 data_path = "data"
 log_path = "log"

templates/ht_streaming_indicators.html

@@ -235,6 +235,8 @@
 				data.append("ruleImport", value);
 			});
 
+			data.append('platform', $('#ruleImportPlatform').data("id"));
+
 			hxtool_ajax_post_request("/api/v1/streaming_indicators/import", data, function() {
 				ioc_datatable.ajax.reload();
 			});
@@ -322,11 +324,18 @@
 	<button class="fe-btn fe-btn--md fe-btn--secondary" id="ruleViewDismiss" aria-label="Dismiss"><span> Dismiss </span></button>
 {{ htModal.widgetFooter() }}
 
-{{ htModal.widgetHeader("Import indicators", modalId="ruleImportPopup", modalSize="small") }}
-	<h3 class='hxtool_typography_h3'>Upload hxtool exported rules</h3>
+{{ htModal.widgetHeader("Import rule(s)", modalId="ruleImportPopup", modalSize="medium") }}
+	<h4 class='hxtool_typography_h3'>Platform</h4>
+		{{ htDropdown.widgetHeader("Microsoft Windows", "ruleImportPlatform", "win") }}
+		{{ htDropdown.widgetItem("Microsoft Windows", "win", elementIcon="fa-windows") }}
+		{{ htDropdown.widgetItem("Apple MacOS", "osx", elementIcon="fa-apple") }}
+		{{ htDropdown.widgetItem("Linux", "linux", elementIcon="fa-linux") }}
+		{{ htDropdown.widgetItem("All", "all", elementIcon="fas fa-globe") }}
+		{{ htDropdown.widgetFooter(elementLabel="Platform selection will be used when the indicator has no platform element in the content") }}
+	<h4 class='hxtool_typography_h3'>Rule file(s)</h4>
 	<input class="fe-btn fe-btn--sm fe-btn--primary active" style='background: transparent;' type='file' id='ruleImportFile' name='ruleImportFile' multiple>
-	<br>
-	<h5>HXTool rule format only! <br>(No STIX/YARA/OpenIOC formats)</h5>
+	<br />
+	<span class="fe-input-hint-text">HXTool rule and OpenIOC 1.1 format only! (STIX/TAXII/YARA are not supported)</span>
 {{ htModal.widgetMiddle() }}
 	<button class="fe-btn fe-btn--md fe-btn--secondary" id="ruleImportCancel" aria-label="Cancel"><span> Cancel </span></button>
 	<button class="fe-btn fe-btn--md fe-btn--primary" id="ruleImportSubmit" aria-label="Submit"><span> Submit </span></button>