From 525a164c8cfd7cb06cd17efe4d240ab8d29861f1 Mon Sep 17 00:00:00 2001 From: Elazar Broad Date: Tue, 6 Apr 2021 16:52:27 -0400 Subject: [PATCH 1/7] Update docker instructions --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index dd4b7759..987254cf 100644 --- a/README.md +++ b/README.md @@ -33,13 +33,14 @@ Configuration for HXTool is held in the `conf.json` file, documentation is in [R ### Docker To build a Docker image from the HXTool source, execute the following: ```bash -docker build -t hxtool:latest . +docker build --pull -t hxtool:latest . ``` To run HXTool once the image build process is complete, execute the following: ```bash -docker run -p 8080:8080 -d --name hxtool hxtool:latest +docker run -p 8080:8080/tcp -d --cap-add=IPC_LOCK --name hxtool hxtool:latest ``` +IPC_LOCK is needed for the GNOME keyring daemon. See [README.DOCKER](README.DOCKER) ## Contribution From e930eb6a7a5fdcdcc120f0dbbada26ae04a01d2e Mon Sep 17 00:00:00 2001 From: Elazar Broad Date: Tue, 4 May 2021 13:20:01 -0400 Subject: [PATCH 2/7] Fix syntax warning on Python 3.7+ --- hxtool_apicache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hxtool_apicache.py b/hxtool_apicache.py index e74bc730..d3916fb7 100755 --- a/hxtool_apicache.py +++ b/hxtool_apicache.py @@ -87,7 +87,7 @@ def apicache_processor(self, currOffset, objectType, records, myCache, refresh_i hxtool_global.apicache['data'][objectType]['stats']['timeline'].append(myStats) # Show log if we have updates or new records - if s_update is not 0 or s_add is not 0: + if s_update != 0 or s_add != 0: self.logger.info("{}: [{}] {} records updated, {} records added in {} seconds".format(self.profile_id, objectType, s_update, s_add, (s_end - s_start).total_seconds())) return currOffset From 1eb69d6ab27d2ebb6e28baad25160440d128a40d Mon Sep 17 00:00:00 2001 From: Elazar Broad Date: Tue, 4 May 2021 13:33:20 -0400 Subject: [PATCH 3/7] Sort audits by ID descending --- hxtool_mongodb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hxtool_mongodb.py b/hxtool_mongodb.py index 7315175e..386aafba 100755 --- a/hxtool_mongodb.py +++ b/hxtool_mongodb.py @@ -92,7 +92,7 @@ def auditQuery(self, query, sort = None, qlimit = 1000): return self._db_audits.find(query).limit(qlimit) def auditGetCollections(self): - pipeline = [{'$group': {'count': {'$sum': 1}, '_id': {'bulk_acquisition_id': '$bulk_acquisition_id'}}}, {'$limit': 1000}] + pipeline = [{'$group': {'count': {'$sum': 1}, '_id': {'bulk_acquisition_id': '$bulk_acquisition_id'}}}, {'$limit': 1000}, {'$sort': {'_id': 1}}] return self._db_audits.aggregate(pipeline) def profileCreate(self, hx_name, hx_host, hx_port): From a169126962c20efb57f52cebe5d5d0bd9d6cfefd Mon Sep 17 00:00:00 2001 From: Elazar Broad Date: Tue, 4 May 2021 13:55:41 -0400 Subject: [PATCH 4/7] Move platform selection up, no need to do it for every loop iteration --- hxtool_api.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/hxtool_api.py b/hxtool_api.py index 2ab0d92e..3d720971 100755 --- a/hxtool_api.py +++ b/hxtool_api.py @@ -1316,16 +1316,17 @@ def hxtool_api_indicators_export(hx_api_object): def hxtool_api_indicators_import(hx_api_object): files = request.files.getlist('ruleImport') + import_platform = request.form.get('platform', '') + if import_platform == "all": + import_platform = ['win', 'osx', 'linux'] + else: + import_platform = [import_platform] + for file in files: file_content = file.read().decode(default_encoding) try: iocs = json.loads(file_content) except json.decoder.JSONDecodeError: - import_platform = request.form.get('platform', '') - if import_platform == "all": - import_platform = ['win', 'osx', 'linux'] - else: - import_platform = [import_platform] iocs = openioc_to_hxioc(file_content, import_platform) if iocs is None: app.logger.warn(format_activity_log(msg="rule action fail", reason="{} is not a valid Endpoint Security (HX) JSON or OpenIOC 1.1 indicator." .format(file.filename), action="import", user=session['ht_user'], controller=session['hx_ip'])) From 89c3944dad0a721967ef2dcde8aff9f0301ffc0a Mon Sep 17 00:00:00 2001 From: Elazar Broad Date: Tue, 4 May 2021 14:42:17 -0400 Subject: [PATCH 5/7] Add realtime and OpenIOC 1.1 support for streaming indicators --- hxtool_api.py | 46 +++++++++++++++++++++++--- templates/ht_streaming_indicators.html | 15 ++++++--- 2 files changed, 53 insertions(+), 8 deletions(-) diff --git a/hxtool_api.py b/hxtool_api.py index 3d720971..733f05a3 100755 --- a/hxtool_api.py +++ b/hxtool_api.py @@ -1089,7 +1089,13 @@ def hxtool_api_streaming_indicators_export(hx_api_object): @valid_session_required def hxtool_api_streaming_indicators_import(hx_api_object): files = request.files.getlist('ruleImport') - + + import_platform = request.form.get('platform', '') + if import_platform == "all": + import_platform = ['win', 'osx', 'linux'] + else: + import_platform = [import_platform] + for file in files: # we may have a zip file, or a list of one or more files if (zipfile.is_zipfile(file)): @@ -1100,18 +1106,41 @@ def hxtool_api_streaming_indicators_import(hx_api_object): # for each zip file for zfile in zfiles: with myzip.open(zfile) as myfile: - iocs = json.loads(myfile.read()) + file_content = myfile.read().decode(default_encoding) + try: + iocs = json.loads(file_content) + except json.decoder.JSONDecodeError: + iocs = openioc_to_hxioc(file_content, import_platform) + if iocs is None: + app.logger.warn(format_activity_log(msg="rule action fail", reason="{} is not a valid Endpoint Security (HX) JSON or OpenIOC 1.1 indicator." .format(file.filename), action="import", user=session['ht_user'], controller=session['hx_ip'])) + continue + hxtool_handle_streaming_indicator_import(hx_api_object=hx_api_object, iocs=iocs) else: # not a zip file file.seek(0) # rewind to the start of the file - iocs = json.loads(file.read().decode(default_encoding)) + file_content = file.read().decode(default_encoding) + try: + iocs = json.loads(file_content) + except json.decoder.JSONDecodeError: + iocs = openioc_to_hxioc(file_content, import_platform) + if iocs is None: + app.logger.warn(format_activity_log(msg="rule action fail", reason="{} is not a valid Endpoint Security (HX) JSON or OpenIOC 1.1 indicator." .format(file.filename), action="import", user=session['ht_user'], controller=session['hx_ip'])) + continue + hxtool_handle_streaming_indicator_import(hx_api_object=hx_api_object, iocs=iocs) return(app.response_class(response=json.dumps("OK"), status=200, mimetype='application/json')) def hxtool_handle_streaming_indicator_import(hx_api_object, iocs): for (_, ioc) in iocs.items(): + # Add category for IOCs that don't have, i.e. OpenIOC 1.1 import + if not ioc.get('category', False): + ioc['category'] = 'Custom' + if not ioc.get('description', False): + ioc['description'] = "" + if not ioc.get('uri_name', False): + ioc['uri_name'] = _ # only import custom if ioc['category'].lower() == 'custom': # create the new indicator. If this is an update, the orginal will be removed below. @@ -1123,6 +1152,15 @@ def hxtool_handle_streaming_indicator_import(hx_api_object, iocs): description='{0}\n\nImported from {1}'.format(ioc['description'], ioc['uri_name'])) if ret: new_ioc_id = response_data['id'] + + # Convert realtime format to streaming format + if not 'conditions' in ioc: + ioc['conditions'] = [] + if 'presence' in ioc: + ioc['conditions'].extend(ioc['presence']) + if 'execution' in ioc: + ioc['conditions'].extend(ioc['execution']) + for condition in ioc['conditions']: mytests = {"tests": []} for test in condition: @@ -1146,7 +1184,7 @@ def hxtool_handle_streaming_indicator_import(hx_api_object, iocs): app.logger.info(format_activity_log(msg="streaming rule action", reason="imported indicator", action="import", name=ioc['name'], user=session['ht_user'], controller=session['hx_ip'])) else: # failed to create IOC - app.logger.info(format_activity_log(msg="streaming rule action", reason="unable to import indicator", action="import", name=iocs['name'], user=session['ht_user'], controller=session['hx_ip'])) + app.logger.info(format_activity_log(msg="streaming rule action", reason="unable to import indicator", action="import", name=ioc['name'], user=session['ht_user'], controller=session['hx_ip'])) return diff --git a/templates/ht_streaming_indicators.html b/templates/ht_streaming_indicators.html index 7bf3258f..fe0653ee 100755 --- a/templates/ht_streaming_indicators.html +++ b/templates/ht_streaming_indicators.html @@ -322,11 +322,18 @@ {{ htModal.widgetFooter() }} -{{ htModal.widgetHeader("Import indicators", modalId="ruleImportPopup", modalSize="small") }} -

Upload hxtool exported rules

+{{ htModal.widgetHeader("Import rule(s)", modalId="ruleImportPopup", modalSize="medium") }} +

Platform

+ {{ htDropdown.widgetHeader("Microsoft Windows", "ruleImportPlatform", "win") }} + {{ htDropdown.widgetItem("Microsoft Windows", "win", elementIcon="fa-windows") }} + {{ htDropdown.widgetItem("Apple MacOS", "osx", elementIcon="fa-apple") }} + {{ htDropdown.widgetItem("Linux", "linux", elementIcon="fa-linux") }} + {{ htDropdown.widgetItem("All", "all", elementIcon="fas fa-globe") }} + {{ htDropdown.widgetFooter(elementLabel="Platform selection will be used when the indicator has no platform element in the content") }} +

Rule file(s)

-
-
HXTool rule format only!
(No STIX/YARA/OpenIOC formats)
+
+ HXTool rule and OpenIOC 1.1 format only! (STIX/TAXII/YARA are not supported) {{ htModal.widgetMiddle() }} From ccba225fbba1348c031277f089a650092a8e417c Mon Sep 17 00:00:00 2001 From: Elazar Broad Date: Tue, 4 May 2021 14:43:39 -0400 Subject: [PATCH 6/7] OpenIOC for streaming contd. --- templates/ht_streaming_indicators.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/ht_streaming_indicators.html b/templates/ht_streaming_indicators.html index fe0653ee..c4a2675b 100755 --- a/templates/ht_streaming_indicators.html +++ b/templates/ht_streaming_indicators.html @@ -235,6 +235,8 @@ data.append("ruleImport", value); }); + data.append('platform', $('#ruleImportPlatform').data("id")); + hxtool_ajax_post_request("/api/v1/streaming_indicators/import", data, function() { ioc_datatable.ajax.reload(); }); From 24cd61d64bd013f7da4430f74cc13ca4e535fe0c Mon Sep 17 00:00:00 2001 From: Elazar Broad Date: Tue, 4 May 2021 14:43:59 -0400 Subject: [PATCH 7/7] Bump version to 4.7 --- README.md | 2 +- hxtool_vars.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 987254cf..ee32bde3 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ HXTool is a web-based, standalone tool that can be used with FireEye Endpoint Se HXTool provides additional features not directly available in the product by leveraging FireEye Endpoint Security's rich API. ### Version -4.7-pre +4.7 ## Installation To install HXTool: diff --git a/hxtool_vars.py b/hxtool_vars.py index b786bfe6..c500d556 100644 --- a/hxtool_vars.py +++ b/hxtool_vars.py @@ -5,7 +5,7 @@ default_encoding = 'utf-8' HXTOOL_API_VERSION = 1 -__version__ = "4.7-pre" +__version__ = "4.7" hxtool_schema_version = 40 data_path = "data" log_path = "log"