From 9df32765e84236ef235668abb35c6771264e4c7c Mon Sep 17 00:00:00 2001 From: Riley Priddle Date: Thu, 21 Apr 2022 21:41:53 +0300 Subject: [PATCH] turned down some logging added auditing extension for pushing requests to saas --- .github/workflows/pipeline.yml | 2 + pointsecio/__init__.py | 2 +- pointsecio/auditor.py | 59 +++++++++++++++++------------ pointsecio/handlers.py | 12 +++--- pointsecio/middleware/swagger_ui.py | 6 +-- pointsecio/options.py | 2 +- pointsecio/sender.py | 25 ++++++------ tests/api/test_bootstrap.py | 3 +- tests/api/test_parameters.py | 6 +-- tests/api/test_responses.py | 3 +- 10 files changed, 66 insertions(+), 54 deletions(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 910b74c..d6fff38 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -3,6 +3,7 @@ on: push: branches: - main + - dev pull_request: jobs: test: @@ -22,6 +23,7 @@ jobs: pip install "tox<4" "tox-gh-actions<3" "setuptools<58" "coveralls<4" - name: Test with tox run: tox + # - name: Coveralls # run: coveralls --service github # env: diff --git a/pointsecio/__init__.py b/pointsecio/__init__.py index 5cb2b6b..c227d5f 100644 --- a/pointsecio/__init__.py +++ b/pointsecio/__init__.py @@ -39,4 +39,4 @@ Api = FlaskApi # This version is replaced during release process. -__version__ = '2022.0.dev1' +__version__ = '2022.4.dev1' diff --git a/pointsecio/auditor.py b/pointsecio/auditor.py index df69d42..728eb3a 100644 --- a/pointsecio/auditor.py +++ b/pointsecio/auditor.py @@ -11,13 +11,15 @@ class auditor: def __init__(self, - url='https://ingest.eu-west-1.dev.platform.pointsec.io', + url='https://ingest.eu-west-1.dev.platform.pointsec.io/ingest/request', + api_key='5WqBxkOi3m6F1fDRryrR654xalAwz67815Rfe0ds', debug=False, backup_logs=True, network_timeout=10.0, number_of_retries=4, retry_timeout=2, logs_drain_timeout=5): + self.api_key = api_key self.startThread = True self.requests_session = requests.Session() self.url = url @@ -46,9 +48,10 @@ def __init__(self, 'class': 'pointsecio.handlers.PointsecHandler', 'level': 'DEBUG', 'formatter': 'pointsecFormat', - 'token': '<>', + 'token': self.token, 'logs_drain_timeout': 5, - 'url': 'https://ingest.eu-west-1.dev.platform.pointsec.io/ingest/request', + 'url': self.url, + 'api_key': self.api_key, 'retries_no': 4, 'retry_timeout': 2, } @@ -62,6 +65,9 @@ def __init__(self, } } + def set_token(self, token_secret): + self.token = token_secret + def create(self, response, token): self.token = token @@ -70,31 +76,34 @@ def create(self, response, token): logging.config.dictConfig(self.LOGGING) self.logger = logging.getLogger('pointsecLogger') - jsonResponse = ['application/problem+json', 'application/json'] - if response.content_type.lower() in jsonResponse: - body = response.get_json() - else: - body = response.response payload = { - 'status_code': response.status_code, - 'body': str(body), - 'dateAdded': int((datetime.datetime.utcnow()).timestamp()), - 'method': request.method, - 'content_length': response.content_length, - 'request_headers': dict(request.headers), - 'response_headers': dict(response.headers), - 'content_type': response.content_type, - 'path': request.path, - 'args': dict(request.args), - 'full_path': request.full_path, - 'url': request.url, - 'pathParameters': request.view_args + "version": "1.1", + "dateCreated": int((datetime.datetime.utcnow()).timestamp() * 1000), + "req": { + "url": request.base_url, + "headers": dict(request.headers), + "path": request.path, + "method": request.method, + "oPath": request.url_rule.rule if request.url_rule is not None else "", + "fPath": request.full_path, + "args": dict(request.args), + "ip": request.remote_addr, + 'pathParams': request.view_args + + }, + "resp": { + "status_code": response.status_code, + "content_len": response.content_length, + "content_enc": response.content_encoding, + "body": response.get_json() if response.is_json else response.response, + "headers": dict(response.headers), + "content_type": response.content_type + } } - print(request.__dir__()) try: - self.logger.info(json.dumps(payload)) - except Exception as e: - print(payload) + if self.token: + self.logger.info(json.dumps(payload)) + except TypeError as e: print(str(e)) print("created log") diff --git a/pointsecio/handlers.py b/pointsecio/handlers.py index 2c281e7..a1475a8 100644 --- a/pointsecio/handlers.py +++ b/pointsecio/handlers.py @@ -100,10 +100,11 @@ def get_path_parameter_types(self): class PointsecHandler(logging.Handler): def __init__(self, + api_key, token, + url, pointsec_type="python", logs_drain_timeout=3, - url="https://27rznp5fh5.execute-api.eu-west-2.amazonaws.com/dev/mock", debug=False, backup_logs=True, network_timeout=10.0, @@ -118,6 +119,7 @@ def __init__(self, self.pointsec_sender = PointsecSender( token=token, url=url, + api_key=api_key, logs_drain_timeout=logs_drain_timeout, debug=debug, backup_logs=backup_logs, @@ -187,10 +189,10 @@ def format_message(self, message): except json.decoder.JSONDecodeError: return {'ignore': True} - requiredInBody = ['method'] - for item in requiredInBody: - if item not in payload: - return {'ignore': True} + # requiredInBody = ['req', 'res'] + # for item in requiredInBody: + # if item not in payload: + # return {'ignore': True} # return_json = payload return payload diff --git a/pointsecio/middleware/swagger_ui.py b/pointsecio/middleware/swagger_ui.py index 415a808..9fe78e0 100644 --- a/pointsecio/middleware/swagger_ui.py +++ b/pointsecio/middleware/swagger_ui.py @@ -109,9 +109,9 @@ def add_openapi_json(self): Adds openapi json to {base_path}/openapi.json (or {base_path}/swagger.json for swagger2) """ - logger.info( - "Adding spec json: %s/%s", self.base_path, self.options.openapi_spec_path - ) + # logger.info( + # "Adding spec json: %s/%s", self.base_path, self.options.openapi_spec_path + # ) self.router.add_route( methods=["GET"], path=self.options.openapi_spec_path, diff --git a/pointsecio/options.py b/pointsecio/options.py index 7cd963a..e99ab61 100644 --- a/pointsecio/options.py +++ b/pointsecio/options.py @@ -80,7 +80,7 @@ def openapi_console_ui_available(self): """ if (self._options.get('swagger_ui', True) and self.openapi_console_ui_from_dir is None): - logger.warning(NO_UI_MSG) + # logger.warning(NO_UI_MSG) return False return self._options.get('swagger_ui', True) diff --git a/pointsecio/sender.py b/pointsecio/sender.py index 614e321..edce031 100644 --- a/pointsecio/sender.py +++ b/pointsecio/sender.py @@ -12,11 +12,11 @@ from .logger import get_stdout_logger -loger4.basicConfig(filename="here.log", - filemode='a', - format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s', - datefmt='%H:%M:%S', - level=loger4.DEBUG) +# loger4.basicConfig(filename="here.log", +# filemode='a', +# format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s', +# datefmt='%H:%M:%S', +# level=loger4.DEBUG) MAX_BULK_SIZE_IN_BYTES = 1 * 1024 * 1024 # 1 MB @@ -32,7 +32,9 @@ def backup_logs(logs, logger): class PointsecSender: def __init__(self, - token, url='https://ingest.eu-west-1.dev.platform.pointsec.io', + token, + api_key, + url, logs_drain_timeout=5, debug=False, backup_logs=True, @@ -40,7 +42,8 @@ def __init__(self, number_of_retries=4, retry_timeout=2): self.token = token - # self.url = '{}/?token={}'.format(url, token) + self.api_key = api_key + self.url = url self.logs_drain_timeout = logs_drain_timeout self.stdout_logger = get_stdout_logger(debug) self.backup_logs = backup_logs @@ -114,7 +117,7 @@ def _flush_queue(self): should_backup_to_disk = True headers = {"Content-type": "text/plain", - 'x-api-key': 'IN2Oj3Wif32oixWUy2BdP3KUeR9nhJYPa6WGn6fq', + 'x-api-key': self.api_key, 'x-ps-api-key': self.token} for current_try in range(self.number_of_retries): @@ -127,7 +130,7 @@ def _flush_queue(self): # self.stdout_logger.info(str(response.status_code)) if response.status_code != 200: if response.status_code == 400: - self.stdout_logger.info( + self.stdout_logger.debug( 'Got 400 code from pointsec.io. This means that ' 'some of your logs are too big, or badly ' 'formatted. response: %s', response.text) @@ -136,13 +139,13 @@ def _flush_queue(self): break if response.status_code == 401: - self.stdout_logger.info( + self.stdout_logger.debug( 'You are not authorized with pointsec.io! Token ' 'OK? dropping logs...') should_backup_to_disk = False break else: - self.stdout_logger.info( + self.stdout_logger.debug( 'Got %s while sending logs to pointsec.io, ' 'Try (%s/%s). Response: %s', response.status_code, diff --git a/tests/api/test_bootstrap.py b/tests/api/test_bootstrap.py index 96c2fe2..3f94a87 100644 --- a/tests/api/test_bootstrap.py +++ b/tests/api/test_bootstrap.py @@ -285,8 +285,7 @@ def test_resolve_method(simple_app): def test_resolve_classmethod(simple_app): app_client = simple_app.app.test_client() - # type: flask.Response - resp = app_client.get('/v1.0/resolver-test/classmethod') + resp = app_client.get('/v1.0/resolver-test/classmethod') # type: flask.Response assert resp.data.decode('utf-8', 'replace') == '"DummyClass"\n' diff --git a/tests/api/test_parameters.py b/tests/api/test_parameters.py index cd3abc4..8b6adf7 100644 --- a/tests/api/test_parameters.py +++ b/tests/api/test_parameters.py @@ -189,16 +189,14 @@ def test_path_parameter_someint__bad(simple_app): def test_path_parameter_somefloat(simple_app, arg, result): assert isinstance(arg, str) # sanity check app_client = simple_app.app.test_client() - # type: flask.Response - resp = app_client.get(f'/v1.0/test-float-path/{arg}') + resp = app_client.get(f'/v1.0/test-float-path/{arg}') # type: flask.Response assert resp.data.decode('utf-8', 'replace') == f'"{result}"\n' def test_path_parameter_somefloat__bad(simple_app): # non-float values will not match Flask route app_client = simple_app.app.test_client() - # type: flask.Response - resp = app_client.get('/v1.0/test-float-path/123,45') + resp = app_client.get('/v1.0/test-float-path/123,45') # type: flask.Response assert resp.status_code == 404 diff --git a/tests/api/test_responses.py b/tests/api/test_responses.py index cba8337..68bd61c 100644 --- a/tests/api/test_responses.py +++ b/tests/api/test_responses.py @@ -87,8 +87,7 @@ def test_produce_decorator(simple_app): def test_returning_flask_response_tuple(simple_app): app_client = simple_app.app.test_client() - # type: flask.Response - result = app_client.get('/v1.0/flask_response_tuple') + result = app_client.get('/v1.0/flask_response_tuple') # type: flask.Response assert result.status_code == 201 assert result.content_type == 'application/json' result_data = json.loads(result.data.decode('utf-8', 'replace'))