turned down some logging

added auditing extension for pushing requests to saas

.github/workflows/pipeline.yml

@@ -3,6 +3,7 @@ on:
   push:
     branches:
       - main
+      - dev
   pull_request:
 jobs:
   test:
@@ -22,6 +23,7 @@ jobs:
           pip install "tox<4" "tox-gh-actions<3" "setuptools<58" "coveralls<4"
       - name: Test with tox
         run: tox
+
     # - name: Coveralls
     #   run: coveralls --service github
     #   env:

pointsecio/__init__.py

@@ -39,4 +39,4 @@
 Api = FlaskApi
 
 # This version is replaced during release process.
-__version__ = '2022.0.dev1'
+__version__ = '2022.4.dev1'

pointsecio/auditor.py

@@ -11,13 +11,15 @@
 
 class auditor:
     def __init__(self,
-                 url='https://ingest.eu-west-1.dev.platform.pointsec.io',
+                 url='https://ingest.eu-west-1.dev.platform.pointsec.io/ingest/request',
+                 api_key='5WqBxkOi3m6F1fDRryrR654xalAwz67815Rfe0ds',
                  debug=False,
                  backup_logs=True,
                  network_timeout=10.0,
                  number_of_retries=4,
                  retry_timeout=2,
                  logs_drain_timeout=5):
+        self.api_key = api_key
         self.startThread = True
         self.requests_session = requests.Session()
         self.url = url
@@ -46,9 +48,10 @@ def __init__(self,
                     'class': 'pointsecio.handlers.PointsecHandler',
                     'level': 'DEBUG',
                     'formatter': 'pointsecFormat',
-                    'token': '<<pointsec-TOKEN>>',
+                    'token': self.token,
                     'logs_drain_timeout': 5,
-                    'url': 'https://ingest.eu-west-1.dev.platform.pointsec.io/ingest/request',
+                    'url': self.url,
+                    'api_key': self.api_key,
                     'retries_no': 4,
                     'retry_timeout': 2,
                 }
@@ -62,6 +65,9 @@ def __init__(self,
             }
         }
 
+    def set_token(self, token_secret):
+        self.token = token_secret
+
     def create(self, response, token):
 
         self.token = token
@@ -70,31 +76,34 @@ def create(self, response, token):
             logging.config.dictConfig(self.LOGGING)
             self.logger = logging.getLogger('pointsecLogger')
 
-        jsonResponse = ['application/problem+json', 'application/json']
-        if response.content_type.lower() in jsonResponse:
-            body = response.get_json()
-        else:
-            body = response.response
         payload = {
-            'status_code': response.status_code,
-            'body': str(body),
-            'dateAdded': int((datetime.datetime.utcnow()).timestamp()),
-            'method': request.method,
-            'content_length': response.content_length,
-            'request_headers': dict(request.headers),
-            'response_headers': dict(response.headers),
-            'content_type': response.content_type,
-            'path': request.path,
-            'args': dict(request.args),
-            'full_path': request.full_path,
-            'url': request.url,
-            'pathParameters': request.view_args
+            "version": "1.1",
+            "dateCreated": int((datetime.datetime.utcnow()).timestamp() * 1000),
+            "req": {
+                "url": request.base_url,
+                "headers": dict(request.headers),
+                "path": request.path,
+                "method": request.method,
+                "oPath": request.url_rule.rule if request.url_rule is not None else "",
+                "fPath": request.full_path,
+                "args": dict(request.args),
+                "ip": request.remote_addr,
+                'pathParams': request.view_args
+
+            },
+            "resp": {
+                "status_code": response.status_code,
+                "content_len": response.content_length,
+                "content_enc": response.content_encoding,
+                "body": response.get_json() if response.is_json else response.response,
+                "headers": dict(response.headers),
+                "content_type": response.content_type
+            }
         }
-        print(request.__dir__())
         try:
-            self.logger.info(json.dumps(payload))
-        except Exception as e:
-            print(payload)
+            if self.token:
+                self.logger.info(json.dumps(payload))
+        except TypeError as e:
             print(str(e))
         print("created log")
 

pointsecio/handlers.py

@@ -100,10 +100,11 @@ def get_path_parameter_types(self):
 class PointsecHandler(logging.Handler):
 
     def __init__(self,
+                 api_key,
                  token,
+                 url,
                  pointsec_type="python",
                  logs_drain_timeout=3,
-                 url="https://27rznp5fh5.execute-api.eu-west-2.amazonaws.com/dev/mock",
                  debug=False,
                  backup_logs=True,
                  network_timeout=10.0,
@@ -118,6 +119,7 @@ def __init__(self,
         self.pointsec_sender = PointsecSender(
             token=token,
             url=url,
+            api_key=api_key,
             logs_drain_timeout=logs_drain_timeout,
             debug=debug,
             backup_logs=backup_logs,
@@ -187,10 +189,10 @@ def format_message(self, message):
         except json.decoder.JSONDecodeError:
             return {'ignore': True}
 
-        requiredInBody = ['method']
-        for item in requiredInBody:
-            if item not in payload:
-                return {'ignore': True}
+        # requiredInBody = ['req', 'res']
+        # for item in requiredInBody:
+        #     if item not in payload:
+        #         return {'ignore': True}
 
         # return_json = payload
         return payload

pointsecio/middleware/swagger_ui.py

@@ -109,9 +109,9 @@ def add_openapi_json(self):
         Adds openapi json to {base_path}/openapi.json
              (or {base_path}/swagger.json for swagger2)
         """
-        logger.info(
-            "Adding spec json: %s/%s", self.base_path, self.options.openapi_spec_path
-        )
+        # logger.info(
+        #     "Adding spec json: %s/%s", self.base_path, self.options.openapi_spec_path
+        # )
         self.router.add_route(
             methods=["GET"],
             path=self.options.openapi_spec_path,

pointsecio/options.py

@@ -80,7 +80,7 @@ def openapi_console_ui_available(self):
         """
         if (self._options.get('swagger_ui', True) and
                 self.openapi_console_ui_from_dir is None):
-            logger.warning(NO_UI_MSG)
+            # logger.warning(NO_UI_MSG)
             return False
         return self._options.get('swagger_ui', True)
 

pointsecio/sender.py

@@ -12,11 +12,11 @@
 
 from .logger import get_stdout_logger
 
-loger4.basicConfig(filename="here.log",
-                            filemode='a',
-                            format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
-                            datefmt='%H:%M:%S',
-                            level=loger4.DEBUG)
+# loger4.basicConfig(filename="here.log",
+#                             filemode='a',
+#                             format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
+#                             datefmt='%H:%M:%S',
+#                             level=loger4.DEBUG)
 
 
 MAX_BULK_SIZE_IN_BYTES = 1 * 1024 * 1024  # 1 MB
@@ -32,15 +32,18 @@ def backup_logs(logs, logger):
 
 class PointsecSender:
     def __init__(self,
-                 token, url='https://ingest.eu-west-1.dev.platform.pointsec.io',
+                 token,
+                 api_key,
+                 url,
                  logs_drain_timeout=5,
                  debug=False,
                  backup_logs=True,
                  network_timeout=10.0,
                  number_of_retries=4,
                  retry_timeout=2):
         self.token = token
-        # self.url = '{}/?token={}'.format(url, token)
+        self.api_key = api_key
+        self.url = url
         self.logs_drain_timeout = logs_drain_timeout
         self.stdout_logger = get_stdout_logger(debug)
         self.backup_logs = backup_logs
@@ -114,7 +117,7 @@ def _flush_queue(self):
 
             should_backup_to_disk = True
             headers = {"Content-type": "text/plain",
-                       'x-api-key': 'IN2Oj3Wif32oixWUy2BdP3KUeR9nhJYPa6WGn6fq',
+                       'x-api-key': self.api_key,
                        'x-ps-api-key': self.token}
 
             for current_try in range(self.number_of_retries):
@@ -127,7 +130,7 @@ def _flush_queue(self):
                     # self.stdout_logger.info(str(response.status_code))
                     if response.status_code != 200:
                         if response.status_code == 400:
-                            self.stdout_logger.info(
+                            self.stdout_logger.debug(
                                 'Got 400 code from pointsec.io. This means that '
                                 'some of your logs are too big, or badly '
                                 'formatted. response: %s', response.text)
@@ -136,13 +139,13 @@ def _flush_queue(self):
                             break
 
                         if response.status_code == 401:
-                            self.stdout_logger.info(
+                            self.stdout_logger.debug(
                                 'You are not authorized with pointsec.io! Token '
                                 'OK? dropping logs...')
                             should_backup_to_disk = False
                             break
                         else:
-                            self.stdout_logger.info(
+                            self.stdout_logger.debug(
                                 'Got %s while sending logs to pointsec.io, '
                                 'Try (%s/%s). Response: %s',
                                 response.status_code,

tests/api/test_bootstrap.py

@@ -285,8 +285,7 @@ def test_resolve_method(simple_app):
 
 def test_resolve_classmethod(simple_app):
     app_client = simple_app.app.test_client()
-    # type: flask.Response
-    resp = app_client.get('/v1.0/resolver-test/classmethod')
+    resp = app_client.get('/v1.0/resolver-test/classmethod')  # type: flask.Response
     assert resp.data.decode('utf-8', 'replace') == '"DummyClass"\n'
 
 

tests/api/test_parameters.py

@@ -189,16 +189,14 @@ def test_path_parameter_someint__bad(simple_app):
 def test_path_parameter_somefloat(simple_app, arg, result):
     assert isinstance(arg, str)  # sanity check
     app_client = simple_app.app.test_client()
-    # type: flask.Response
-    resp = app_client.get(f'/v1.0/test-float-path/{arg}')
+    resp = app_client.get(f'/v1.0/test-float-path/{arg}')  # type: flask.Response
     assert resp.data.decode('utf-8', 'replace') == f'"{result}"\n'
 
 
 def test_path_parameter_somefloat__bad(simple_app):
     # non-float values will not match Flask route
     app_client = simple_app.app.test_client()
-    # type: flask.Response
-    resp = app_client.get('/v1.0/test-float-path/123,45')
+    resp = app_client.get('/v1.0/test-float-path/123,45')  # type: flask.Response
     assert resp.status_code == 404
 
 

tests/api/test_responses.py

@@ -87,8 +87,7 @@ def test_produce_decorator(simple_app):
 def test_returning_flask_response_tuple(simple_app):
     app_client = simple_app.app.test_client()
 
-    # type: flask.Response
-    result = app_client.get('/v1.0/flask_response_tuple')
+    result = app_client.get('/v1.0/flask_response_tuple')  # type: flask.Response
     assert result.status_code == 201
     assert result.content_type == 'application/json'
     result_data = json.loads(result.data.decode('utf-8', 'replace'))