From a4efcb41ec828731883ae82660b59b2f6d652c32 Mon Sep 17 00:00:00 2001 From: riley priddle Date: Wed, 31 Jul 2024 15:56:38 +0100 Subject: [PATCH 1/4] added fixes to logging pii --- firetail/auditor.py | 54 ++++++++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/firetail/auditor.py b/firetail/auditor.py index 7f7d6fb..bdd482b 100644 --- a/firetail/auditor.py +++ b/firetail/auditor.py @@ -1,3 +1,4 @@ +from functools import lru_cache import hashlib import json import logging @@ -42,7 +43,6 @@ def __init__( self.requests_session = requests.Session() self.url = url self.token = token - self.auth_token = None self.logs_drain_timeout = logs_drain_timeout self.stdout_logger = get_stdout_logger(debug) self.backup_logs = backup_logs @@ -90,37 +90,46 @@ def init_app(self, app, token): def set_token(self, token_secret): self.token = token_secret - def sha1_hash(self, value): + @staticmethod + def sha1_hash(value): hash_object = hashlib.sha1(value.encode("utf-8")) return "sha1:" + hash_object.hexdigest() + @staticmethod + def get_ttl_hash(seconds=600): + return round(time.time() / seconds) + + @lru_cache + def decode_token(token, ttl_hash=None): + return jwt.decode( + token, + options={"verify_signature": False, "verify_exp": False}, + ) + def clean_pii(self, payload): + oauth = False + auth_token = None clean_headers = self.scrub_headers - if "req" in payload and "headers" in payload["req"]: - for k, v in payload["req"]["headers"].items(): - if k.lower() in clean_headers: - if k.lower() == "authorization" and "bearer " in v.lower(): - self.oauth = True - v = v.split(" ")[1] - self.auth_token = v - payload["req"]["headers"][k] = self.sha1_hash(v) - if "res" in payload and "headers" in payload["res"]: - for k, v in payload["res"]["headers"].items(): + + if req_headers := payload["req"].get("headers"): + for k, v in req_headers.items(): + if k.lower() in clean_headers and ("authorization", "bearer ") in k.lower(): + oauth = True + auth_token = v.split(" ")[1] if " " in v else None + payload["req"]["headers"][k] = "{SANITIZED_HEADER:" + self.sha1_hash(v) + "}" + + if res_headers := payload["res"].get("headers"): + for k, v in res_headers.items(): if k.lower() in clean_headers: - payload["req"]["headers"][k] = self.sha1_hash(v) + payload["res"]["headers"][k] = "{SANITIZED_HEADER:" + self.sha1_hash(v) + "}" - if self.oauth and self.enrich_oauth: + if auth_token and oauth and self.enrich_oauth: try: - jwt_decoded = jwt.decode( - self.auth_token, - options={"verify_signature": False, "verify_exp": False}, - ) + jwt_decoded = self.decode_token(auth_token, ttl_hash=self.get_ttl_hash()) except jwt.exceptions.DecodeError: - self.oauth = False - if self.oauth: + oauth = False + if oauth: payload["oauth"] = {"sub": jwt_decoded["sub"]} - if "email" in jwt_decoded: - payload["oauth"]["email"] = jwt_decoded["email"] return payload def format_headers(self, req_headers): @@ -154,7 +163,6 @@ def create(self, response, token, diff=-1, scrub_headers=None, debug=False): "resource": request.url_rule.rule if request.url_rule is not None else request.path, "method": request.method, "body": request.get_data(as_text=True), - "ip": request.remote_addr, }, "response": { From 6ed62eaccef7cbd2b819dcff1c026394957a6b9e Mon Sep 17 00:00:00 2001 From: rileyfiretail <107564215+rileyfiretail@users.noreply.github.com> Date: Thu, 1 Aug 2024 17:48:01 +0100 Subject: [PATCH 2/4] Apply suggestions from code review Co-authored-by: Joshua O'Sullivan --- firetail/auditor.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/firetail/auditor.py b/firetail/auditor.py index bdd482b..31e2608 100644 --- a/firetail/auditor.py +++ b/firetail/auditor.py @@ -99,7 +99,7 @@ def sha1_hash(value): def get_ttl_hash(seconds=600): return round(time.time() / seconds) - @lru_cache + @lru_cache(maxsize=128) def decode_token(token, ttl_hash=None): return jwt.decode( token, @@ -111,25 +111,23 @@ def clean_pii(self, payload): auth_token = None clean_headers = self.scrub_headers - if req_headers := payload["req"].get("headers"): - for k, v in req_headers.items(): - if k.lower() in clean_headers and ("authorization", "bearer ") in k.lower(): - oauth = True - auth_token = v.split(" ")[1] if " " in v else None + for k, v in payload["req"].get("headers", {}).items(): + if k.lower() == "authorization" and "bearer " in v.lower(): + oauth = True + auth_token = v.split(" ")[1] if " " in v else None + if k.lower() in self.scrub_headers: payload["req"]["headers"][k] = "{SANITIZED_HEADER:" + self.sha1_hash(v) + "}" - if res_headers := payload["res"].get("headers"): - for k, v in res_headers.items(): - if k.lower() in clean_headers: - payload["res"]["headers"][k] = "{SANITIZED_HEADER:" + self.sha1_hash(v) + "}" + for k, v in payload["res"].get("headers", {}).items(): + if k.lower() in self.scrub_headers: + payload["res"]["headers"][k] = "{SANITIZED_HEADER:" + self.sha1_hash(v) + "}" - if auth_token and oauth and self.enrich_oauth: + if auth_token not in [None, ""] and oauth and self.enrich_oauth: try: jwt_decoded = self.decode_token(auth_token, ttl_hash=self.get_ttl_hash()) - except jwt.exceptions.DecodeError: - oauth = False - if oauth: payload["oauth"] = {"sub": jwt_decoded["sub"]} + except jwt.exceptions.DecodeError: + pass return payload def format_headers(self, req_headers): From a85437d54bc65b2a9fe6fbb565039ed1f73eebc9 Mon Sep 17 00:00:00 2001 From: rileyfiretail <107564215+rileyfiretail@users.noreply.github.com> Date: Thu, 1 Aug 2024 17:51:30 +0100 Subject: [PATCH 3/4] Update firetail/auditor.py --- firetail/auditor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/firetail/auditor.py b/firetail/auditor.py index 31e2608..ef7ade2 100644 --- a/firetail/auditor.py +++ b/firetail/auditor.py @@ -109,7 +109,6 @@ def decode_token(token, ttl_hash=None): def clean_pii(self, payload): oauth = False auth_token = None - clean_headers = self.scrub_headers for k, v in payload["req"].get("headers", {}).items(): if k.lower() == "authorization" and "bearer " in v.lower(): From 85cd6d4b1c2f3baabea80a6a391a33a7b95214af Mon Sep 17 00:00:00 2001 From: riley priddle Date: Thu, 1 Aug 2024 18:03:57 +0100 Subject: [PATCH 4/4] fixed import order --- firetail/auditor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firetail/auditor.py b/firetail/auditor.py index ef7ade2..0c7ba71 100644 --- a/firetail/auditor.py +++ b/firetail/auditor.py @@ -1,9 +1,9 @@ -from functools import lru_cache import hashlib import json import logging import logging.config import time +from functools import lru_cache import jwt import requests