From 038b7107349abb6535f993bb5ed0571681500607 Mon Sep 17 00:00:00 2001 From: Arman Jasuja Date: Fri, 5 Jul 2024 10:32:10 +0400 Subject: [PATCH 01/10] Added ability to filter by key --- logredactor/redacting_filter.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/logredactor/redacting_filter.py b/logredactor/redacting_filter.py index 08f1a2e..bd6387a 100644 --- a/logredactor/redacting_filter.py +++ b/logredactor/redacting_filter.py @@ -4,12 +4,14 @@ class RedactingFilter(logging.Filter): # Do not try and redact the built in values. With the wrong regex it can break the logging - ignore_keys = [ + ignore_keys = set( 'name', 'levelname', 'levelno', 'pathname', 'filename', 'module', 'exc_info', 'exc_text', 'stack_info', 'lineno', 'funcName', 'created', 'msecs', 'relativeCreated', 'thread', 'threadName', 'process', 'processName', 'args', - ] + ) + + keys = set() # Specify keys to redact by def __init__(self, patterns, default_mask='****'): super(RedactingFilter, self).__init__() @@ -35,7 +37,7 @@ def redact(self, content): if content: if isinstance(content, dict): for k, v in content.items(): - content[k] = self.redact(v) + content[k] = self.redact(v) if k not in self.keys else self._default_mask elif isinstance(content, (list, tuple)): for i, v in enumerate(content): From 47fd9aed292475e8bb17dd6ba5d6be677b1480cc Mon Sep 17 00:00:00 2001 From: Arman Jasuja Date: Fri, 5 Jul 2024 11:17:03 +0400 Subject: [PATCH 02/10] Added unit tests --- logredactor/redacting_filter.py | 20 +++++++++++--------- tests/test_redacting_filter.py | 31 +++++++++++-------------------- 2 files changed, 22 insertions(+), 29 deletions(-) diff --git a/logredactor/redacting_filter.py b/logredactor/redacting_filter.py index bd6387a..cd7f3c2 100644 --- a/logredactor/redacting_filter.py +++ b/logredactor/redacting_filter.py @@ -4,44 +4,46 @@ class RedactingFilter(logging.Filter): # Do not try and redact the built in values. With the wrong regex it can break the logging - ignore_keys = set( + ignore_keys = { 'name', 'levelname', 'levelno', 'pathname', 'filename', 'module', 'exc_info', 'exc_text', 'stack_info', 'lineno', 'funcName', 'created', 'msecs', 'relativeCreated', 'thread', 'threadName', 'process', 'processName', 'args', - ) + } - keys = set() # Specify keys to redact by - - def __init__(self, patterns, default_mask='****'): + def __init__(self, patterns, default_mask='****', mask_keys=set()): super(RedactingFilter, self).__init__() self._patterns = patterns self._default_mask = str(default_mask) + self._mask_keys = set(mask_keys) def filter(self, record): d = vars(record) for k, content in d.items(): if k not in self.ignore_keys: - d[k] = self.redact(content) + d[k] = self.redact(content, k) # Also clean any contents in args if isinstance(record.args, dict): for k in record.args.keys(): - record.args[k] = self.redact(record.args[k]) + record.args[k] = self.redact(record.args[k], k) else: record.args = tuple(self.redact(arg) for arg in record.args) return True - def redact(self, content): + def redact(self, content, key=None): if content: if isinstance(content, dict): for k, v in content.items(): - content[k] = self.redact(v) if k not in self.keys else self._default_mask + content[k] = self.redact(v) elif isinstance(content, (list, tuple)): for i, v in enumerate(content): content[i] = self.redact(v) + + elif key in self._mask_keys: + content = self._default_mask else: content = isinstance(content, str) and content or str(content) diff --git a/tests/test_redacting_filter.py b/tests/test_redacting_filter.py index eecb36d..49f2fe2 100644 --- a/tests/test_redacting_filter.py +++ b/tests/test_redacting_filter.py @@ -3,7 +3,6 @@ import logging import logredactor - @pytest.fixture def logger_setup(request): def get_logger(filters): @@ -12,7 +11,8 @@ def get_logger(filters): logger.addFilter( logredactor.RedactingFilter( filters, - default_mask='****' + default_mask='****', + mask_keys={'phonenumber'} ) ) return logger @@ -38,6 +38,12 @@ def test_arg_dict(caplog, logger_setup): assert caplog.records[0].message == "foo ****" +def test_arg_dict_with_key_to_remove(caplog, logger_setup): + logger = logger_setup([re.compile(r'\d{3}')]) + logger.warning("foo %s", {'phonenumber': '123'}) + assert caplog.records[0].message == "foo {'phonenumber': '****'}" + + def test_extra_string_value(caplog, logger_setup): logger = logger_setup([re.compile(r'\d{3}')]) logger.warning("foo", extra={'bar': '123 too'}) @@ -73,22 +79,7 @@ def test_extra_do_not_redact_key(caplog, logger_setup): assert caplog.records[0].thing987 == "foobar" -def test_extra_nested_dict_with_list(caplog, logger_setup): +def test_extra_do_redact_specific_key(caplog, logger_setup): logger = logger_setup([re.compile(r'\d{3}')]) - extra_data = { - 'bar': { - 'thing': ['one', '456'], - }, - } - logger.warning("foo", extra=extra_data) - assert caplog.records[0].bar['thing'][0] == 'one' - assert caplog.records[0].bar['thing'][1] == '****' - - -def test_match_group(caplog, logger_setup): - # Nothing in the code has to change - # But this shows the use of a Positive Lookbehind - # https://www.regextutorial.org/positive-and-negative-lookbehind-assertions.php - logger = logger_setup([re.compile(r'(?<=api_key=)[\w-]+')]) - logger.warning("example.com?api_key=this-is-my-key&sort=price") - assert caplog.records[0].message == "example.com?api_key=****&sort=price" + logger.warning("foo", extra={'phonenumber': 'foobar'}) + assert caplog.records[0].phonenumber == "****" \ No newline at end of file From e4495516692b62093ea03eba063c67cc442a822b Mon Sep 17 00:00:00 2001 From: Arman Jasuja Date: Fri, 5 Jul 2024 11:19:16 +0400 Subject: [PATCH 03/10] Updated unit tests --- tests/test_redacting_filter.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_redacting_filter.py b/tests/test_redacting_filter.py index 49f2fe2..afee83a 100644 --- a/tests/test_redacting_filter.py +++ b/tests/test_redacting_filter.py @@ -3,6 +3,7 @@ import logging import logredactor + @pytest.fixture def logger_setup(request): def get_logger(filters): @@ -79,6 +80,28 @@ def test_extra_do_not_redact_key(caplog, logger_setup): assert caplog.records[0].thing987 == "foobar" +def test_extra_nested_dict_with_list(caplog, logger_setup): + logger = logger_setup([re.compile(r'\d{3}')]) + extra_data = { + 'bar': { + 'thing': ['one', '456'], + }, + } + logger.warning("foo", extra=extra_data) + assert caplog.records[0].bar['thing'][0] == 'one' + assert caplog.records[0].bar['thing'][1] == '****' + + +def test_match_group(caplog, logger_setup): + # Nothing in the code has to change + # But this shows the use of a Positive Lookbehind + # https://www.regextutorial.org/positive-and-negative-lookbehind-assertions.php + logger = logger_setup([re.compile(r'(?<=api_key=)[\w-]+')]) + logger.warning("example.com?api_key=this-is-my-key&sort=price") + assert caplog.records[0].message == "example.com?api_key=****&sort=price" + + + def test_extra_do_redact_specific_key(caplog, logger_setup): logger = logger_setup([re.compile(r'\d{3}')]) logger.warning("foo", extra={'phonenumber': 'foobar'}) From aa6eb59059e43463fc66591eaf093e9589f596d8 Mon Sep 17 00:00:00 2001 From: Arman Jasuja Date: Fri, 5 Jul 2024 11:20:03 +0400 Subject: [PATCH 04/10] Minor lint fix --- tests/test_redacting_filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_redacting_filter.py b/tests/test_redacting_filter.py index afee83a..f760da4 100644 --- a/tests/test_redacting_filter.py +++ b/tests/test_redacting_filter.py @@ -105,4 +105,4 @@ def test_match_group(caplog, logger_setup): def test_extra_do_redact_specific_key(caplog, logger_setup): logger = logger_setup([re.compile(r'\d{3}')]) logger.warning("foo", extra={'phonenumber': 'foobar'}) - assert caplog.records[0].phonenumber == "****" \ No newline at end of file + assert caplog.records[0].phonenumber == "****" From d25b378dfd46eb3f22270033d299045c1dac6d1a Mon Sep 17 00:00:00 2001 From: Arman Jasuja Date: Fri, 5 Jul 2024 11:33:39 +0400 Subject: [PATCH 05/10] Lint fix --- tests/test_redacting_filter.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_redacting_filter.py b/tests/test_redacting_filter.py index f760da4..8226b5c 100644 --- a/tests/test_redacting_filter.py +++ b/tests/test_redacting_filter.py @@ -13,7 +13,7 @@ def get_logger(filters): logredactor.RedactingFilter( filters, default_mask='****', - mask_keys={'phonenumber'} + mask_keys={'phonenumber',} ) ) return logger @@ -101,7 +101,6 @@ def test_match_group(caplog, logger_setup): assert caplog.records[0].message == "example.com?api_key=****&sort=price" - def test_extra_do_redact_specific_key(caplog, logger_setup): logger = logger_setup([re.compile(r'\d{3}')]) logger.warning("foo", extra={'phonenumber': 'foobar'}) From 5812685b32c584f95f66fb4348661e260454959c Mon Sep 17 00:00:00 2001 From: Arman Jasuja Date: Fri, 5 Jul 2024 11:34:36 +0400 Subject: [PATCH 06/10] Fix default value for mask key --- logredactor/redacting_filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logredactor/redacting_filter.py b/logredactor/redacting_filter.py index cd7f3c2..a420cd1 100644 --- a/logredactor/redacting_filter.py +++ b/logredactor/redacting_filter.py @@ -11,7 +11,7 @@ class RedactingFilter(logging.Filter): 'processName', 'args', } - def __init__(self, patterns, default_mask='****', mask_keys=set()): + def __init__(self, patterns, default_mask='****', mask_keys={}): super(RedactingFilter, self).__init__() self._patterns = patterns self._default_mask = str(default_mask) From 4ebae38b53bb316eee50042cacccd6d5a31a1d53 Mon Sep 17 00:00:00 2001 From: Arman Jasuja Date: Fri, 5 Jul 2024 11:56:35 +0400 Subject: [PATCH 07/10] Review comment fix --- logredactor/redacting_filter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/logredactor/redacting_filter.py b/logredactor/redacting_filter.py index a420cd1..3c5803c 100644 --- a/logredactor/redacting_filter.py +++ b/logredactor/redacting_filter.py @@ -11,7 +11,9 @@ class RedactingFilter(logging.Filter): 'processName', 'args', } - def __init__(self, patterns, default_mask='****', mask_keys={}): + def __init__(self, patterns, default_mask='****', mask_keys=None): + if mask_keys is None: + mask_keys = {} super(RedactingFilter, self).__init__() self._patterns = patterns self._default_mask = str(default_mask) From f318af8d04151323ec337927be256f5009183c38 Mon Sep 17 00:00:00 2001 From: Arman Jasuja <114303348+armurox@users.noreply.github.com> Date: Fri, 5 Jul 2024 14:48:36 +0400 Subject: [PATCH 08/10] Update logredactor/redacting_filter.py Co-authored-by: Danny Crasto --- logredactor/redacting_filter.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/logredactor/redacting_filter.py b/logredactor/redacting_filter.py index 3c5803c..eeb0dd9 100644 --- a/logredactor/redacting_filter.py +++ b/logredactor/redacting_filter.py @@ -12,8 +12,6 @@ class RedactingFilter(logging.Filter): } def __init__(self, patterns, default_mask='****', mask_keys=None): - if mask_keys is None: - mask_keys = {} super(RedactingFilter, self).__init__() self._patterns = patterns self._default_mask = str(default_mask) From 6996b70174e76a3b6864e1b63cca858ce3d7b2cb Mon Sep 17 00:00:00 2001 From: Arman Jasuja <114303348+armurox@users.noreply.github.com> Date: Fri, 5 Jul 2024 14:48:44 +0400 Subject: [PATCH 09/10] Update logredactor/redacting_filter.py Co-authored-by: Danny Crasto --- logredactor/redacting_filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logredactor/redacting_filter.py b/logredactor/redacting_filter.py index eeb0dd9..fb8a879 100644 --- a/logredactor/redacting_filter.py +++ b/logredactor/redacting_filter.py @@ -15,7 +15,7 @@ def __init__(self, patterns, default_mask='****', mask_keys=None): super(RedactingFilter, self).__init__() self._patterns = patterns self._default_mask = str(default_mask) - self._mask_keys = set(mask_keys) + self._mask_keys = set(mask_keys or {}) def filter(self, record): d = vars(record) From 69d328ea65d3bfd4b21586b489fea23671984636 Mon Sep 17 00:00:00 2001 From: Arman Jasuja <114303348+armurox@users.noreply.github.com> Date: Fri, 5 Jul 2024 14:48:48 +0400 Subject: [PATCH 10/10] Update logredactor/redacting_filter.py Co-authored-by: Danny Crasto --- logredactor/redacting_filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logredactor/redacting_filter.py b/logredactor/redacting_filter.py index fb8a879..3fc2c36 100644 --- a/logredactor/redacting_filter.py +++ b/logredactor/redacting_filter.py @@ -42,7 +42,7 @@ def redact(self, content, key=None): for i, v in enumerate(content): content[i] = self.redact(v) - elif key in self._mask_keys: + elif key and key in self._mask_keys: content = self._default_mask else: