From 3c206f0777f0f3343746c9cc30c62fc75931d00b Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Sat, 27 Jun 2026 23:16:34 -0400 Subject: [PATCH 01/11] Allow per-recognizer analyzer tuning from YAML --- CHANGELOG.md | 1 + docs/analyzer/analyzer_engine_provider.md | 20 +- docs/tutorial/08_no_code.md | 8 + .../presidio_analyzer/analyzer_engine.py | 62 +++++- .../analyzer_engine_provider.py | 4 + .../conf/default_analyzer_full.yaml | 6 + .../input_validation/schemas.py | 63 +++++++ .../tests/test_analyzer_engine.py | 176 ++++++++++++++++++ .../tests/test_analyzer_engine_provider.py | 48 ++++- .../tests/test_configuration_validator.py | 79 +++++++- 10 files changed, 450 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9ebd1bb44..c23f99077d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ All notable changes to this project will be documented in this file. ### Analyzer #### Added +- Analyzer YAML now accepts `recognizer_score_thresholds` for per-recognizer and per-entity score cutoffs, keyed by `RecognizerResult.RECOGNIZER_NAME_KEY`, while preserving the global `default_score_threshold` fallback when no override matches. - UK Driving Licence Number (`UK_DRIVING_LICENCE`) recognizer with pattern matching and context support (#1857) (Thanks @tee-jagz) - German PII recognizers for `DE_TAX_ID`, `DE_TAX_NUMBER`, `DE_PASSPORT`, `DE_ID_CARD`, `DE_SOCIAL_SECURITY`, `DE_HEALTH_INSURANCE`, `DE_KFZ`, `DE_HANDELSREGISTER`, and `DE_PLZ`; all are disabled by default (#1909) (Thanks @MvdB) - `SE_PERSONNUMMER` recognizer for Swedish personal identity and coordination numbers, plus Swedish Organisationsnummer recognition; both are disabled by default (#1912, #1918) (Thanks @goveebee) diff --git a/docs/analyzer/analyzer_engine_provider.md b/docs/analyzer/analyzer_engine_provider.md index 68dd2b0ae6..346f0537a6 100644 --- a/docs/analyzer/analyzer_engine_provider.md +++ b/docs/analyzer/analyzer_engine_provider.md @@ -70,10 +70,22 @@ recognizer_registry: The configuration file contains the following parameters: - - `supported_languages`: A list of supported languages that the analyzer will support. - - `default_score_threshold`: A score that determines the minimal threshold for detection. - - `nlp_configuration`: Configuration given to the NLP engine which will detect the PIIs and extract features for the downstream logic. - - `recognizer_registry`: All the recognizers that will be used by the analyzer. +- `supported_languages`: A list of supported languages that the analyzer will support. +- `default_score_threshold`: A score that determines the minimal threshold for detection. +- `recognizer_score_thresholds`: Optional per-recognizer thresholds keyed by the recognizer name from `RecognizerResult.RECOGNIZER_NAME_KEY`. Use `default` for a recognizer-wide fallback, then add entity names for overrides. +- `nlp_configuration`: Configuration given to the NLP engine which will detect the PIIs and extract features for the downstream logic. +- `recognizer_registry`: All the recognizers that will be used by the analyzer. + +Example: + +```yaml +recognizer_score_thresholds: + CreditCardRecognizer: + default: 0.4 + CREDIT_CARD: 0.7 + SpacyRecognizer: + PERSON: 0.6 +``` !!! note "Note" diff --git a/docs/tutorial/08_no_code.md b/docs/tutorial/08_no_code.md index 1312f6135f..819e30ee92 100644 --- a/docs/tutorial/08_no_code.md +++ b/docs/tutorial/08_no_code.md @@ -39,9 +39,17 @@ supported_languages: - en - es default_score_threshold: 0.4 +recognizer_score_thresholds: + CreditCardRecognizer: + default: 0.4 + CREDIT_CARD: 0.7 + SpacyRecognizer: + PERSON: 0.6 """ ``` +Use this when a recognizer needs a lower or higher cutoff than the global `default_score_threshold`, with entity-specific overrides taking priority over the recognizer default. + ### Recognizer Registry parameters ([default file](https://github.com/data-privacy-stack/presidio/blob/main/presidio-analyzer/presidio_analyzer/conf/default_recognizers.yaml)) diff --git a/presidio-analyzer/presidio_analyzer/analyzer_engine.py b/presidio-analyzer/presidio_analyzer/analyzer_engine.py index b68d24d6ab..fdc69f2a89 100644 --- a/presidio-analyzer/presidio_analyzer/analyzer_engine.py +++ b/presidio-analyzer/presidio_analyzer/analyzer_engine.py @@ -2,7 +2,7 @@ import logging import os from collections import Counter -from typing import List, Optional +from typing import Dict, List, Optional import regex as re @@ -56,6 +56,7 @@ def __init__( default_score_threshold: float = 0, supported_languages: List[str] = None, context_aware_enhancer: Optional[ContextAwareEnhancer] = None, + recognizer_score_thresholds: Optional[Dict[str, Dict[str, float]]] = None, ): if not supported_languages: supported_languages = ["en"] @@ -102,6 +103,9 @@ def __init__( self.log_decision_process = log_decision_process self.default_score_threshold = default_score_threshold + self.recognizer_score_thresholds = self.__normalize_recognizer_score_thresholds( + recognizer_score_thresholds + ) if not context_aware_enhancer: logger.debug( @@ -340,11 +344,65 @@ def __remove_low_scores( :return: List[RecognizerResult] """ if score_threshold is None: - score_threshold = self.default_score_threshold + return [ + result + for result in results + if result.score >= self.__get_result_score_threshold(result) + ] new_results = [result for result in results if result.score >= score_threshold] return new_results + @staticmethod + def __normalize_recognizer_score_thresholds( + recognizer_score_thresholds: Optional[Dict[str, Dict[str, float]]] + ) -> Dict[str, Dict[str, float]]: + """Normalize shorthand threshold values into the nested mapping shape.""" + normalized_thresholds: Dict[str, Dict[str, float]] = {} + + for recognizer_name, recognizer_thresholds in ( + recognizer_score_thresholds or {} + ).items(): + if isinstance(recognizer_thresholds, (int, float)) and not isinstance( + recognizer_thresholds, bool + ): + normalized_thresholds[recognizer_name] = { + "default": recognizer_thresholds + } + elif isinstance(recognizer_thresholds, dict): + normalized_thresholds[recognizer_name] = recognizer_thresholds + else: + raise ValueError( + "recognizer_score_thresholds values must be a number " + "or a dictionary" + ) + + return normalized_thresholds + + def __get_result_score_threshold(self, result: RecognizerResult) -> float: + """Resolve the threshold to apply for a single recognizer result.""" + if not self.recognizer_score_thresholds: + return self.default_score_threshold + + metadata = result.recognition_metadata or {} + recognizer_name = metadata.get(RecognizerResult.RECOGNIZER_NAME_KEY) + if not recognizer_name: + return self.default_score_threshold + + recognizer_thresholds = self.recognizer_score_thresholds.get(recognizer_name) + if not recognizer_thresholds: + return self.default_score_threshold + + entity_threshold = recognizer_thresholds.get(result.entity_type) + if entity_threshold is not None: + return entity_threshold + + recognizer_default_threshold = recognizer_thresholds.get("default") + if recognizer_default_threshold is not None: + return recognizer_default_threshold + + return self.default_score_threshold + @staticmethod def _remove_allow_list( results: List[RecognizerResult], diff --git a/presidio-analyzer/presidio_analyzer/analyzer_engine_provider.py b/presidio-analyzer/presidio_analyzer/analyzer_engine_provider.py index d03430a1f9..49f0d9ac62 100644 --- a/presidio-analyzer/presidio_analyzer/analyzer_engine_provider.py +++ b/presidio-analyzer/presidio_analyzer/analyzer_engine_provider.py @@ -88,6 +88,9 @@ def create_engine(self) -> AnalyzerEngine: nlp_engine = self._load_nlp_engine() supported_languages = self.configuration.get("supported_languages", ["en"]) default_score_threshold = self.configuration.get("default_score_threshold", 0) + recognizer_score_thresholds = self.configuration.get( + "recognizer_score_thresholds", {} + ) registry = self._load_recognizer_registry( supported_languages=supported_languages, nlp_engine=nlp_engine @@ -98,6 +101,7 @@ def create_engine(self) -> AnalyzerEngine: registry=registry, supported_languages=supported_languages, default_score_threshold=default_score_threshold, + recognizer_score_thresholds=recognizer_score_thresholds, ) return analyzer diff --git a/presidio-analyzer/presidio_analyzer/conf/default_analyzer_full.yaml b/presidio-analyzer/presidio_analyzer/conf/default_analyzer_full.yaml index 15c89d7b92..c5be4e13d5 100644 --- a/presidio-analyzer/presidio_analyzer/conf/default_analyzer_full.yaml +++ b/presidio-analyzer/presidio_analyzer/conf/default_analyzer_full.yaml @@ -1,6 +1,12 @@ supported_languages: - en default_score_threshold: 0 +# recognizer_score_thresholds: +# CreditCardRecognizer: +# default: 0.4 +# CREDIT_CARD: 0.7 +# SpacyRecognizer: +# PERSON: 0.6 nlp_configuration: nlp_engine_name: spacy models: diff --git a/presidio-analyzer/presidio_analyzer/input_validation/schemas.py b/presidio-analyzer/presidio_analyzer/input_validation/schemas.py index d5c1050d49..794c66e191 100644 --- a/presidio-analyzer/presidio_analyzer/input_validation/schemas.py +++ b/presidio-analyzer/presidio_analyzer/input_validation/schemas.py @@ -44,6 +44,61 @@ def validate_score_threshold(threshold: float) -> float: ) return threshold + @staticmethod + def validate_recognizer_score_thresholds( + thresholds: Dict[str, Any], + ) -> Dict[str, Dict[str, float]]: + """Validate recognizer-specific score thresholds.""" + if not isinstance(thresholds, dict): + raise ValueError("recognizer_score_thresholds must be a dictionary") + + validated_thresholds: Dict[str, Dict[str, float]] = {} + for recognizer_name, recognizer_thresholds in thresholds.items(): + if ( + not isinstance(recognizer_name, str) + or recognizer_name.strip() != recognizer_name + or not recognizer_name + ): + raise ValueError( + "recognizer_score_thresholds keys must be non-empty strings" + ) + + if isinstance(recognizer_thresholds, (int, float)) and not isinstance( + recognizer_thresholds, bool + ): + validated_thresholds[recognizer_name] = { + "default": ConfigurationValidator.validate_score_threshold( + recognizer_thresholds + ) + } + continue + + if not isinstance(recognizer_thresholds, dict): + raise ValueError( + f"Thresholds for recognizer '{recognizer_name}' " + f"must be a dictionary" + ) + + validated_recognizer_thresholds: Dict[str, float] = {} + for threshold_name, threshold_value in recognizer_thresholds.items(): + if ( + not isinstance(threshold_name, str) + or threshold_name.strip() != threshold_name + or not threshold_name + ): + raise ValueError( + "recognizer_score_thresholds nested keys must be " + "non-empty strings" + ) + + validated_recognizer_thresholds[threshold_name] = ( + ConfigurationValidator.validate_score_threshold(threshold_value) + ) + + validated_thresholds[recognizer_name] = validated_recognizer_thresholds + + return validated_thresholds + @staticmethod def validate_nlp_configuration(config: Dict[str, Any]) -> Dict[str, Any]: """Validate NLP configuration structure. @@ -112,6 +167,7 @@ def validate_analyzer_configuration(config: Dict[str, Any]) -> Dict[str, Any]: valid_keys = { "supported_languages", "default_score_threshold", + "recognizer_score_thresholds", "nlp_configuration", "recognizer_registry", } @@ -135,6 +191,13 @@ def validate_analyzer_configuration(config: Dict[str, Any]) -> Dict[str, Any]: config["default_score_threshold"] ) + if "recognizer_score_thresholds" in config: + config["recognizer_score_thresholds"] = ( + ConfigurationValidator.validate_recognizer_score_thresholds( + config["recognizer_score_thresholds"] + ) + ) + # Validate nested configurations if "nlp_configuration" in config: ConfigurationValidator.validate_nlp_configuration( diff --git a/presidio-analyzer/tests/test_analyzer_engine.py b/presidio-analyzer/tests/test_analyzer_engine.py index 9e21e93c43..2be93bf1b2 100644 --- a/presidio-analyzer/tests/test_analyzer_engine.py +++ b/presidio-analyzer/tests/test_analyzer_engine.py @@ -1,3 +1,5 @@ +# ruff: noqa: D103,E501,W291 + import copy import re from abc import ABC @@ -57,6 +59,76 @@ def unit_test_guid(): return "00000000-0000-0000-0000-000000000000" +class ThresholdRecognizer(EntityRecognizer, ABC): + """Static recognizer used to exercise score threshold precedence.""" + + def __init__(self): + """Seed fixed results for threshold precedence assertions.""" + self._results = [ + RecognizerResult("PERSON", 0, 4, 0.55), + RecognizerResult("CREDIT_CARD", 5, 9, 0.65), + RecognizerResult("URL", 10, 13, 0.75), + RecognizerResult("DATE_TIME", 14, 18, 0.9), + ] + super().__init__( + supported_entities=["PERSON", "CREDIT_CARD", "URL", "DATE_TIME"], + name="ThresholdRecognizer", + ) + + def load(self): + """Keep the test recognizer lightweight.""" + return None + + def analyze(self, text: str, entities: List[str], nlp_artifacts: NlpArtifacts): + """Return deterministic results for threshold filtering tests.""" + return [ + RecognizerResult( + result.entity_type, result.start, result.end, result.score + ) + for result in self._results + ] + + +class FallbackRecognizer(EntityRecognizer, ABC): + """Recognizer that exercises the default fallback threshold path.""" + + def __init__(self): + """Seed a result that only the global default should filter.""" + self._results = [RecognizerResult("PHONE_NUMBER", 20, 26, 0.75)] + super().__init__( + supported_entities=["PHONE_NUMBER"], + name="FallbackRecognizer", + ) + + def load(self): + """Keep the test recognizer lightweight.""" + return None + + def analyze(self, text: str, entities: List[str], nlp_artifacts: NlpArtifacts): + """Return deterministic results for threshold filtering tests.""" + return [ + RecognizerResult( + result.entity_type, result.start, result.end, result.score + ) + for result in self._results + ] + + +def _build_threshold_analyzer( + recognizer_score_thresholds=None, default_score_threshold=0.8 +): + """Create an analyzer with deterministic recognizers for threshold tests.""" + registry = RecognizerRegistry() + registry.add_recognizer(ThresholdRecognizer()) + registry.add_recognizer(FallbackRecognizer()) + return AnalyzerEngine( + registry=registry, + nlp_engine=NlpEngineMock(), + default_score_threshold=default_score_threshold, + recognizer_score_thresholds=recognizer_score_thresholds, + ) + + def test_simple(): dic = { "text": "John Smith drivers license is AC432223", @@ -558,6 +630,110 @@ def test_when_default_threshold_is_zero_then_all_results_pass( assert len(results) == 2 +def test_recognizer_threshold_precedence(): + """Recognizer-specific and entity-specific thresholds should win.""" + analyzer_engine = _build_threshold_analyzer( + recognizer_score_thresholds={ + "ThresholdRecognizer": { + "default": 0.4, + "PERSON": 0.6, + "CREDIT_CARD": 0.7, + } + } + ) + + results = analyzer_engine.analyze( + text="Threshold config", + language="en", + entities=["PERSON", "CREDIT_CARD", "URL", "DATE_TIME", "PHONE_NUMBER"], + ) + + scores = {result.entity_type: result.score for result in results} + + assert scores == {"URL": 0.75, "DATE_TIME": 0.9} + + +def test_explicit_score_threshold_overrides_config(): + """A request-level score threshold should override config thresholds.""" + analyzer_engine = _build_threshold_analyzer( + recognizer_score_thresholds={ + "ThresholdRecognizer": { + "default": 0.4, + "PERSON": 0.6, + "CREDIT_CARD": 0.7, + } + } + ) + + results = analyzer_engine.analyze( + text="Threshold config", + language="en", + entities=["PERSON", "CREDIT_CARD", "URL", "DATE_TIME", "PHONE_NUMBER"], + score_threshold=0.8, + ) + + scores = {result.entity_type: result.score for result in results} + + assert scores == {"DATE_TIME": 0.9} + + +def test_numeric_threshold_shorthand_applies(): + """Numeric shorthand should behave like a recognizer default threshold.""" + class NumericThresholdRecognizer(EntityRecognizer, ABC): + """Recognizer with a single score below the global default.""" + + def __init__(self): + """Set up the recognizer metadata for threshold lookup.""" + super().__init__( + supported_entities=["URL"], + name="NumericThresholdRecognizer", + ) + + def load(self): + """Keep the test recognizer lightweight.""" + return None + + def analyze( + self, text: str, entities: List[str], nlp_artifacts: NlpArtifacts + ): + """Return a single deterministic result.""" + return [RecognizerResult("URL", 0, 8, 0.75)] + + registry = RecognizerRegistry() + registry.add_recognizer(NumericThresholdRecognizer()) + + analyzer_engine = AnalyzerEngine( + registry=registry, + nlp_engine=NlpEngineMock(), + default_score_threshold=0.8, + recognizer_score_thresholds={"NumericThresholdRecognizer": 0.4}, + ) + + results = analyzer_engine.analyze( + text="bing.com", + language="en", + entities=["URL"], + ) + + assert len(results) == 1 + assert results[0].entity_type == "URL" + + +def test_empty_recognizer_thresholds_use_default(): + """An empty threshold map should keep the global default behavior.""" + analyzer_engine = _build_threshold_analyzer(recognizer_score_thresholds={}) + + results = analyzer_engine.analyze( + text="Threshold config", + language="en", + entities=["PERSON", "CREDIT_CARD", "URL", "DATE_TIME", "PHONE_NUMBER"], + ) + + scores = {result.entity_type: result.score for result in results} + + assert scores == {"DATE_TIME": 0.9} + + def test_when_get_supported_fields_then_return_all_languages( analyzer_engine_simple, unit_test_guid ): diff --git a/presidio-analyzer/tests/test_analyzer_engine_provider.py b/presidio-analyzer/tests/test_analyzer_engine_provider.py index ed58e4cfaf..2547e1935b 100644 --- a/presidio-analyzer/tests/test_analyzer_engine_provider.py +++ b/presidio-analyzer/tests/test_analyzer_engine_provider.py @@ -1,12 +1,15 @@ +# ruff: noqa: D103,D205,E501,F541,F841,W293 + import re from pathlib import Path from typing import List from unittest.mock import patch -from presidio_analyzer import AnalyzerEngineProvider, RecognizerResult, PatternRecognizer -from presidio_analyzer.nlp_engine import SpacyNlpEngine, NlpArtifacts - - +import pytest +import yaml +from install_nlp_models import _install_models_from_nlp_config, install_models +from presidio_analyzer import AnalyzerEngineProvider, RecognizerResult +from presidio_analyzer.nlp_engine import NlpArtifacts, SpacyNlpEngine from presidio_analyzer.predefined_recognizers import ( AzureAILanguageRecognizer, CreditCardRecognizer, @@ -14,10 +17,6 @@ StanzaRecognizer, ) -import pytest - -from install_nlp_models import install_models, _install_models_from_nlp_config - def get_full_paths(analyzer_yaml, nlp_engine_yaml=None, recognizer_registry_yaml=None): this_path = Path(__file__).parent.absolute() @@ -38,6 +37,7 @@ def test_analyzer_engine_provider_default_configuration(mandatory_recognizers): engine.registry.global_regex_flags == re.DOTALL | re.MULTILINE | re.IGNORECASE ) assert engine.default_score_threshold == 0 + assert engine.recognizer_score_thresholds == {} names = [recognizer.name for recognizer in engine.registry.recognizers] for predefined_recognizer in mandatory_recognizers: assert predefined_recognizer in names @@ -96,11 +96,41 @@ def test_analyzer_engine_provider_configuration_file(): assert engine.nlp_engine.engine_name == "spacy" +def test_analyzer_engine_provider_passes_recognizer_score_thresholds(tmp_path): + """Numeric shorthand should normalize and apply through the provider.""" + analyzer_yaml, _, _ = get_full_paths("conf/test_analyzer_engine.yaml") + + with open(analyzer_yaml) as file: + configuration = yaml.safe_load(file) + + configuration["default_score_threshold"] = 0.9 + configuration["recognizer_score_thresholds"] = {"CreditCardRecognizer": 0.4} + + threshold_yaml = tmp_path / "analyzer_with_thresholds.yaml" + threshold_yaml.write_text(yaml.safe_dump(configuration, sort_keys=False)) + + provider = AnalyzerEngineProvider(threshold_yaml) + engine = provider.create_engine() + + assert engine.recognizer_score_thresholds == { + "CreditCardRecognizer": {"default": 0.4} + } + + results = engine.analyze( + text=" Credit card: 4095-2609-9393-4932", + language="en", + entities=["CREDIT_CARD"], + ) + + assert len(results) == 1 + + def test_analyzer_engine_provider_defaults(mandatory_recognizers): provider = AnalyzerEngineProvider() engine = provider.create_engine() assert engine.supported_languages == ["en"] assert engine.default_score_threshold == 0 + assert engine.recognizer_score_thresholds == {} recognizer_registry = engine.registry assert ( recognizer_registry.global_regex_flags @@ -366,8 +396,8 @@ def test_analyzer_engine_provider_get_configuration_with_nonexistent_file(): def test_analyzer_engine_provider_get_configuration_with_invalid_yaml(): """Test get_configuration handles invalid YAML gracefully.""" - import tempfile import os + import tempfile # Create a temporary file with invalid YAML with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f: diff --git a/presidio-analyzer/tests/test_configuration_validator.py b/presidio-analyzer/tests/test_configuration_validator.py index ad5ed5687a..6916b12a68 100644 --- a/presidio-analyzer/tests/test_configuration_validator.py +++ b/presidio-analyzer/tests/test_configuration_validator.py @@ -1,9 +1,9 @@ """Tests for the Pydantic-based validation system using existing adapted classes.""" -import pytest +# ruff: noqa: E501 +import pytest from presidio_analyzer.input_validation import ConfigurationValidator - # ========== Language Code Validation Tests ========== def test_validate_language_codes_valid(): @@ -318,6 +318,81 @@ def test_configuration_validator_analyzer_config_valid(): assert validated == valid_config +def test_configuration_validator_analyzer_config_valid_with_recognizer_score_thresholds(): + """Validator should accept numeric shorthand and nested overrides.""" + valid_config = { + "supported_languages": ["en"], + "default_score_threshold": 0.5, + "recognizer_score_thresholds": { + "CreditCardRecognizer": 0.4, + "SpacyRecognizer": { + "PERSON": 0.6, + }, + }, + } + + validated = ConfigurationValidator.validate_analyzer_configuration(valid_config) + assert validated == { + "supported_languages": ["en"], + "default_score_threshold": 0.5, + "recognizer_score_thresholds": { + "CreditCardRecognizer": {"default": 0.4}, + "SpacyRecognizer": {"PERSON": 0.6}, + }, + } + + +@pytest.mark.parametrize( + "invalid_config,expected_message", + [ + ( + { + "supported_languages": ["en"], + "recognizer_score_thresholds": { + 123: {"default": 0.4}, + }, + }, + "non-empty strings", + ), + ( + { + "supported_languages": ["en"], + "recognizer_score_thresholds": { + "CreditCardRecognizer": ["default", 0.4], + }, + }, + "must be a dictionary", + ), + ( + { + "supported_languages": ["en"], + "recognizer_score_thresholds": { + "CreditCardRecognizer": {"": 0.4}, + }, + }, + "nested keys must be non-empty strings", + ), + ( + { + "supported_languages": ["en"], + "recognizer_score_thresholds": { + "CreditCardRecognizer": {"default": 1.5}, + }, + }, + "must be between 0.0 and 1.0", + ), + ], +) +def test_configuration_validator_analyzer_config_invalid_recognizer_score_thresholds( + invalid_config, expected_message +): + """Validator should reject malformed threshold overrides.""" + with pytest.raises(ValueError) as exc_info: + ConfigurationValidator.validate_analyzer_configuration(invalid_config) + + assert expected_message in str(exc_info.value) + + def test_analyzer_config_minimal(): """Test minimal valid analyzer configuration.""" valid_config = { From e458cd45d4a02b3a4a67b78a9bd4ccace1c0386e Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Sat, 27 Jun 2026 23:32:19 -0400 Subject: [PATCH 02/11] Keep boolean flags from slipping into threshold config --- .../presidio_analyzer/analyzer_engine.py | 30 ++++++++++++++++++- .../input_validation/schemas.py | 6 ++++ .../tests/test_analyzer_engine.py | 8 +++++ .../tests/test_configuration_validator.py | 9 ++++++ 4 files changed, 52 insertions(+), 1 deletion(-) diff --git a/presidio-analyzer/presidio_analyzer/analyzer_engine.py b/presidio-analyzer/presidio_analyzer/analyzer_engine.py index fdc69f2a89..e248798a8d 100644 --- a/presidio-analyzer/presidio_analyzer/analyzer_engine.py +++ b/presidio-analyzer/presidio_analyzer/analyzer_engine.py @@ -363,6 +363,15 @@ def __normalize_recognizer_score_thresholds( for recognizer_name, recognizer_thresholds in ( recognizer_score_thresholds or {} ).items(): + if ( + not isinstance(recognizer_name, str) + or recognizer_name.strip() != recognizer_name + or not recognizer_name + ): + raise ValueError( + "recognizer_score_thresholds keys must be non-empty strings" + ) + if isinstance(recognizer_thresholds, (int, float)) and not isinstance( recognizer_thresholds, bool ): @@ -370,7 +379,26 @@ def __normalize_recognizer_score_thresholds( "default": recognizer_thresholds } elif isinstance(recognizer_thresholds, dict): - normalized_thresholds[recognizer_name] = recognizer_thresholds + normalized_thresholds[recognizer_name] = {} + for threshold_name, threshold_value in recognizer_thresholds.items(): + if ( + not isinstance(threshold_name, str) + or threshold_name.strip() != threshold_name + or not threshold_name + ): + raise ValueError( + "recognizer_score_thresholds nested keys must be " + "non-empty strings" + ) + if not isinstance(threshold_value, (int, float)) or isinstance( + threshold_value, bool + ): + raise ValueError( + "recognizer_score_thresholds values must be numeric" + ) + normalized_thresholds[recognizer_name][ + threshold_name + ] = threshold_value else: raise ValueError( "recognizer_score_thresholds values must be a number " diff --git a/presidio-analyzer/presidio_analyzer/input_validation/schemas.py b/presidio-analyzer/presidio_analyzer/input_validation/schemas.py index 794c66e191..d8d5909395 100644 --- a/presidio-analyzer/presidio_analyzer/input_validation/schemas.py +++ b/presidio-analyzer/presidio_analyzer/input_validation/schemas.py @@ -90,6 +90,12 @@ def validate_recognizer_score_thresholds( "recognizer_score_thresholds nested keys must be " "non-empty strings" ) + if not isinstance(threshold_value, (int, float)) or isinstance( + threshold_value, bool + ): + raise ValueError( + "recognizer_score_thresholds values must be numeric" + ) validated_recognizer_thresholds[threshold_name] = ( ConfigurationValidator.validate_score_threshold(threshold_value) diff --git a/presidio-analyzer/tests/test_analyzer_engine.py b/presidio-analyzer/tests/test_analyzer_engine.py index 2be93bf1b2..edb612efc7 100644 --- a/presidio-analyzer/tests/test_analyzer_engine.py +++ b/presidio-analyzer/tests/test_analyzer_engine.py @@ -719,6 +719,14 @@ def analyze( assert results[0].entity_type == "URL" +def test_direct_threshold_config_rejects_boolean_values(): + """Direct constructor config should reject boolean threshold values.""" + with pytest.raises(ValueError, match="values must be numeric"): + _build_threshold_analyzer( + recognizer_score_thresholds={"ThresholdRecognizer": {"PERSON": True}} + ) + + def test_empty_recognizer_thresholds_use_default(): """An empty threshold map should keep the global default behavior.""" analyzer_engine = _build_threshold_analyzer(recognizer_score_thresholds={}) diff --git a/presidio-analyzer/tests/test_configuration_validator.py b/presidio-analyzer/tests/test_configuration_validator.py index 6916b12a68..f8fa319211 100644 --- a/presidio-analyzer/tests/test_configuration_validator.py +++ b/presidio-analyzer/tests/test_configuration_validator.py @@ -381,6 +381,15 @@ def test_configuration_validator_analyzer_config_valid_with_recognizer_score_thr }, "must be between 0.0 and 1.0", ), + ( + { + "supported_languages": ["en"], + "recognizer_score_thresholds": { + "CreditCardRecognizer": {"default": True}, + }, + }, + "values must be numeric", + ), ], ) def test_configuration_validator_analyzer_config_invalid_recognizer_score_thresholds( From 2c5cb34aec29cdf204508f9d50dcf9876700beac Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Sat, 27 Jun 2026 23:36:45 -0400 Subject: [PATCH 03/11] Keep analyzer docs aligned with the new threshold contract --- presidio-analyzer/presidio_analyzer/analyzer_engine.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/presidio-analyzer/presidio_analyzer/analyzer_engine.py b/presidio-analyzer/presidio_analyzer/analyzer_engine.py index e248798a8d..ff95b0a82e 100644 --- a/presidio-analyzer/presidio_analyzer/analyzer_engine.py +++ b/presidio-analyzer/presidio_analyzer/analyzer_engine.py @@ -40,6 +40,8 @@ class AnalyzerEngine: defines whether the decision process within the analyzer should be logged or not. :param default_score_threshold: Minimum confidence value for detected entities to be returned + :param recognizer_score_thresholds: Optional recognizer and entity-specific + score thresholds applied when no request-level score threshold is provided. :param supported_languages: List of possible languages this engine could be run on. Used for loading the right NLP models and recognizers for these languages. :param context_aware_enhancer: instance of type ContextAwareEnhancer for enhancing From 08da64e062d861193ac327c7bcf77f4e6064ce88 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Sat, 27 Jun 2026 23:43:01 -0400 Subject: [PATCH 04/11] Keep direct threshold config aligned with YAML validation --- .../presidio_analyzer/analyzer_engine.py | 21 ++++++++++++++----- .../tests/test_analyzer_engine.py | 14 +++++++++++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/presidio-analyzer/presidio_analyzer/analyzer_engine.py b/presidio-analyzer/presidio_analyzer/analyzer_engine.py index ff95b0a82e..70a501bda2 100644 --- a/presidio-analyzer/presidio_analyzer/analyzer_engine.py +++ b/presidio-analyzer/presidio_analyzer/analyzer_engine.py @@ -15,6 +15,7 @@ ContextAwareEnhancer, LemmaContextAwareEnhancer, ) +from presidio_analyzer.input_validation import ConfigurationValidator from presidio_analyzer.nlp_engine import NlpArtifacts, NlpEngine, NlpEngineProvider from presidio_analyzer.recognizer_registry import ( RecognizerRegistry, @@ -360,11 +361,17 @@ def __normalize_recognizer_score_thresholds( recognizer_score_thresholds: Optional[Dict[str, Dict[str, float]]] ) -> Dict[str, Dict[str, float]]: """Normalize shorthand threshold values into the nested mapping shape.""" + if recognizer_score_thresholds is None: + return {} + if not isinstance(recognizer_score_thresholds, dict): + raise ValueError("recognizer_score_thresholds must be a dictionary") + normalized_thresholds: Dict[str, Dict[str, float]] = {} - for recognizer_name, recognizer_thresholds in ( - recognizer_score_thresholds or {} - ).items(): + for ( + recognizer_name, + recognizer_thresholds, + ) in recognizer_score_thresholds.items(): if ( not isinstance(recognizer_name, str) or recognizer_name.strip() != recognizer_name @@ -378,7 +385,9 @@ def __normalize_recognizer_score_thresholds( recognizer_thresholds, bool ): normalized_thresholds[recognizer_name] = { - "default": recognizer_thresholds + "default": ConfigurationValidator.validate_score_threshold( + recognizer_thresholds + ) } elif isinstance(recognizer_thresholds, dict): normalized_thresholds[recognizer_name] = {} @@ -400,7 +409,9 @@ def __normalize_recognizer_score_thresholds( ) normalized_thresholds[recognizer_name][ threshold_name - ] = threshold_value + ] = ConfigurationValidator.validate_score_threshold( + threshold_value + ) else: raise ValueError( "recognizer_score_thresholds values must be a number " diff --git a/presidio-analyzer/tests/test_analyzer_engine.py b/presidio-analyzer/tests/test_analyzer_engine.py index edb612efc7..b81394d510 100644 --- a/presidio-analyzer/tests/test_analyzer_engine.py +++ b/presidio-analyzer/tests/test_analyzer_engine.py @@ -727,6 +727,20 @@ def test_direct_threshold_config_rejects_boolean_values(): ) +def test_direct_threshold_config_rejects_out_of_range_values(): + """Direct constructor config should reject out-of-range threshold values.""" + with pytest.raises(ValueError, match="between 0.0 and 1.0"): + _build_threshold_analyzer( + recognizer_score_thresholds={"ThresholdRecognizer": {"PERSON": 1.5}} + ) + + +def test_direct_threshold_config_rejects_non_dict_top_level_values(): + """Direct constructor config should reject non-dictionary threshold payloads.""" + with pytest.raises(ValueError, match="must be a dictionary"): + _build_threshold_analyzer(recognizer_score_thresholds=0.4) + + def test_empty_recognizer_thresholds_use_default(): """An empty threshold map should keep the global default behavior.""" analyzer_engine = _build_threshold_analyzer(recognizer_score_thresholds={}) From 5de75b890ad8ebd38c504cadc9300511aa0006da Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Sat, 27 Jun 2026 23:50:13 -0400 Subject: [PATCH 05/11] Preserve lenient recognizers when duplicate spans collide --- docs/analyzer/analyzer_engine_provider.md | 2 +- .../presidio_analyzer/analyzer_engine.py | 5 ++- .../tests/test_analyzer_engine.py | 45 +++++++++++++++++++ 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/docs/analyzer/analyzer_engine_provider.md b/docs/analyzer/analyzer_engine_provider.md index 346f0537a6..7724459e83 100644 --- a/docs/analyzer/analyzer_engine_provider.md +++ b/docs/analyzer/analyzer_engine_provider.md @@ -74,7 +74,7 @@ The configuration file contains the following parameters: - `default_score_threshold`: A score that determines the minimal threshold for detection. - `recognizer_score_thresholds`: Optional per-recognizer thresholds keyed by the recognizer name from `RecognizerResult.RECOGNIZER_NAME_KEY`. Use `default` for a recognizer-wide fallback, then add entity names for overrides. - `nlp_configuration`: Configuration given to the NLP engine which will detect the PIIs and extract features for the downstream logic. -- `recognizer_registry`: All the recognizers that will be used by the analyzer. +- `recognizer_registry`: All the recognizers that will be used by the analyzer. Example: diff --git a/presidio-analyzer/presidio_analyzer/analyzer_engine.py b/presidio-analyzer/presidio_analyzer/analyzer_engine.py index 70a501bda2..bcfe0135ec 100644 --- a/presidio-analyzer/presidio_analyzer/analyzer_engine.py +++ b/presidio-analyzer/presidio_analyzer/analyzer_engine.py @@ -261,9 +261,10 @@ def analyze( json.dumps([str(result.to_dict()) for result in results]), ) - # Remove duplicates or low score results - results = EntityRecognizer.remove_duplicates(results) + # Filter low-score results before deduplication so recognizer-specific + # thresholds do not get lost when duplicate spans collapse. results = self.__remove_low_scores(results, score_threshold) + results = EntityRecognizer.remove_duplicates(results) if allow_list: results = self._remove_allow_list( diff --git a/presidio-analyzer/tests/test_analyzer_engine.py b/presidio-analyzer/tests/test_analyzer_engine.py index b81394d510..4ab40c61c7 100644 --- a/presidio-analyzer/tests/test_analyzer_engine.py +++ b/presidio-analyzer/tests/test_analyzer_engine.py @@ -114,6 +114,22 @@ def analyze(self, text: str, entities: List[str], nlp_artifacts: NlpArtifacts): ] +class DuplicateThresholdRecognizer(EntityRecognizer, ABC): + """Recognizer that emits a duplicate span for threshold ordering tests.""" + + def __init__(self, name): + """Use the recognizer name as the threshold lookup key.""" + super().__init__(supported_entities=["PERSON"], name=name) + + def load(self): + """Keep the test recognizer lightweight.""" + return None + + def analyze(self, text: str, entities: List[str], nlp_artifacts: NlpArtifacts): + """Return a duplicate result with a configurable recognizer name.""" + return [RecognizerResult("PERSON", 0, 4, 0.5)] + + def _build_threshold_analyzer( recognizer_score_thresholds=None, default_score_threshold=0.8 ): @@ -741,6 +757,35 @@ def test_direct_threshold_config_rejects_non_dict_top_level_values(): _build_threshold_analyzer(recognizer_score_thresholds=0.4) +def test_duplicate_results_keep_recognizers_that_meet_their_thresholds(): + """Threshold filtering should run before duplicate collapse.""" + registry = RecognizerRegistry() + registry.add_recognizer(DuplicateThresholdRecognizer("StrictRecognizer")) + registry.add_recognizer(DuplicateThresholdRecognizer("LenientRecognizer")) + + analyzer_engine = AnalyzerEngine( + registry=registry, + nlp_engine=NlpEngineMock(), + default_score_threshold=0.0, + recognizer_score_thresholds={ + "StrictRecognizer": {"PERSON": 0.9}, + "LenientRecognizer": {"PERSON": 0.4}, + }, + ) + + results = analyzer_engine.analyze( + text="John", + language="en", + entities=["PERSON"], + ) + + assert len(results) == 1 + assert ( + results[0].recognition_metadata[RecognizerResult.RECOGNIZER_NAME_KEY] + == "LenientRecognizer" + ) + + def test_empty_recognizer_thresholds_use_default(): """An empty threshold map should keep the global default behavior.""" analyzer_engine = _build_threshold_analyzer(recognizer_score_thresholds={}) From 21606fc17b27c90214b675c6a6d992e625f3dd58 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Sun, 28 Jun 2026 00:03:00 -0400 Subject: [PATCH 06/11] Keep threshold validation errors tied to the config key --- .../input_validation/schemas.py | 29 ++++++++++++++++--- .../tests/test_configuration_validator.py | 4 +-- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/presidio-analyzer/presidio_analyzer/input_validation/schemas.py b/presidio-analyzer/presidio_analyzer/input_validation/schemas.py index d8d5909395..c7e10c9a07 100644 --- a/presidio-analyzer/presidio_analyzer/input_validation/schemas.py +++ b/presidio-analyzer/presidio_analyzer/input_validation/schemas.py @@ -67,15 +67,17 @@ def validate_recognizer_score_thresholds( recognizer_thresholds, bool ): validated_thresholds[recognizer_name] = { - "default": ConfigurationValidator.validate_score_threshold( - recognizer_thresholds + "default": ConfigurationValidator._validate_named_threshold( + recognizer_name=recognizer_name, + threshold_name="default", + threshold_value=recognizer_thresholds, ) } continue if not isinstance(recognizer_thresholds, dict): raise ValueError( - f"Thresholds for recognizer '{recognizer_name}' " + f"recognizer_score_thresholds for recognizer '{recognizer_name}' " f"must be a dictionary" ) @@ -98,13 +100,32 @@ def validate_recognizer_score_thresholds( ) validated_recognizer_thresholds[threshold_name] = ( - ConfigurationValidator.validate_score_threshold(threshold_value) + ConfigurationValidator._validate_named_threshold( + recognizer_name=recognizer_name, + threshold_name=threshold_name, + threshold_value=threshold_value, + ) ) validated_thresholds[recognizer_name] = validated_recognizer_thresholds return validated_thresholds + @staticmethod + def _validate_named_threshold( + recognizer_name: str, + threshold_name: str, + threshold_value: float, + ) -> float: + """Validate one named recognizer threshold and keep config context.""" + try: + return ConfigurationValidator.validate_score_threshold(threshold_value) + except ValueError as exc: + raise ValueError( + "recognizer_score_thresholds value for " + f"'{recognizer_name}.{threshold_name}' {exc}" + ) from exc + @staticmethod def validate_nlp_configuration(config: Dict[str, Any]) -> Dict[str, Any]: """Validate NLP configuration structure. diff --git a/presidio-analyzer/tests/test_configuration_validator.py b/presidio-analyzer/tests/test_configuration_validator.py index f8fa319211..499d8639f9 100644 --- a/presidio-analyzer/tests/test_configuration_validator.py +++ b/presidio-analyzer/tests/test_configuration_validator.py @@ -361,7 +361,7 @@ def test_configuration_validator_analyzer_config_valid_with_recognizer_score_thr "CreditCardRecognizer": ["default", 0.4], }, }, - "must be a dictionary", + "recognizer_score_thresholds", ), ( { @@ -379,7 +379,7 @@ def test_configuration_validator_analyzer_config_valid_with_recognizer_score_thr "CreditCardRecognizer": {"default": 1.5}, }, }, - "must be between 0.0 and 1.0", + "recognizer_score_thresholds", ), ( { From 94b050b5147337a7ab8fd38d0a6c45d27f841556 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Sun, 28 Jun 2026 00:16:26 -0400 Subject: [PATCH 07/11] Keep constructor threshold hints aligned with shared validation --- .../presidio_analyzer/analyzer_engine.py | 70 +++---------------- 1 file changed, 9 insertions(+), 61 deletions(-) diff --git a/presidio-analyzer/presidio_analyzer/analyzer_engine.py b/presidio-analyzer/presidio_analyzer/analyzer_engine.py index bcfe0135ec..0b57d027f7 100644 --- a/presidio-analyzer/presidio_analyzer/analyzer_engine.py +++ b/presidio-analyzer/presidio_analyzer/analyzer_engine.py @@ -2,7 +2,7 @@ import logging import os from collections import Counter -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Union import regex as re @@ -25,6 +25,7 @@ logger = logging.getLogger("presidio-analyzer") REGEX_TIMEOUT_SECONDS = int(os.environ.get("REGEX_TIMEOUT_SECONDS", 60)) +RecognizerScoreThresholds = Dict[str, Union[float, Dict[str, float]]] class AnalyzerEngine: """ @@ -59,7 +60,7 @@ def __init__( default_score_threshold: float = 0, supported_languages: List[str] = None, context_aware_enhancer: Optional[ContextAwareEnhancer] = None, - recognizer_score_thresholds: Optional[Dict[str, Dict[str, float]]] = None, + recognizer_score_thresholds: Optional[RecognizerScoreThresholds] = None, ): if not supported_languages: supported_languages = ["en"] @@ -106,8 +107,8 @@ def __init__( self.log_decision_process = log_decision_process self.default_score_threshold = default_score_threshold - self.recognizer_score_thresholds = self.__normalize_recognizer_score_thresholds( - recognizer_score_thresholds + self.recognizer_score_thresholds = ( + self.__normalize_recognizer_score_thresholds(recognizer_score_thresholds) ) if not context_aware_enhancer: @@ -359,67 +360,14 @@ def __remove_low_scores( @staticmethod def __normalize_recognizer_score_thresholds( - recognizer_score_thresholds: Optional[Dict[str, Dict[str, float]]] + recognizer_score_thresholds: Optional[RecognizerScoreThresholds] ) -> Dict[str, Dict[str, float]]: """Normalize shorthand threshold values into the nested mapping shape.""" if recognizer_score_thresholds is None: return {} - if not isinstance(recognizer_score_thresholds, dict): - raise ValueError("recognizer_score_thresholds must be a dictionary") - - normalized_thresholds: Dict[str, Dict[str, float]] = {} - - for ( - recognizer_name, - recognizer_thresholds, - ) in recognizer_score_thresholds.items(): - if ( - not isinstance(recognizer_name, str) - or recognizer_name.strip() != recognizer_name - or not recognizer_name - ): - raise ValueError( - "recognizer_score_thresholds keys must be non-empty strings" - ) - - if isinstance(recognizer_thresholds, (int, float)) and not isinstance( - recognizer_thresholds, bool - ): - normalized_thresholds[recognizer_name] = { - "default": ConfigurationValidator.validate_score_threshold( - recognizer_thresholds - ) - } - elif isinstance(recognizer_thresholds, dict): - normalized_thresholds[recognizer_name] = {} - for threshold_name, threshold_value in recognizer_thresholds.items(): - if ( - not isinstance(threshold_name, str) - or threshold_name.strip() != threshold_name - or not threshold_name - ): - raise ValueError( - "recognizer_score_thresholds nested keys must be " - "non-empty strings" - ) - if not isinstance(threshold_value, (int, float)) or isinstance( - threshold_value, bool - ): - raise ValueError( - "recognizer_score_thresholds values must be numeric" - ) - normalized_thresholds[recognizer_name][ - threshold_name - ] = ConfigurationValidator.validate_score_threshold( - threshold_value - ) - else: - raise ValueError( - "recognizer_score_thresholds values must be a number " - "or a dictionary" - ) - - return normalized_thresholds + return ConfigurationValidator.validate_recognizer_score_thresholds( + recognizer_score_thresholds + ) def __get_result_score_threshold(self, result: RecognizerResult) -> float: """Resolve the threshold to apply for a single recognizer result.""" From adac24d541810afa34dd557823b60757b8b1d763 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Thu, 9 Jul 2026 13:28:34 -0400 Subject: [PATCH 08/11] fix(analyzer): reject boolean score thresholds --- .../presidio_analyzer/analyzer_engine.py | 2 +- .../input_validation/schemas.py | 6 +++-- .../tests/test_configuration_validator.py | 22 +++++++++++++++++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/presidio-analyzer/presidio_analyzer/analyzer_engine.py b/presidio-analyzer/presidio_analyzer/analyzer_engine.py index 0b57d027f7..9b100dc0ef 100644 --- a/presidio-analyzer/presidio_analyzer/analyzer_engine.py +++ b/presidio-analyzer/presidio_analyzer/analyzer_engine.py @@ -15,7 +15,7 @@ ContextAwareEnhancer, LemmaContextAwareEnhancer, ) -from presidio_analyzer.input_validation import ConfigurationValidator +from presidio_analyzer.input_validation.schemas import ConfigurationValidator from presidio_analyzer.nlp_engine import NlpArtifacts, NlpEngine, NlpEngineProvider from presidio_analyzer.recognizer_registry import ( RecognizerRegistry, diff --git a/presidio-analyzer/presidio_analyzer/input_validation/schemas.py b/presidio-analyzer/presidio_analyzer/input_validation/schemas.py index c7e10c9a07..f620f96507 100644 --- a/presidio-analyzer/presidio_analyzer/input_validation/schemas.py +++ b/presidio-analyzer/presidio_analyzer/input_validation/schemas.py @@ -38,6 +38,8 @@ def validate_score_threshold(threshold: float) -> float: :param threshold: score threshold to validate. """ + if not isinstance(threshold, (int, float)) or isinstance(threshold, bool): + raise ValueError(f"Score threshold must be numeric, got: {threshold}") if not 0.0 <= threshold <= 1.0: raise ValueError( f"Score threshold must be between 0.0 and 1.0, got: {threshold}" @@ -221,8 +223,8 @@ def validate_analyzer_configuration(config: Dict[str, Any]) -> Dict[str, Any]: if "recognizer_score_thresholds" in config: config["recognizer_score_thresholds"] = ( ConfigurationValidator.validate_recognizer_score_thresholds( - config["recognizer_score_thresholds"] - ) + config["recognizer_score_thresholds"] + ) ) # Validate nested configurations diff --git a/presidio-analyzer/tests/test_configuration_validator.py b/presidio-analyzer/tests/test_configuration_validator.py index 499d8639f9..b3f753caa6 100644 --- a/presidio-analyzer/tests/test_configuration_validator.py +++ b/presidio-analyzer/tests/test_configuration_validator.py @@ -106,6 +106,14 @@ def test_validate_score_threshold_way_above(): assert "must be between 0.0 and 1.0" in str(exc_info.value) +@pytest.mark.parametrize("threshold", [True, False, "0.5", None, [], {}]) +def test_validate_score_threshold_non_numeric(threshold): + """Test score threshold rejects booleans and non-numeric values.""" + with pytest.raises(ValueError) as exc_info: + ConfigurationValidator.validate_score_threshold(threshold) + assert "must be numeric" in str(exc_info.value) + + # ========== NLP Configuration Validation Tests ========== def test_configuration_validator_nlp_config_valid(): @@ -444,6 +452,20 @@ def test_configuration_validator_analyzer_config_invalid_threshold(): assert "must be between 0.0 and 1.0" in str(exc_info.value) +@pytest.mark.parametrize("threshold", [True, "0.5"]) +def test_configuration_validator_analyzer_config_non_numeric_threshold(threshold): + """Test ConfigurationValidator rejects non-numeric score thresholds.""" + invalid_config = { + "supported_languages": ["en"], + "default_score_threshold": threshold, + } + + with pytest.raises(ValueError) as exc_info: + ConfigurationValidator.validate_analyzer_configuration(invalid_config) + + assert "must be numeric" in str(exc_info.value) + + def test_analyzer_config_not_dict(): """Test analyzer configuration that is not a dictionary.""" with pytest.raises(ValueError) as exc_info: From 2c4918252103ae51eafcddd6b3f6a17b3d19a1a0 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Fri, 10 Jul 2026 02:02:53 -0400 Subject: [PATCH 09/11] fix(analyzer): move thresholds onto recognizers (#1572) --- CHANGELOG.md | 2 +- docs/analyzer/analyzer_engine_provider.md | 17 +- docs/analyzer/recognizer_registry_provider.md | 4 + docs/tutorial/08_no_code.md | 13 +- .../presidio_analyzer/analyzer_engine.py | 55 ++--- .../analyzer_engine_provider.py | 4 - .../conf/default_analyzer_full.yaml | 9 +- .../conf/default_recognizers.yaml | 3 + .../presidio_analyzer/entity_recognizer.py | 16 +- .../input_validation/schemas.py | 113 ++-------- .../yaml_recognizer_models.py | 5 + .../recognizer_registry.py | 8 +- .../recognizers_loader_utils.py | 25 ++- .../presidio_analyzer/score_thresholds.py | 30 +++ .../tests/conf/test_analyzer_engine.yaml | 5 +- .../tests/conf/test_recognizer_registry.yaml | 7 +- .../tests/test_analyzer_engine.py | 203 +++++++++--------- .../tests/test_analyzer_engine_provider.py | 90 +++++++- .../tests/test_configuration_validator.py | 114 ++++------ .../tests/test_entity_recognizer.py | 49 ++++- .../tests/test_recognizer_registry.py | 75 +++++++ .../test_recognizer_registry_provider.py | 77 +++++++ .../tests/test_recognizers_loader_utils.py | 117 ++++++++++ .../tests/test_yaml_recognizer_models.py | 69 ++++++ 24 files changed, 756 insertions(+), 354 deletions(-) create mode 100644 presidio-analyzer/presidio_analyzer/score_thresholds.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c23f99077d..990d35c9dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,7 @@ All notable changes to this project will be documented in this file. ### Analyzer #### Added -- Analyzer YAML now accepts `recognizer_score_thresholds` for per-recognizer and per-entity score cutoffs, keyed by `RecognizerResult.RECOGNIZER_NAME_KEY`, while preserving the global `default_score_threshold` fallback when no override matches. +- Recognizer registry YAML entries now accept `score_thresholds` for recognizer-wide and entity-specific score cutoffs, with the analyzer's `default_score_threshold` as the fallback. - UK Driving Licence Number (`UK_DRIVING_LICENCE`) recognizer with pattern matching and context support (#1857) (Thanks @tee-jagz) - German PII recognizers for `DE_TAX_ID`, `DE_TAX_NUMBER`, `DE_PASSPORT`, `DE_ID_CARD`, `DE_SOCIAL_SECURITY`, `DE_HEALTH_INSURANCE`, `DE_KFZ`, `DE_HANDELSREGISTER`, and `DE_PLZ`; all are disabled by default (#1909) (Thanks @MvdB) - `SE_PERSONNUMMER` recognizer for Swedish personal identity and coordination numbers, plus Swedish Organisationsnummer recognition; both are disabled by default (#1912, #1918) (Thanks @goveebee) diff --git a/docs/analyzer/analyzer_engine_provider.md b/docs/analyzer/analyzer_engine_provider.md index 7724459e83..f78ef16501 100644 --- a/docs/analyzer/analyzer_engine_provider.md +++ b/docs/analyzer/analyzer_engine_provider.md @@ -63,6 +63,9 @@ recognizer_registry: - en supported_entity: IT_FISCAL_CODE type: predefined + score_thresholds: + default: 0.4 + CREDIT_CARD: 0.7 - name: ItFiscalCodeRecognizer type: predefined @@ -72,20 +75,10 @@ The configuration file contains the following parameters: - `supported_languages`: A list of supported languages that the analyzer will support. - `default_score_threshold`: A score that determines the minimal threshold for detection. -- `recognizer_score_thresholds`: Optional per-recognizer thresholds keyed by the recognizer name from `RecognizerResult.RECOGNIZER_NAME_KEY`. Use `default` for a recognizer-wide fallback, then add entity names for overrides. - `nlp_configuration`: Configuration given to the NLP engine which will detect the PIIs and extract features for the downstream logic. -- `recognizer_registry`: All the recognizers that will be used by the analyzer. +- `recognizer_registry`: All the recognizers that will be used by the analyzer. Each recognizer entry can define `score_thresholds`, using `default` as its fallback and entity names for overrides. -Example: - -```yaml -recognizer_score_thresholds: - CreditCardRecognizer: - default: 0.4 - CREDIT_CARD: 0.7 - SpacyRecognizer: - PERSON: 0.6 -``` +The threshold precedence is an explicit `analyze(score_threshold=...)` value, an entity-specific recognizer threshold, the recognizer's `default`, then `default_score_threshold`. !!! note "Note" diff --git a/docs/analyzer/recognizer_registry_provider.md b/docs/analyzer/recognizer_registry_provider.md index 4fd39e5c3a..f290fc950f 100644 --- a/docs/analyzer/recognizer_registry_provider.md +++ b/docs/analyzer/recognizer_registry_provider.md @@ -56,6 +56,9 @@ The recognizer list comprises of both the predefined and custom recognizers, for - language: it - language: pl type: predefined + score_thresholds: + default: 0.4 + CREDIT_CARD: 0.7 - name: UsBankRecognizer supported_languages: @@ -107,6 +110,7 @@ The recognizer list comprises of both the predefined and custom recognizers, for - `supported_entity`: the detected entity associated by the recognizer. - `deny_list`: A list of words to detect, in case the recognizer uses a predefined list of words. - `deny_list_score`: confidence score for a term identified using a deny-list. + - `score_thresholds`: optional score cutoffs for this recognizer. Use `default` as the recognizer-wide cutoff and entity names for overrides. An explicit request threshold takes priority, followed by an entity override, this recognizer default, and the analyzer's `default_score_threshold`. !!! tip "Configuration Tip: Agglutinative languages (e.g., Korean)" diff --git a/docs/tutorial/08_no_code.md b/docs/tutorial/08_no_code.md index 819e30ee92..b7c362d5dd 100644 --- a/docs/tutorial/08_no_code.md +++ b/docs/tutorial/08_no_code.md @@ -39,17 +39,9 @@ supported_languages: - en - es default_score_threshold: 0.4 -recognizer_score_thresholds: - CreditCardRecognizer: - default: 0.4 - CREDIT_CARD: 0.7 - SpacyRecognizer: - PERSON: 0.6 """ ``` -Use this when a recognizer needs a lower or higher cutoff than the global `default_score_threshold`, with entity-specific overrides taking priority over the recognizer default. - ### Recognizer Registry parameters ([default file](https://github.com/data-privacy-stack/presidio/blob/main/presidio-analyzer/presidio_analyzer/conf/default_recognizers.yaml)) @@ -71,6 +63,9 @@ recognizer_registry: - language: es context: [tarjeta, credito, visa, mastercard, cc, amex, discover, jcb, diners, maestro, instapayment] type: predefined + score_thresholds: + default: 0.4 + CREDIT_CARD: 0.7 - name: DateRecognizer supported_languages: @@ -127,6 +122,8 @@ recognizer_registry: """ ``` +Each recognizer can set a `default` cutoff and entity-specific overrides in `score_thresholds`. An explicit request threshold takes priority, followed by an entity override, the recognizer default, and the analyzer's `default_score_threshold`. + ### NLP Engine parameters ([default file](https://github.com/data-privacy-stack/presidio/blob/main/presidio-analyzer/presidio_analyzer/conf/default.yaml)) diff --git a/presidio-analyzer/presidio_analyzer/analyzer_engine.py b/presidio-analyzer/presidio_analyzer/analyzer_engine.py index 9b100dc0ef..de8294c6e3 100644 --- a/presidio-analyzer/presidio_analyzer/analyzer_engine.py +++ b/presidio-analyzer/presidio_analyzer/analyzer_engine.py @@ -2,7 +2,7 @@ import logging import os from collections import Counter -from typing import Dict, List, Optional, Union +from typing import List, Optional import regex as re @@ -15,7 +15,6 @@ ContextAwareEnhancer, LemmaContextAwareEnhancer, ) -from presidio_analyzer.input_validation.schemas import ConfigurationValidator from presidio_analyzer.nlp_engine import NlpArtifacts, NlpEngine, NlpEngineProvider from presidio_analyzer.recognizer_registry import ( RecognizerRegistry, @@ -25,7 +24,7 @@ logger = logging.getLogger("presidio-analyzer") REGEX_TIMEOUT_SECONDS = int(os.environ.get("REGEX_TIMEOUT_SECONDS", 60)) -RecognizerScoreThresholds = Dict[str, Union[float, Dict[str, float]]] + class AnalyzerEngine: """ @@ -42,8 +41,6 @@ class AnalyzerEngine: defines whether the decision process within the analyzer should be logged or not. :param default_score_threshold: Minimum confidence value for detected entities to be returned - :param recognizer_score_thresholds: Optional recognizer and entity-specific - score thresholds applied when no request-level score threshold is provided. :param supported_languages: List of possible languages this engine could be run on. Used for loading the right NLP models and recognizers for these languages. :param context_aware_enhancer: instance of type ContextAwareEnhancer for enhancing @@ -60,7 +57,6 @@ def __init__( default_score_threshold: float = 0, supported_languages: List[str] = None, context_aware_enhancer: Optional[ContextAwareEnhancer] = None, - recognizer_score_thresholds: Optional[RecognizerScoreThresholds] = None, ): if not supported_languages: supported_languages = ["en"] @@ -107,9 +103,6 @@ def __init__( self.log_decision_process = log_decision_process self.default_score_threshold = default_score_threshold - self.recognizer_score_thresholds = ( - self.__normalize_recognizer_score_thresholds(recognizer_score_thresholds) - ) if not context_aware_enhancer: logger.debug( @@ -264,7 +257,7 @@ def analyze( # Filter low-score results before deduplication so recognizer-specific # thresholds do not get lost when duplicate spans collapse. - results = self.__remove_low_scores(results, score_threshold) + results = self.__remove_low_scores(results, score_threshold, recognizers) results = EntityRecognizer.remove_duplicates(results) if allow_list: @@ -339,49 +332,45 @@ def _enhance_using_context( return results def __remove_low_scores( - self, results: List[RecognizerResult], score_threshold: float = None + self, + results: List[RecognizerResult], + score_threshold: float = None, + recognizers: Optional[List[EntityRecognizer]] = None, ) -> List[RecognizerResult]: """ Remove results for which the confidence is lower than the threshold. :param results: List of RecognizerResult :param score_threshold: float value for minimum possible confidence + :param recognizers: recognizers selected for this request :return: List[RecognizerResult] """ if score_threshold is None: + recognizers_by_id = { + recognizer.id: recognizer for recognizer in recognizers or [] + } return [ result for result in results - if result.score >= self.__get_result_score_threshold(result) + if result.score + >= self.__get_result_score_threshold(result, recognizers_by_id) ] new_results = [result for result in results if result.score >= score_threshold] return new_results - @staticmethod - def __normalize_recognizer_score_thresholds( - recognizer_score_thresholds: Optional[RecognizerScoreThresholds] - ) -> Dict[str, Dict[str, float]]: - """Normalize shorthand threshold values into the nested mapping shape.""" - if recognizer_score_thresholds is None: - return {} - return ConfigurationValidator.validate_recognizer_score_thresholds( - recognizer_score_thresholds - ) - - def __get_result_score_threshold(self, result: RecognizerResult) -> float: + def __get_result_score_threshold( + self, + result: RecognizerResult, + recognizers_by_id: dict, + ) -> float: """Resolve the threshold to apply for a single recognizer result.""" - if not self.recognizer_score_thresholds: - return self.default_score_threshold - metadata = result.recognition_metadata or {} - recognizer_name = metadata.get(RecognizerResult.RECOGNIZER_NAME_KEY) - if not recognizer_name: - return self.default_score_threshold - - recognizer_thresholds = self.recognizer_score_thresholds.get(recognizer_name) - if not recognizer_thresholds: + recognizer_id = metadata.get(RecognizerResult.RECOGNIZER_IDENTIFIER_KEY) + recognizer = recognizers_by_id.get(recognizer_id) + if recognizer is None: return self.default_score_threshold + recognizer_thresholds = recognizer.score_thresholds entity_threshold = recognizer_thresholds.get(result.entity_type) if entity_threshold is not None: diff --git a/presidio-analyzer/presidio_analyzer/analyzer_engine_provider.py b/presidio-analyzer/presidio_analyzer/analyzer_engine_provider.py index 49f0d9ac62..d03430a1f9 100644 --- a/presidio-analyzer/presidio_analyzer/analyzer_engine_provider.py +++ b/presidio-analyzer/presidio_analyzer/analyzer_engine_provider.py @@ -88,9 +88,6 @@ def create_engine(self) -> AnalyzerEngine: nlp_engine = self._load_nlp_engine() supported_languages = self.configuration.get("supported_languages", ["en"]) default_score_threshold = self.configuration.get("default_score_threshold", 0) - recognizer_score_thresholds = self.configuration.get( - "recognizer_score_thresholds", {} - ) registry = self._load_recognizer_registry( supported_languages=supported_languages, nlp_engine=nlp_engine @@ -101,7 +98,6 @@ def create_engine(self) -> AnalyzerEngine: registry=registry, supported_languages=supported_languages, default_score_threshold=default_score_threshold, - recognizer_score_thresholds=recognizer_score_thresholds, ) return analyzer diff --git a/presidio-analyzer/presidio_analyzer/conf/default_analyzer_full.yaml b/presidio-analyzer/presidio_analyzer/conf/default_analyzer_full.yaml index c5be4e13d5..a3788b4015 100644 --- a/presidio-analyzer/presidio_analyzer/conf/default_analyzer_full.yaml +++ b/presidio-analyzer/presidio_analyzer/conf/default_analyzer_full.yaml @@ -1,12 +1,6 @@ supported_languages: - en default_score_threshold: 0 -# recognizer_score_thresholds: -# CreditCardRecognizer: -# default: 0.4 -# CREDIT_CARD: 0.7 -# SpacyRecognizer: -# PERSON: 0.6 nlp_configuration: nlp_engine_name: spacy models: @@ -63,6 +57,9 @@ recognizer_registry: - language: en context: [credit, card, visa, mastercard, cc, amex, discover, jcb, diners, maestro, instapayment] type: predefined + # score_thresholds: + # default: 0.4 + # CREDIT_CARD: 0.7 - name: UsBankRecognizer type: predefined diff --git a/presidio-analyzer/presidio_analyzer/conf/default_recognizers.yaml b/presidio-analyzer/presidio_analyzer/conf/default_recognizers.yaml index 0ad50b1603..d0ddf815bf 100644 --- a/presidio-analyzer/presidio_analyzer/conf/default_recognizers.yaml +++ b/presidio-analyzer/presidio_analyzer/conf/default_recognizers.yaml @@ -36,6 +36,9 @@ recognizers: - language: it - language: pl type: predefined + # score_thresholds: + # default: 0.4 + # CREDIT_CARD: 0.7 - name: UsBankRecognizer supported_languages: diff --git a/presidio-analyzer/presidio_analyzer/entity_recognizer.py b/presidio-analyzer/presidio_analyzer/entity_recognizer.py index 18a6fc3d65..178d94e63c 100644 --- a/presidio-analyzer/presidio_analyzer/entity_recognizer.py +++ b/presidio-analyzer/presidio_analyzer/entity_recognizer.py @@ -1,8 +1,9 @@ import logging from abc import abstractmethod -from typing import TYPE_CHECKING, ClassVar, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Tuple from presidio_analyzer import RecognizerResult +from presidio_analyzer.score_thresholds import normalize_score_thresholds if TYPE_CHECKING: from presidio_analyzer.nlp_engine import NlpArtifacts @@ -33,6 +34,7 @@ class EntityRecognizer: recognizers may set it per instance; predefined recognizers should prefer the class-level :attr:`COUNTRY_CODE`. Values are stripped, lower-cased, and must match :attr:`COUNTRY_CODE` when both are set. + :param score_thresholds: Optional default and entity-specific score thresholds. """ MIN_SCORE = 0 @@ -53,6 +55,7 @@ def __init__( version: str = "0.0.1", context: Optional[List[str]] = None, country_code: Optional[str] = None, + score_thresholds: Any = None, ): self.supported_entities = supported_entities @@ -67,6 +70,7 @@ def __init__( self.version = version self.is_loaded = False self.context = context if context else [] + self.score_thresholds = score_thresholds self._country_code = self._resolve_country_code(country_code) @@ -74,6 +78,16 @@ def __init__( logger.info("Loaded recognizer: %s", self.name) self.is_loaded = True + @property + def score_thresholds(self) -> Dict[str, float]: + """Return a defensive copy of this recognizer's score thresholds.""" + return self._score_thresholds.copy() + + @score_thresholds.setter + def score_thresholds(self, value: Any) -> None: + """Validate and store this recognizer's score thresholds.""" + self._score_thresholds = normalize_score_thresholds(value) + @classmethod def _resolve_country_code(cls, passed: Optional[str]) -> Optional[str]: """Reconcile a constructor-passed country code with the class attribute. diff --git a/presidio-analyzer/presidio_analyzer/input_validation/schemas.py b/presidio-analyzer/presidio_analyzer/input_validation/schemas.py index f620f96507..cf9887a8ac 100644 --- a/presidio-analyzer/presidio_analyzer/input_validation/schemas.py +++ b/presidio-analyzer/presidio_analyzer/input_validation/schemas.py @@ -3,6 +3,11 @@ from pydantic import ValidationError +from presidio_analyzer.score_thresholds import ( + normalize_score_thresholds, + validate_score_threshold, +) + from . import validate_language_codes from .yaml_recognizer_models import RecognizerRegistryConfig @@ -38,95 +43,7 @@ def validate_score_threshold(threshold: float) -> float: :param threshold: score threshold to validate. """ - if not isinstance(threshold, (int, float)) or isinstance(threshold, bool): - raise ValueError(f"Score threshold must be numeric, got: {threshold}") - if not 0.0 <= threshold <= 1.0: - raise ValueError( - f"Score threshold must be between 0.0 and 1.0, got: {threshold}" - ) - return threshold - - @staticmethod - def validate_recognizer_score_thresholds( - thresholds: Dict[str, Any], - ) -> Dict[str, Dict[str, float]]: - """Validate recognizer-specific score thresholds.""" - if not isinstance(thresholds, dict): - raise ValueError("recognizer_score_thresholds must be a dictionary") - - validated_thresholds: Dict[str, Dict[str, float]] = {} - for recognizer_name, recognizer_thresholds in thresholds.items(): - if ( - not isinstance(recognizer_name, str) - or recognizer_name.strip() != recognizer_name - or not recognizer_name - ): - raise ValueError( - "recognizer_score_thresholds keys must be non-empty strings" - ) - - if isinstance(recognizer_thresholds, (int, float)) and not isinstance( - recognizer_thresholds, bool - ): - validated_thresholds[recognizer_name] = { - "default": ConfigurationValidator._validate_named_threshold( - recognizer_name=recognizer_name, - threshold_name="default", - threshold_value=recognizer_thresholds, - ) - } - continue - - if not isinstance(recognizer_thresholds, dict): - raise ValueError( - f"recognizer_score_thresholds for recognizer '{recognizer_name}' " - f"must be a dictionary" - ) - - validated_recognizer_thresholds: Dict[str, float] = {} - for threshold_name, threshold_value in recognizer_thresholds.items(): - if ( - not isinstance(threshold_name, str) - or threshold_name.strip() != threshold_name - or not threshold_name - ): - raise ValueError( - "recognizer_score_thresholds nested keys must be " - "non-empty strings" - ) - if not isinstance(threshold_value, (int, float)) or isinstance( - threshold_value, bool - ): - raise ValueError( - "recognizer_score_thresholds values must be numeric" - ) - - validated_recognizer_thresholds[threshold_name] = ( - ConfigurationValidator._validate_named_threshold( - recognizer_name=recognizer_name, - threshold_name=threshold_name, - threshold_value=threshold_value, - ) - ) - - validated_thresholds[recognizer_name] = validated_recognizer_thresholds - - return validated_thresholds - - @staticmethod - def _validate_named_threshold( - recognizer_name: str, - threshold_name: str, - threshold_value: float, - ) -> float: - """Validate one named recognizer threshold and keep config context.""" - try: - return ConfigurationValidator.validate_score_threshold(threshold_value) - except ValueError as exc: - raise ValueError( - "recognizer_score_thresholds value for " - f"'{recognizer_name}.{threshold_name}' {exc}" - ) from exc + return validate_score_threshold(threshold) @staticmethod def validate_nlp_configuration(config: Dict[str, Any]) -> Dict[str, Any]: @@ -164,9 +81,15 @@ def validate_recognizer_registry_configuration( try: # Use Pydantic model for validation validated_config = RecognizerRegistryConfig(**config) - return ConfigurationValidator._dump_recognizer_registry_configuration( - validated_config + dumped_config = ( + ConfigurationValidator._dump_recognizer_registry_configuration( + validated_config + ) ) + for recognizer in dumped_config["recognizers"]: + if isinstance(recognizer, dict): + normalize_score_thresholds(recognizer.get("score_thresholds")) + return dumped_config except ValidationError as e: raise ValueError("Invalid recognizer registry configuration") from e @@ -196,7 +119,6 @@ def validate_analyzer_configuration(config: Dict[str, Any]) -> Dict[str, Any]: valid_keys = { "supported_languages", "default_score_threshold", - "recognizer_score_thresholds", "nlp_configuration", "recognizer_registry", } @@ -220,13 +142,6 @@ def validate_analyzer_configuration(config: Dict[str, Any]) -> Dict[str, Any]: config["default_score_threshold"] ) - if "recognizer_score_thresholds" in config: - config["recognizer_score_thresholds"] = ( - ConfigurationValidator.validate_recognizer_score_thresholds( - config["recognizer_score_thresholds"] - ) - ) - # Validate nested configurations if "nlp_configuration" in config: ConfigurationValidator.validate_nlp_configuration( diff --git a/presidio-analyzer/presidio_analyzer/input_validation/yaml_recognizer_models.py b/presidio-analyzer/presidio_analyzer/input_validation/yaml_recognizer_models.py index 3b58101d92..bf950e7432 100644 --- a/presidio-analyzer/presidio_analyzer/input_validation/yaml_recognizer_models.py +++ b/presidio-analyzer/presidio_analyzer/input_validation/yaml_recognizer_models.py @@ -81,6 +81,10 @@ class BaseRecognizerConfig(BaseModel): supported_entities: Optional[List[str]] = Field( default=None, description="List of supported entities for this recognizer" ) + score_thresholds: Optional[Any] = Field( + default=None, + description="Default and entity-specific score thresholds", + ) @field_validator("supported_language") @classmethod @@ -435,6 +439,7 @@ def parse_recognizers( continue if isinstance(recognizer, dict): + recognizer = recognizer.copy() recognizer_type = recognizer.get("type") # Validate conflicting custom-only fields if explicitly predefined diff --git a/presidio-analyzer/presidio_analyzer/recognizer_registry/recognizer_registry.py b/presidio-analyzer/presidio_analyzer/recognizer_registry/recognizer_registry.py index 059f194f8a..a47e3f7dbb 100644 --- a/presidio-analyzer/presidio_analyzer/recognizer_registry/recognizer_registry.py +++ b/presidio-analyzer/presidio_analyzer/recognizer_registry/recognizer_registry.py @@ -22,6 +22,7 @@ RecognizerConfigurationLoader, RecognizerListLoader, ) +from presidio_analyzer.score_thresholds import normalize_score_thresholds logger = logging.getLogger("presidio-analyzer") @@ -314,7 +315,12 @@ def add_pattern_recognizer_from_dict(self, recognizer_dict: Dict) -> None: >>> registry.add_pattern_recognizer_from_dict(recognizer) """ # noqa: E501 - recognizer = PatternRecognizer.from_dict(recognizer_dict) + recognizer_config = recognizer_dict.copy() + score_thresholds = normalize_score_thresholds( + recognizer_config.pop("score_thresholds", None) + ) + recognizer = PatternRecognizer.from_dict(recognizer_config) + recognizer.score_thresholds = score_thresholds self.add_recognizer(recognizer) def add_recognizers_from_yaml(self, yml_path: Union[str, Path]) -> None: diff --git a/presidio-analyzer/presidio_analyzer/recognizer_registry/recognizers_loader_utils.py b/presidio-analyzer/presidio_analyzer/recognizer_registry/recognizers_loader_utils.py index d41d6398c7..c433e8bf1f 100644 --- a/presidio-analyzer/presidio_analyzer/recognizer_registry/recognizers_loader_utils.py +++ b/presidio-analyzer/presidio_analyzer/recognizer_registry/recognizers_loader_utils.py @@ -20,6 +20,7 @@ import yaml from presidio_analyzer import EntityRecognizer, PatternRecognizer +from presidio_analyzer.score_thresholds import normalize_score_thresholds logger = logging.getLogger("presidio-analyzer") @@ -393,9 +394,13 @@ def get( "supported_languages", "class_name", "country_code", + "score_thresholds", } - custom_to_exclude = {"enabled", "type", "class_name"} + custom_to_exclude = {"enabled", "type", "class_name", "score_thresholds"} for recognizer_conf in predefined: + score_thresholds = normalize_score_thresholds( + recognizer_conf.get("score_thresholds") + ) for language_conf in RecognizerListLoader._get_recognizer_languages( recognizer_conf=recognizer_conf, supported_languages=supported_languages ): @@ -421,19 +426,25 @@ def get( new_conf, language_conf, recognizer_cls ) - recognizer_instances.append(recognizer_cls(**kwargs)) + recognizer = recognizer_cls(**kwargs) + recognizer.score_thresholds = score_thresholds + recognizer_instances.append(recognizer) for recognizer_conf in custom: if RecognizerListLoader.is_recognizer_enabled(recognizer_conf): + score_thresholds = normalize_score_thresholds( + recognizer_conf.get("score_thresholds") + ) new_conf = RecognizerListLoader._filter_recognizer_fields( recognizer_conf, to_exclude=custom_to_exclude ) - recognizer_instances.extend( - RecognizerListLoader._create_custom_recognizers( - recognizer_conf=new_conf, - supported_languages=supported_languages, - ) + custom_recognizers = RecognizerListLoader._create_custom_recognizers( + recognizer_conf=new_conf, + supported_languages=supported_languages, ) + for recognizer in custom_recognizers: + recognizer.score_thresholds = score_thresholds + recognizer_instances.extend(custom_recognizers) for recognizer_conf in recognizer_instances: if isinstance(recognizer_conf, PatternRecognizer): diff --git a/presidio-analyzer/presidio_analyzer/score_thresholds.py b/presidio-analyzer/presidio_analyzer/score_thresholds.py new file mode 100644 index 0000000000..b809b8ccef --- /dev/null +++ b/presidio-analyzer/presidio_analyzer/score_thresholds.py @@ -0,0 +1,30 @@ +"""Validation helpers for recognizer score thresholds.""" + +from collections.abc import Mapping +from typing import Any, Dict + + +def validate_score_threshold(threshold: Any) -> float: + """Validate a score threshold without coercing its input type.""" + if not isinstance(threshold, (int, float)) or isinstance(threshold, bool): + raise ValueError(f"Score threshold must be numeric, got: {threshold}") + if not 0.0 <= threshold <= 1.0: + raise ValueError( + f"Score threshold must be between 0.0 and 1.0, got: {threshold}" + ) + return threshold + + +def normalize_score_thresholds(score_thresholds: Any) -> Dict[str, float]: + """Validate and defensively copy one recognizer's score thresholds.""" + if score_thresholds is None: + return {} + if not isinstance(score_thresholds, Mapping): + raise ValueError("score_thresholds must be a mapping") + + normalized = {} + for entity, threshold in score_thresholds.items(): + if not isinstance(entity, str) or not entity or entity.strip() != entity: + raise ValueError("score_thresholds keys must be non-empty strings") + normalized[entity] = validate_score_threshold(threshold) + return normalized diff --git a/presidio-analyzer/tests/conf/test_analyzer_engine.yaml b/presidio-analyzer/tests/conf/test_analyzer_engine.yaml index 9d68dea705..e13b08113f 100644 --- a/presidio-analyzer/tests/conf/test_analyzer_engine.yaml +++ b/presidio-analyzer/tests/conf/test_analyzer_engine.yaml @@ -6,6 +6,9 @@ recognizer_registry: - en supported_entity: CREDIT_CARD type: predefined + score_thresholds: + default: 0.4 + CREDIT_CARD: 0.7 - name: ItFiscalCodeRecognizer type: predefined @@ -90,4 +93,4 @@ nlp_configuration: low_confidence_score_multiplier: 0.4 low_score_entity_names: - - ID \ No newline at end of file + - ID diff --git a/presidio-analyzer/tests/conf/test_recognizer_registry.yaml b/presidio-analyzer/tests/conf/test_recognizer_registry.yaml index c81c197150..2ca7f895e9 100644 --- a/presidio-analyzer/tests/conf/test_recognizer_registry.yaml +++ b/presidio-analyzer/tests/conf/test_recognizer_registry.yaml @@ -5,6 +5,9 @@ recognizers: - en supported_entity: IT_FISCAL_CODE type: predefined + score_thresholds: + default: 0.4 + CREDIT_CARD: 0.8 - name: ItFiscalCodeRecognizer type: predefined @@ -27,6 +30,8 @@ recognizers: deny_list_score: 1 type: custom enabled: true + score_thresholds: + ZIP: 0.6 - name: "ZipCodeRecognizer" supported_language: "de" @@ -54,4 +59,4 @@ recognizers: supported_languages: - en - - es \ No newline at end of file + - es diff --git a/presidio-analyzer/tests/test_analyzer_engine.py b/presidio-analyzer/tests/test_analyzer_engine.py index 4ab40c61c7..6aed1e468d 100644 --- a/presidio-analyzer/tests/test_analyzer_engine.py +++ b/presidio-analyzer/tests/test_analyzer_engine.py @@ -62,7 +62,7 @@ def unit_test_guid(): class ThresholdRecognizer(EntityRecognizer, ABC): """Static recognizer used to exercise score threshold precedence.""" - def __init__(self): + def __init__(self, score_thresholds=None, name="ThresholdRecognizer"): """Seed fixed results for threshold precedence assertions.""" self._results = [ RecognizerResult("PERSON", 0, 4, 0.55), @@ -72,7 +72,8 @@ def __init__(self): ] super().__init__( supported_entities=["PERSON", "CREDIT_CARD", "URL", "DATE_TIME"], - name="ThresholdRecognizer", + name=name, + score_thresholds=score_thresholds, ) def load(self): @@ -117,9 +118,13 @@ def analyze(self, text: str, entities: List[str], nlp_artifacts: NlpArtifacts): class DuplicateThresholdRecognizer(EntityRecognizer, ABC): """Recognizer that emits a duplicate span for threshold ordering tests.""" - def __init__(self, name): + def __init__(self, name, score_thresholds): """Use the recognizer name as the threshold lookup key.""" - super().__init__(supported_entities=["PERSON"], name=name) + super().__init__( + supported_entities=["PERSON"], + name=name, + score_thresholds=score_thresholds, + ) def load(self): """Keep the test recognizer lightweight.""" @@ -130,18 +135,44 @@ def analyze(self, text: str, entities: List[str], nlp_artifacts: NlpArtifacts): return [RecognizerResult("PERSON", 0, 4, 0.5)] -def _build_threshold_analyzer( - recognizer_score_thresholds=None, default_score_threshold=0.8 -): +class ContextThresholdRecognizer(DuplicateThresholdRecognizer): + """Recognizer which raises its result score during context enhancement.""" + + def enhance_using_context(self, *args, **kwargs): + """Raise the score above the configured cutoff.""" + results = kwargs["raw_recognizer_results"] + results[0].score = 0.8 + return results + + +class MissingIdentifierRecognizer(DuplicateThresholdRecognizer): + """Recognizer which replaces producer metadata during context enhancement.""" + + def __init__(self, identifier): + """Choose whether the final producer identifier is missing or unmatched.""" + self.identifier = identifier + super().__init__("MissingIdentifierRecognizer", {"PERSON": 0.1}) + + def enhance_using_context(self, *args, **kwargs): + """Replace the identifier after recognizer execution.""" + results = kwargs["raw_recognizer_results"] + results[0].recognition_metadata = ( + {} + if self.identifier is None + else {RecognizerResult.RECOGNIZER_IDENTIFIER_KEY: self.identifier} + ) + return results + + +def _build_threshold_analyzer(score_thresholds=None, default_score_threshold=0.8): """Create an analyzer with deterministic recognizers for threshold tests.""" registry = RecognizerRegistry() - registry.add_recognizer(ThresholdRecognizer()) + registry.add_recognizer(ThresholdRecognizer(score_thresholds)) registry.add_recognizer(FallbackRecognizer()) return AnalyzerEngine( registry=registry, nlp_engine=NlpEngineMock(), default_score_threshold=default_score_threshold, - recognizer_score_thresholds=recognizer_score_thresholds, ) @@ -649,13 +680,7 @@ def test_when_default_threshold_is_zero_then_all_results_pass( def test_recognizer_threshold_precedence(): """Recognizer-specific and entity-specific thresholds should win.""" analyzer_engine = _build_threshold_analyzer( - recognizer_score_thresholds={ - "ThresholdRecognizer": { - "default": 0.4, - "PERSON": 0.6, - "CREDIT_CARD": 0.7, - } - } + {"default": 0.4, "PERSON": 0.6, "CREDIT_CARD": 0.7} ) results = analyzer_engine.analyze( @@ -672,13 +697,7 @@ def test_recognizer_threshold_precedence(): def test_explicit_score_threshold_overrides_config(): """A request-level score threshold should override config thresholds.""" analyzer_engine = _build_threshold_analyzer( - recognizer_score_thresholds={ - "ThresholdRecognizer": { - "default": 0.4, - "PERSON": 0.6, - "CREDIT_CARD": 0.7, - } - } + {"default": 0.4, "PERSON": 0.6, "CREDIT_CARD": 0.7} ) results = analyzer_engine.analyze( @@ -693,112 +712,102 @@ def test_explicit_score_threshold_overrides_config(): assert scores == {"DATE_TIME": 0.9} -def test_numeric_threshold_shorthand_applies(): - """Numeric shorthand should behave like a recognizer default threshold.""" - class NumericThresholdRecognizer(EntityRecognizer, ABC): - """Recognizer with a single score below the global default.""" - - def __init__(self): - """Set up the recognizer metadata for threshold lookup.""" - super().__init__( - supported_entities=["URL"], - name="NumericThresholdRecognizer", - ) - - def load(self): - """Keep the test recognizer lightweight.""" - return None - - def analyze( - self, text: str, entities: List[str], nlp_artifacts: NlpArtifacts - ): - """Return a single deterministic result.""" - return [RecognizerResult("URL", 0, 8, 0.75)] - +def test_duplicate_results_keep_recognizers_that_meet_their_thresholds(): + """Threshold filtering should run before duplicate collapse.""" registry = RecognizerRegistry() - registry.add_recognizer(NumericThresholdRecognizer()) + strict = DuplicateThresholdRecognizer("SameRecognizer", {"PERSON": 0.9}) + lenient = DuplicateThresholdRecognizer("SameRecognizer", {"PERSON": 0.4}) + registry.add_recognizer(strict) + registry.add_recognizer(lenient) analyzer_engine = AnalyzerEngine( registry=registry, nlp_engine=NlpEngineMock(), - default_score_threshold=0.8, - recognizer_score_thresholds={"NumericThresholdRecognizer": 0.4}, + default_score_threshold=0.0, ) results = analyzer_engine.analyze( - text="bing.com", + text="John", language="en", - entities=["URL"], + entities=["PERSON"], ) assert len(results) == 1 - assert results[0].entity_type == "URL" + assert strict.id != lenient.id + assert results[0].recognition_metadata[ + RecognizerResult.RECOGNIZER_IDENTIFIER_KEY + ] == lenient.id -def test_direct_threshold_config_rejects_boolean_values(): - """Direct constructor config should reject boolean threshold values.""" - with pytest.raises(ValueError, match="values must be numeric"): - _build_threshold_analyzer( - recognizer_score_thresholds={"ThresholdRecognizer": {"PERSON": True}} - ) - +def test_empty_recognizer_thresholds_use_default(): + """An empty threshold map should keep the global default behavior.""" + analyzer_engine = _build_threshold_analyzer(score_thresholds={}) -def test_direct_threshold_config_rejects_out_of_range_values(): - """Direct constructor config should reject out-of-range threshold values.""" - with pytest.raises(ValueError, match="between 0.0 and 1.0"): - _build_threshold_analyzer( - recognizer_score_thresholds={"ThresholdRecognizer": {"PERSON": 1.5}} - ) + results = analyzer_engine.analyze( + text="Threshold config", + language="en", + entities=["PERSON", "CREDIT_CARD", "URL", "DATE_TIME", "PHONE_NUMBER"], + ) + scores = {result.entity_type: result.score for result in results} -def test_direct_threshold_config_rejects_non_dict_top_level_values(): - """Direct constructor config should reject non-dictionary threshold payloads.""" - with pytest.raises(ValueError, match="must be a dictionary"): - _build_threshold_analyzer(recognizer_score_thresholds=0.4) + assert scores == {"DATE_TIME": 0.9} -def test_duplicate_results_keep_recognizers_that_meet_their_thresholds(): - """Threshold filtering should run before duplicate collapse.""" +def test_context_enhancement_runs_before_recognizer_threshold_filtering(): + """A context-enhanced score should be compared with the final score.""" registry = RecognizerRegistry() - registry.add_recognizer(DuplicateThresholdRecognizer("StrictRecognizer")) - registry.add_recognizer(DuplicateThresholdRecognizer("LenientRecognizer")) - - analyzer_engine = AnalyzerEngine( + registry.add_recognizer( + ContextThresholdRecognizer("ContextThresholdRecognizer", {"PERSON": 0.7}) + ) + analyzer = AnalyzerEngine( registry=registry, nlp_engine=NlpEngineMock(), - default_score_threshold=0.0, - recognizer_score_thresholds={ - "StrictRecognizer": {"PERSON": 0.9}, - "LenientRecognizer": {"PERSON": 0.4}, - }, + default_score_threshold=0.9, ) - results = analyzer_engine.analyze( - text="John", - language="en", - entities=["PERSON"], - ) + results = analyzer.analyze("John", "en", entities=["PERSON"]) - assert len(results) == 1 - assert ( - results[0].recognition_metadata[RecognizerResult.RECOGNIZER_NAME_KEY] - == "LenientRecognizer" + assert [result.score for result in results] == [0.8] + + +@pytest.mark.parametrize("identifier", [None, "not-a-selected-recognizer"]) +def test_missing_or_unmatched_recognizer_identifier_uses_engine_default(identifier): + """Unknown producer provenance should use the engine fallback.""" + registry = RecognizerRegistry() + registry.add_recognizer(MissingIdentifierRecognizer(identifier)) + analyzer = AnalyzerEngine( + registry=registry, + nlp_engine=NlpEngineMock(), + default_score_threshold=0.8, ) + assert analyzer.analyze("John", "en", entities=["PERSON"]) == [] -def test_empty_recognizer_thresholds_use_default(): - """An empty threshold map should keep the global default behavior.""" - analyzer_engine = _build_threshold_analyzer(recognizer_score_thresholds={}) - results = analyzer_engine.analyze( - text="Threshold config", - language="en", - entities=["PERSON", "CREDIT_CARD", "URL", "DATE_TIME", "PHONE_NUMBER"], +def test_ad_hoc_recognizer_uses_its_score_thresholds(): + """Request-local recognizers should participate in identifier lookup.""" + ad_hoc = PatternRecognizer( + supported_entity="ROCKET", + patterns=[Pattern("rocket", "rocket", 0.5)], + ) + ad_hoc.score_thresholds = {"default": 0.4} + registry = RecognizerRegistry() + registry.add_recognizer(FallbackRecognizer()) + analyzer = AnalyzerEngine( + registry=registry, + nlp_engine=NlpEngineMock(), + default_score_threshold=0.8, ) - scores = {result.entity_type: result.score for result in results} + results = analyzer.analyze( + "rocket", + "en", + entities=["ROCKET"], + ad_hoc_recognizers=[ad_hoc], + ) - assert scores == {"DATE_TIME": 0.9} + assert [result.entity_type for result in results] == ["ROCKET"] def test_when_get_supported_fields_then_return_all_languages( diff --git a/presidio-analyzer/tests/test_analyzer_engine_provider.py b/presidio-analyzer/tests/test_analyzer_engine_provider.py index 2547e1935b..59df104e11 100644 --- a/presidio-analyzer/tests/test_analyzer_engine_provider.py +++ b/presidio-analyzer/tests/test_analyzer_engine_provider.py @@ -37,7 +37,10 @@ def test_analyzer_engine_provider_default_configuration(mandatory_recognizers): engine.registry.global_regex_flags == re.DOTALL | re.MULTILINE | re.IGNORECASE ) assert engine.default_score_threshold == 0 - assert engine.recognizer_score_thresholds == {} + assert all( + recognizer.score_thresholds == {} + for recognizer in engine.registry.recognizers + ) names = [recognizer.name for recognizer in engine.registry.recognizers] for predefined_recognizer in mandatory_recognizers: assert predefined_recognizer in names @@ -92,19 +95,28 @@ def test_analyzer_engine_provider_configuration_file(): and recognizer.supported_language == "es" ][0] assert spanish_recognizer.context == ["tarjeta", "credito"] + credit_card = next( + recognizer + for recognizer in recognizer_registry.recognizers + if recognizer.name == "CreditCardRecognizer" + ) + assert credit_card.score_thresholds == { + "default": 0.4, + "CREDIT_CARD": 0.7, + } assert isinstance(engine.nlp_engine, SpacyNlpEngine) assert engine.nlp_engine.engine_name == "spacy" -def test_analyzer_engine_provider_passes_recognizer_score_thresholds(tmp_path): - """Numeric shorthand should normalize and apply through the provider.""" +def test_analyzer_engine_provider_inline_recognizer_thresholds_affect_output(tmp_path): analyzer_yaml, _, _ = get_full_paths("conf/test_analyzer_engine.yaml") with open(analyzer_yaml) as file: configuration = yaml.safe_load(file) configuration["default_score_threshold"] = 0.9 - configuration["recognizer_score_thresholds"] = {"CreditCardRecognizer": 0.4} + credit_card = configuration["recognizer_registry"]["recognizers"][0] + credit_card["score_thresholds"] = {"default": 0.4} threshold_yaml = tmp_path / "analyzer_with_thresholds.yaml" threshold_yaml.write_text(yaml.safe_dump(configuration, sort_keys=False)) @@ -112,9 +124,12 @@ def test_analyzer_engine_provider_passes_recognizer_score_thresholds(tmp_path): provider = AnalyzerEngineProvider(threshold_yaml) engine = provider.create_engine() - assert engine.recognizer_score_thresholds == { - "CreditCardRecognizer": {"default": 0.4} - } + loaded_credit_card = next( + recognizer + for recognizer in engine.registry.recognizers + if recognizer.name == "CreditCardRecognizer" + ) + assert loaded_credit_card.score_thresholds == {"default": 0.4} results = engine.analyze( text=" Credit card: 4095-2609-9393-4932", @@ -125,12 +140,62 @@ def test_analyzer_engine_provider_passes_recognizer_score_thresholds(tmp_path): assert len(results) == 1 +def test_analyzer_engine_provider_external_registry_thresholds_affect_output(tmp_path): + analyzer_yaml = tmp_path / "analyzer.yaml" + analyzer_yaml.write_text( + yaml.safe_dump( + { + "supported_languages": ["en"], + "default_score_threshold": 0.9, + "nlp_configuration": { + "nlp_engine_name": "spacy", + "models": [ + {"lang_code": "en", "model_name": "en_core_web_lg"} + ], + }, + } + ) + ) + registry_yaml = tmp_path / "registry.yaml" + registry_yaml.write_text( + yaml.safe_dump( + { + "supported_languages": ["en"], + "recognizers": [ + { + "name": "RocketRecognizer", + "type": "custom", + "supported_entity": "ROCKET", + "supported_language": "en", + "patterns": [ + {"name": "rocket", "regex": "rocket", "score": 0.5} + ], + "score_thresholds": {"default": 0.4}, + } + ], + } + ) + ) + provider = AnalyzerEngineProvider( + analyzer_engine_conf_file=analyzer_yaml, + recognizer_registry_conf_file=registry_yaml, + ) + + engine = provider.create_engine() + results = engine.analyze("rocket", "en", entities=["ROCKET"]) + + assert [result.entity_type for result in results] == ["ROCKET"] + + def test_analyzer_engine_provider_defaults(mandatory_recognizers): provider = AnalyzerEngineProvider() engine = provider.create_engine() assert engine.supported_languages == ["en"] assert engine.default_score_threshold == 0 - assert engine.recognizer_score_thresholds == {} + assert all( + recognizer.score_thresholds == {} + for recognizer in engine.registry.recognizers + ) recognizer_registry = engine.registry assert ( recognizer_registry.global_regex_flags @@ -580,6 +645,15 @@ def test_analyzer_engine_provider_inline_sections_take_priority_over_per_section # The registry from the inline section has more than the 6 recognizers # in test_recognizer_registry.yaml, confirming the inline section won. assert len(engine.registry.recognizers) > 6 + credit_card = next( + recognizer + for recognizer in engine.registry.recognizers + if recognizer.name == "CreditCardRecognizer" + ) + assert credit_card.score_thresholds == { + "default": 0.4, + "CREDIT_CARD": 0.7, + } def test_analyzer_engine_provider_multiple_languages_support(): diff --git a/presidio-analyzer/tests/test_configuration_validator.py b/presidio-analyzer/tests/test_configuration_validator.py index b3f753caa6..73d5172b6f 100644 --- a/presidio-analyzer/tests/test_configuration_validator.py +++ b/presidio-analyzer/tests/test_configuration_validator.py @@ -1,5 +1,5 @@ """Tests for the Pydantic-based validation system using existing adapted classes.""" -# ruff: noqa: E501 +# ruff: noqa: D103,E501 import pytest from presidio_analyzer.input_validation import ConfigurationValidator @@ -326,88 +326,58 @@ def test_configuration_validator_analyzer_config_valid(): assert validated == valid_config -def test_configuration_validator_analyzer_config_valid_with_recognizer_score_thresholds(): - """Validator should accept numeric shorthand and nested overrides.""" +def test_analyzer_config_rejects_removed_recognizer_score_thresholds_key(): + with pytest.raises(ValueError, match="Unknown configuration key"): + ConfigurationValidator.validate_analyzer_configuration( + { + "supported_languages": ["en"], + "recognizer_score_thresholds": {"CreditCardRecognizer": 0.4}, + } + ) + + +def test_registry_config_preserves_valid_score_thresholds(): valid_config = { "supported_languages": ["en"], - "default_score_threshold": 0.5, - "recognizer_score_thresholds": { - "CreditCardRecognizer": 0.4, - "SpacyRecognizer": { - "PERSON": 0.6, - }, - }, + "recognizers": [ + { + "name": "CreditCardRecognizer", + "type": "predefined", + "score_thresholds": {"default": 0.4, "CREDIT_CARD": 0.7}, + } + ], } - validated = ConfigurationValidator.validate_analyzer_configuration(valid_config) - assert validated == { - "supported_languages": ["en"], - "default_score_threshold": 0.5, - "recognizer_score_thresholds": { - "CreditCardRecognizer": {"default": 0.4}, - "SpacyRecognizer": {"PERSON": 0.6}, - }, + validated = ConfigurationValidator.validate_recognizer_registry_configuration( + valid_config + ) + + assert validated["recognizers"][0]["score_thresholds"] == { + "default": 0.4, + "CREDIT_CARD": 0.7, } @pytest.mark.parametrize( - "invalid_config,expected_message", - [ - ( - { - "supported_languages": ["en"], - "recognizer_score_thresholds": { - 123: {"default": 0.4}, - }, - }, - "non-empty strings", - ), - ( - { - "supported_languages": ["en"], - "recognizer_score_thresholds": { - "CreditCardRecognizer": ["default", 0.4], - }, - }, - "recognizer_score_thresholds", - ), - ( - { - "supported_languages": ["en"], - "recognizer_score_thresholds": { - "CreditCardRecognizer": {"": 0.4}, - }, - }, - "nested keys must be non-empty strings", - ), - ( - { - "supported_languages": ["en"], - "recognizer_score_thresholds": { - "CreditCardRecognizer": {"default": 1.5}, - }, - }, - "recognizer_score_thresholds", - ), - ( - { - "supported_languages": ["en"], - "recognizer_score_thresholds": { - "CreditCardRecognizer": {"default": True}, - }, - }, - "values must be numeric", - ), - ], + "score_thresholds", + [False, 0, "", [], {"default": True}, {"default": "0.4"}], ) -def test_configuration_validator_analyzer_config_invalid_recognizer_score_thresholds( - invalid_config, expected_message +def test_registry_config_rejects_uncoerced_invalid_score_thresholds( + score_thresholds, ): - """Validator should reject malformed threshold overrides.""" - with pytest.raises(ValueError) as exc_info: - ConfigurationValidator.validate_analyzer_configuration(invalid_config) + config = { + "supported_languages": ["en"], + "recognizers": [ + { + "name": "CreditCardRecognizer", + "type": "predefined", + "score_thresholds": score_thresholds, + } + ], + } - assert expected_message in str(exc_info.value) + with pytest.raises(ValueError): + ConfigurationValidator.validate_recognizer_registry_configuration(config) def test_analyzer_config_minimal(): diff --git a/presidio-analyzer/tests/test_entity_recognizer.py b/presidio-analyzer/tests/test_entity_recognizer.py index 3bb695eb5d..6035ea1652 100644 --- a/presidio-analyzer/tests/test_entity_recognizer.py +++ b/presidio-analyzer/tests/test_entity_recognizer.py @@ -1,4 +1,8 @@ -from presidio_analyzer import EntityRecognizer, RecognizerResult, AnalysisExplanation +# ruff: noqa: D103,E501,I001 + +import pytest + +from presidio_analyzer import AnalysisExplanation, EntityRecognizer, RecognizerResult def test_when_to_dict_then_return_correct_dictionary(): @@ -120,8 +124,6 @@ def test_when_remove_duplicates_contained_shorter_length_results_removed(): results = EntityRecognizer.remove_duplicates(arr) assert len(results) == 1 -import pytest - sanitizer_test_set = [ [" a|b:c ::-", [("-", ""), (" ", ""), (":", ""), ("|", "")], "abc"], ["def", "", "def"], @@ -138,3 +140,44 @@ def test_sanitize_value(input_text, params, expected_output): :return: True/False """ assert EntityRecognizer.sanitize_value(input_text, params) == expected_output + + +def test_score_thresholds_default_to_empty_mapping(): + recognizer = EntityRecognizer(["ENTITY"]) + + assert recognizer.score_thresholds == {} + + +def test_score_thresholds_constructor_and_setter_defensively_copy(): + thresholds = {"default": 0.4, "ENTITY": 0.7} + recognizer = EntityRecognizer(["ENTITY"], score_thresholds=thresholds) + thresholds["ENTITY"] = 0.1 + returned = recognizer.score_thresholds + returned["ENTITY"] = 0.2 + + assert recognizer.score_thresholds == {"default": 0.4, "ENTITY": 0.7} + + recognizer.score_thresholds = {"ENTITY": 0.5} + assert recognizer.score_thresholds == {"ENTITY": 0.5} + + +@pytest.mark.parametrize("thresholds", [False, True, 0, "", "0.4", []]) +def test_score_thresholds_reject_non_mapping_values(thresholds): + with pytest.raises(ValueError, match="must be a mapping"): + EntityRecognizer(["ENTITY"], score_thresholds=thresholds) + + +@pytest.mark.parametrize( + "thresholds", + [ + {"ENTITY": False}, + {"ENTITY": "0.4"}, + {"ENTITY": -0.1}, + {"ENTITY": 1.1}, + {"": 0.4}, + {" ENTITY": 0.4}, + ], +) +def test_score_thresholds_reject_invalid_entries(thresholds): + with pytest.raises(ValueError): + EntityRecognizer(["ENTITY"], score_thresholds=thresholds) diff --git a/presidio-analyzer/tests/test_recognizer_registry.py b/presidio-analyzer/tests/test_recognizer_registry.py index e0c6b8db53..a7b2d596b3 100644 --- a/presidio-analyzer/tests/test_recognizer_registry.py +++ b/presidio-analyzer/tests/test_recognizer_registry.py @@ -1,3 +1,5 @@ +# ruff: noqa: D103,D205,E501,F841,I001 + from pathlib import Path import pytest @@ -162,6 +164,79 @@ def test_add_recognizer_from_dict(): assert registry.recognizers[0].name == "Zip code Recognizer" +def test_add_recognizer_from_dict_attaches_thresholds_without_mutating_input( + monkeypatch, +): + registry = RecognizerRegistry() + recognizer = { + "name": "Zip code Recognizer", + "supported_language": "de", + "patterns": [{"name": "zip", "regex": r"\d{5}", "score": 0.5}], + "supported_entity": "ZIP", + "score_thresholds": {"default": 0.4, "ZIP": 0.7}, + } + original = recognizer.copy() + received = {} + from_dict = PatternRecognizer.from_dict + + def capture_from_dict(config): + received.update(config) + return from_dict(config) + + monkeypatch.setattr(PatternRecognizer, "from_dict", capture_from_dict) + + registry.add_pattern_recognizer_from_dict(recognizer) + + assert "score_thresholds" not in received + assert recognizer == original + assert registry.recognizers[0].score_thresholds == { + "default": 0.4, + "ZIP": 0.7, + } + + +def test_add_recognizers_from_yaml_attaches_thresholds(tmp_path): + yaml_path = tmp_path / "recognizers.yaml" + yaml_path.write_text( + """recognizers: +- name: Zip code Recognizer + supported_language: de + supported_entity: ZIP + patterns: + - name: zip + regex: '\\d{5}' + score: 0.5 + score_thresholds: + default: 0.4 + ZIP: 0.7 +""" + ) + registry = RecognizerRegistry() + + registry.add_recognizers_from_yaml(yaml_path) + + assert registry.recognizers[0].score_thresholds == { + "default": 0.4, + "ZIP": 0.7, + } + + +@pytest.mark.parametrize("score_thresholds", [False, 0, "", []]) +def test_add_recognizer_from_dict_rejects_falsey_non_mapping_thresholds( + score_thresholds, +): + recognizer = { + "name": "Zip code Recognizer", + "supported_language": "de", + "patterns": [{"name": "zip", "regex": r"\d{5}", "score": 0.5}], + "supported_entity": "ZIP", + "score_thresholds": score_thresholds, + } + + with pytest.raises(ValueError, match="must be a mapping"): + RecognizerRegistry().add_pattern_recognizer_from_dict(recognizer) + + def test_recognizer_registry_add_from_yaml_file(): this_path = Path(__file__).parent.absolute() test_yaml = Path(this_path, "conf/recognizers.yaml") diff --git a/presidio-analyzer/tests/test_recognizer_registry_provider.py b/presidio-analyzer/tests/test_recognizer_registry_provider.py index 17771a1d3e..bc294c7f43 100644 --- a/presidio-analyzer/tests/test_recognizer_registry_provider.py +++ b/presidio-analyzer/tests/test_recognizer_registry_provider.py @@ -1,3 +1,5 @@ +# ruff: noqa: D103,D200,D205,E501,F841,I001 + import pytest import re from pathlib import Path @@ -38,6 +40,81 @@ def test_recognizer_registry_provider_configuration_file(): assert [recognizer.supported_language for recognizer in recognizer_registry.recognizers if recognizer.name == "ExampleCustomRecognizer"] == ["en", "es"] spanish_recognizer = [recognizer for recognizer in recognizer_registry.recognizers if recognizer.name == "ExampleCustomRecognizer" and recognizer.supported_language == "es"][0] assert spanish_recognizer.context == ["tarjeta", "credito"] + credit_card = next( + recognizer + for recognizer in recognizer_registry.recognizers + if recognizer.name == "CreditCardRecognizer" + ) + assert credit_card.score_thresholds == { + "default": 0.4, + "CREDIT_CARD": 0.8, + } + assert all( + recognizer.score_thresholds == {"ZIP": 0.6} + for recognizer in recognizer_registry.recognizers + if recognizer.name == "ExampleCustomRecognizer" + ) + + +def test_recognizer_registry_provider_inline_thresholds_attach_to_instance(): + provider = RecognizerRegistryProvider( + registry_configuration={ + "supported_languages": ["en"], + "recognizers": [ + { + "name": "CreditCardRecognizer", + "type": "predefined", + "supported_language": "en", + "score_thresholds": {"default": 0.4}, + } + ], + } + ) + + recognizer = provider.create_recognizer_registry().recognizers[0] + + assert recognizer.score_thresholds == {"default": 0.4} + + +def test_recognizer_registry_provider_omitted_thresholds_default_to_empty(): + provider = RecognizerRegistryProvider( + registry_configuration={ + "supported_languages": ["en"], + "recognizers": [ + { + "name": "CreditCardRecognizer", + "type": "predefined", + "supported_language": "en", + } + ], + } + ) + + recognizer = provider.create_recognizer_registry().recognizers[0] + + assert recognizer.score_thresholds == {} + + +@pytest.mark.parametrize( + "score_thresholds", + [True, False, 0, "", "0.4", [], {"": 0.4}, {"default": True}, {"default": 1.1}], +) +def test_recognizer_registry_provider_rejects_invalid_score_thresholds( + score_thresholds, +): + with pytest.raises(ValueError): + RecognizerRegistryProvider( + registry_configuration={ + "supported_languages": ["en"], + "recognizers": [ + { + "name": "CreditCardRecognizer", + "type": "predefined", + "score_thresholds": score_thresholds, + } + ], + } + ) def test_recognizer_registry_provider_configuration_file_load_predefined(mandatory_recognizers): diff --git a/presidio-analyzer/tests/test_recognizers_loader_utils.py b/presidio-analyzer/tests/test_recognizers_loader_utils.py index 89a178b182..7a5bf2b7cd 100644 --- a/presidio-analyzer/tests/test_recognizers_loader_utils.py +++ b/presidio-analyzer/tests/test_recognizers_loader_utils.py @@ -1,3 +1,6 @@ +# ruff: noqa: D103,D200,D205,E501,F841,I001 + +import copy import functools import re from pathlib import Path @@ -78,6 +81,120 @@ def __init__(self, **kwargs): ) +def load_recognizers(recognizers, languages=("en",)): + return list(RecognizerListLoader.get(recognizers, languages, 26)) + + +def test_predefined_score_thresholds_attach_without_mutating_config(): + config = [ + { + "name": "CreditCardRecognizer", + "type": "predefined", + "supported_language": "en", + "score_thresholds": {"default": 0.4, "CREDIT_CARD": 0.7}, + } + ] + original = copy.deepcopy(config) + + recognizers = load_recognizers(config) + + assert recognizers[0].score_thresholds == { + "default": 0.4, + "CREDIT_CARD": 0.7, + } + assert config == original + + +def test_custom_multilanguage_score_thresholds_attach_to_each_instance(): + config = [ + { + "name": "custom_thresholds", + "type": "custom", + "supported_entity": "CUSTOM", + "supported_languages": [{"language": "en"}, {"language": "es"}], + "patterns": [{"name": "custom", "regex": "x", "score": 0.5}], + "score_thresholds": {"default": 0.4, "CUSTOM": 0.6}, + } + ] + original = copy.deepcopy(config) + + recognizers = load_recognizers(config, ("en", "es")) + + assert {recognizer.supported_language for recognizer in recognizers} == {"en", "es"} + assert all( + recognizer.score_thresholds == {"default": 0.4, "CUSTOM": 0.6} + for recognizer in recognizers + ) + assert config == original + + +@pytest.mark.parametrize("score_thresholds", [None, {}]) +def test_loader_missing_or_empty_score_thresholds_default_to_empty(score_thresholds): + config = { + "name": "CreditCardRecognizer", + "type": "predefined", + "supported_language": "en", + } + if score_thresholds is not None: + config["score_thresholds"] = score_thresholds + + recognizer = load_recognizers([config])[0] + + assert recognizer.score_thresholds == {} + + +def test_loader_explicit_none_score_thresholds_defaults_to_empty(): + recognizer = load_recognizers( + [ + { + "name": "CreditCardRecognizer", + "type": "predefined", + "supported_language": "en", + "score_thresholds": None, + } + ] + )[0] + + assert recognizer.score_thresholds == {} + + +@pytest.mark.parametrize("score_thresholds", [False, 0, "", []]) +def test_loader_rejects_falsey_non_mapping_score_thresholds(score_thresholds): + config = [ + { + "name": "CreditCardRecognizer", + "type": "predefined", + "supported_language": "en", + "score_thresholds": score_thresholds, + } + ] + + with pytest.raises(ValueError, match="must be a mapping"): + load_recognizers(config) + + +def test_same_name_and_language_entries_keep_distinct_thresholds_and_ids(): + config = [ + { + "name": "CreditCardRecognizer", + "type": "predefined", + "supported_language": "en", + "score_thresholds": {"default": threshold}, + } + for threshold in (0.4, 0.8) + ] + + recognizers = load_recognizers(config) + + assert [recognizer.score_thresholds for recognizer in recognizers] == [ + {"default": 0.4}, + {"default": 0.8}, + ] + assert recognizers[0].name == recognizers[1].name + assert recognizers[0].supported_language == recognizers[1].supported_language + assert recognizers[0].id != recognizers[1].id + + def test_cleanup_none_removes_entity_keys(): """Test that explicit None values for entity keys are removed.""" kwargs = prepare( diff --git a/presidio-analyzer/tests/test_yaml_recognizer_models.py b/presidio-analyzer/tests/test_yaml_recognizer_models.py index 72415031d8..e6b6eb7bab 100644 --- a/presidio-analyzer/tests/test_yaml_recognizer_models.py +++ b/presidio-analyzer/tests/test_yaml_recognizer_models.py @@ -1,4 +1,5 @@ """Tests for YAML recognizer configuration models.""" +# ruff: noqa: D103,E501,F841,I001 import pytest from presidio_analyzer.input_validation.yaml_recognizer_models import ( @@ -853,3 +854,71 @@ def test_config_model_map_fallback_to_predefined(): assert isinstance(recognizer, PredefinedRecognizerConfig) assert recognizer.name == "MySpacy" assert recognizer.class_name == "SpacyRecognizer" + + +@pytest.mark.parametrize( + "raw_thresholds", + [True, "0.4", ["default", 0.4], {"default": "0.4"}, {"default": 0.4}], +) +def test_base_recognizer_score_thresholds_preserve_raw_values(raw_thresholds): + config = BaseRecognizerConfig( + name="CreditCardRecognizer", score_thresholds=raw_thresholds + ) + + assert config.model_dump()["score_thresholds"] == raw_thresholds + assert type(config.model_dump()["score_thresholds"]) is type(raw_thresholds) + + +@pytest.mark.parametrize( + "recognizer", + [ + { + "name": "CreditCardRecognizer", + "type": "predefined", + "score_thresholds": {"default": 0.4}, + }, + { + "name": "custom_thresholds", + "type": "custom", + "supported_entity": "CUSTOM", + "supported_language": "en", + "patterns": [{"name": "custom", "regex": "x", "score": 0.5}], + "score_thresholds": {"CUSTOM": 0.6}, + }, + { + "name": "HuggingFaceNerRecognizer", + "type": "predefined", + "supported_language": "en", + "score_thresholds": {"PERSON": 0.7}, + }, + { + "name": "GLiNERRecognizer", + "type": "predefined", + "supported_language": "en", + "score_thresholds": {"PERSON": 0.8}, + }, + ], +) +def test_registry_model_dump_preserves_score_thresholds_for_every_entry_type( + recognizer, +): + original = recognizer["score_thresholds"].copy() + + dumped = RecognizerRegistryConfig(recognizers=[recognizer]).model_dump() + + assert dumped["recognizers"][0]["score_thresholds"] == original + + +def test_registry_model_does_not_mutate_recognizer_input_when_inferring_type(): + recognizer = { + "name": "custom_thresholds", + "supported_entity": "CUSTOM", + "supported_language": "en", + "patterns": [{"name": "custom", "regex": "x", "score": 0.5}], + "score_thresholds": {"default": 0.4}, + } + original = recognizer.copy() + + RecognizerRegistryConfig(recognizers=[recognizer]) + + assert recognizer == original From d1f5528cbd96ab38874e668b0f542a30ff070737 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Mon, 13 Jul 2026 11:41:33 -0400 Subject: [PATCH 10/11] fix(analyzer): tighten threshold consumer contract --- docs/analyzer/analyzer_engine_provider.md | 2 +- docs/analyzer/recognizer_registry_provider.md | 2 +- docs/tutorial/08_no_code.md | 2 +- .../presidio_analyzer/analyzer_engine.py | 33 ++-- .../presidio_analyzer/entity_recognizer.py | 11 +- .../presidio_analyzer/score_thresholds.py | 18 ++- .../tests/test_analyzer_engine.py | 148 +++++++++++------- 7 files changed, 134 insertions(+), 82 deletions(-) diff --git a/docs/analyzer/analyzer_engine_provider.md b/docs/analyzer/analyzer_engine_provider.md index f78ef16501..61b38d6742 100644 --- a/docs/analyzer/analyzer_engine_provider.md +++ b/docs/analyzer/analyzer_engine_provider.md @@ -78,7 +78,7 @@ The configuration file contains the following parameters: - `nlp_configuration`: Configuration given to the NLP engine which will detect the PIIs and extract features for the downstream logic. - `recognizer_registry`: All the recognizers that will be used by the analyzer. Each recognizer entry can define `score_thresholds`, using `default` as its fallback and entity names for overrides. -The threshold precedence is an explicit `analyze(score_threshold=...)` value, an entity-specific recognizer threshold, the recognizer's `default`, then `default_score_threshold`. +Supplying `analyze(score_threshold=...)` bypasses recognizer-level thresholds for that request and applies the supplied threshold to every result. When omitted, precedence is an entity-specific recognizer threshold, the recognizer's `default`, then `default_score_threshold`. !!! note "Note" diff --git a/docs/analyzer/recognizer_registry_provider.md b/docs/analyzer/recognizer_registry_provider.md index f290fc950f..9c95824838 100644 --- a/docs/analyzer/recognizer_registry_provider.md +++ b/docs/analyzer/recognizer_registry_provider.md @@ -110,7 +110,7 @@ The recognizer list comprises of both the predefined and custom recognizers, for - `supported_entity`: the detected entity associated by the recognizer. - `deny_list`: A list of words to detect, in case the recognizer uses a predefined list of words. - `deny_list_score`: confidence score for a term identified using a deny-list. - - `score_thresholds`: optional score cutoffs for this recognizer. Use `default` as the recognizer-wide cutoff and entity names for overrides. An explicit request threshold takes priority, followed by an entity override, this recognizer default, and the analyzer's `default_score_threshold`. + - `score_thresholds`: optional score thresholds for this recognizer. Use `default` as the recognizer-wide threshold and entity names for overrides. Supplying `analyze(score_threshold=...)` bypasses recognizer-level thresholds for that request. When omitted, precedence is an entity override, this recognizer default, then the analyzer's `default_score_threshold`. !!! tip "Configuration Tip: Agglutinative languages (e.g., Korean)" diff --git a/docs/tutorial/08_no_code.md b/docs/tutorial/08_no_code.md index b7c362d5dd..d717761b75 100644 --- a/docs/tutorial/08_no_code.md +++ b/docs/tutorial/08_no_code.md @@ -122,7 +122,7 @@ recognizer_registry: """ ``` -Each recognizer can set a `default` cutoff and entity-specific overrides in `score_thresholds`. An explicit request threshold takes priority, followed by an entity override, the recognizer default, and the analyzer's `default_score_threshold`. +Each recognizer can set a `default` threshold and entity-specific overrides in `score_thresholds`. Supplying `analyze(score_threshold=...)` bypasses recognizer-level thresholds for that request. When omitted, precedence is an entity override, the recognizer default, then the analyzer's `default_score_threshold`. ### NLP Engine parameters diff --git a/presidio-analyzer/presidio_analyzer/analyzer_engine.py b/presidio-analyzer/presidio_analyzer/analyzer_engine.py index de8294c6e3..e7d7e6b8c9 100644 --- a/presidio-analyzer/presidio_analyzer/analyzer_engine.py +++ b/presidio-analyzer/presidio_analyzer/analyzer_engine.py @@ -2,7 +2,7 @@ import logging import os from collections import Counter -from typing import List, Optional +from typing import Dict, List, Optional import regex as re @@ -171,8 +171,10 @@ def analyze( :param entities: List of PII entities that should be looked for in the text. If entities=None then all entities are looked for. :param correlation_id: cross call ID for this request - :param score_threshold: A minimum value for which - to return an identified entity + :param score_threshold: An explicit threshold for every result. When + supplied, this bypasses recognizer-level score thresholds for this + request. When omitted, the analyzer uses an entity-specific recognizer + threshold, the recognizer's default threshold, then the engine default. :param return_decision_process: Whether the analysis decision process steps returned in the response. :param ad_hoc_recognizers: List of recognizers which will be used only @@ -334,7 +336,7 @@ def _enhance_using_context( def __remove_low_scores( self, results: List[RecognizerResult], - score_threshold: float = None, + score_threshold: Optional[float] = None, recognizers: Optional[List[EntityRecognizer]] = None, ) -> List[RecognizerResult]: """ @@ -346,8 +348,9 @@ def __remove_low_scores( :return: List[RecognizerResult] """ if score_threshold is None: - recognizers_by_id = { - recognizer.id: recognizer for recognizer in recognizers or [] + recognizers_by_id: Dict[str, Dict[str, float]] = { + recognizer.id: recognizer.score_thresholds + for recognizer in recognizers or [] } return [ result @@ -362,15 +365,25 @@ def __remove_low_scores( def __get_result_score_threshold( self, result: RecognizerResult, - recognizers_by_id: dict, + recognizers_by_id: Dict[str, Dict[str, float]], ) -> float: """Resolve the threshold to apply for a single recognizer result.""" metadata = result.recognition_metadata or {} recognizer_id = metadata.get(RecognizerResult.RECOGNIZER_IDENTIFIER_KEY) - recognizer = recognizers_by_id.get(recognizer_id) - if recognizer is None: + if recognizer_id is None: + logger.debug( + "Recognizer identifier is missing; using the engine default " + "score threshold" + ) + return self.default_score_threshold + recognizer_thresholds = recognizers_by_id.get(recognizer_id) + if recognizer_thresholds is None: + logger.debug( + "Recognizer identifier %s did not match a selected recognizer; " + "using the engine default score threshold", + recognizer_id, + ) return self.default_score_threshold - recognizer_thresholds = recognizer.score_thresholds entity_threshold = recognizer_thresholds.get(result.entity_type) if entity_threshold is not None: diff --git a/presidio-analyzer/presidio_analyzer/entity_recognizer.py b/presidio-analyzer/presidio_analyzer/entity_recognizer.py index 178d94e63c..dd74cd248f 100644 --- a/presidio-analyzer/presidio_analyzer/entity_recognizer.py +++ b/presidio-analyzer/presidio_analyzer/entity_recognizer.py @@ -1,6 +1,6 @@ import logging from abc import abstractmethod -from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, ClassVar, Dict, List, Optional, Tuple from presidio_analyzer import RecognizerResult from presidio_analyzer.score_thresholds import normalize_score_thresholds @@ -55,7 +55,7 @@ def __init__( version: str = "0.0.1", context: Optional[List[str]] = None, country_code: Optional[str] = None, - score_thresholds: Any = None, + score_thresholds: Optional[Dict[str, float]] = None, ): self.supported_entities = supported_entities @@ -84,8 +84,11 @@ def score_thresholds(self) -> Dict[str, float]: return self._score_thresholds.copy() @score_thresholds.setter - def score_thresholds(self, value: Any) -> None: - """Validate and store this recognizer's score thresholds.""" + def score_thresholds(self, value: Optional[Dict[str, float]]) -> None: + """Validate and store this recognizer's score thresholds. + + :param value: The default and entity-specific score thresholds. + """ self._score_thresholds = normalize_score_thresholds(value) @classmethod diff --git a/presidio-analyzer/presidio_analyzer/score_thresholds.py b/presidio-analyzer/presidio_analyzer/score_thresholds.py index b809b8ccef..b41c14d7dc 100644 --- a/presidio-analyzer/presidio_analyzer/score_thresholds.py +++ b/presidio-analyzer/presidio_analyzer/score_thresholds.py @@ -1,11 +1,15 @@ """Validation helpers for recognizer score thresholds.""" from collections.abc import Mapping -from typing import Any, Dict +from typing import Dict -def validate_score_threshold(threshold: Any) -> float: - """Validate a score threshold without coercing its input type.""" +def validate_score_threshold(threshold: object) -> float: + """Validate a score threshold without coercing its input type. + + :param threshold: The value to validate. + :return: The validated score threshold. + """ if not isinstance(threshold, (int, float)) or isinstance(threshold, bool): raise ValueError(f"Score threshold must be numeric, got: {threshold}") if not 0.0 <= threshold <= 1.0: @@ -15,8 +19,12 @@ def validate_score_threshold(threshold: Any) -> float: return threshold -def normalize_score_thresholds(score_thresholds: Any) -> Dict[str, float]: - """Validate and defensively copy one recognizer's score thresholds.""" +def normalize_score_thresholds(score_thresholds: object) -> Dict[str, float]: + """Validate and defensively copy one recognizer's score thresholds. + + :param score_thresholds: The threshold mapping to validate. + :return: A normalized copy of the score thresholds. + """ if score_thresholds is None: return {} if not isinstance(score_thresholds, Mapping): diff --git a/presidio-analyzer/tests/test_analyzer_engine.py b/presidio-analyzer/tests/test_analyzer_engine.py index 6aed1e468d..e567c1e5ab 100644 --- a/presidio-analyzer/tests/test_analyzer_engine.py +++ b/presidio-analyzer/tests/test_analyzer_engine.py @@ -83,9 +83,7 @@ def load(self): def analyze(self, text: str, entities: List[str], nlp_artifacts: NlpArtifacts): """Return deterministic results for threshold filtering tests.""" return [ - RecognizerResult( - result.entity_type, result.start, result.end, result.score - ) + RecognizerResult(result.entity_type, result.start, result.end, result.score) for result in self._results ] @@ -108,9 +106,7 @@ def load(self): def analyze(self, text: str, entities: List[str], nlp_artifacts: NlpArtifacts): """Return deterministic results for threshold filtering tests.""" return [ - RecognizerResult( - result.entity_type, result.start, result.end, result.score - ) + RecognizerResult(result.entity_type, result.start, result.end, result.score) for result in self._results ] @@ -139,7 +135,7 @@ class ContextThresholdRecognizer(DuplicateThresholdRecognizer): """Recognizer which raises its result score during context enhancement.""" def enhance_using_context(self, *args, **kwargs): - """Raise the score above the configured cutoff.""" + """Raise the score above the configured threshold.""" results = kwargs["raw_recognizer_results"] results[0].score = 0.8 return results @@ -204,6 +200,7 @@ def test_when_analyze_with_predefined_recognizers_then_return_results( assert len(results) == 1 assert_result(results[0], "CREDIT_CARD", 14, 33, max_score) + @pytest.mark.parametrize( "registry_config,analyzer_lang,expectation", [ @@ -212,22 +209,27 @@ def test_when_analyze_with_predefined_recognizers_then_return_results( ({"supported_languages": ["es", "de"]}, None, pytest.raises(ValueError)), ({"supported_languages": ["es", "de"]}, ["de", "es"], nullcontext()), (None, None, nullcontext()), - ] + ], ) -def test_when_analyze_with_unsupported_language_must_match(registry_config, analyzer_lang, expectation): +def test_when_analyze_with_unsupported_language_must_match( + registry_config, analyzer_lang, expectation +): with expectation: - registry = RecognizerRegistryProvider(registry_configuration=registry_config).create_recognizer_registry() + registry = RecognizerRegistryProvider( + registry_configuration=registry_config + ).create_recognizer_registry() AnalyzerEngine( registry=registry, supported_languages=analyzer_lang, nlp_engine=NlpEngineMock(), ) -def test_when_analyze_with_defaults_success( -): + +def test_when_analyze_with_defaults_success(): registry = RecognizerRegistryProvider().create_recognizer_registry() AnalyzerEngine(registry=registry) + def test_when_analyze_with_multiple_predefined_recognizers_then_succeed( loaded_registry, unit_test_guid, spacy_nlp_engine, max_score ): @@ -390,7 +392,7 @@ def test_when_regex_allow_list_specified(loaded_analyzer_engine): assert_result(results[0], "URL", 0, 8, 0.5) results = loaded_analyzer_engine.analyze( - text=text, language="en", allow_list=["bing"], allow_list_match = "regex" + text=text, language="en", allow_list=["bing"], allow_list_match="regex" ) assert len(results) == 2 assert text[results[0].start : results[0].end] == "microsoft.com" @@ -408,13 +410,15 @@ def test_when_regex_allow_list_specified_but_none_in_file(loaded_analyzer_engine assert_result(results[0], "URL", 0, 8, 0.5) results = loaded_analyzer_engine.analyze( - text=text, language="en", allow_list=["microsoft"], allow_list_match = "regex" + text=text, language="en", allow_list=["microsoft"], allow_list_match="regex" ) assert len(results) == 1 assert_result(results[0], "URL", 0, 8, 0.5) -def test_when_regex_allow_list_specified_multiple_items_with_missing_flags(loaded_analyzer_engine): +def test_when_regex_allow_list_specified_multiple_items_with_missing_flags( + loaded_analyzer_engine, +): text = "bing.com is his favorite website, microsoft.com is his second favorite, azure.com is his third favorite" results = loaded_analyzer_engine.analyze( text=text, @@ -424,7 +428,10 @@ def test_when_regex_allow_list_specified_multiple_items_with_missing_flags(loade assert_result(results[0], "URL", 0, 8, 0.5) results = loaded_analyzer_engine.analyze( - text=text, language="en", allow_list=["bing", "microsoft"], allow_list_match = "regex", + text=text, + language="en", + allow_list=["bing", "microsoft"], + allow_list_match="regex", ) assert len(results) == 1 assert text[results[0].start : results[0].end] == "azure.com" @@ -440,12 +447,20 @@ def test_when_regex_allow_list_specified_with_regex_flags(loaded_analyzer_engine assert_result(results[0], "URL", 0, 8, 0.5) results = loaded_analyzer_engine.analyze( - text=text, language="en", allow_list=["BING", "MICROSOFT", "AZURE"], allow_list_match = "regex", regex_flags=0 + text=text, + language="en", + allow_list=["BING", "MICROSOFT", "AZURE"], + allow_list_match="regex", + regex_flags=0, ) assert len(results) == 3 results = loaded_analyzer_engine.analyze( - text=text, language="en", allow_list=["BING", "MICROSOFT", "AZURE"], allow_list_match = "regex", regex_flags=re.IGNORECASE + text=text, + language="en", + allow_list=["BING", "MICROSOFT", "AZURE"], + allow_list_match="regex", + regex_flags=re.IGNORECASE, ) assert len(results) == 0 @@ -538,13 +553,13 @@ def test_when_entities_is_none_then_return_all_fields(loaded_registry): def test_when_entities_is_none_all_recognizers_loaded_then_return_all_fields( - spacy_nlp_engine, + spacy_nlp_engine, ): analyze_engine = AnalyzerEngine( registry=RecognizerRegistry(), nlp_engine=spacy_nlp_engine ) threshold = 0 - text = "My name is Sharon and I live in Seattle." "Domain: microsoft.com " + text = "My name is Sharon and I live in Seattle.Domain: microsoft.com " response = analyze_engine.analyze( text=text, score_threshold=threshold, language="en" ) @@ -677,39 +692,41 @@ def test_when_default_threshold_is_zero_then_all_results_pass( assert len(results) == 2 -def test_recognizer_threshold_precedence(): - """Recognizer-specific and entity-specific thresholds should win.""" - analyzer_engine = _build_threshold_analyzer( - {"default": 0.4, "PERSON": 0.6, "CREDIT_CARD": 0.7} - ) - - results = analyzer_engine.analyze( - text="Threshold config", - language="en", - entities=["PERSON", "CREDIT_CARD", "URL", "DATE_TIME", "PHONE_NUMBER"], - ) - - scores = {result.entity_type: result.score for result in results} - - assert scores == {"URL": 0.75, "DATE_TIME": 0.9} - - -def test_explicit_score_threshold_overrides_config(): - """A request-level score threshold should override config thresholds.""" +@pytest.mark.parametrize( + "score_thresholds, default_score_threshold, request_score_threshold, expected", + [ + ( + {"default": 0.85, "PERSON": 0.95, "CREDIT_CARD": 0.95}, + 0.1, + 0.8, + {"DATE_TIME": 0.9}, + ), + ( + {"default": 0.4, "PERSON": 0.6}, + 0.3, + None, + {"CREDIT_CARD": 0.65, "URL": 0.75, "DATE_TIME": 0.9}, + ), + ({"default": 0.8}, 0.3, None, {"DATE_TIME": 0.9}), + ({}, 0.8, None, {"DATE_TIME": 0.9}), + ], +) +def test_competing_threshold_sources_resolve_in_precedence_order( + score_thresholds, default_score_threshold, request_score_threshold, expected +): + """The decisive threshold source should determine the returned results.""" analyzer_engine = _build_threshold_analyzer( - {"default": 0.4, "PERSON": 0.6, "CREDIT_CARD": 0.7} + score_thresholds, default_score_threshold ) results = analyzer_engine.analyze( text="Threshold config", language="en", - entities=["PERSON", "CREDIT_CARD", "URL", "DATE_TIME", "PHONE_NUMBER"], - score_threshold=0.8, + entities=["PERSON", "CREDIT_CARD", "URL", "DATE_TIME"], + score_threshold=request_score_threshold, ) - scores = {result.entity_type: result.score for result in results} - - assert scores == {"DATE_TIME": 0.9} + assert {result.entity_type: result.score for result in results} == expected def test_duplicate_results_keep_recognizers_that_meet_their_thresholds(): @@ -734,9 +751,10 @@ def test_duplicate_results_keep_recognizers_that_meet_their_thresholds(): assert len(results) == 1 assert strict.id != lenient.id - assert results[0].recognition_metadata[ - RecognizerResult.RECOGNIZER_IDENTIFIER_KEY - ] == lenient.id + assert ( + results[0].recognition_metadata[RecognizerResult.RECOGNIZER_IDENTIFIER_KEY] + == lenient.id + ) def test_empty_recognizer_thresholds_use_default(): @@ -771,8 +789,16 @@ def test_context_enhancement_runs_before_recognizer_threshold_filtering(): assert [result.score for result in results] == [0.8] -@pytest.mark.parametrize("identifier", [None, "not-a-selected-recognizer"]) -def test_missing_or_unmatched_recognizer_identifier_uses_engine_default(identifier): +@pytest.mark.parametrize( + "identifier, log_message", + [ + (None, "Recognizer identifier is missing"), + ("not-a-selected-recognizer", "did not match"), + ], +) +def test_missing_or_unmatched_recognizer_identifier_uses_engine_default( + identifier, log_message, caplog +): """Unknown producer provenance should use the engine fallback.""" registry = RecognizerRegistry() registry.add_recognizer(MissingIdentifierRecognizer(identifier)) @@ -782,7 +808,10 @@ def test_missing_or_unmatched_recognizer_identifier_uses_engine_default(identifi default_score_threshold=0.8, ) - assert analyzer.analyze("John", "en", entities=["PERSON"]) == [] + with caplog.at_level("DEBUG", logger="presidio-analyzer"): + assert analyzer.analyze("John", "en", entities=["PERSON"]) == [] + + assert log_message in caplog.text def test_ad_hoc_recognizer_uses_its_score_thresholds(): @@ -1005,7 +1034,9 @@ def test_entities_filter_for_ad_hoc_removes_recognizer(loaded_analyzer_engine): assert "MR" not in [resp.entity_type for resp in responses2] -def test_ad_hoc_with_context_support_higher_confidence(spacy_nlp_engine, zip_code_recognizer): +def test_ad_hoc_with_context_support_higher_confidence( + spacy_nlp_engine, zip_code_recognizer +): text = "Mr. John Smith's zip code is 10023" analyzer_engine = AnalyzerEngine(nlp_engine=spacy_nlp_engine) @@ -1088,7 +1119,9 @@ def analyze(self, text: str, entities: List[str], nlp_artifacts: NlpArtifacts): ) -def test_when_recognizer_overrides_enhance_score_then_it_get_boosted_once(spacy_nlp_engine): +def test_when_recognizer_overrides_enhance_score_then_it_get_boosted_once( + spacy_nlp_engine, +): class MockRecognizer(EntityRecognizer, ABC): def analyze(self, text: str, entities: List[str], nlp_artifacts: NlpArtifacts): return [ @@ -1191,12 +1224,8 @@ def test_when_regex_allow_list_times_out_then_result_is_kept(loaded_analyzer_eng """Test that a timed-out allow list regex keeps the result (conservative behavior).""" text = "bing.com is his favorite website" - with patch( - "presidio_analyzer.analyzer_engine.REGEX_TIMEOUT_SECONDS", 0.001 - ): - with patch( - "presidio_analyzer.analyzer_engine.re.compile" - ) as mock_compile: + with patch("presidio_analyzer.analyzer_engine.REGEX_TIMEOUT_SECONDS", 0.001): + with patch("presidio_analyzer.analyzer_engine.re.compile") as mock_compile: mock_compiled = mock_compile.return_value mock_compiled.search.side_effect = TimeoutError("regex timed out") @@ -1245,4 +1274,3 @@ def test_when_regex_allow_list_is_all_empty_entries_then_results_are_kept(): ) assert filtered == results - From fe2b416bbbdf6a0424a8c51483111097ddfdfed9 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Mon, 13 Jul 2026 12:02:54 -0400 Subject: [PATCH 11/11] Retry external CI flakes