diff --git a/docs/detectors.md b/docs/detectors.md index 3123155..3366ab7 100644 --- a/docs/detectors.md +++ b/docs/detectors.md @@ -16,7 +16,7 @@ This document describes the minimal API, implementation guidance, a short exampl ```python class CoreDetectorConfig(CoreConfig): - comp_type: str = "detectors" + component_type: str = "detectors" method_type: str = "core_detector" parser: str = "" diff --git a/setup.py b/setup.py index 4e86b68..7dcd355 100644 --- a/setup.py +++ b/setup.py @@ -25,4 +25,11 @@ def gather_dependencies(toml_path: str = "pyproject.toml") -> list[str]: author="voice", author_email="voice@example.com", install_requires=gather_dependencies(), + data_files=[( + "src/tools/workspace/templates/data", + [ + "src/tools/workspace/templates/data/logs.json", + "src/tools/workspace/templates/data/parsed_log.json", + ] + )] ) diff --git a/src/detectmatelibrary/common/_config/__init__.py b/src/detectmatelibrary/common/_config/__init__.py index 7c0265b..0ee1ed8 100644 --- a/src/detectmatelibrary/common/_config/__init__.py +++ b/src/detectmatelibrary/common/_config/__init__.py @@ -9,6 +9,14 @@ from typing import Any, Dict from copy import deepcopy +from numpy.random import choice +import string + + +def random_id(length: int = 10) -> str: + characters = [s for s in string.ascii_letters + string.digits] + return "".join(str(choice(characters)) for _ in range(length)) + class BasicConfig(BaseModel): """Base configuration class with helper methods.""" @@ -16,7 +24,7 @@ class BasicConfig(BaseModel): model_config = ConfigDict(extra="forbid") method_type: str = "default_method_type" - comp_type: str = "default_type" + component_type: str = "default_type" auto_config: bool = False @@ -33,13 +41,13 @@ def update_config(self, new_config: Dict[str, Any]) -> None: def from_dict(cls, data: Dict[str, Any], method_id: str) -> Self: aux = cls() config_ = ConfigMethods.get_method( - deepcopy(data), comp_type=aux.comp_type, method_id=method_id + deepcopy(data), component_type=aux.component_type, method_id=method_id ) ConfigMethods.check_type(config_, method_type=aux.method_type) return cls(**ConfigMethods.process(config_)) - def to_dict(self, method_id: str) -> Dict[str, Any]: + def to_dict(self, method_id: str = random_id()) -> Dict[str, Any]: """Convert the config back to YAML-compatible dictionary format. This is the inverse of from_dict() and ensures yaml -> pydantic -> yaml preservation. @@ -48,7 +56,7 @@ def to_dict(self, method_id: str) -> Dict[str, Any]: method_id: The method identifier to use in the output structure Returns: - Dictionary with structure: {comp_type: {method_id: config_data}} + Dictionary with structure: {component_type: {method_id: config_data}} """ # Build the config in the format expected by from_dict result: Dict[str, Any] = { @@ -63,7 +71,7 @@ def to_dict(self, method_id: str) -> Dict[str, Any]: for field_name, field_value in self: # Skip meta fields - if field_name in ("comp_type", "method_type", "auto_config"): + if field_name in ("component_type", "method_type", "auto_config"): continue # Handle EventsConfig specially @@ -96,9 +104,9 @@ def to_dict(self, method_id: str) -> Dict[str, Any]: if events_data is not None: result["events"] = events_data - # Wrap in the comp_type and method_id structure + # Wrap in the component_type and method_id structure return { - self.comp_type: { + self.component_type: { method_id: result } } diff --git a/src/detectmatelibrary/common/_config/_compile.py b/src/detectmatelibrary/common/_config/_compile.py index 5198629..5a864e8 100644 --- a/src/detectmatelibrary/common/_config/_compile.py +++ b/src/detectmatelibrary/common/_config/_compile.py @@ -41,9 +41,9 @@ def _classify_variables( class MethodNotFoundError(Exception): - def __init__(self, method_id: str, comp_type: str) -> None: + def __init__(self, method_id: str, component_type: str) -> None: super().__init__( - f"Method '{method_id}' of type '{comp_type}' not found in configuration." + f"Method '{method_id}' of type '{component_type}' not found in configuration." ) @@ -72,15 +72,15 @@ def __init__(self) -> None: class ConfigMethods: @staticmethod def get_method( - config: Dict[str, Dict[str, Dict[str, Any]]], method_id: str, comp_type: str + config: Dict[str, Dict[str, Dict[str, Any]]], method_id: str, component_type: str ) -> Dict[str, Any]: - if comp_type not in config: - raise TypeNotFoundError(comp_type) + if component_type not in config: + raise TypeNotFoundError(component_type) - args = config[comp_type] + args = config[component_type] if method_id not in args: - raise MethodNotFoundError(method_id, comp_type) + raise MethodNotFoundError(method_id, component_type) return args[method_id] diff --git a/src/detectmatelibrary/common/detector.py b/src/detectmatelibrary/common/detector.py index 58f839e..e224f68 100644 --- a/src/detectmatelibrary/common/detector.py +++ b/src/detectmatelibrary/common/detector.py @@ -90,7 +90,7 @@ def get_global_variables( class CoreDetectorConfig(CoreConfig): - comp_type: str = "detectors" + component_type: str = "detectors" method_type: str = "core_detector" parser: str = "" @@ -112,7 +112,7 @@ def __init__( super().__init__( name=name, - type_=config.comp_type, # type: ignore + type_=config.component_type, # type: ignore config=config, # type: ignore args_buffer=ArgsBuffer(mode=buffer_mode, size=buffer_size), input_schema=ParserSchema, diff --git a/src/detectmatelibrary/common/parser.py b/src/detectmatelibrary/common/parser.py index ca4e5c4..4bfc170 100644 --- a/src/detectmatelibrary/common/parser.py +++ b/src/detectmatelibrary/common/parser.py @@ -12,7 +12,7 @@ class CoreParserConfig(CoreConfig): - comp_type: str = "parsers" + component_type: str = "parsers" method_type: str = "core_parser" log_format: str | None = None diff --git a/src/tools/workspace/create_workspace.py b/src/tools/workspace/create_workspace.py index 4045713..09e286b 100644 --- a/src/tools/workspace/create_workspace.py +++ b/src/tools/workspace/create_workspace.py @@ -123,7 +123,13 @@ def create_workspace(type_: str, name: str, target_dir: Path) -> None: create_tests(type_=type_, name=name, workspace_root=workspace_root, pkg_name=pkg_name) # Copy data - copy_file(PROJECT_ROOT / DATA_FILES[type_], workspace_root / "data.json") + try: + copy_file(PROJECT_ROOT / DATA_FILES[type_], workspace_root / "data.json") + except FileNotFoundError: # Copy data from pip instead + copy_file( + PROJECT_ROOT / DATA_FILES[type_].replace("src/", "site-packages/"), + workspace_root / "data.json" + ) # Copy meta/root files for file_name in META_FILES: diff --git a/tests/test_common/test_config.py b/tests/test_common/test_config.py index 804a41a..7d53d97 100644 --- a/tests/test_common/test_config.py +++ b/tests/test_common/test_config.py @@ -29,7 +29,7 @@ def load_test_config() -> dict: class TestConfigMethods: def test_get_method(self): config = ConfigMethods.get_method( - config_test, method_id="example_parser", comp_type="parsers" + config_test, method_id="example_parser", component_type="parsers" ) assert config["method_type"] == "ExampleParser" assert not config["auto_config"] @@ -40,18 +40,18 @@ def test_get_method(self): def test_method_not_found(self): with pytest.raises(MethodNotFoundError): ConfigMethods.get_method( - config_test, method_id="non_existent", comp_type="parsers" + config_test, method_id="non_existent", component_type="parsers" ) def test_type_not_found(self): with pytest.raises(TypeNotFoundError): ConfigMethods.get_method( - config_test, method_id="example_parser", comp_type="non_existent_type" + config_test, method_id="example_parser", component_type="non_existent_type" ) def test_check_type(self): config = ConfigMethods.get_method( - config_test, method_id="example_parser", comp_type="parsers" + config_test, method_id="example_parser", component_type="parsers" ) ConfigMethods.check_type(config, method_type="ExampleParser") @@ -60,7 +60,7 @@ def test_check_type(self): def test_process_simple(self): config = ConfigMethods.process(ConfigMethods.get_method( - config_test, method_id="example_parser", comp_type="parsers" + config_test, method_id="example_parser", component_type="parsers" )) assert config["method_type"] == "ExampleParser" @@ -71,7 +71,7 @@ def test_process_simple(self): def test_process_auto_config(self): config = ConfigMethods.process(ConfigMethods.get_method( - config_test, method_id="detector_auto", comp_type="detectors" + config_test, method_id="detector_auto", component_type="detectors" )) assert config["method_type"] == "ExampleDetector" @@ -82,13 +82,13 @@ def test_process_auto_config(self): def test_process_auto_config_false(self): with pytest.warns(MissingParamsWarning): ConfigMethods.process(ConfigMethods.get_method( - config_test, method_id="detector_wrong", comp_type="detectors" + config_test, method_id="detector_wrong", component_type="detectors" )) def test_process_auto_config_warning(self): with pytest.warns(AutoConfigWarning): ConfigMethods.process(ConfigMethods.get_method( - config_test, method_id="detector_weird", comp_type="detectors" + config_test, method_id="detector_weird", component_type="detectors" )) @@ -96,7 +96,7 @@ class TestParamsFormat: def test_correct_format(self): config_test = load_test_config() config = ConfigMethods.process(ConfigMethods.get_method( - config_test, method_id="detector_variables", comp_type="detectors" + config_test, method_id="detector_variables", component_type="detectors" )) assert config["method_type"] == "ExampleDetector" @@ -118,7 +118,7 @@ def test_correct_format(self): def test_correct_format2(self): config = ConfigMethods.process(ConfigMethods.get_method( - config_test, method_id="detector_variables2", comp_type="detectors" + config_test, method_id="detector_variables2", component_type="detectors" )) assert config["method_type"] == "ExampleDetector" @@ -138,7 +138,7 @@ def test_correct_format2(self): def test_return_none_if_not_found(self): config_test = load_test_config() config = ConfigMethods.process(ConfigMethods.get_method( - config_test, method_id="detector_variables", comp_type="detectors" + config_test, method_id="detector_variables", component_type="detectors" )) assert isinstance(config["events"][1], _EventConfig) @@ -147,7 +147,7 @@ def test_return_none_if_not_found(self): def test_get_dict(self): config_test = load_test_config() config = ConfigMethods.process(ConfigMethods.get_method( - config_test, method_id="detector_variables", comp_type="detectors" + config_test, method_id="detector_variables", component_type="detectors" )) variables = config["events"][1].get_all() @@ -156,13 +156,13 @@ def test_get_dict(self): def test_incorrect_format(self): with pytest.raises(ValidationError): ConfigMethods.process(ConfigMethods.get_method( - config_test, method_id="detector_incorrect_format1", comp_type="detectors" + config_test, method_id="detector_incorrect_format1", component_type="detectors" )) class MockupParserConfig(BasicConfig): method_type: str = "ExampleParser" - comp_type: str = "parsers" + component_type: str = "parsers" auto_config: bool = False log_format: str = "" @@ -171,7 +171,7 @@ class MockupParserConfig(BasicConfig): class MockuptDetectorConfig(BasicConfig): method_type: str = "ExampleDetector" - comp_type: str = "detectors" + component_type: str = "detectors" parser: str = "" diff --git a/tests/test_common/test_config_roundtrip.py b/tests/test_common/test_config_roundtrip.py index 764b668..24359ff 100644 --- a/tests/test_common/test_config_roundtrip.py +++ b/tests/test_common/test_config_roundtrip.py @@ -9,7 +9,7 @@ class MockupParserConfig(BasicConfig): method_type: str = "ExampleParser" - comp_type: str = "parsers" + component_type: str = "parsers" auto_config: bool = False log_format: str = "" depth: int = -1 @@ -17,7 +17,7 @@ class MockupParserConfig(BasicConfig): class MockupDetectorConfig(BasicConfig): method_type: str = "ExampleDetector" - comp_type: str = "detectors" + component_type: str = "detectors" auto_config: bool = False parser: str = "" events: EventsConfig | None = None @@ -202,7 +202,7 @@ def test_true_roundtrip_preservation(self): # The two configs should be identical assert config1.method_type == config2.method_type - assert config1.comp_type == config2.comp_type + assert config1.component_type == config2.component_type assert config1.auto_config == config2.auto_config assert config1.parser == config2.parser diff --git a/tests/test_common/test_core.py b/tests/test_common/test_core.py index 6fdc137..f8d0183 100644 --- a/tests/test_common/test_core.py +++ b/tests/test_common/test_core.py @@ -25,7 +25,7 @@ class MockConfigWithTraining(CoreConfig): default_args = { "method_type": "default_method_type", - "comp_type": "default_type", + "component_type": "default_type", "auto_config": False, "start_id": 10, "data_use_training": None, diff --git a/uv.lock b/uv.lock index 5823ee0..c673bc1 100644 --- a/uv.lock +++ b/uv.lock @@ -221,7 +221,7 @@ wheels = [ [[package]] name = "detectmatelibrary" -version = "0.1.0" +version = "0.2.0" source = { editable = "." } dependencies = [ { name = "kafka-python" },