diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ff966b7..b651b0e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -58,7 +58,7 @@ repos: rev: 1.8.6 hooks: - id: bandit - exclude: ^tests/ + exclude: (^tests/|.*/test_.*|^test_.*) # Unused code detection - repo: https://github.com/jendrikseipp/vulture diff --git a/README.md b/README.md index 67d2b04..ac8ae4f 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Main library to run the different components in DetectMate. The library contains the next components: -* **Readers**: insert logs into the system. +* **Readers**: insert logs into the system. * **Parsers**: parse the logs receive from the reader. * **Detectors**: return alerts if anomalies are detected. * **Schemas**: standard data classes use in DetectMate. @@ -26,13 +26,16 @@ uv run prek install ``` ### Step 2: Install Protobuf dependencies -To installed in linux do: + +To install in Linux do: + ```bash sudo apt install -y protobuf-compiler protoc --version ``` -This dependency is only need if a proto files is modifiy. To compile the proto file do: -``` + +This dependency is only needed if a proto file is modified. To compile the proto file do: +```bash protoc --proto_path=src/schemas/ --python_out=src/schemas/ src/schemas/schemas.proto ``` @@ -48,3 +51,48 @@ Run the tests with coverage (add --cov-report=html to generate an HTML report): ```bash uv run pytest --cov=. --cov-report=term-missing ``` + +## Workspace generator (`mate create`) + +DetectMateLibrary includes a small CLI helper to bootstrap standalone workspaces +for custom parsers and detectors. This is useful if you want to develop and test +components in isolation while still using the same library and schemas. + +### Usage + +The CLI entry point is `mate` with a `create` command: + +```bash +mate create --type --name --dir +``` + +| Option | Description | +|----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `--type` | Component type to generate:
- `parser`: CoreParser-based template
- `detector`: CoreDetector-based template | +| `--name` | Name of the component and package:
- Creates package dir: `//`
- Creates main file: `.py`
- Derives class names: `` and `Config` | +| `--dir` | Directory where the workspace will be created | + + +### What gets generated + +For example: + +```bash +mate create --type parser --name custom_parser --dir ./workspaces/custom_parser +``` + +will create: + +```text +workspaces/custom_parser/ # workspace root +├── custom_parser/ # Python package +│ ├── __init__.py +│ └── custom_parser.py # CoreParser-based template +├── tests/ +│ └── test_custom_parser.py # generated from template to test custom_parser +├── LICENSE.md # copied from main project +├── .gitignore # copied from main project +├── .pre-commit-config.yaml # copied from main project +├── pyproject.toml # minimal project + dev extras +└── README.md # setup instructions +``` diff --git a/pyproject.toml b/pyproject.toml index fa2f9b1..7322744 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,6 @@ dependencies = [ "regex>=2025.11.3", ] - [project.optional-dependencies] # add dependencies in this section with: uv add --optional dev # install with all the dev dependencies: uv pip install -e .[dev] @@ -29,3 +28,6 @@ build-backend = "setuptools.build_meta" [tool.setuptools.packages.find] where = ["src"] + +[project.scripts] +mate = "tools.workspace.create_workspace:main" diff --git a/src/detectmatelibrary/utils/time_format_handler.py b/src/detectmatelibrary/utils/time_format_handler.py index 129b4b5..5f2fcd0 100644 --- a/src/detectmatelibrary/utils/time_format_handler.py +++ b/src/detectmatelibrary/utils/time_format_handler.py @@ -40,7 +40,7 @@ def _parse_with_format(self, time_str: str, fmt: str) -> str | None: try: # handle syslog-like formats that lack a year by prepending the current year if fmt == "%b %d %H:%M:%S" and re.match(r"^[A-Za-z]{3} \d{1,2} ", time_str): - year = datetime.utcnow().year + year = datetime.now().year dt = datetime.strptime(f"{year} {time_str}", f"%Y {fmt}") else: dt = datetime.strptime(time_str, fmt) diff --git a/src/tools/__init__.py b/src/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/tools/workspace/__init__.py b/src/tools/workspace/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/tools/workspace/create_workspace.py b/src/tools/workspace/create_workspace.py new file mode 100644 index 0000000..eb582c1 --- /dev/null +++ b/src/tools/workspace/create_workspace.py @@ -0,0 +1,175 @@ +import argparse +import shutil +import sys +from pathlib import Path + +from .utils import create_readme, create_pyproject, normalize_package_name + +# resolve paths relative to this file +BASE_DIR = Path(__file__).resolve().parent.parent # tools/ +PROJECT_ROOT = BASE_DIR.parent.parent # root of project +TEMPLATE_DIR = BASE_DIR / "workspace" / "templates" + +META_FILES = ["LICENSE.md", ".gitignore", ".pre-commit-config.yaml"] + + +def copy_file(src: Path, dst: Path) -> None: + """Copy file while ensuring destination directory exists.""" + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst) + + +def camelize(name: str) -> str: + """Convert names like "custom_parser" or "customParser" to + "CustomParser".""" + if "_" in name or "-" in name: + parts = name.replace("-", "_").split("_") + return "".join(p.capitalize() for p in parts if p) + return name[0].upper() + name[1:] + + +def create_tests(type_: str, name: str, workspace_root: Path, pkg_name: str) -> None: + """Create a tests/ directory with a basic pytest file for the component. + + - Reads template tests from src/tools/workspace/templates/test_templates/ + - Rewrites the import to point to . + - Renames CustomParser/CustomDetector to the camelized class name + """ + + tests_dir = workspace_root / "tests" + tests_dir.mkdir(parents=True, exist_ok=True) + template_file = TEMPLATE_DIR / "test_templates" / f"test_Custom{type_.capitalize()}.py" + test_file = tests_dir / f"test_{name}.py" + + if not template_file.exists(): + print(f"WARNING: Test template {template_file} not found. Creating empty test file.", file=sys.stderr) + test_file.touch() + return + + template_content = template_file.read_text() + base_class = f"Custom{type_.capitalize()}" # base names in the template (CustomParser/CustomDetector) + # The exact import line in the template, e.g: + # from ..CustomParser import CustomParser, CustomParserConfig + # from ..CustomDetector import CustomDetector, CustomDetectorConfig + original_import = f"from ..{base_class} import {base_class}, {base_class}Config" + new_class = camelize(name) + # new import line for the generated workspace: + # from . import , Config + new_import = f"from {pkg_name}.{normalize_package_name(name)} import {new_class}, {new_class}Config" + # replace the import line + content = template_content.replace(original_import, new_import) + # replace the remaining occurrences of CustomParser/CustomDetector + # with the new class name (inside the tests) + content = content.replace(base_class, new_class) + content = content.rstrip() + "\n" + + test_file.write_text(content) + print(f"- Added tests file: {test_file}") + + +def create_workspace(type_: str, name: str, target_dir: Path) -> None: + """Main workspace creation logic. + + - `target_dir` is the workspace root (from --dir) + - Code lives in a subpackage: // + - Meta files (LICENSE, .gitignore, etc.) + README.md + pyproject.toml live in workspace root + """ + + # Workspace root + workspace_root = Path(target_dir).expanduser().resolve() + workspace_root.mkdir(parents=True, exist_ok=True) + + # Package directory inside the workspace + pkg_name = normalize_package_name(name) + pkg_dir = workspace_root / pkg_name + + # Fail if the package directory already exists + if pkg_dir.exists(): + print(f"ERROR: Target directory already exists: {pkg_dir}", file=sys.stderr) + sys.exit(1) + + print(f"Creating workspace at: {pkg_dir}") + pkg_dir.mkdir(parents=True, exist_ok=False) + + # Template selection + template_file = TEMPLATE_DIR / f"Custom{type_.capitalize()}.py" + module_name = normalize_package_name(name) + target_code_file = pkg_dir / f"{module_name}.py" + + if not template_file.exists(): + print(f"WARNING: Template not found: {template_file}. Creating empty file.", file=sys.stderr) + target_code_file.touch() + else: + template_content = template_file.read_text() + + # Replace default class name inside template + original_class = f"Custom{type_.capitalize()}" + new_class = camelize(name) + template_content = template_content.replace(original_class, new_class) + + target_code_file.write_text(template_content) + + print(f"- Added implementation file: {target_code_file}") + + # Make the package importable + (pkg_dir / "__init__.py").touch() + + create_tests(type_=type_, name=name, workspace_root=workspace_root, pkg_name=pkg_name) + + # Copy meta/root files + for file_name in META_FILES: + src = PROJECT_ROOT / file_name + dst = workspace_root / file_name + + if src.exists(): + copy_file(src, dst) + print(f"- Copied {file_name}") + else: + print(f"! Warning: {file_name} not found in project root.") + + # Create pyproject.toml + create_pyproject(name, type_, workspace_root) + print(f"- Created pyproject.toml in {workspace_root}") + + # Create README + create_readme(name, type_, target_code_file, workspace_root) + print(f"- Created README.md in {workspace_root}") + print("\nWorkspace created successfully!") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Create a new workspace") + + subparsers = parser.add_subparsers(dest="command", required=True) + create_cmd = subparsers.add_parser("create", help="Create a workspace") + + create_cmd.add_argument( + "--type", + required=True, + choices=["parser", "detector"], + help="Type of component to generate", + ) + + create_cmd.add_argument( + "--name", + required=True, + help="Name for the generated file (e.g., customParser)", + ) + + create_cmd.add_argument( + "--dir", + required=True, + help="Directory where the workspace will be created", + ) + + args = parser.parse_args() + + if args.command == "create": + target_dir = Path(args.dir).expanduser().resolve() + create_workspace(args.type, args.name, target_dir) + else: + parser.print_help() + + +if __name__ == "__main__": + main() diff --git a/src/tools/workspace/templates/CustomDetector.py b/src/tools/workspace/templates/CustomDetector.py new file mode 100644 index 0000000..6eb0111 --- /dev/null +++ b/src/tools/workspace/templates/CustomDetector.py @@ -0,0 +1,56 @@ +from typing import Any, List + +from detectmatelibrary.common.detector import CoreDetector, CoreDetectorConfig +from detectmatelibrary.utils.data_buffer import BufferMode +from detectmatelibrary import schemas + + +class CustomDetectorConfig(CoreDetectorConfig): + """Configuration for CustomDetector.""" + # You can change this to whatever method_type you need + method_type: str = "custom_detector" + + +class CustomDetector(CoreDetector): + """Template detector implementation based on CoreDetector. + + Replace this docstring with a description of what your detector does + and how it should be used. + """ + + def __init__( + self, + name: str = "CustomDetector", + config: CustomDetectorConfig | dict[str, Any] = CustomDetectorConfig(), + ) -> None: + + # Allow passing either a config instance or a plain dict + if isinstance(config, dict): + config = CustomDetectorConfig.from_dict(config, name) + + super().__init__(name=name, buffer_mode=BufferMode.NO_BUF, config=config) + self._call_count = 0 + + def detect( + self, + input_: List[schemas.ParserSchema] | schemas.ParserSchema, + output_: schemas.DetectorSchema, + ) -> bool | None: + """Run detection on parser output and populate the detector schema. + + :param input_: One or many ParserSchema instances + :param output_: DetectorSchema instance to be mutated in-place + :return: Detection result (True/False) or None + """ + + output_["description"] = "Dummy detection process" # Description of the detection + + # Alternating pattern: True, False, True, False, etc + self._call_count += 1 + pattern = [True, False] + result = pattern[self._call_count % len(pattern)] + if result: + output_["score"] = 1.0 # Score of the detector + output_["alertsObtain"]["type"] = "Anomaly detected by CustomDetector" # Additional info + + return result diff --git a/src/tools/workspace/templates/CustomParser.py b/src/tools/workspace/templates/CustomParser.py new file mode 100644 index 0000000..38fa2cb --- /dev/null +++ b/src/tools/workspace/templates/CustomParser.py @@ -0,0 +1,45 @@ +from typing import Any + +from detectmatelibrary.common.parser import CoreParser, CoreParserConfig +from detectmatelibrary import schemas + + +class CustomParserConfig(CoreParserConfig): + """Configuration for CustomParser.""" + # You can change this to whatever method_type you need + method_type: str = "custom_parser" + + +class CustomParser(CoreParser): + """Template parser implementation based on CoreParser. + + Replace this docstring with a description of what your parser does. + """ + + def __init__( + self, + name: str = "CustomParser", + config: CustomParserConfig | dict[str, Any] = CustomParserConfig(), + ) -> None: + # Allow passing either a config instance or a plain dict + if isinstance(config, dict): + config = CustomParserConfig.from_dict(config, name) + + super().__init__(name=name, config=config) + + def parse( + self, + input_: schemas.LogSchema, + output_: schemas.ParserSchema, + ) -> None: + """Parse a single log entry and populate the output schema. + + :param input_: Input log schema instance + :param output_: Parser output schema instance to be mutated in- + place + """ + + # Dummy implementation example (replace with real logic) + output_["EventID"] = 2 # Number of the log template + output_["variables"].extend(["dummy_variable"]) # Variables found in the log + output_["template"] = "This is a dummy template" # Log template diff --git a/src/tools/workspace/templates/__init__.py b/src/tools/workspace/templates/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/tools/workspace/templates/test_templates/__init__.py b/src/tools/workspace/templates/test_templates/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/tools/workspace/templates/test_templates/test_CustomDetector.py b/src/tools/workspace/templates/test_templates/test_CustomDetector.py new file mode 100644 index 0000000..a0e6414 --- /dev/null +++ b/src/tools/workspace/templates/test_templates/test_CustomDetector.py @@ -0,0 +1,37 @@ +from typing import Any +from detectmatelibrary import schemas +from ..CustomDetector import CustomDetector, CustomDetectorConfig + + +default_args = { + "detectors": { + "CustomDetector": { + "method_type": "custom_detector", + "auto_config": False, + "params": {}, + } + } +} + + +class TestCustomDetector: + def test_initialize_default(self) -> None: + detector = CustomDetector(name="CustomDetector", config=default_args) + assert isinstance(detector, CustomDetector) + assert detector.name == "CustomDetector" + assert isinstance(detector.config, CustomDetectorConfig) + + def test_run_detect_method(self) -> None: + detector = CustomDetector() + data = schemas.ParserSchema({"log": "test log"}) + output: Any = schemas.DetectorSchema() + + result = detector.detect(data, output) + + assert output.description == "Dummy detection process" + if result: + assert output.score == 1.0 + assert "Anomaly detected" in output.alertsObtain["type"] + else: + assert output.score == 0.0 + assert len(output.alertsObtain) == 0 diff --git a/src/tools/workspace/templates/test_templates/test_CustomParser.py b/src/tools/workspace/templates/test_templates/test_CustomParser.py new file mode 100644 index 0000000..855cd6d --- /dev/null +++ b/src/tools/workspace/templates/test_templates/test_CustomParser.py @@ -0,0 +1,32 @@ +from typing import Any +from detectmatelibrary import schemas +from ..CustomParser import CustomParser, CustomParserConfig + + +default_args = { + "parsers": { + "CustomParser": { + "auto_config": False, + "method_type": "custom_parser", + "params": {}, + } + } +} + + +class TestCustomParser: + def test_initialize_default(self) -> None: + parser = CustomParser(name="CustomParser", config=default_args) + assert isinstance(parser, CustomParser) + assert parser.name == "CustomParser" + assert isinstance(parser.config, CustomParserConfig) + + def test_run_parse_method(self) -> None: + parser = CustomParser() + input_data = schemas.LogSchema({"log": "test log"}) + output_data: Any = schemas.ParserSchema() + + parser.parse(input_data, output_data) + + assert output_data.variables == ["dummy_variable"] + assert output_data.template == "This is a dummy template" diff --git a/src/tools/workspace/utils.py b/src/tools/workspace/utils.py new file mode 100644 index 0000000..516ee66 --- /dev/null +++ b/src/tools/workspace/utils.py @@ -0,0 +1,191 @@ +import textwrap +import re +from pathlib import Path + + +def normalize(name: str) -> str: + """Normalize name for project name (PEP 621 style).""" + return re.sub(r"[-_.]+", "-", name).lower() + + +def normalize_package_name(name: str) -> str: + """Normalize to a valid Python package name: lowercase, underscores only.""" + name = name.lower() + name = re.sub(r"[^a-z0-9]+", "_", name) # replace non-alphanumerics w/ _ + name = re.sub(r"_+", "_", name) # collapse repeated _ + name = name.strip("_") + return name + + +def create_readme(name: str, ws_type: str, target_impl: Path, target_dir: Path) -> None: + """Create a README.md file for the generated workspace. + + Parameters: + name (str): Name of the workspace. + ws_type (str): Type of workspace (parser/detector). + target_impl (Path): Path to the main implementation file. + target_dir (Path): Directory where the README will be created. + """ + # Import path to the implementation module, e.g. "MyCoolThing.MyCoolThing" + impl_module = f"{target_dir.name}.{target_impl.stem}" + + readme_text = textwrap.dedent( + f""" + # {name} + + This is an auto-generated workspace for implementing your custom {ws_type}. + The directory containing this `README.md` is referred to as the *workspace root* below. + + ## Contents + + - `{normalize(name)}/{target_impl.name}`: starting point for your `{ws_type}` implementation. + - `tests/test_{target_impl.name}`: unit tests for your `{ws_type}`. + - `LICENSE.md`: EUPL-1.2 license copied from the main project. + - `.pre-commit-config.yaml`: recommended pre-commit hook configuration. + - `.gitignore`: standard ignore rules. + - `pyproject.toml`: Python project metadata, dependencies, and dev extras. + + ## Recommended setup (uv + prek) + + We recommend using [`uv`](https://github.com/astral-sh/uv) to manage the environment + and dependencies, and [`prek`](https://github.com/j178/prek) to manage Git + pre-commit hooks. `prek` is configured via the existing `.pre-commit-config.yaml` + and can be installed as part of the `dev` extras. + + ### 1. Create and activate a virtual environment with uv + + From the workspace root (the directory containing this `README.md`): + + ```bash + cd + + # Create a virtual environment (if you don't have one yet) + uv venv + + # Activate it + source .venv/bin/activate # Linux/macOS + # .venv\\Scripts\\activate # Windows + ``` + + ### 2. Install the project and dev dependencies + + ```bash + uv pip install -e .[dev] + ``` + + ### 3. Install and run Git hooks with prek (optional but recommended) + + With the virtual environment activated: + + ```bash + # Install Git hooks from .pre-commit-config.yaml using prek + prek install + + # You can run all hooks on the full codebase with: + # prek run --all-files + ``` + + After this, hooks will run automatically on each commit. + + ## Alternative setup (pip instead of uv) + + If you prefer plain `pip`, you can set things up like this instead: + + ```bash + cd + + # Create a virtual environment + python -m venv .venv + + # Activate it + source .venv/bin/activate # Linux/macOS + # .venv\\Scripts\\activate # Windows + + # Install the project in editable mode with dev dependencies + pip install -e .[dev] + ``` + + With `pip`, `prek` will still be available from the virtual environment, + and you can use the same `prek install` command to install hooks. + + ## Next steps + + Open `{target_impl.name}` and implement your custom {ws_type}. + + ## (Optional) Run it as a Service + + You can run your {ws_type} as a Service using the DetectMateService, which is added as + an optional dependency in the `dev` extras. + + For this, create a settings file (e.g., `service_settings.yaml`) in the workspace root, + which could look like this: + + ```yaml + component_name: {name} + component_type: {impl_module}.{name} + component_config_class: {impl_module}.{name}Config + log_level: DEBUG + log_dir: ./logs + manager_addr: ipc:///tmp/{name.lower()}_cmd.ipc + engine_addr: ipc:///tmp/{name.lower()}_engine.ipc + ``` + + Then start your {ws_type} service with: + + ```bash + detectmate start --settings service_settings.yaml + ``` + + For more info about DetectMate Service, see https://github.com/ait-detectmate/DetectMateService. + + Make sure you run this command from within the virtual environment where you installed + this workspace (e.g. after `uv venv && source .venv/bin/activate`). + """ + ).strip() + "\n" + + (target_dir / "README.md").write_text(readme_text) + + +def create_pyproject(name: str, ws_type: str, target_dir: Path) -> None: + """Create a minimal pyproject.toml file for the generated workspace. + + - Uses the --name argument (normalized) as the project name. + - Leaves a dependencies list ready for you to fill with the necessary libraries. + """ + + package_name = normalize_package_name(name) + + pyproject_text = textwrap.dedent( + f""" + [project] + name = "{normalize(name)}" + version = "0.1.0" + description = "Generated {ws_type} workspace '{name}'" + readme = "README.md" + requires-python = ">=3.12" + + # Add the libraries your workspace needs below + dependencies = [ + "detectmatelibrary @ git+https://github.com/ait-detectmate/DetectMateLibrary.git", + ] + + [project.optional-dependencies] + # Add dependencies in this section with: uv add --optional dev + # Install with all the dev dependencies: uv pip install -e .[dev] + dev = [ + "detectmateservice @ git+https://github.com/ait-detectmate/DetectMateService.git", + "prek>=0.2.8", + "pytest>=8.4.2", + ] + + [build-system] + requires = ["setuptools>=64", "wheel"] + build-backend = "setuptools.build_meta" + + [tool.setuptools] + # Treat the '{package_name}' directory as the package + packages = ["{package_name}"] + """ + ).strip() + "\n" + + (target_dir / "pyproject.toml").write_text(pyproject_text) diff --git a/tests/test_utils/test_log_format_utils.py b/tests/test_utils/test_log_format_utils.py index 6913f0b..b79d6b8 100644 --- a/tests/test_utils/test_log_format_utils.py +++ b/tests/test_utils/test_log_format_utils.py @@ -39,7 +39,7 @@ def test_epoch_seconds_and_millis(): def test_syslog_without_year_assumes_current_year(): # Expect that a string like 'Nov 11 12:13:14' is parsed with current year s = "Nov 11 12:13:14" - year = datetime.utcnow().year + year = datetime.now().year dt = datetime.strptime(f"{year} {s}", "%Y %b %d %H:%M:%S") dt = dt.replace(tzinfo=timezone.utc) assert tfh.parse_timestamp(s) == str(int(dt.timestamp())) diff --git a/tests/test_workspace/test_create_workspace.py b/tests/test_workspace/test_create_workspace.py new file mode 100644 index 0000000..ddca99f --- /dev/null +++ b/tests/test_workspace/test_create_workspace.py @@ -0,0 +1,205 @@ +import os +import sys +import subprocess +import pytest +from pathlib import Path +from tools.workspace.utils import normalize_package_name + + +# Path to the CLI entry point +CLI = ["mate"] # installed as console script + + +@pytest.fixture +def temp_dir(tmp_path: Path) -> Path: + # Creates an isolated directory for each test (workspace root) + return tmp_path + + +def test_create_parser_workspace(temp_dir: Path): + ws_name = "myParser" + workspace_root = temp_dir + pkg_name = normalize_package_name(ws_name) # myparser + module_name = normalize_package_name(ws_name) # myparser + pkg_dir = workspace_root / pkg_name + tests_dir = workspace_root / "tests" + + # Run the CLI tool + subprocess.check_call([ + *CLI, + "create", + "--type", "parser", + "--name", ws_name, + "--dir", str(workspace_root), + ]) + + # Workspace root exists + assert workspace_root.exists() + + # Package directory exists + assert pkg_dir.exists() + + # Meta files live in workspace root + assert (workspace_root / "LICENSE.md").exists() + assert (workspace_root / ".gitignore").exists() + assert (workspace_root / ".pre-commit-config.yaml").exists() + assert (workspace_root / "README.md").exists() + + # Python files live in package directory + py_files = list(pkg_dir.glob("*.py")) + assert len(py_files) == 2 # __init__.py + myParser.py + assert (pkg_dir / f"{module_name}.py").exists() + assert (pkg_dir / "__init__.py").exists() + assert tests_dir.exists() + assert (tests_dir / f"test_{ws_name}.py").exists() + + +def test_create_detector_workspace(temp_dir: Path): + ws_name = "myDetector" + workspace_root = temp_dir + pkg_name = normalize_package_name(ws_name) # mydetector + module_name = normalize_package_name(ws_name) # mydetector + pkg_dir = workspace_root / pkg_name + tests_dir = workspace_root / "tests" + + subprocess.check_call([ + *CLI, + "create", + "--type", "detector", + "--name", ws_name, + "--dir", str(workspace_root), + ]) + + assert workspace_root.exists() + assert pkg_dir.exists() + + assert (workspace_root / "LICENSE.md").exists() + assert (workspace_root / ".gitignore").exists() + assert (workspace_root / ".pre-commit-config.yaml").exists() + assert (workspace_root / "README.md").exists() + + py_files = list(pkg_dir.glob("*.py")) + assert len(py_files) == 2 # __init__.py + myDetector.py + assert (pkg_dir / f"{module_name}.py").exists() + assert (pkg_dir / "__init__.py").exists() + assert tests_dir.exists() + assert (tests_dir / f"test_{ws_name}.py").exists() + + +def test_create_workspace_with_dash_name(temp_dir: Path): + ws_name = "custom-parser" + workspace_root = temp_dir + pkg_name = normalize_package_name(ws_name) # custom_parser + module_name = normalize_package_name(ws_name) # custom_parser + pkg_dir = workspace_root / pkg_name + tests_dir = workspace_root / "tests" + test_file = tests_dir / f"test_{ws_name}.py" + + subprocess.check_call([ + *CLI, + "create", + "--type", "parser", + "--name", ws_name, + "--dir", str(workspace_root), + ]) + + assert workspace_root.exists() + assert pkg_dir.exists() + assert tests_dir.exists() + assert test_file.exists() + assert (pkg_dir / "__init__.py").exists() + assert (pkg_dir / f"{module_name}.py").exists() + + # check that the generated test imports use the normalized names + content = test_file.read_text() + assert f"from {pkg_name}.{module_name} import " in content + + +def test_fail_if_dir_exists(temp_dir: Path): + ws_name = "existing" + workspace_root = temp_dir + pkg_dir = workspace_root / ws_name + + # Pre-create the package directory to force failure + pkg_dir.mkdir(parents=True, exist_ok=True) + + # Should fail because the package directory already exists + result = subprocess.run([ + *CLI, + "create", + "--type", "parser", + "--name", ws_name, + "--dir", str(workspace_root), + ], capture_output=True, text=True) + + assert result.returncode != 0 + assert "already exists" in result.stderr + + +def test_generated_detector_tests_pass(temp_dir: Path): + """Run pytest inside the generated workspace on the generated detector test + file.""" + + ws_name = "MyCoolThing" + workspace_root = temp_dir + pkg_dir = workspace_root / "mycoolthing" + tests_dir = workspace_root / "tests" + test_file = tests_dir / f"test_{ws_name}.py" + + subprocess.check_call([ + *CLI, + "create", + "--type", "detector", + "--name", ws_name, + "--dir", str(workspace_root), + ]) + + assert workspace_root.exists() + assert pkg_dir.exists() + assert tests_dir.exists() + assert test_file.exists() + + # run pytest on the generated test file + # and make sure the workspace root is on sys.path so "import mycoolthing" works + old_cwd = os.getcwd() + old_sys_path = list(sys.path) + try: + os.chdir(workspace_root) + sys.path.insert(0, str(workspace_root)) + result = pytest.main(["-q", str(test_file.relative_to(workspace_root))]) + assert result == 0 + finally: + os.chdir(old_cwd) + sys.path[:] = old_sys_path + + +def test_generated_parser_tests_pass(temp_dir: Path): + ws_name = "MyCoolParser" + workspace_root = temp_dir + pkg_dir = workspace_root / "mycoolparser" + tests_dir = workspace_root / "tests" + test_file = tests_dir / f"test_{ws_name}.py" + + subprocess.check_call([ + *CLI, + "create", + "--type", "parser", + "--name", ws_name, + "--dir", str(workspace_root), + ]) + + assert workspace_root.exists() + assert pkg_dir.exists() + assert tests_dir.exists() + assert test_file.exists() + + old_cwd = os.getcwd() + old_sys_path = list(sys.path) + try: + os.chdir(workspace_root) + sys.path.insert(0, str(workspace_root)) + result = pytest.main(["-q", str(test_file.relative_to(workspace_root))]) + assert result == 0 + finally: + os.chdir(old_cwd) + sys.path[:] = old_sys_path diff --git a/tests/test_workspace/test_utils.py b/tests/test_workspace/test_utils.py new file mode 100644 index 0000000..8804c3d --- /dev/null +++ b/tests/test_workspace/test_utils.py @@ -0,0 +1,21 @@ +import pytest +from tools.workspace.utils import normalize_package_name + + +@pytest.mark.parametrize("input_name, expected", [ + ("custom-parser", "custom_parser"), + ("custom.parser", "custom_parser"), + ("CUSTOM__Thing", "custom_thing"), + ("a--b..c", "a_b_c"), +]) +def test_normalize_package_name_basic_cases(input_name: str, expected: str) -> None: + assert normalize_package_name(input_name) == expected + + +@pytest.mark.parametrize("input_name", [ + "_leading", "trailing_", "--both--", "...", "__" +]) +def test_normalize_package_name_strips_outer_underscores(input_name: str) -> None: + result = normalize_package_name(input_name) + assert not result.startswith("_") + assert not result.endswith("_") diff --git a/tools/workspace/create_workspace.py b/tools/workspace/create_workspace.py deleted file mode 100644 index 7931b30..0000000 --- a/tools/workspace/create_workspace.py +++ /dev/null @@ -1 +0,0 @@ -# script that sets up the workspace for the DetectMateLibrary project diff --git a/tools/workspace/templates/CustomDetector.py b/tools/workspace/templates/CustomDetector.py deleted file mode 100644 index 0f5dc4a..0000000 --- a/tools/workspace/templates/CustomDetector.py +++ /dev/null @@ -1 +0,0 @@ -# template for a custom detector diff --git a/tools/workspace/templates/CustomParser.py b/tools/workspace/templates/CustomParser.py deleted file mode 100644 index facad0a..0000000 --- a/tools/workspace/templates/CustomParser.py +++ /dev/null @@ -1 +0,0 @@ -# template for a custom parser