From d2b9f8e1b6e1b51dd76684fc847a5daa5780eea3 Mon Sep 17 00:00:00 2001 From: Qing <44231502+byemaxx@users.noreply.github.com> Date: Wed, 15 Jul 2026 16:05:26 -0400 Subject: [PATCH 1/5] Unify MetaUmbra manifest annotation workflow --- .gitignore | 1 + Docs/Annotation_CLI.md | 152 +- Docs/Annotation_Result_Schema.md | 207 +- Docs/MetaX_Cookbook.md | 50 +- README.md | 2 +- metax/cli/annotate.py | 287 +-- metax/gui/main_gui.py | 513 +++-- metax/gui/metax_gui/main_window.ui | 24 +- metax/gui/metax_gui/ui_main_window.py | 20 +- metax/gui/unit_specific_settings_dialog.py | 64 +- .../peptide_annotator/annotation_workflow.py | 18 +- .../genome_selection_manifest.py | 19 + metax/peptide_annotator/manifest_otf.py | 13 + metax/peptide_annotator/pep_table_to_otf.py | 13 +- .../unit_specific_manifest.py | 83 +- metax/peptide_annotator/unit_specific_otf.py | 58 +- metax/workflow_recorder/__init__.py | 6 +- metax/workflow_recorder/recorder.py | 47 +- .../genome_selection_manifest.v1.json | 46 + tests/test_annotation_cli.py | 988 --------- tests/test_genome_selection_manifest.py | 68 + tests/test_manifest_annotation_cli.py | 127 ++ tests/test_manifest_otf_backend.py | 43 + tests/test_metaumbra_contract_integration.py | 59 + tests/test_pep_table_to_otf_dataframe.py | 11 + tests/test_peptide_annotator_layout.py | 116 ++ .../test_unit_specific_gui_settings_dialog.py | 542 ----- tests/test_unit_specific_manifest.py | 148 -- tests/test_unit_specific_otf.py | 1800 ----------------- tests/test_workflow_recorder.py | 32 +- 30 files changed, 1292 insertions(+), 4265 deletions(-) create mode 100644 metax/peptide_annotator/genome_selection_manifest.py create mode 100644 metax/peptide_annotator/manifest_otf.py create mode 100644 tests/fixtures/genome_selection_manifest.v1.json delete mode 100644 tests/test_annotation_cli.py create mode 100644 tests/test_genome_selection_manifest.py create mode 100644 tests/test_manifest_annotation_cli.py create mode 100644 tests/test_manifest_otf_backend.py create mode 100644 tests/test_metaumbra_contract_integration.py create mode 100644 tests/test_peptide_annotator_layout.py delete mode 100644 tests/test_unit_specific_gui_settings_dialog.py delete mode 100644 tests/test_unit_specific_manifest.py delete mode 100644 tests/test_unit_specific_otf.py diff --git a/.gitignore b/.gitignore index 44d87525..e65b36f0 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ venv conda-recipe report_output*/ metax/data/example_data/Example_OTF_auto_report/ +.test-temp/ diff --git a/Docs/Annotation_CLI.md b/Docs/Annotation_CLI.md index ce8825b1..3c10cd1b 100644 --- a/Docs/Annotation_CLI.md +++ b/Docs/Annotation_CLI.md @@ -1,144 +1,40 @@ -# MetaX Annotation CLI +# Annotation CLI -`metax-annotate` provides Qt-free global and unit-specific peptide-to-OTF annotation. - -## Installation - -The default package contains the complete annotation backend: +`metax-annotate` is a Qt-free annotation entry point with three explicit genome-selection sources. It never guesses the source from a filename or table columns. ```bash -python -m pip install MetaXTools -``` - -Analyzer, Report, and desktop installation profiles are described in the -[README](../README.md#installation). - -## Global annotation - -Run MetaUmbra genome scoring and generate an OTF from DIA-NN parquet: - -```bash -metax-annotate \ - --mode global \ +python -m metax.cli.annotate \ + --input-source metaumbra-manifest \ --peptide-table report.parquet \ - --digested-genome-folders digested_genomes \ + --metaumbra-manifest results/genome_selection_manifest.json \ --taxafunc-db MetaX_taxafunc.db \ + --digested-genome-folders UHGP_digested \ + --genome-threshold auto \ --output OTF.tsv \ - --selection-mode metaumbra \ - --diann-intensity-col Precursor.Normalised \ - --result-json annotation_result.json + --result-json result.json ``` -Use `--selection-mode provided` with `--selected-genomes` or -`--genome-list-file` to supply genomes directly. Use `automatic` for MetaX's -internal genome ranking, or `metaumbra-only` to stop after genome scoring. - -When `--selection-mode` is omitted, selected genome IDs or a genome-list file -imply `provided`; otherwise digest directories imply `metaumbra`; otherwise the -mode is `automatic`. Explicit modes take precedence, and contradictory genome -sources are rejected instead of ignored. - -Exactly one peptide-mapping source is required for OTF generation: -`--peptide-db` or `--digested-genome-folders`. - -## Unit-specific annotation - -Use a MetaUmbra unit-specific manifest: +MetaX automatic selection remains available for non-MetaUmbra inputs: ```bash -metax-annotate \ - --mode unit-specific \ - --peptide-table report.parquet \ - --unit-specific-manifest unit_specific_manifest.json \ - --digested-genome-folders digested_genomes \ - --taxafunc-db MetaX_taxafunc.db \ - --output OTF_unit_specific.tsv \ - --result-json annotation_result.json +python -m metax.cli.annotate --input-source metax-automatic \ + --peptide-table peptides.tsv --taxafunc-db MetaX_taxafunc.db \ + --digested-genome-folders UHGP_digested \ + --intensity-col-prefix Intensity --output OTF.tsv ``` -## YAML or JSON configuration +For an explicit genome set, use `--input-source genome-list` with either `--genome-list-file genomes.txt` or `--selected-genomes g1 g2`. +For delimited non-MetaUmbra peptide tables, `--intensity-col-prefix` identifies the input sample columns; the GUI forwards the editable **Prefix of Intensity Column** value to the same backend. -CLI arguments override configuration-file values: +`--genome-threshold` accepts `auto`, `q0.05`, or `q0.01`. `auto` uses `selection.default_genome_threshold` from the manifest. There is no global/unit-specific mode selector and MetaX does not infer genome IDs from a TSV. -```yaml -api_version: "1.0" -mode: global - -inputs: - peptide_table: report.parquet - digested_genome_folders: - - digested_genomes - taxafunc_db: MetaX_taxafunc.db - -output: - otf: OTF.tsv - result_json: annotation_result.json - -options: - selection_mode: metaumbra - duplicate_peptide_handling_mode: sum - diann_intensity_col: Precursor.Normalised -``` +The annotation backend always performs the same sequence: -Run it with: - -```bash -metax-annotate --config annotation.yaml -``` - -Relative paths loaded from a configuration file are resolved from the -configuration file's directory. Relative paths supplied directly on the command -line remain relative to the current working directory. - -## Result JSON - -When `--result-json` is supplied, MetaX writes schema `2.0` atomically. The -top-level sections are: - -```text -run, inputs, parameters, stages, genome_selection, metrics, outputs, diagnostics -``` - -Use `run.status` and `run.exit_code` for the outcome, `outputs` for generated -files, `metrics` for mapping/QC values, and `diagnostics` for warnings or errors. -Scientific outputs are safely renamed when their requested path already exists, -and `outputs` reports the actual paths. The result JSON itself keeps its requested -stable path and is atomically replaced. -See [Annotation Result Schema](Annotation_Result_Schema.md) for the complete -field reference and examples. - -## Exit codes - -| Code | Meaning | -| ---: | --- | -| 0 | Successful execution | -| 2 | Invalid arguments or configuration | -| 3 | Missing input, database, directory, or configuration file | -| 4 | Required optional dependency is unavailable | -| 5 | Annotation or external scoring failed | -| 130 | Execution was cancelled | - -On interruption, MetaX terminates active MetaUmbra or digested-scan process -trees before writing the cancellation result JSON. - -## Python subprocess - -Invoke MetaX with the same Python environment in which it is installed: - -```python -import subprocess -import sys - -completed = subprocess.run( - [ - sys.executable, - "-m", - "metax.cli.annotate", - "--config", - "annotation.yaml", - ], - check=False, -) -``` +1. Validate `metaumbra.genome_selection_manifest.v1`. + A unit with no genomes at the selected threshold is rejected before annotation. +2. Resolve every manifest sample ID to a DIA-NN run or prepared intensity column. +3. Scan the union of selected genome digest files once. +4. Annotate each analysis unit using its samples and selected genomes. +5. Merge unit outputs while retaining `analysis_unit_id`. -Read the process exit code first, then inspect result JSON for structured details. +The result JSON uses `metax.annotation_result.v2` and records `input_source`. Manifest runs additionally record the manifest path/schema, number of units, selected threshold, sample mapping, and per-unit summary; non-manifest runs set those manifest-only fields to null. Exit codes are `0` success, `2` invalid configuration, `3` missing input, `4` missing optional dependency, `5` annotation failure, and `130` cancellation. diff --git a/Docs/Annotation_Result_Schema.md b/Docs/Annotation_Result_Schema.md index 824d62a8..5c5c9d07 100644 --- a/Docs/Annotation_Result_Schema.md +++ b/Docs/Annotation_Result_Schema.md @@ -1,195 +1,36 @@ -# Annotation Result Schema +# Annotation result schema -This document defines the machine-readable result written by -`metax-annotate --result-json`. The current schema version is `2.0`; the -annotation workflow API remains `1.0`. - -## Top-level structure - -| Section | Purpose | -| --- | --- | -| `schema_version` | Result-document schema version | -| `run` | Outcome, identity, timing, mode, and software versions | -| `inputs` | Input files, formats, sizes, and DIA-NN sample mapping | -| `parameters` | Effective parameters after defaults and CLI overrides | -| `stages` | Status and duration of individual processing stages | -| `genome_selection` | Selection method, thresholds, counts, and genome IDs | -| `metrics` | Input, mapping, output, filter, and per-sample QC metrics | -| `outputs` | Files and directories actually generated by the run | -| `diagnostics` | Warnings and structured error information | - -## Successful global result - -```json -{ - "schema_version": "2.0", - "run": { - "id": "c4d5a2f0-0000-4000-8000-000000000000", - "status": "success", - "exit_code": 0, - "mode": "global", - "started_at": "2026-07-14T15:20:00.000-04:00", - "finished_at": "2026-07-14T15:23:15.200-04:00", - "duration_seconds": 195.2, - "software": { - "metax_version": "2.5.0", - "workflow_api_version": "1.0", - "python_version": "3.13.5", - "platform": "Windows-11", - "metaumbra_version": "1.3.7" - } - }, - "inputs": { - "peptide_table": { - "path": "report.parquet", - "format": "diann_parquet", - "size_bytes": 123456, - "modified_at": "2026-07-14T15:00:00-04:00", - "diann": { - "intensity_column": "Precursor.Normalised", - "run_to_sample_column": { - "sample.raw": "Intensity_sample" - } - } - } - }, - "parameters": { - "selection_mode": "metaumbra", - "duplicate_peptide_handling_mode": "sum", - "metaumbra": { - "single_peptide_error_rate_upper_bound": 0.3, - "genome_qvalue_cutoff": 0.05 - } - }, - "stages": { - "input_preparation": { - "status": "success", - "duration_seconds": 2.1 - }, - "metaumbra_scoring": { - "status": "success", - "duration_seconds": 169.0 - }, - "peptide_mapping_and_annotation": { - "status": "success", - "duration_seconds": 20.4 - } - }, - "genome_selection": { - "method": "metaumbra", - "source": "metax_temp/OTF_metaumbra_genome_presence.tsv", - "qvalue_cutoff": 0.05, - "genomes_evaluated": 4744, - "genomes_selected": 222, - "selected_genome_ids": ["MGYG000001315"] - }, - "metrics": { - "input": { - "rows": 106563, - "runs": 3, - "prepared_peptides": 33546 - }, - "mapping": { - "peptides_before_mapping": 33546, - "peptides_after_mapping": 30906, - "unmapped_peptides": 2640, - "mapping_rate": 0.9213 - }, - "output": { - "rows": 30906, - "columns": 72, - "unique_sequences": 30906, - "sample_columns": ["Intensity_sample"], - "samples": { - "Intensity_sample": { - "detected_peptides": 25000, - "zero_or_missing_peptides": 5906, - "zero_or_missing_fraction": 0.1911, - "total_intensity": 123456.0, - "median_nonzero_intensity": 12.3 - } - } - } - }, - "outputs": { - "otf": { - "path": "OTF.tsv", - "format": "tsv", - "rows": 30906, - "columns": 72 - }, - "annotation_summary": { - "path": "OTF_info.txt", - "format": "text" - } - }, - "diagnostics": { - "warnings": [], - "error": null - } -} -``` - -Fields that do not apply to the selected mode are omitted. Output entries are -only written for artifacts that were actually produced. - -`parameters.selection_mode` and `genome_selection.method` contain the same -effective selection mode. `genome_selection.source` identifies explicit genome -IDs, a genome-list file, MetaUmbra output, or automatic genome ranking. - -## Unit-specific additions - -Unit-specific results use `run.mode: "unit-specific"` and add: - -- `metrics.units.completed` and `metrics.units.skipped`; -- `outputs.unit_summary`; -- `outputs.unit_sample_mapping`; -- `outputs.per_unit_directory` when per-unit outputs are retained. - -`genome_selection.method` is `unit_specific_manifest` and records the selected -manifest threshold. - -## Failure result - -Handled failures retain the same top-level structure: +All annotation sources write a JSON object with schema version `metax.annotation_result.v2`. ```json { - "schema_version": "2.0", - "run": { - "status": "error", - "exit_code": 3, - "mode": "global" - }, + "schema_version": "metax.annotation_result.v2", + "input_source": "metaumbra-manifest", + "run": {"status": "success", "exit_code": 0, "software": {}}, + "manifest": { + "path": "results/genome_selection_manifest.json", + "schema_version": "metaumbra.genome_selection_manifest.v1" + }, + "number_of_units": 2, + "selected_threshold": "q0.05", "inputs": {}, "parameters": {}, - "stages": { - "annotation": { - "status": "error", - "duration_seconds": 0.001 - } + "stages": {}, + "genome_selection": { + "method": "metaumbra_genome_selection_manifest", + "threshold": "q0.05" }, - "genome_selection": {}, "metrics": {}, - "outputs": {}, - "diagnostics": { - "warnings": [], - "error": { - "stage": "annotation", - "category": "input", - "type": "FileNotFoundError", - "message": "Peptide table not found: missing.tsv" - } - } + "outputs": { + "otf": {"path": "OTF.tsv", "format": "tsv"}, + "sample_mapping": {"path": "OTF_artifacts/unit_sample_column_mapping.tsv"}, + "unit_summary": {"path": "OTF_artifacts/unit_annotation_summary.tsv"} + }, + "per_unit_summary": {}, + "diagnostics": {"warnings": [], "error": null} } ``` -Error categories are `configuration`, `input`, `dependency`, `processing`, or -`cancelled`. Tracebacks are not included in the public result document. - -## Compatibility +Manifest-driven OTF output always retains `analysis_unit_id`. A one-unit manifest therefore produces the same table schema as a multi-unit manifest, using `__global__` for the all-samples unit. Keeping the column avoids a schema branch and lets Analyzer use the same unit-aware peptide identity for both one-unit and multi-unit results. -`schema_version` describes this JSON document. `workflow_api_version` describes -the accepted annotation configuration contract. Consumers should reject an -unsupported schema major version and may accept additive changes within the same -major version. +`input_source` is one of `metaumbra-manifest`, `metax-automatic`, or `genome-list`. For the latter two, `manifest`, `number_of_units`, `selected_threshold`, and `per_unit_summary` are null; their existing MetaX OTF schema is retained. diff --git a/Docs/MetaX_Cookbook.md b/Docs/MetaX_Cookbook.md index e51ceda3..e5875a8a 100644 --- a/Docs/MetaX_Cookbook.md +++ b/Docs/MetaX_Cookbook.md @@ -826,17 +826,9 @@ These peptide results use metagenome-assembled genomes (MAGs) as the reference d - **Duplicate peptide handling**: Controls how repeated peptide rows are combined before annotation. Available options are `sum`, `max`, `min`, `mean`, `first`, and `keep`. -#### Genome Selection Modes +#### Genome Selection Manifest -Peptide Direct to OTF has three genome-selection modes: - -- **Run MetaUmbra scoring, then annotate OTFs**: This is the default workflow. MetaX runs `MetaUmbra score` in an isolated process, writes the intermediate genome-presence table under `metax_temp`, selects genomes by the configured MetaUmbra q-value cutoff, scans the digested genome folder for peptide-to-protein matches from those genomes, and then annotates the final OTF table. - -- **Run MetaUmbra scoring only**: Enable **Stop after MetaUmbra** when you only want the MetaUmbra genome-presence table. The output path changes to a genome-presence TSV, and MetaX does not require the Protein to TaxaFunc database for this mode. - -- **Use selected genome list**: Open or paste a genome list, or load a MetaUmbra genome-presence result. MetaX skips MetaUmbra scoring and directly scans the digested genome folder for the selected genomes. This is useful when you already reviewed the selected genomes or want to reuse the same genome set across runs. - -MetaUmbra scoring currently requires a tab-separated peptide table. When a DIA-NN parquet file is selected, MetaX first prepares a temporary tab-separated peptide table in `metax_temp` before running MetaUmbra. +For MetaUmbra results, Peptide Direct to OTF uses one `genome_selection_manifest.json` for every analysis-unit layout. A run with all samples pooled is represented by `__global__`; per-sample and metadata-grouped runs use the same schema and annotation backend. MetaX automatic genome selection and explicit custom genome lists remain available as separate, clearly labeled sources. #### DIA-NN Parquet Preparation @@ -848,60 +840,54 @@ When the input is a DIA-NN parquet file, MetaX reads only the required columns a - `Evidence` and `Q.Value` are required in the normal Peptide Direct to OTF window and are preserved for MetaUmbra scoring. - Run names are cleaned into safe sample column names. The selected DIA-NN intensity source is recorded in conversion metadata, but `Precursor.Normalised` or `Precursor.Quantity` is not embedded in the sample-column names. -The same global workflow is available without Qt through the shared annotation backend. For automation, prefer the module entry point so MetaX runs in the caller's active Python environment: +The same manifest workflow is available without Qt. For automation, prefer the module entry point so MetaX runs in the caller's active Python environment: ```bash python -m metax.cli.annotate \ - --mode global \ + --input-source metaumbra-manifest \ --peptide-table report.parquet \ + --metaumbra-manifest genome_selection_manifest.json \ --digested-genome-folders digested_genomes/ \ --taxafunc-db MetaX_taxafunc.db \ --output OTF.tsv \ - --selection-mode metaumbra \ + --genome-threshold auto \ --result-json annotation_result.json ``` -Global mode also accepts `--selection-mode provided` with `--selected-genomes` or `--genome-list-file`, and `--selection-mode automatic` for the existing MetaX genome-ranking path. See [MetaX annotation CLI and automation contract](Annotation_CLI.md) for configuration files, installation profiles, result JSON, workflow API version, and exit codes. - -### 2. MetaUmbra Unit-Specific Direct-to-OTF Annotation +See [MetaX annotation CLI and automation contract](Annotation_CLI.md) for configuration files, result JSON, exit codes, and the complete option list. -MetaX can consume a MetaUmbra `unit_specific_manifest.json` as the preferred backend interface for unit-specific OTF annotation. In this mode, MetaX uses `sample_columns` from each analysis unit to split the peptide intensity table, and uses `genome_ids_q005` or `genome_ids_q001` to restrict peptide-to-protein mapping per unit. If `--genome-threshold` is not provided, the manifest `default_genome_threshold` is used. +Choose `--input-source metax-automatic` for MetaX's non-MetaUmbra automatic selection, or `--input-source genome-list --genome-list-file genomes.txt` for a custom list. For wide delimited tables, set `--intensity-col-prefix` when the sample columns do not use the default `Intensity` prefix. The GUI exposes the same three choices in **Genome selection source** and forwards its editable intensity-prefix field. -This backend is additive to the normal/global Peptide Direct to OTF workflow. When unit-specific mode is disabled, MetaX uses the selected normal mode: MetaUmbra genome scoring, a user-provided genome list, or MetaUmbra scoring-only output. Unit-specific mode does not run the normal global genome-selection path; each analysis unit receives its own genome list directly from the MetaUmbra manifest. +### 2. MetaUmbra Manifest Direct-to-OTF Annotation -The unit-specific distinct-genome filter defaults to `0`, so MetaX trusts the manifest-selected genome list. Set `--distinct-genome-threshold` to a value greater than `0` only when you want an additional MetaX-side filter requiring that many distinct peptides per genome after mapping. +MetaX reads `sample_ids` and `genome_ids_q005` or `genome_ids_q001` from every manifest unit. With `--genome-threshold auto`, it uses the manifest default threshold. The selected genome union is scanned once, then matches are restricted to each unit. Sample names are matched against peptide-table columns after normalizing common prefixes and raw-file suffixes. -Sample columns are matched from manifest `sample_columns` to peptide-table columns in this order: exact name, `Intensity_` prefix, configured output prefix, configured input prefix, stripped `Intensity_`, stripped output prefix, stripped input prefix, leading underscores removed, and raw-file basename without `.raw`, `.mzML`, or `.mzXML`. Use `--input-sample-col-prefix` for inputs such as `LFQ intensity sample_1`. +The merged OTF table always includes `analysis_unit_id` and the original `Sequence`. MetaX derives a unit-aware peptide evidence identity internally; `UnitSpecificSequence` is not emitted. Downstream code must not deduplicate multi-unit output by `Sequence` alone. -The merged unit-specific OTF table includes `analysis_unit_id` and the original `Sequence` column. MetaX internally derives the unit-specific peptide evidence ID as `analysis_unit_id + "||" + Sequence` when downstream analysis needs a unique peptide identity; `UnitSpecificSequence` is not written by default. Do not deduplicate unit-specific output by `Sequence` alone. Downstream final OTF identity remains Taxon + Function. +The GUI exposes the same manifest path, threshold, validation, and sample-column matching settings. Annotation accepts either a wide peptide-intensity table or long-format DIA-NN parquet with `Run`, `Stripped.Sequence`, and `Precursor.Normalised` or `Precursor.Quantity`. -In the GUI, select the MetaUmbra `unit_specific_manifest.json` and genome threshold in the main Peptide Direct to OTF window. The Unit-specific Settings dialog does not select a separate manifest or threshold; it configures sample-column matching behavior and missing/empty unit handling, and validates the selected manifest against the current peptide table when possible. Unit-specific mode disables the legacy global genome scoring controls, and the duplicate peptide handling selector still applies. A manual manifest builder is not implemented yet. - -Unit-specific annotation accepts either a wide peptide-intensity table with one sample intensity column per manifest sample or a long-format DIA-NN parquet containing `Run`, `Stripped.Sequence`, and either `Precursor.Normalised` or `Precursor.Quantity`. Long-format parquet input is pivoted automatically, and common raw-file suffixes such as `.raw`, `.mzML`, and `.mzXML` are ignored when matching `Run` values to manifest samples. - -The default unit-specific execution path is disk-backed. Per-unit temporary files are written under `_artifacts/per_unit/unit_otf/`, merged into the final OTF table by streaming append, and then cleaned up. The final artifacts include: +The execution path is disk-backed. Per-unit temporary files are streamed into the final table and cleaned up. Final artifacts include: - The merged OTF table selected in **OTFs Save To**. - `_info.txt`, with input parameters and annotation summary. - `_artifacts/unit_annotation_summary.tsv`, with one row per analysis unit. - `_artifacts/unit_sample_column_mapping.tsv`, with manifest sample to peptide-table column mapping. -For downstream analysis, unit-specific public count columns use these meanings: +For downstream analysis, public count columns use these meanings: - `peptide_num`: unique biological `Sequence` count. -- `peptide_feature_num`: unique unit-specific peptide feature count. +- `peptide_feature_num`: unique analysis-unit peptide feature count. Example: ```bash python -m metax.cli.annotate \ - --mode unit-specific \ - --peptide-table report.tsv \ - --unit-specific-manifest unit_specific_manifest.json \ + --peptide-table report.parquet \ + --metaumbra-manifest genome_selection_manifest.json \ --genome-threshold q0.05 \ --taxafunc-db MetaX_taxafunc.db \ --digested-genome-folders digested_genomes/ \ - --output OTF_unit_specific.tsv \ + --output OTF.tsv \ --peptide-col Sequence \ --input-sample-col-prefix "LFQ intensity " \ --duplicate-peptide-handling-mode sum \ diff --git a/README.md b/README.md index bf0c5e6a..9b17c2fb 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ Show OTFS intensity in groups(samples), e.g., **Species-KO** OTF Heatmap - Refer to the MetaX Cookbook for detailed instructions on how to use MetaX with the graphical user interface.
- `Command-line version`: - - See the [Annotation CLI integration guide](Docs/Annotation_CLI.md) for global and unit-specific commands, configuration files, result JSON, exit codes, and subprocess integration. + - See the [Annotation CLI integration guide](Docs/Annotation_CLI.md) for unified MetaUmbra manifest input plus explicit MetaX automatic and custom genome-list sources. - Read the example documentation in the [Notebook](https://github.com/byemaxx/MetaX/blob/main/Docs/example.ipynb) for analysis examples. diff --git a/metax/cli/annotate.py b/metax/cli/annotate.py index b27e5d45..33e0185e 100644 --- a/metax/cli/annotate.py +++ b/metax/cli/annotate.py @@ -20,7 +20,7 @@ OptionalDependencyUnavailable, ) from metax.peptide_annotator.peptide_table_prepare import DIANN_INTENSITY_CANDIDATES -from metax.peptide_annotator.unit_specific_otf import UnitSpecificOTFAnnotator +from metax.peptide_annotator.manifest_otf import ManifestOTFAnnotator from metax.utils.version import __version__ @@ -38,10 +38,10 @@ def error(self, message: str) -> None: raise AnnotationConfigurationError(message) -ANNOTATION_RESULT_SCHEMA_VERSION = "2.0" +ANNOTATION_RESULT_SCHEMA_VERSION = "metax.annotation_result.v2" _CONFIG_PATH_FIELDS = { "peptide_table", - "unit_specific_manifest", + "metaumbra_manifest", "taxafunc_db", "output", "peptide_db", @@ -181,22 +181,24 @@ def build_parser(defaults: Mapping[str, Any] | None = None) -> argparse.Argument prog="metax-annotate", allow_abbrev=False, description=( - "Run MetaX global or unit-specific peptide-to-OTF annotation. " + "Run peptide-to-OTF annotation from a MetaUmbra manifest, MetaX automatic selection, or an explicit genome list. " "CLI arguments override values loaded with --config." ), ) parser.add_argument("--config", default=defaults.get("config_path"), help="YAML or JSON workflow configuration") - parser.add_argument( - "--mode", - choices=["global", "unit-specific"], - default=_default(defaults, "mode", "global"), - help="Annotation mode (default: global)", - ) parser.add_argument("--result-json", default=defaults.get("result_json"), help="Write the machine-readable execution result to this JSON file") parser.add_argument("--peptide-table", default=defaults.get("peptide_table"), help="Input peptide intensity table") - parser.add_argument("--unit-specific-manifest", default=defaults.get("unit_specific_manifest"), help="MetaUmbra unit_specific_manifest.json") + parser.add_argument( + "--input-source", + choices=["metaumbra-manifest", "metax-automatic", "genome-list"], + default=_default(defaults, "input_source", "metaumbra-manifest"), + help="Explicit genome-selection source; no file-content mode inference is performed", + ) + parser.add_argument("--metaumbra-manifest", default=defaults.get("metaumbra_manifest"), help="MetaUmbra genome_selection_manifest.json") + parser.add_argument("--genome-list-file", default=defaults.get("genome_list_file"), help="Plain text/TSV/CSV genome list for --input-source genome-list") + parser.add_argument("--selected-genomes", nargs="+", default=defaults.get("selected_genomes"), help="Genome IDs for --input-source genome-list") parser.add_argument("--taxafunc-db", default=defaults.get("taxafunc_db"), help="MetaX taxa-function annotation SQLite database") - parser.add_argument("--output", default=defaults.get("output"), help="Output OTF TSV (or MetaUmbra result in metaumbra-only mode)") + parser.add_argument("--output", default=defaults.get("output"), help="Output OTF TSV") parser.add_argument("--peptide-db", default=defaults.get("peptide_db"), help="SQLite peptide-to-protein database") parser.add_argument( "--digested-genome-folders", @@ -204,24 +206,20 @@ def build_parser(defaults: Mapping[str, Any] | None = None) -> argparse.Argument default=defaults.get("digested_genome_folders"), help="One or more folders containing digested genome TSV files", ) + parser.add_argument("--genome-threshold", choices=["auto", "q0.05", "q0.01"], default=_default(defaults, "genome_threshold", "auto"), help="Genome threshold; auto uses the manifest default") + parser.add_argument("--peptide-col", default=_default(defaults, "peptide_col", "Sequence")) parser.add_argument( - "--selection-mode", - choices=["metaumbra", "provided", "automatic", "metaumbra-only"], - default=defaults.get("selection_mode"), - help="Global genome-selection strategy", + "--intensity-col-prefix", + default=_default(defaults, "intensity_col_prefix", "Intensity"), + help="Input intensity-column prefix for metax-automatic and genome-list sources", ) - parser.add_argument("--genome-list-file", default=defaults.get("genome_list_file"), help="Plain genome list or MetaUmbra genome_presence table") - parser.add_argument("--selected-genomes", nargs="+", default=defaults.get("selected_genomes"), help="Explicit genome IDs for global provided mode") - parser.add_argument("--genome-threshold", default=_default(defaults, "genome_threshold", "auto"), help="Unit-specific threshold: q0.05, q0.01, or auto") - parser.add_argument("--peptide-col", default=_default(defaults, "peptide_col", "Sequence")) - parser.add_argument("--intensity-col-prefix", default=_default(defaults, "intensity_col_prefix", "Intensity"), help="Global input sample/intensity column prefix") parser.add_argument( "--output-sample-col-prefix", default=_default(defaults, "output_sample_col_prefix", "Intensity_"), choices=["Intensity_"], - help="Canonical unit-specific output sample prefix", + help="Canonical output sample prefix", ) - parser.add_argument("--input-sample-col-prefix", default=defaults.get("input_sample_col_prefix"), help="Optional unit-specific input prefix to strip when matching manifest samples") + parser.add_argument("--input-sample-col-prefix", default=defaults.get("input_sample_col_prefix"), help="Optional input prefix to strip when matching manifest samples") parser.add_argument("--table-separator", default=_default(defaults, "table_separator", r"\t")) parser.add_argument("--lca-threshold", type=float, default=_default(defaults, "lca_threshold", 1.0)) parser.add_argument( @@ -234,9 +232,7 @@ def build_parser(defaults: Mapping[str, Any] | None = None) -> argparse.Argument parser.add_argument("--exclude-protein-startwith", default=defaults.get("exclude_protein_startwith")) parser.add_argument("--protein-separator", default=_default(defaults, "protein_separator", ";")) parser.add_argument("--protein-genome-separator", default=_default(defaults, "protein_genome_separator", "_")) - parser.add_argument("--protein-peptide-coverage-cutoff", type=float, default=_default(defaults, "protein_peptide_coverage_cutoff", 1.0)) parser.add_argument("--save-per-unit-outputs", action=argparse.BooleanOptionalAction, default=_default(defaults, "save_per_unit_outputs", False)) - parser.add_argument("--include-unit-specific-sequence", action=argparse.BooleanOptionalAction, default=_default(defaults, "include_unit_specific_sequence", False)) parser.add_argument( "--duplicate-peptide-handling-mode", choices=["sum", "max", "min", "mean", "first", "keep"], @@ -248,10 +244,6 @@ def build_parser(defaults: Mapping[str, Any] | None = None) -> argparse.Argument parser.add_argument("--merge-chunksize", type=int, default=_default(defaults, "merge_chunksize", 100_000)) parser.add_argument("--collect-unique-stats", action=argparse.BooleanOptionalAction, default=_default(defaults, "collect_unique_stats", False)) parser.add_argument("--diann-intensity-col", choices=DIANN_INTENSITY_CANDIDATES, default=defaults.get("diann_intensity_col")) - parser.add_argument("--metaumbra-peptide-score-col", default=_default(defaults, "metaumbra_peptide_score_col", "Evidence")) - parser.add_argument("--metaumbra-peptide-error-col", default=_default(defaults, "metaumbra_peptide_error_col", "Q.Value")) - parser.add_argument("--metaumbra-single-peptide-error-rate-upper-bound", type=float, default=_default(defaults, "metaumbra_single_peptide_error_rate_upper_bound", 0.3)) - parser.add_argument("--metaumbra-genome-qvalue-cutoff", type=float, default=_default(defaults, "metaumbra_genome_qvalue_cutoff", 0.05)) return parser @@ -292,20 +284,6 @@ def _validate_scientific_parameters(args: argparse.Namespace) -> None: ) if args.n_jobs is not None and args.n_jobs < 1: raise AnnotationConfigurationError("--n-jobs must be greater than or equal to 1") - if args.mode == "global": - if not 0 < args.protein_peptide_coverage_cutoff <= 1: - raise AnnotationConfigurationError( - "--protein-peptide-coverage-cutoff must be greater than 0 and at most 1" - ) - if not 0 < args.metaumbra_single_peptide_error_rate_upper_bound <= 1: - raise AnnotationConfigurationError( - "--metaumbra-single-peptide-error-rate-upper-bound must be greater " - "than 0 and at most 1" - ) - if not 0 < args.metaumbra_genome_qvalue_cutoff <= 1: - raise AnnotationConfigurationError( - "--metaumbra-genome-qvalue-cutoff must be greater than 0 and at most 1" - ) def _run_metadata( @@ -358,8 +336,8 @@ def _argument_inputs(args: argparse.Namespace | None) -> dict[str, Any]: path_fields = { "config": (getattr(args, "config", None), None), "peptide_table": (getattr(args, "peptide_table", None), None), - "unit_specific_manifest": ( - getattr(args, "unit_specific_manifest", None), + "metaumbra_manifest": ( + getattr(args, "metaumbra_manifest", None), "json", ), "taxafunc_database": (getattr(args, "taxafunc_db", None), "sqlite"), @@ -393,28 +371,19 @@ def _effective_parameters(args: argparse.Namespace | None, mode: str | None) -> "duplicate_peptide_handling_mode", "n_jobs", "diann_intensity_col", - ] - global_only = [ - "selection_mode", "intensity_col_prefix", - "protein_peptide_coverage_cutoff", - "metaumbra_peptide_score_col", - "metaumbra_peptide_error_col", - "metaumbra_single_peptide_error_rate_upper_bound", - "metaumbra_genome_qvalue_cutoff", ] - unit_only = [ + manifest_parameters = [ "genome_threshold", "input_sample_col_prefix", "output_sample_col_prefix", "save_per_unit_outputs", - "include_unit_specific_sequence", "on_missing_sample", "on_empty_unit", "merge_chunksize", "collect_unique_stats", ] - names = common + (unit_only if mode == "unit-specific" else global_only) + names = common + manifest_parameters return { name: getattr(args, name) for name in names @@ -464,21 +433,21 @@ def _write_result_json(path: str | None, result: Mapping[str, Any]) -> None: temporary_path.replace(result_path) -def _run_unit_specific(args: argparse.Namespace) -> tuple[Any, list[str]]: +def _run_manifest(args: argparse.Namespace) -> tuple[Any, list[str]]: _require(args.peptide_table, "--peptide-table") - _require(args.unit_specific_manifest, "--unit-specific-manifest") + _require(args.metaumbra_manifest, "--metaumbra-manifest") _require(args.taxafunc_db, "--taxafunc-db") _require(args.output, "--output") if bool(args.peptide_db) == bool(args.digested_genome_folders): raise AnnotationConfigurationError( - "Unit-specific mode requires exactly one of --peptide-db or --digested-genome-folders" + "Exactly one of --peptide-db or --digested-genome-folders is required" ) genome_threshold = None if args.genome_threshold == "auto" else args.genome_threshold with warnings.catch_warnings(record=True) as recorded: warnings.simplefilter("always") - result = UnitSpecificOTFAnnotator( + result = ManifestOTFAnnotator( peptide_table_path=args.peptide_table, - unit_specific_manifest_path=args.unit_specific_manifest, + metaumbra_manifest_path=args.metaumbra_manifest, taxafunc_anno_db_path=args.taxafunc_db, output_path=args.output, db_path=args.peptide_db, @@ -495,7 +464,6 @@ def _run_unit_specific(args: argparse.Namespace) -> tuple[Any, list[str]]: protein_separator=args.protein_separator, protein_genome_separator=args.protein_genome_separator, save_per_unit_outputs=args.save_per_unit_outputs, - include_unit_specific_sequence=args.include_unit_specific_sequence, duplicate_peptide_handling_mode=args.duplicate_peptide_handling_mode, on_missing_sample=args.on_missing_sample, on_empty_unit=args.on_empty_unit, @@ -507,38 +475,53 @@ def _run_unit_specific(args: argparse.Namespace) -> tuple[Any, list[str]]: return result, [str(item.message) for item in recorded] -def _run_global(args: argparse.Namespace) -> Any: +def _run_non_manifest(args: argparse.Namespace) -> tuple[Any, list[str]]: _require(args.peptide_table, "--peptide-table") + _require(args.taxafunc_db, "--taxafunc-db") _require(args.output, "--output") - return GlobalOTFAnnotator( - peptide_table_path=args.peptide_table, - output_path=args.output, - taxafunc_anno_db_path=args.taxafunc_db, - db_path=args.peptide_db, - digested_genome_folders=args.digested_genome_folders, - selection_mode=args.selection_mode, - selected_genomes=args.selected_genomes, - genome_list_path=args.genome_list_file, - peptide_col=args.peptide_col, - intensity_col_prefix=args.intensity_col_prefix, - table_separator=_decode_separator(args.table_separator), - protein_peptide_coverage_cutoff=args.protein_peptide_coverage_cutoff, - lca_threshold=args.lca_threshold, - genome_mode=args.genome_mode, - distinct_genome_threshold=args.distinct_genome_threshold, - exclude_protein_startwith=args.exclude_protein_startwith, - protein_separator=args.protein_separator, - protein_genome_separator=args.protein_genome_separator, - duplicate_peptide_handling_mode=args.duplicate_peptide_handling_mode, - n_jobs=args.n_jobs, - diann_intensity_col=args.diann_intensity_col, - metaumbra_peptide_score_col=args.metaumbra_peptide_score_col, - metaumbra_peptide_error_col=args.metaumbra_peptide_error_col, - metaumbra_single_peptide_error_rate_upper_bound=( - args.metaumbra_single_peptide_error_rate_upper_bound - ), - metaumbra_genome_qvalue_cutoff=args.metaumbra_genome_qvalue_cutoff, - ).run() + if bool(args.peptide_db) == bool(args.digested_genome_folders): + raise AnnotationConfigurationError( + "Exactly one of --peptide-db or --digested-genome-folders is required" + ) + selection_mode = "automatic" if args.input_source == "metax-automatic" else "provided" + if selection_mode == "provided" and not args.genome_list_file and not args.selected_genomes: + raise AnnotationConfigurationError( + "--input-source genome-list requires --genome-list-file or --selected-genomes" + ) + if selection_mode == "automatic" and (args.genome_list_file or args.selected_genomes): + raise AnnotationConfigurationError( + "Genome-list options are only valid with --input-source genome-list" + ) + with warnings.catch_warnings(record=True) as recorded: + warnings.simplefilter("always") + result = GlobalOTFAnnotator( + peptide_table_path=args.peptide_table, + output_path=args.output, + taxafunc_anno_db_path=args.taxafunc_db, + db_path=args.peptide_db, + digested_genome_folders=_normalise_digest_folders(args.digested_genome_folders), + selection_mode=selection_mode, + selected_genomes=args.selected_genomes, + genome_list_path=args.genome_list_file, + peptide_col=args.peptide_col, + intensity_col_prefix=args.intensity_col_prefix, + table_separator=_decode_separator(args.table_separator), + lca_threshold=args.lca_threshold, + genome_mode=args.genome_mode, + distinct_genome_threshold=args.distinct_genome_threshold, + exclude_protein_startwith=args.exclude_protein_startwith, + protein_separator=args.protein_separator, + protein_genome_separator=args.protein_genome_separator, + duplicate_peptide_handling_mode=args.duplicate_peptide_handling_mode, + n_jobs=args.n_jobs, + diann_intensity_col=args.diann_intensity_col, + selected_genome_source=( + args.genome_list_file or "CLI --selected-genomes" + if selection_mode == "provided" + else None + ), + ).run() + return result, [str(item.message) for item in recorded] def _success_result( @@ -573,19 +556,6 @@ def _success_result( getattr(run_result, "warnings", []) ) - if mode == "global": - result["inputs"] = dict(getattr(run_result, "inputs", {})) - if args.config: - result["inputs"]["config"] = _path_input(args.config) - result["parameters"] = dict(getattr(run_result, "parameters", {})) - result["stages"] = dict(getattr(run_result, "stages", {})) - result["genome_selection"] = dict( - getattr(run_result, "genome_selection", {}) - ) - result["metrics"] = dict(getattr(run_result, "metrics", {})) - result["outputs"] = dict(getattr(run_result, "outputs", {})) - return result - run_output_path = getattr(run_result, "output_path", args.output) artifacts_dir = Path(run_output_path).with_name( f"{Path(run_output_path).stem}_artifacts" @@ -594,13 +564,13 @@ def _success_result( summary_path = getattr(run_result, "summary_path", None) info_path = getattr(run_result, "info_path", None) result["stages"] = { - "unit_specific_annotation": { + "manifest_otf_annotation": { "status": "success", "duration_seconds": round(annotation_duration, 6), } } result["genome_selection"] = { - "method": "unit_specific_manifest", + "method": "metaumbra_genome_selection_manifest", "threshold": getattr( run_result, "selected_genome_threshold", @@ -640,10 +610,63 @@ def _success_result( outputs["per_unit_directory"] = {"path": str(per_unit_dir)} sample_mapping_path = artifacts_dir / "unit_sample_column_mapping.tsv" if sample_mapping_path.is_file(): - outputs["unit_sample_mapping"] = _path_input( + outputs["sample_mapping"] = _path_input( sample_mapping_path, format_name="tsv" ) result["outputs"] = outputs + result["manifest"] = { + "path": str(args.metaumbra_manifest), + "schema_version": getattr(run_result, "manifest_schema_version", None), + } + result["number_of_units"] = int(getattr(run_result, "completed_units", 0)) + int( + getattr(run_result, "skipped_units", 0) + ) + result["selected_threshold"] = getattr(run_result, "selected_genome_threshold", args.genome_threshold) + result["per_unit_summary"] = outputs.get("unit_summary") + result["input_source"] = mode + return result + + +def _success_result_non_manifest( + mode: str, + run_result: Any, + warning_messages: list[str], + *, + args: argparse.Namespace, + started_at: str, + started_perf: float, + run_id: str, +) -> dict[str, Any]: + result = _empty_result( + mode=mode, + exit_code=ExitCode.SUCCESS, + started_at=started_at, + started_perf=started_perf, + run_id=run_id, + args=args, + ) + result["run"] = _run_metadata( + mode=mode, + exit_code=ExitCode.SUCCESS, + started_at=started_at, + started_perf=started_perf, + run_id=run_id, + software=getattr(run_result, "software", {}), + ) + result["inputs"] = dict(getattr(run_result, "inputs", {})) + result["parameters"] = dict(getattr(run_result, "parameters", {})) + result["stages"] = dict(getattr(run_result, "stages", {})) + result["genome_selection"] = dict(getattr(run_result, "genome_selection", {})) + result["metrics"] = dict(getattr(run_result, "metrics", {})) + result["outputs"] = dict(getattr(run_result, "outputs", {})) + result["diagnostics"]["warnings"] = warning_messages + list( + getattr(run_result, "warnings", []) + ) + result["input_source"] = mode + result["manifest"] = None + result["number_of_units"] = None + result["selected_threshold"] = None + result["per_unit_summary"] = None return result @@ -671,9 +694,9 @@ def main(argv: list[str] | None = None) -> int: stage_started = time.perf_counter() parser = build_parser(config) args = parser.parse_args(argv) + mode = args.input_source if pre_args.config: args.config = config["config_path"] - mode = args.mode _validate_scientific_parameters(args) result_json_path = args.result_json if result_json_path and not Path(result_json_path).parent.is_dir(): @@ -682,22 +705,40 @@ def main(argv: list[str] | None = None) -> int: ) failure_stage = "annotation" stage_started = time.perf_counter() - if mode == "unit-specific": - run_result, warning_messages = _run_unit_specific(args) + if args.input_source == "metaumbra-manifest": + if args.genome_list_file or args.selected_genomes: + raise AnnotationConfigurationError( + "Genome-list options are only valid with --input-source genome-list" + ) + run_result, warning_messages = _run_manifest(args) else: - run_result = _run_global(args) - warning_messages = [] + if args.metaumbra_manifest: + raise AnnotationConfigurationError( + "--metaumbra-manifest is only valid with --input-source metaumbra-manifest" + ) + run_result, warning_messages = _run_non_manifest(args) annotation_duration = time.perf_counter() - stage_started - result = _success_result( - mode, - run_result, - warning_messages, - args=args, - started_at=started_at, - started_perf=started_perf, - run_id=run_id, - annotation_duration=annotation_duration, - ) + if args.input_source == "metaumbra-manifest": + result = _success_result( + mode, + run_result, + warning_messages, + args=args, + started_at=started_at, + started_perf=started_perf, + run_id=run_id, + annotation_duration=annotation_duration, + ) + else: + result = _success_result_non_manifest( + mode, + run_result, + warning_messages, + args=args, + started_at=started_at, + started_perf=started_perf, + run_id=run_id, + ) failure_stage = "result_serialization" stage_started = time.perf_counter() _write_result_json(result_json_path, result) diff --git a/metax/gui/main_gui.py b/metax/gui/main_gui.py index f7391a5c..dd241e16 100644 --- a/metax/gui/main_gui.py +++ b/metax/gui/main_gui.py @@ -114,23 +114,24 @@ search_linked_taxa_func_index, taxa_func_display_item_has_link, ) -from metax.gui.unit_specific_settings_dialog import UnitSpecificGuiConfig, UnitSpecificSettingsDialog +from metax.gui.unit_specific_settings_dialog import ManifestGuiConfig, ManifestSettingsDialog from metax.workflow_recorder import ( AnalysisStep, WorkflowRecorder, deseq2_step, + direct_otf_step, gui_action_step, limma_step, method_call_step, register_current_python_kernel, set_multi_tables_step, taxafunc_analyzer_step, - unit_specific_otf_step, + manifest_otf_step, ) from metax.peptide_annotator.metalab2otf import MetaLab2OTF from metax.peptide_annotator.peptable_annotator import PeptideAnnotator -from metax.peptide_annotator.annotation_workflow import GlobalOTFAnnotator, read_genome_list_file +from metax.peptide_annotator.annotation_workflow import GlobalOTFAnnotator, read_plain_genome_list_file from metax.peptide_annotator.peptide_table_prepare import ( PeptideTableSchema, available_diann_intensity_columns, @@ -138,7 +139,8 @@ is_parquet_path, resolve_diann_parquet_schema, ) -from metax.peptide_annotator.unit_specific_otf import UnitSpecificOTFAnnotator +from metax.peptide_annotator.manifest_otf import ManifestOTFAnnotator +from metax.peptide_annotator.genome_selection_manifest import load_genome_selection_manifest from metax.database_builder.database_builder_own import build_db from metax.database_updater.database_updater import run_db_update @@ -276,7 +278,7 @@ def __init__(self, MainWindow): self.last_path = QDir.homePath() # init last path as home path self.pep_direct_to_otf_selected_genomes = [] self.pep_direct_to_otf_selected_genome_source = "" - self.unit_specific_gui_config = UnitSpecificGuiConfig() + self.unit_specific_gui_config = ManifestGuiConfig() self.metaumbra_gui_process = None # init the check update status @@ -483,16 +485,54 @@ def __init__(self, MainWindow): unit_specific_settings_button = self._get_unit_specific_settings_button() if unit_specific_settings_button is not None: unit_specific_settings_button.clicked.connect(self.open_pep_direct_to_otf_unit_specific_settings) - if hasattr(self, "checkBox_pep_direct_to_otf_use_unit_specific_annotate"): - self.checkBox_pep_direct_to_otf_use_unit_specific_annotate.toggled.connect( - self.update_pep_direct_to_otf_mode_state - ) - self.checkBox_pep_direct_to_otf_use_selected_genome_list.toggled.connect( - self.update_pep_direct_to_otf_mode_state + self.comboBox_pep_direct_to_otf_input_source = QtWidgets.QComboBox() + self.comboBox_pep_direct_to_otf_input_source.addItem( + "MetaUmbra genome selection manifest", "metaumbra-manifest" + ) + self.comboBox_pep_direct_to_otf_input_source.addItem( + "MetaX automatic genome selection", "metax-automatic" ) - self.checkBox_pep_direct_to_otf_stop_after_metaumbra.toggled.connect( + self.comboBox_pep_direct_to_otf_input_source.addItem( + "Custom genome list", "genome-list" + ) + self.comboBox_pep_direct_to_otf_input_source.setToolTip( + "Select the genome source explicitly; MetaX does not infer a mode from file contents." + ) + self.label_pep_direct_to_otf_input_source = QtWidgets.QLabel("Genome selection source") + if hasattr(self, "gridLayout_74"): + source_row = self.gridLayout_74.rowCount() + self.gridLayout_74.addWidget(self.label_pep_direct_to_otf_input_source, source_row, 0, 1, 2) + self.gridLayout_74.addWidget(self.comboBox_pep_direct_to_otf_input_source, source_row, 2, 1, 2) + self.comboBox_pep_direct_to_otf_input_source.currentIndexChanged.connect( self.update_pep_direct_to_otf_mode_state ) + self.label_pep_direct_to_otf_manifest_summary = QtWidgets.QLabel( + "Select a MetaUmbra genome selection manifest to preview its analysis units." + ) + self.label_pep_direct_to_otf_manifest_summary.setWordWrap(True) + self.label_pep_direct_to_otf_manifest_summary.setProperty("subtle", True) + self.label_pep_direct_to_otf_manifest_summary.setFrameShape(QtWidgets.QFrame.StyledPanel) + self.label_pep_direct_to_otf_manifest_summary.setMargin(6) + self.label_pep_direct_to_otf_manifest_summary.setSizePolicy( + QtWidgets.QSizePolicy.Expanding, + QtWidgets.QSizePolicy.Fixed, + ) + self.label_pep_direct_to_otf_manifest_summary.setMaximumHeight(52) + if hasattr(self, "lineEdit_pep_direct_to_otf_unit_specific_manifest_path"): + self.lineEdit_pep_direct_to_otf_unit_specific_manifest_path.textChanged.connect( + self._update_pep_direct_to_otf_manifest_summary + ) + if hasattr(self, "checkBox_pep_direct_to_otf_use_unit_specific_annotate"): + self.checkBox_pep_direct_to_otf_use_unit_specific_annotate.setChecked(True) + self.checkBox_pep_direct_to_otf_use_unit_specific_annotate.setVisible(False) + for legacy_widget_name in ( + "checkBox_pep_direct_to_otf_use_selected_genome_list", + "checkBox_pep_direct_to_otf_stop_after_metaumbra", + ): + legacy_widget = getattr(self, legacy_widget_name, None) + if legacy_widget is not None: + legacy_widget.setVisible(False) + self._arrange_pep_direct_to_otf_layout() self.update_pep_direct_to_otf_mode_state() self._last_pep_direct_to_otf_peptide_table_signature = '' self.lineEdit_pep_direct_to_otf_peptide_path.textChanged.connect( @@ -2239,6 +2279,102 @@ def _set_widgets_enabled(self, widget_names, enabled): if widget is not None: widget.setEnabled(enabled) + def _arrange_pep_direct_to_otf_layout(self) -> None: + """Lay out annotation controls in input-to-output order.""" + layout = self.gridLayout_74 + if not hasattr(self, "label_pep_direct_to_otf_custom_genome_list"): + self.label_pep_direct_to_otf_custom_genome_list = QtWidgets.QLabel("Custom genome list") + if not hasattr(self, "horizontalLayout_pep_direct_to_otf_custom_genome_list"): + self.gridLayout_76.removeWidget(self.pushButton_pep_direct_to_otf_open_genome_list_file) + self.gridLayout_76.removeWidget(self.pushButton_pep_direct_to_otf_open_window_paste_gnome_list) + for index in range(self.gridLayout_76.count() - 1, -1, -1): + item = self.gridLayout_76.itemAt(index) + if item is not None and item.layout() is self.horizontalLayout_135: + self.gridLayout_76.takeAt(index) + self.horizontalLayout_pep_direct_to_otf_custom_genome_list = QtWidgets.QHBoxLayout() + self.horizontalLayout_pep_direct_to_otf_custom_genome_list.setSpacing(8) + self.horizontalLayout_pep_direct_to_otf_custom_genome_list.addWidget( + self.pushButton_pep_direct_to_otf_open_genome_list_file + ) + self.horizontalLayout_pep_direct_to_otf_custom_genome_list.addWidget( + self.pushButton_pep_direct_to_otf_open_window_paste_gnome_list + ) + self.horizontalLayout_pep_direct_to_otf_custom_genome_list.addLayout(self.horizontalLayout_135) + widgets = [ + self.label_pep_direct_to_otf_input_source, + self.comboBox_pep_direct_to_otf_input_source, + self.label_220, + self.lineEdit_pep_direct_to_otf_peptide_path, + self.pushButton_open_pep_direct_to_otf_peptide_path, + self.label_unit_specific_manifest, + self.label_pep_direct_to_otf_custom_genome_list, + self.lineEdit_pep_direct_to_otf_unit_specific_manifest_path, + self.pushButton_open_pep_direct_to_otf_unit_specific_manifest_path, + self.pushButton_pep_direct_to_otf_open_metaumbra_gui, + self.label_pep_direct_to_otf_manifest_summary, + self.label_222, + self.lineEdit_pep_direct_to_otf_digestied_genome_pep_path, + self.pushButton_open_pep_direct_to_otf_digestied_pep_db_path, + self.label_223, + self.lineEdit_pep_direct_to_otf_pro2taxafunc_db_path, + self.pushButton_open_pep_direct_to_otf_pro2taxafunc_db_path, + self.label_224, + self.lineEdit_pep_direct_to_otf_output_path, + self.pushButton_open_pep_direct_to_otf_output_path, + self.checkBox_8, + self.scrollArea_pep_direct_to_otf_settings, + self.pushButton_run_pep_direct_to_otf, + self.checkBox_pep_direct_to_otf_use_unit_specific_annotate, + ] + for widget in widgets: + layout.removeWidget(widget) + for index in range(layout.count() - 1, -1, -1): + item = layout.itemAt(index) + if item is not None and item.layout() in ( + self.horizontalLayout_140, + self.horizontalLayout_pep_direct_to_otf_custom_genome_list, + self.gridLayout_78, + ): + layout.takeAt(index) + + layout.addWidget(self.label_pep_direct_to_otf_input_source, 0, 0, 1, 2) + layout.addWidget(self.comboBox_pep_direct_to_otf_input_source, 0, 2, 1, 2) + layout.addWidget(self.label_220, 1, 0, 1, 2) + layout.addWidget(self.lineEdit_pep_direct_to_otf_peptide_path, 1, 2) + layout.addWidget(self.pushButton_open_pep_direct_to_otf_peptide_path, 1, 3) + layout.addWidget(self.label_unit_specific_manifest, 2, 0, 1, 2) + layout.addWidget(self.lineEdit_pep_direct_to_otf_unit_specific_manifest_path, 2, 2) + layout.addWidget(self.pushButton_open_pep_direct_to_otf_unit_specific_manifest_path, 2, 3) + layout.addWidget(self.label_pep_direct_to_otf_custom_genome_list, 2, 0, 1, 2) + layout.addLayout(self.horizontalLayout_pep_direct_to_otf_custom_genome_list, 2, 2, 1, 2) + layout.addWidget(self.pushButton_pep_direct_to_otf_open_metaumbra_gui, 3, 0, 1, 2) + layout.addLayout(self.horizontalLayout_140, 3, 2, 1, 2) + layout.addWidget(self.label_pep_direct_to_otf_manifest_summary, 4, 0, 1, 4) + layout.addWidget(self.label_222, 5, 0, 1, 2) + layout.addWidget(self.lineEdit_pep_direct_to_otf_digestied_genome_pep_path, 5, 2) + layout.addWidget(self.pushButton_open_pep_direct_to_otf_digestied_pep_db_path, 5, 3) + layout.addWidget(self.label_223, 6, 0, 1, 2) + layout.addWidget(self.lineEdit_pep_direct_to_otf_pro2taxafunc_db_path, 6, 2) + layout.addWidget(self.pushButton_open_pep_direct_to_otf_pro2taxafunc_db_path, 6, 3) + layout.addWidget(self.label_224, 7, 0, 1, 2) + layout.addWidget(self.lineEdit_pep_direct_to_otf_output_path, 7, 2) + layout.addWidget(self.pushButton_open_pep_direct_to_otf_output_path, 7, 3) + layout.addLayout(self.gridLayout_78, 8, 2, 1, 2) + layout.addWidget(self.checkBox_8, 9, 0, 1, 2) + layout.addWidget(self.scrollArea_pep_direct_to_otf_settings, 10, 0, 1, 4) + layout.addWidget(self.pushButton_run_pep_direct_to_otf, 11, 0, 1, 4) + if not hasattr(self, "pep_direct_to_otf_bottom_spacer"): + self.pep_direct_to_otf_bottom_spacer = QtWidgets.QSpacerItem( + 0, + 0, + QtWidgets.QSizePolicy.Minimum, + QtWidgets.QSizePolicy.Expanding, + ) + layout.addItem(self.pep_direct_to_otf_bottom_spacer, 12, 0, 1, 4) + layout.setColumnStretch(2, 1) + layout.setHorizontalSpacing(12) + layout.setVerticalSpacing(8) + def _get_unit_specific_settings_button(self): widget = getattr(self, "pushButton_pep_direct_to_otf_unit_specific_settings", None) if widget is not None: @@ -2302,45 +2438,48 @@ def _set_otf_annotation_controls_enabled(self, enabled: bool): ) def update_pep_direct_to_otf_mode_state(self): - use_unit_specific = ( - hasattr(self, "checkBox_pep_direct_to_otf_use_unit_specific_annotate") - and self.checkBox_pep_direct_to_otf_use_unit_specific_annotate.isChecked() - ) - - if use_unit_specific: - with QtCore.QSignalBlocker(self.checkBox_pep_direct_to_otf_use_selected_genome_list): - self.checkBox_pep_direct_to_otf_use_selected_genome_list.setChecked(False) - with QtCore.QSignalBlocker(self.checkBox_pep_direct_to_otf_stop_after_metaumbra): - self.checkBox_pep_direct_to_otf_stop_after_metaumbra.setChecked(False) - - self.checkBox_pep_direct_to_otf_use_selected_genome_list.setEnabled(False) - self.checkBox_pep_direct_to_otf_stop_after_metaumbra.setEnabled(False) - - self._set_selected_genome_list_controls_enabled(False) - self._set_metaumbra_scoring_controls_enabled(False) - self._set_unit_specific_controls_enabled(True) - self._set_otf_annotation_controls_enabled(True) - - self.label_224.setText('OTFs Save To') - self.lineEdit_pep_direct_to_otf_output_path.setStatusTip('The path for save the OTF result') - if hasattr(self.lineEdit_pep_direct_to_otf_output_path, 'default_filename'): - self.lineEdit_pep_direct_to_otf_output_path.default_filename = 'OTF_direct_anno.tsv' - return - - self._set_unit_specific_controls_enabled(False) - - use_selected = self.checkBox_pep_direct_to_otf_use_selected_genome_list.isChecked() - stop_after = self.checkBox_pep_direct_to_otf_stop_after_metaumbra.isChecked() - - self.checkBox_pep_direct_to_otf_use_selected_genome_list.setEnabled(not stop_after) - self.checkBox_pep_direct_to_otf_stop_after_metaumbra.setEnabled(not use_selected) - + source = "metaumbra-manifest" + if hasattr(self, "comboBox_pep_direct_to_otf_input_source"): + source = str(self.comboBox_pep_direct_to_otf_input_source.currentData() or source) + use_manifest = source == "metaumbra-manifest" + use_selected = source == "genome-list" + with QtCore.QSignalBlocker(self.checkBox_pep_direct_to_otf_use_unit_specific_annotate): + self.checkBox_pep_direct_to_otf_use_unit_specific_annotate.setChecked(use_manifest) + with QtCore.QSignalBlocker(self.checkBox_pep_direct_to_otf_use_selected_genome_list): + self.checkBox_pep_direct_to_otf_use_selected_genome_list.setChecked(use_selected) + with QtCore.QSignalBlocker(self.checkBox_pep_direct_to_otf_stop_after_metaumbra): + self.checkBox_pep_direct_to_otf_stop_after_metaumbra.setChecked(False) + self._set_unit_specific_controls_enabled(use_manifest) self._set_selected_genome_list_controls_enabled(use_selected) - self._set_metaumbra_scoring_controls_enabled(not use_selected) - if not use_selected and stop_after and hasattr(self, "doubleSpinBox_pep_direct_to_otf_protein_coverage_cutoff"): - self.doubleSpinBox_pep_direct_to_otf_protein_coverage_cutoff.setEnabled(False) - self._set_otf_annotation_controls_enabled(not stop_after) - self.update_pep_direct_to_otf_output_mode() + self._set_metaumbra_scoring_controls_enabled(False) + self._set_otf_annotation_controls_enabled(True) + for widget_name in ( + "label_unit_specific_manifest", + "lineEdit_pep_direct_to_otf_unit_specific_manifest_path", + "pushButton_open_pep_direct_to_otf_unit_specific_manifest_path", + "pushButton_pep_direct_to_otf_unit_specific_settings", + "label_pep_direct_to_otf_use_unit_specific_genome_threshold", + "comboBox_pep_direct_to_otf_use_unit_specific_genome_threshold", + "pushButton_pep_direct_to_otf_open_metaumbra_gui", + ): + widget = getattr(self, widget_name, None) + if widget is not None: + widget.setVisible(use_manifest) + for widget_name in ( + "label_pep_direct_to_otf_custom_genome_list", + "pushButton_pep_direct_to_otf_open_genome_list_file", + "pushButton_pep_direct_to_otf_open_window_paste_gnome_list", + "pushButton_pep_direct_to_otf_reset_selected_genome_list", + "label__pep_direct_to_otf_selected_genome_num", + ): + widget = getattr(self, widget_name, None) + if widget is not None: + widget.setVisible(use_selected) + self.label_pep_direct_to_otf_manifest_summary.setVisible(use_manifest) + self.label_224.setText('OTFs Save To') + self.lineEdit_pep_direct_to_otf_output_path.setStatusTip('The path for the OTF result') + if hasattr(self.lineEdit_pep_direct_to_otf_output_path, 'default_filename'): + self.lineEdit_pep_direct_to_otf_output_path.default_filename = 'OTF_direct_anno.tsv' def update_pep_direct_to_otf_output_mode(self): stop_after_metaumbra = self.checkBox_pep_direct_to_otf_stop_after_metaumbra.isChecked() @@ -3190,7 +3329,7 @@ def set_lineEdit_pep_direct_to_otf_output_path(self): def set_lineEdit_pep_direct_to_otf_unit_specific_manifest_path(self): file_path, _ = QFileDialog.getOpenFileName( self.MainWindow, - 'Open MetaUmbra unit-specific manifest JSON', + 'Open MetaUmbra genome selection manifest JSON', self.last_path, 'JSON files (*.json);;All files (*)', ) @@ -3199,6 +3338,29 @@ def set_lineEdit_pep_direct_to_otf_unit_specific_manifest_path(self): self.last_path = os.path.dirname(file_path) self.unit_specific_gui_config.manifest_path = os.path.normpath(file_path) + def _update_pep_direct_to_otf_manifest_summary(self, manifest_path: str) -> None: + path = str(manifest_path or "").strip() + if not path: + self.label_pep_direct_to_otf_manifest_summary.setText( + "Select a MetaUmbra genome selection manifest to preview its analysis units." + ) + return + try: + manifest = load_genome_selection_manifest(path, genome_threshold="auto", strict=True) + sample_ids = { + sample_id + for unit in manifest.units.values() + for sample_id in unit.sample_ids + } + self.label_pep_direct_to_otf_manifest_summary.setText( + f"Manifest mode: {manifest.unit_definition.get('mode', 'unknown')} | " + f"Samples: {len(sample_ids)} | Analysis units: {len(manifest.units)} | " + f"Genome threshold: {manifest.selected_genome_threshold} | " + f"MetaUmbra version: {manifest.generated_by.get('version', 'unknown')}" + ) + except Exception as exc: + self.label_pep_direct_to_otf_manifest_summary.setText(f"Manifest validation error: {exc}") + def _decode_pep_direct_to_otf_separator(self, separator: str) -> str: separator = separator or '' if separator == r'\t': @@ -3207,8 +3369,8 @@ def _decode_pep_direct_to_otf_separator(self, separator: str) -> str: return r'\s+' return separator - def _get_unit_specific_gui_config_from_controls(self) -> UnitSpecificGuiConfig: - config = getattr(self, "unit_specific_gui_config", UnitSpecificGuiConfig()) + def _get_unit_specific_gui_config_from_controls(self) -> ManifestGuiConfig: + config = getattr(self, "unit_specific_gui_config", ManifestGuiConfig()) manifest_path = "" genome_threshold = config.genome_threshold if hasattr(self, "lineEdit_pep_direct_to_otf_unit_specific_manifest_path"): @@ -3218,7 +3380,7 @@ def _get_unit_specific_gui_config_from_controls(self) -> UnitSpecificGuiConfig: self.comboBox_pep_direct_to_otf_use_unit_specific_genome_threshold.currentText().strip() or genome_threshold ) - return UnitSpecificGuiConfig( + return ManifestGuiConfig( manifest_path=manifest_path or config.manifest_path, genome_threshold=genome_threshold, input_sample_col_prefix=config.input_sample_col_prefix, @@ -3241,7 +3403,7 @@ def open_pep_direct_to_otf_unit_specific_settings(self): if is_diann_parquet(parquet_columns): input_intensity_prefix = None diann_intensity_col = intensity_selector_value or None - dialog = UnitSpecificSettingsDialog( + dialog = ManifestSettingsDialog( parent=self.MainWindow, peptide_table_path=peptide_table_path, peptide_col=self.comboBox_pep_direct_to_otf_peptide_col_name.currentText().strip(), @@ -3250,6 +3412,7 @@ def open_pep_direct_to_otf_unit_specific_settings(self): ), input_intensity_prefix=input_intensity_prefix, diann_intensity_col=diann_intensity_col, + digested_genome_folders=self.lineEdit_pep_direct_to_otf_digestied_genome_pep_path.text().strip(), current_config=config, ) if dialog.exec_() == QDialog.Accepted: @@ -3292,16 +3455,12 @@ def _add_pep_direct_to_otf_selected_genomes(self, genomes: list[str], source: st return added def _read_pep_direct_to_otf_genome_list_file(self, file_path: str) -> list[str]: - qvalue_cutoff = round( - self.doubleSpinBox_pep_direct_to_otf_metaumbra_qvalue_cutoff.value(), - 3, - ) - return read_genome_list_file(file_path, qvalue_cutoff=qvalue_cutoff) + return read_plain_genome_list_file(file_path) def open_pep_direct_to_otf_genome_list_file(self): genome_list_path = QFileDialog.getOpenFileName( self.MainWindow, - 'Select Genome List or MetaUmbra Result', + 'Select Custom Genome List', self.last_path, 'Genome list (*.txt *.tsv *.csv);;All Files (*)' )[0] @@ -3860,13 +4019,13 @@ def _get_pep_direct_to_otf_metaumbra_settings(self) -> dict: ), } - def run_pep_direct_to_otf_unit_specific(self): + def run_pep_direct_to_otf_manifest(self): config = self._get_unit_specific_gui_config_from_controls() peptide_table_path = self.lineEdit_pep_direct_to_otf_peptide_path.text().strip() digested_genome_folder_path = self.lineEdit_pep_direct_to_otf_digestied_genome_pep_path.text().strip() taxafunc_anno_db_path = self.lineEdit_pep_direct_to_otf_pro2taxafunc_db_path.text().strip() output_path = self.lineEdit_pep_direct_to_otf_output_path.text().strip() - unit_specific_manifest_path = config.manifest_path + metaumbra_manifest_path = config.manifest_path peptide_col = self.comboBox_pep_direct_to_otf_peptide_col_name.currentText().strip() table_separator = self._decode_pep_direct_to_otf_separator( self.lineEdit_pep_direct_to_otf_pep_table_sep.text().strip() @@ -3890,7 +4049,7 @@ def run_pep_direct_to_otf_unit_specific(self): ("Digested genome folder", digested_genome_folder_path), ("Protein to TaxaFunc database", taxafunc_anno_db_path), ("OTFs Save To", output_path), - ("Unit-specific manifest", unit_specific_manifest_path), + ("MetaUmbra genome selection manifest", metaumbra_manifest_path), ("Peptide column", peptide_col), ("Separator of peptide table", table_separator), ("Genome separator in protein ID", protein_genome_separator), @@ -3903,7 +4062,7 @@ def run_pep_direct_to_otf_unit_specific(self): file_checks = [ ("Peptide table", peptide_table_path), ("Protein to TaxaFunc database", taxafunc_anno_db_path), - ("Unit-specific manifest", unit_specific_manifest_path), + ("MetaUmbra genome selection manifest", metaumbra_manifest_path), ] for label, path in file_checks: if not os.path.isfile(path): @@ -3940,9 +4099,9 @@ def run_pep_direct_to_otf_unit_specific(self): try: self.logger.write_log( - f'run_pep_direct_to_otf_unit_specific: peptide_table_path:{peptide_table_path} ' + f'run_pep_direct_to_otf_manifest: peptide_table_path:{peptide_table_path} ' f'digested_genome_folder_path:{digested_genome_folder_path} output_path:{output_path} ' - f'taxafunc_anno_db_path:{taxafunc_anno_db_path} manifest:{unit_specific_manifest_path} ' + f'taxafunc_anno_db_path:{taxafunc_anno_db_path} manifest:{metaumbra_manifest_path} ' f'genome_threshold:{genome_threshold or "auto"} ' f'duplicate_peptide_handling_mode:{duplicate_peptide_handling_mode} ' f'n_jobs:{n_jobs if n_jobs is not None else "auto"}' @@ -3950,7 +4109,7 @@ def run_pep_direct_to_otf_unit_specific(self): workflow_params = { "peptide_table_path": peptide_table_path, - "unit_specific_manifest_path": unit_specific_manifest_path, + "metaumbra_manifest_path": metaumbra_manifest_path, "taxafunc_anno_db_path": taxafunc_anno_db_path, "output_path": output_path, "digested_genome_folders": digested_genome_folder_path, @@ -3964,7 +4123,6 @@ def run_pep_direct_to_otf_unit_specific(self): "distinct_genome_threshold": 0, "protein_genome_separator": protein_genome_separator, "save_per_unit_outputs": config.save_per_unit_outputs, - "include_unit_specific_sequence": False, "duplicate_peptide_handling_mode": duplicate_peptide_handling_mode, "on_missing_sample": config.on_missing_sample, "on_empty_unit": config.on_empty_unit, @@ -3974,161 +4132,116 @@ def run_pep_direct_to_otf_unit_specific(self): "diann_intensity_col": diann_intensity_col, } - def pep_direct_to_otf_unit_specific_wrapper(): - annotator = UnitSpecificOTFAnnotator(**workflow_params) + def pep_direct_to_otf_manifest_wrapper(): + annotator = ManifestOTFAnnotator(**workflow_params) return annotator.run() self.run_in_new_window( - pep_direct_to_otf_unit_specific_wrapper, + pep_direct_to_otf_manifest_wrapper, show_msg=True, - workflow_step=unit_specific_otf_step(workflow_params), + workflow_step=manifest_otf_step(workflow_params), ) except Exception as e: - self.logger.write_log(f'run_pep_direct_to_otf_unit_specific error: {e}', 'e') + self.logger.write_log(f'run_pep_direct_to_otf_manifest error: {e}', 'e') QMessageBox.warning(self.MainWindow, 'Warning', f'Error: {e}') - def run_pep_dircet_to_otf(self): - if ( - hasattr(self, "checkBox_pep_direct_to_otf_use_unit_specific_annotate") - and self.checkBox_pep_direct_to_otf_use_unit_specific_annotate.isChecked() - ): - return self.run_pep_direct_to_otf_unit_specific() - + def run_pep_direct_to_otf_non_metaumbra(self, source: str): peptide_table_path = self.lineEdit_pep_direct_to_otf_peptide_path.text().strip() digested_genome_folder_path = self.lineEdit_pep_direct_to_otf_digestied_genome_pep_path.text().strip() - table_separator = self._decode_pep_direct_to_otf_separator(self.lineEdit_pep_direct_to_otf_pep_table_sep.text().strip()) + taxafunc_anno_db_path = self.lineEdit_pep_direct_to_otf_pro2taxafunc_db_path.text().strip() + output_path = self.lineEdit_pep_direct_to_otf_output_path.text().strip() peptide_col = self.comboBox_pep_direct_to_otf_peptide_col_name.currentText().strip() - intensity_selector_value = ( - self.comboBox_pep_direct_to_otf_intensity_column.currentText().strip() + table_separator = self._decode_pep_direct_to_otf_separator( + self.lineEdit_pep_direct_to_otf_pep_table_sep.text().strip() ) - input_intensity_prefix = intensity_selector_value - diann_intensity_col = None - mapper_intensity_col_prefix = input_intensity_prefix - protein_peptide_coverage_cutoff = round(self.doubleSpinBox_pep_direct_to_otf_protein_coverage_cutoff.value(), 3) - output_path = self.lineEdit_pep_direct_to_otf_output_path.text().strip() - taxafunc_anno_db_path = self.lineEdit_pep_direct_to_otf_pro2taxafunc_db_path.text().strip() lca_threshold = round(self.doubleSpinBox_pep_direct_to_otf_LCA_threshold.value(), 3) protein_genome_separator = self.lineEdit_pep_direct_to_otf_genome_separator.text().strip() - duplicate_peptide_handling_mode = self.comboBox_pep_direct_to_otf_duplicate_peptide_handle_mode.currentText().strip() - stop_after_metaumbra = self.checkBox_pep_direct_to_otf_stop_after_metaumbra.isChecked() - use_selected_genome_list = self.checkBox_pep_direct_to_otf_use_selected_genome_list.isChecked() - diann_parquet_detected = False - if self._is_parquet_path(peptide_table_path): - table_separator = '\t' - parquet_columns, _ = self._read_parquet_preview(peptide_table_path) - diann_parquet_detected = is_diann_parquet(parquet_columns) - if diann_parquet_detected: - diann_intensity_col = intensity_selector_value - schema = resolve_diann_parquet_schema( - parquet_columns, - require_score_columns=True, - intensity_col=diann_intensity_col, - ) - peptide_col = schema.peptide_col - mapper_intensity_col_prefix = schema.intensity_col_prefix - self._apply_metaumbra_columns(schema) - - for value in [ - peptide_table_path, - digested_genome_folder_path, - output_path, - table_separator, - peptide_col, - mapper_intensity_col_prefix, - ]: - if value == '': - QMessageBox.warning(self.MainWindow, 'Warning', 'Please set all above paths and values') + duplicate_mode = ( + self.comboBox_pep_direct_to_otf_duplicate_peptide_handle_mode.currentText().strip() or "sum" + ) + required = [ + ("Peptide table", peptide_table_path), + ("Digested genome folder", digested_genome_folder_path), + ("Protein to TaxaFunc database", taxafunc_anno_db_path), + ("OTFs Save To", output_path), + ("Peptide column", peptide_col), + ("Genome separator in protein ID", protein_genome_separator), + ] + for label, value in required: + if not value: + QMessageBox.warning(self.MainWindow, "Warning", f"Please set {label}.") return None - - if not duplicate_peptide_handling_mode: - duplicate_peptide_handling_mode = 'sum' - - if not stop_after_metaumbra and not taxafunc_anno_db_path: - QMessageBox.warning(self.MainWindow, 'Warning', 'Please select Protein to TaxaFunc Database') - return None - - if not use_selected_genome_list and table_separator != '\t': - QMessageBox.warning(self.MainWindow, 'Warning', 'MetaUmbra scoring currently requires a tab-separated peptide table.') - return None - - check_files = [peptide_table_path, digested_genome_folder_path] - if not stop_after_metaumbra: - check_files.append(taxafunc_anno_db_path) - - for file in check_files: - if not os.path.exists(file): - QMessageBox.warning(self.MainWindow, 'Warning', f'File not found: {file}') + for label, path in (("Peptide table", peptide_table_path), ("Protein to TaxaFunc database", taxafunc_anno_db_path)): + if not os.path.isfile(path): + QMessageBox.warning(self.MainWindow, "Warning", f"{label} not found:\n{path}") return None - + if not os.path.isdir(digested_genome_folder_path): + QMessageBox.warning(self.MainWindow, "Warning", f"Digested genome folder not found:\n{digested_genome_folder_path}") + return None output_dir = os.path.dirname(output_path) if output_dir and not os.path.isdir(output_dir): - QMessageBox.warning(self.MainWindow, 'Warning', f'Output directory not found: {output_dir}') + QMessageBox.warning(self.MainWindow, "Warning", f"Output directory not found:\n{output_dir}") return None - - selected_genomes = list(self.pep_direct_to_otf_selected_genomes) - if use_selected_genome_list and not selected_genomes: - QMessageBox.warning(self.MainWindow, 'Warning', 'Please open or paste at least one selected genome.') + selection_mode = "provided" if source == "genome-list" else "automatic" + selected_genomes = list(self.pep_direct_to_otf_selected_genomes) if selection_mode == "provided" else None + if selection_mode == "provided" and not selected_genomes: + QMessageBox.warning(self.MainWindow, "Warning", "Load or paste at least one genome ID.") return None - + diann_intensity_col = None + intensity_col_prefix = self.comboBox_pep_direct_to_otf_intensity_column.currentText().strip() or "Intensity" + if self._is_parquet_path(peptide_table_path): + parquet_columns, _ = self._read_parquet_preview(peptide_table_path) + if is_diann_parquet(parquet_columns): + diann_intensity_col = self.comboBox_pep_direct_to_otf_intensity_column.currentText().strip() or None + # DIA-NN preparation resolves its own canonical sample prefix. + intensity_col_prefix = "Intensity" + workflow_params = { + "peptide_table_path": peptide_table_path, + "output_path": output_path, + "taxafunc_anno_db_path": taxafunc_anno_db_path, + "digested_genome_folders": digested_genome_folder_path, + "selection_mode": selection_mode, + "selected_genomes": selected_genomes, + "selected_genome_source": ( + self.pep_direct_to_otf_selected_genome_source or "GUI genome list" + if selection_mode == "provided" + else None + ), + "peptide_col": peptide_col, + "intensity_col_prefix": intensity_col_prefix, + "table_separator": table_separator, + "lca_threshold": lca_threshold, + "genome_mode": True, + "distinct_genome_threshold": 0, + "protein_genome_separator": protein_genome_separator, + "duplicate_peptide_handling_mode": duplicate_mode, + "diann_intensity_col": diann_intensity_col, + } try: self.logger.write_log( - f'run_pep_dircet_to_otf: peptide_table_path:{peptide_table_path} ' - f'digested_genome_folder_path:{digested_genome_folder_path} output_path:{output_path} ' - f'taxafunc_anno_db_path:{taxafunc_anno_db_path} stop_after_metaumbra:{stop_after_metaumbra} ' - f'use_selected_genome_list:{use_selected_genome_list} ' - f'duplicate_peptide_handling_mode:{duplicate_peptide_handling_mode}' + f"run_pep_direct_to_otf_non_metaumbra: source:{source} " + f"peptide_table_path:{peptide_table_path} output_path:{output_path}" ) - def pep_direct_to_otf_main_wrapper(): - selection_mode = ( - "provided" - if use_selected_genome_list - else "metaumbra-only" - if stop_after_metaumbra - else "metaumbra" - ) - metaumbra_settings = self._get_pep_direct_to_otf_metaumbra_settings() - return GlobalOTFAnnotator( - peptide_table_path=peptide_table_path, - digested_genome_folders=digested_genome_folder_path, - table_separator=table_separator, - peptide_col=peptide_col, - intensity_col_prefix=mapper_intensity_col_prefix, - protein_peptide_coverage_cutoff=protein_peptide_coverage_cutoff, - output_path=output_path, - taxafunc_anno_db_path=taxafunc_anno_db_path, - lca_threshold=lca_threshold, - protein_genome_separator=protein_genome_separator, - selection_mode=selection_mode, - selected_genomes=selected_genomes, - selected_genome_source=( - ( - self.pep_direct_to_otf_selected_genome_source - or "GUI selected genome list" - ) - if use_selected_genome_list - else None - ), - duplicate_peptide_handling_mode=duplicate_peptide_handling_mode, - diann_intensity_col=diann_intensity_col, - metaumbra_peptide_score_col=metaumbra_settings[ - "metaumbra_peptide_score_col" - ], - metaumbra_peptide_error_col=metaumbra_settings[ - "metaumbra_peptide_error_col" - ], - metaumbra_single_peptide_error_rate_upper_bound=metaumbra_settings[ - "metaumbra_single_peptide_error_rate_upper_bound" - ], - metaumbra_genome_qvalue_cutoff=metaumbra_settings[ - "metaumbra_genome_qvalue_cutoff" - ], - ).run() - - self.run_in_new_window(pep_direct_to_otf_main_wrapper, show_msg=True) - except Exception as e: - self.logger.write_log(f'run_pep_dircet_to_otf error: {e}', 'e') - QMessageBox.warning(self.MainWindow, 'Warning', f'Error: {e}') + def direct_otf_wrapper(): + return GlobalOTFAnnotator(**workflow_params).run() + + self.run_in_new_window( + direct_otf_wrapper, + show_msg=True, + workflow_step=direct_otf_step(workflow_params), + ) + except Exception as exc: + self.logger.write_log(f"run_pep_direct_to_otf_non_metaumbra error: {exc}", "e") + QMessageBox.warning(self.MainWindow, "Warning", f"Error: {exc}") + + def run_pep_dircet_to_otf(self): + source = str( + self.comboBox_pep_direct_to_otf_input_source.currentData() or "metaumbra-manifest" + ) + if source == "metaumbra-manifest": + return self.run_pep_direct_to_otf_manifest() + return self.run_pep_direct_to_otf_non_metaumbra(source) diff --git a/metax/gui/metax_gui/main_window.ui b/metax/gui/metax_gui/main_window.ui index 70036350..7f3c3ff2 100644 --- a/metax/gui/metax_gui/main_window.ui +++ b/metax/gui/metax_gui/main_window.ui @@ -351,7 +351,7 @@ - 50 + 70 0 @@ -10756,7 +10756,7 @@ - Pepetide Direct to OTFs (MetaUmbra) + Peptide Direct to OTFs @@ -11018,7 +11018,7 @@ Reset the selected genome list - 🔄 + Reset @@ -11030,7 +11030,7 @@ false - Accepts either a plain genome ID list or a MetaUmbra genome_presence.tsv file. + Load a plain text, TSV, or CSV file containing custom genome IDs. Open Genome List File @@ -11186,8 +11186,14 @@ + + false + + + true + - Use unit-specific annotation + Manifest-driven annotation @@ -11211,7 +11217,7 @@ - false + true @@ -11281,7 +11287,7 @@ - MetaUmbra unit-specific manifest + MetaUmbra genome selection manifest @@ -11322,7 +11328,7 @@ - false + true Open @@ -11343,7 +11349,7 @@ - Unit-specific Settings... + Validate / Settings... diff --git a/metax/gui/metax_gui/ui_main_window.py b/metax/gui/metax_gui/ui_main_window.py index 5669f940..b4cf3761 100644 --- a/metax/gui/metax_gui/ui_main_window.py +++ b/metax/gui/metax_gui/ui_main_window.py @@ -5777,7 +5777,7 @@ def setupUi(self, metaX_main): sizePolicy.setHeightForWidth(self.pushButton_pep_direct_to_otf_reset_selected_genome_list.sizePolicy().hasHeightForWidth()) self.pushButton_pep_direct_to_otf_reset_selected_genome_list.setSizePolicy(sizePolicy) self.pushButton_pep_direct_to_otf_reset_selected_genome_list.setMinimumSize(QtCore.QSize(0, 0)) - self.pushButton_pep_direct_to_otf_reset_selected_genome_list.setMaximumSize(QtCore.QSize(50, 16777215)) + self.pushButton_pep_direct_to_otf_reset_selected_genome_list.setMaximumSize(QtCore.QSize(70, 16777215)) self.pushButton_pep_direct_to_otf_reset_selected_genome_list.setObjectName("pushButton_pep_direct_to_otf_reset_selected_genome_list") self.horizontalLayout_135.addWidget(self.pushButton_pep_direct_to_otf_reset_selected_genome_list) self.gridLayout_76.addLayout(self.horizontalLayout_135, 4, 3, 1, 1) @@ -5843,6 +5843,8 @@ def setupUi(self, metaX_main): self.scrollArea_pep_direct_to_otf_settings.setWidget(self.scrollAreaWidgetContents_11) self.gridLayout_74.addWidget(self.scrollArea_pep_direct_to_otf_settings, 9, 0, 1, 4) self.checkBox_pep_direct_to_otf_use_unit_specific_annotate = QtWidgets.QCheckBox(self.tab_6) + self.checkBox_pep_direct_to_otf_use_unit_specific_annotate.setVisible(False) + self.checkBox_pep_direct_to_otf_use_unit_specific_annotate.setChecked(True) self.checkBox_pep_direct_to_otf_use_unit_specific_annotate.setObjectName("checkBox_pep_direct_to_otf_use_unit_specific_annotate") self.gridLayout_74.addWidget(self.checkBox_pep_direct_to_otf_use_unit_specific_annotate, 5, 0, 1, 1) self.pushButton_pep_direct_to_otf_open_metaumbra_gui = QtWidgets.QPushButton(self.tab_6) @@ -5852,7 +5854,7 @@ def setupUi(self, metaX_main): self.label_222.setObjectName("label_222") self.gridLayout_74.addWidget(self.label_222, 1, 0, 1, 2) self.lineEdit_pep_direct_to_otf_unit_specific_manifest_path = QtWidgets.QLineEdit(self.tab_6) - self.lineEdit_pep_direct_to_otf_unit_specific_manifest_path.setEnabled(False) + self.lineEdit_pep_direct_to_otf_unit_specific_manifest_path.setEnabled(True) self.lineEdit_pep_direct_to_otf_unit_specific_manifest_path.setObjectName("lineEdit_pep_direct_to_otf_unit_specific_manifest_path") self.gridLayout_74.addWidget(self.lineEdit_pep_direct_to_otf_unit_specific_manifest_path, 6, 2, 1, 1) self.label_224 = QtWidgets.QLabel(self.tab_6) @@ -5900,7 +5902,7 @@ def setupUi(self, metaX_main): self.label_220.setObjectName("label_220") self.gridLayout_74.addWidget(self.label_220, 0, 0, 1, 2) self.pushButton_open_pep_direct_to_otf_unit_specific_manifest_path = QtWidgets.QPushButton(self.tab_6) - self.pushButton_open_pep_direct_to_otf_unit_specific_manifest_path.setEnabled(False) + self.pushButton_open_pep_direct_to_otf_unit_specific_manifest_path.setEnabled(True) self.pushButton_open_pep_direct_to_otf_unit_specific_manifest_path.setObjectName("pushButton_open_pep_direct_to_otf_unit_specific_manifest_path") self.gridLayout_74.addWidget(self.pushButton_open_pep_direct_to_otf_unit_specific_manifest_path, 6, 3, 1, 1) self.horizontalLayout_140 = QtWidgets.QHBoxLayout() @@ -7267,8 +7269,8 @@ def retranslateUi(self, metaX_main): self.label__pep_direct_to_otf_selected_genome_num.setStatusTip(_translate("metaX_main", "Number of genomes currently loaded from the selected genome list.")) self.label__pep_direct_to_otf_selected_genome_num.setText(_translate("metaX_main", "Selected genomes: 0")) self.pushButton_pep_direct_to_otf_reset_selected_genome_list.setStatusTip(_translate("metaX_main", "Reset the selected genome list")) - self.pushButton_pep_direct_to_otf_reset_selected_genome_list.setText(_translate("metaX_main", "🔄")) - self.pushButton_pep_direct_to_otf_open_genome_list_file.setStatusTip(_translate("metaX_main", "Accepts either a plain genome ID list or a MetaUmbra genome_presence.tsv file.")) + self.pushButton_pep_direct_to_otf_reset_selected_genome_list.setText(_translate("metaX_main", "Reset")) + self.pushButton_pep_direct_to_otf_open_genome_list_file.setStatusTip(_translate("metaX_main", "Load a plain text, TSV, or CSV file containing custom genome IDs.")) self.pushButton_pep_direct_to_otf_open_genome_list_file.setText(_translate("metaX_main", "Open Genome List File")) self.checkBox_pep_direct_to_otf_use_selected_genome_list.setStatusTip(_translate("metaX_main", "Skip MetaUmbra genome scoring and continue OTF construction using a predefined genome list.")) self.checkBox_pep_direct_to_otf_use_selected_genome_list.setText(_translate("metaX_main", "Use selected genome list")) @@ -7283,7 +7285,7 @@ def retranslateUi(self, metaX_main): self.comboBox_pep_direct_to_otf_duplicate_peptide_handle_mode.setItemText(2, _translate("metaX_main", "min")) self.comboBox_pep_direct_to_otf_duplicate_peptide_handle_mode.setItemText(3, _translate("metaX_main", "mean")) self.comboBox_pep_direct_to_otf_duplicate_peptide_handle_mode.setItemText(4, _translate("metaX_main", "first")) - self.checkBox_pep_direct_to_otf_use_unit_specific_annotate.setText(_translate("metaX_main", "Use unit-specific annotation")) + self.checkBox_pep_direct_to_otf_use_unit_specific_annotate.setText(_translate("metaX_main", "Manifest-driven annotation")) self.pushButton_pep_direct_to_otf_open_metaumbra_gui.setStatusTip(_translate("metaX_main", "Open MetaUmbra GUI for more functions")) self.pushButton_pep_direct_to_otf_open_metaumbra_gui.setText(_translate("metaX_main", "Open MetaUmbra GUI")) self.label_222.setText(_translate("metaX_main", "Digested Genome Folder")) @@ -7296,17 +7298,17 @@ def retranslateUi(self, metaX_main): self.lineEdit_pep_direct_to_otf_peptide_path.setStatusTip(_translate("metaX_main", "Peptite table with intensity from search result(e.g. report.pr_matrix)")) self.pushButton_run_pep_direct_to_otf.setText(_translate("metaX_main", "GO")) self.pushButton_open_pep_direct_to_otf_pro2taxafunc_db_path.setText(_translate("metaX_main", "Open")) - self.label_unit_specific_manifest.setText(_translate("metaX_main", "MetaUmbra unit-specific manifest")) + self.label_unit_specific_manifest.setText(_translate("metaX_main", "MetaUmbra genome selection manifest")) self.lineEdit_pep_direct_to_otf_digestied_genome_pep_path.setStatusTip(_translate("metaX_main", "The folder contains all the digested genome files, with each file containing protein-peptide information.")) self.checkBox_8.setText(_translate("metaX_main", "Show Advanced Settings")) self.lineEdit_pep_direct_to_otf_pro2taxafunc_db_path.setStatusTip(_translate("metaX_main", "The databse file from Database builder of MetaX")) self.label_220.setText(_translate("metaX_main", "Peptide Table")) self.pushButton_open_pep_direct_to_otf_unit_specific_manifest_path.setText(_translate("metaX_main", "Open")) - self.pushButton_pep_direct_to_otf_unit_specific_settings.setText(_translate("metaX_main", "Unit-specific Settings...")) + self.pushButton_pep_direct_to_otf_unit_specific_settings.setText(_translate("metaX_main", "Validate / Settings...")) self.label_pep_direct_to_otf_use_unit_specific_genome_threshold.setText(_translate("metaX_main", "Genome threshold")) self.comboBox_pep_direct_to_otf_use_unit_specific_genome_threshold.setItemText(0, _translate("metaX_main", "q0.05")) self.comboBox_pep_direct_to_otf_use_unit_specific_genome_threshold.setItemText(1, _translate("metaX_main", "q0.01")) - self.tabWidget_6.setTabText(self.tabWidget_6.indexOf(self.tab_6), _translate("metaX_main", "Pepetide Direct to OTFs (MetaUmbra)")) + self.tabWidget_6.setTabText(self.tabWidget_6.indexOf(self.tab_6), _translate("metaX_main", "Peptide Direct to OTFs")) self.toolButton__final_peptide_help.setText(_translate("metaX_main", "?")) self.label_6.setText(_translate("metaX_main", "Peptide Table")) self.label_8.setText(_translate("metaX_main", "LCA Threshold")) diff --git a/metax/gui/unit_specific_settings_dialog.py b/metax/gui/unit_specific_settings_dialog.py index e26fbc7a..121b55c4 100644 --- a/metax/gui/unit_specific_settings_dialog.py +++ b/metax/gui/unit_specific_settings_dialog.py @@ -15,14 +15,14 @@ is_parquet_path, select_diann_intensity_column, ) -from metax.peptide_annotator.unit_specific_manifest import ( - load_unit_specific_manifest, +from metax.peptide_annotator.genome_selection_manifest import ( + load_genome_selection_manifest, resolve_manifest_sample_columns, ) @dataclass -class UnitSpecificGuiConfig: +class ManifestGuiConfig: manifest_path: str = "" genome_threshold: str = "auto" input_sample_col_prefix: str = "" @@ -45,7 +45,7 @@ class UnitSpecificManifestValidationResult: class UnitSpecificValidationResultDialog(QtWidgets.QDialog): def __init__(self, result: UnitSpecificManifestValidationResult, parent=None): super().__init__(parent) - self.setWindowTitle("Unit-specific Manifest Validation") + self.setWindowTitle("MetaUmbra Manifest Validation") self.setModal(True) layout = QtWidgets.QVBoxLayout(self) @@ -111,7 +111,7 @@ def _all_manifest_samples(manifest) -> list[str]: return samples -def validate_unit_specific_manifest_for_gui( +def validate_genome_selection_manifest_for_gui( manifest_path: str, peptide_table_path: str = "", peptide_col: str = "Sequence", @@ -120,6 +120,7 @@ def validate_unit_specific_manifest_for_gui( input_sample_col_prefix: str | None = None, on_missing_sample: str = "error", diann_intensity_col: str | None = None, + digested_genome_folders: str | list[str] | None = None, ) -> UnitSpecificManifestValidationResult: manifest_path = str(manifest_path or "").strip() if on_missing_sample not in {"error", "warn-skip"}: @@ -133,7 +134,7 @@ def validate_unit_specific_manifest_for_gui( if not manifest_path: return UnitSpecificManifestValidationResult( False, - "Please select a unit-specific manifest JSON in the main Peptide Direct to OTF window.", + "Please select a MetaUmbra genome selection manifest in the main Peptide Direct to OTF window.", [], {}, [], @@ -143,7 +144,7 @@ def validate_unit_specific_manifest_for_gui( threshold_arg = None if (genome_threshold or "auto") == "auto" else genome_threshold try: - manifest = load_unit_specific_manifest(manifest_path, genome_threshold=threshold_arg, strict=True) + manifest = load_genome_selection_manifest(manifest_path, genome_threshold=threshold_arg, strict=True) except Exception as exc: return UnitSpecificManifestValidationResult(False, f"Manifest validation failed:\n{exc}", [], {}, []) @@ -284,22 +285,50 @@ def validate_unit_specific_manifest_for_gui( f"({DIANN_RUN_COLUMN}-derived sample columns validated; " f"intensity={intensity_column})" ) + digest_status = "not checked" + if digested_genome_folders: + folders = [digested_genome_folders] if isinstance(digested_genome_folders, str) else list(digested_genome_folders) + folder_paths = [Path(path) for path in folders] + missing_folders = [str(path) for path in folder_paths if not path.is_dir()] + if missing_folders: + return UnitSpecificManifestValidationResult( + False, "Genome digest directories not found:\n" + "\n".join(missing_folders), + manifest_samples, mapped_samples, missing_samples, + ) + selected_genomes = {genome for unit in manifest.units.values() for genome in unit.genome_ids} + missing_genomes = sorted( + genome for genome in selected_genomes + if not any((folder / f"{genome}.tsv").is_file() for folder in folder_paths) + ) + if missing_genomes: + return UnitSpecificManifestValidationResult( + False, + f"Digest files are missing for {len(missing_genomes)} selected genomes:\n" + + "\n".join(missing_genomes[:20]), + manifest_samples, mapped_samples, missing_samples, + ) + digest_status = f"valid ({len(selected_genomes)} selected genome files)" + provenance_warning = "\n".join(f"Manifest warning: {item}" for item in manifest.warnings) message = ( "Manifest schema: valid\n" + f"Manifest mode: {manifest.unit_definition.get('mode')}\n" + f"MetaUmbra version: {manifest.generated_by.get('version', 'unknown')}\n" f"Selected genome threshold: {manifest.selected_genome_threshold}\n" f"Units: {len(manifest.units)}\n" f"Manifest samples: {len(manifest_samples)}\n" + f"Genome digest source: {digest_status}\n" + ("\n".join(header_status_lines) + "\n" if header_status_lines else "") + f"Mapped peptide table samples: {mapped_text}\n" f"Missing samples: {missing_text}\n" "Per-unit summary:\n" + "\n".join(unit_lines) + + ("\n" + provenance_warning if provenance_warning else "") ) ok = not missing_samples or on_missing_sample == "warn-skip" return UnitSpecificManifestValidationResult(ok, message, manifest_samples, mapped_samples, missing_samples) -class UnitSpecificSettingsDialog(QtWidgets.QDialog): +class ManifestSettingsDialog(QtWidgets.QDialog): def __init__( self, parent=None, @@ -308,17 +337,19 @@ def __init__( peptide_table_separator: str = "\t", input_intensity_prefix: str | None = None, diann_intensity_col: str | None = None, - current_config: UnitSpecificGuiConfig | None = None, + digested_genome_folders: str | list[str] | None = None, + current_config: ManifestGuiConfig | None = None, ): super().__init__(parent) - self.setWindowTitle("Unit-specific Settings") + self.setWindowTitle("Manifest Annotation Settings") self.resize(720, 360) self.peptide_table_path = peptide_table_path self.peptide_col = peptide_col self.peptide_table_separator = peptide_table_separator self.input_intensity_prefix = input_intensity_prefix self.diann_intensity_col = diann_intensity_col - self._config = current_config or UnitSpecificGuiConfig() + self.digested_genome_folders = digested_genome_folders + self._config = current_config or ManifestGuiConfig() self._build_ui() self._load_config(self._config) @@ -333,7 +364,7 @@ def _build_ui(self) -> None: self.lineEdit_current_manifest_path = QtWidgets.QLineEdit(manifest_tab) self.lineEdit_current_manifest_path.setReadOnly(True) - form.addRow("Current manifest JSON", self.lineEdit_current_manifest_path) + form.addRow("MetaUmbra genome selection manifest", self.lineEdit_current_manifest_path) self.lineEdit_input_prefix = QtWidgets.QLineEdit(manifest_tab) form.addRow("Input sample column prefix", self.lineEdit_input_prefix) @@ -374,7 +405,7 @@ def _build_ui(self) -> None: buttons.rejected.connect(self.reject) layout.addWidget(buttons) - def _load_config(self, config: UnitSpecificGuiConfig) -> None: + def _load_config(self, config: ManifestGuiConfig) -> None: self.lineEdit_current_manifest_path.setText(config.manifest_path) self.lineEdit_input_prefix.setText(config.input_sample_col_prefix or "") self.comboBox_on_missing_sample.setCurrentText(config.on_missing_sample or "error") @@ -383,7 +414,7 @@ def _load_config(self, config: UnitSpecificGuiConfig) -> None: self.spinBox_n_jobs.setValue(0 if config.n_jobs is None else max(1, int(config.n_jobs))) def _validate_manifest(self) -> bool: - result = validate_unit_specific_manifest_for_gui( + result = validate_genome_selection_manifest_for_gui( manifest_path=self.lineEdit_current_manifest_path.text().strip(), peptide_table_path=self.peptide_table_path, peptide_col=self.peptide_col, @@ -392,6 +423,7 @@ def _validate_manifest(self) -> bool: input_sample_col_prefix=self.lineEdit_input_prefix.text().strip() or None, on_missing_sample=self.comboBox_on_missing_sample.currentText().strip(), diann_intensity_col=self.diann_intensity_col, + digested_genome_folders=self.digested_genome_folders, ) UnitSpecificValidationResultDialog(result, self).exec_() return result.ok @@ -402,9 +434,9 @@ def accept(self) -> None: self._config = self.get_config() super().accept() - def get_config(self) -> UnitSpecificGuiConfig: + def get_config(self) -> ManifestGuiConfig: n_jobs_value = self.spinBox_n_jobs.value() - return UnitSpecificGuiConfig( + return ManifestGuiConfig( manifest_path=self.lineEdit_current_manifest_path.text().strip(), genome_threshold=self._config.genome_threshold, input_sample_col_prefix=self.lineEdit_input_prefix.text(), diff --git a/metax/peptide_annotator/annotation_workflow.py b/metax/peptide_annotator/annotation_workflow.py index bcb73bde..eb359d88 100644 --- a/metax/peptide_annotator/annotation_workflow.py +++ b/metax/peptide_annotator/annotation_workflow.py @@ -92,6 +92,19 @@ def read_genome_list_file( return parse_genome_text("\n".join(dataframe["genome_id"].dropna().astype(str))) +def read_plain_genome_list_file(file_path: str | Path) -> list[str]: + """Read a user genome list without interpreting it as MetaUmbra output.""" + file_path = Path(file_path) + text = file_path.read_text(encoding="utf-8-sig") + first_line = next((line for line in text.splitlines() if line.strip()), "") + delimiter = "\t" if "\t" in first_line else "," + columns = [column.strip() for column in first_line.split(delimiter)] + if "genome_id" not in columns: + return parse_genome_text(text) + dataframe = pd.read_csv(file_path, sep=delimiter) + return parse_genome_text("\n".join(dataframe["genome_id"].dropna().astype(str))) + + def _parse_version(version_text: str) -> tuple[int, int, int] | None: match = re.search(r"(\d+)\.(\d+)\.(\d+)", version_text) if not match: @@ -558,10 +571,7 @@ def _resolve_selected_genomes(self) -> list[str]: genomes = list(self.selected_genomes) if self.genome_list_path is not None: genomes.extend( - read_genome_list_file( - self.genome_list_path, - qvalue_cutoff=self.metaumbra_genome_qvalue_cutoff, - ) + read_plain_genome_list_file(self.genome_list_path) ) return _normalise_genomes(genomes) diff --git a/metax/peptide_annotator/genome_selection_manifest.py b/metax/peptide_annotator/genome_selection_manifest.py new file mode 100644 index 00000000..0f38c3c3 --- /dev/null +++ b/metax/peptide_annotator/genome_selection_manifest.py @@ -0,0 +1,19 @@ +"""Canonical MetaUmbra genome-selection manifest API.""" + +from .unit_specific_manifest import ( + SCHEMA_VERSION, + GenomeSelectionManifest, + GenomeSelectionUnitSpec, + load_genome_selection_manifest, + resolve_manifest_sample_columns, + write_unit_sample_column_mapping, +) + +__all__ = [ + "SCHEMA_VERSION", + "GenomeSelectionManifest", + "GenomeSelectionUnitSpec", + "load_genome_selection_manifest", + "resolve_manifest_sample_columns", + "write_unit_sample_column_mapping", +] diff --git a/metax/peptide_annotator/manifest_otf.py b/metax/peptide_annotator/manifest_otf.py new file mode 100644 index 00000000..b09a2a5a --- /dev/null +++ b/metax/peptide_annotator/manifest_otf.py @@ -0,0 +1,13 @@ +"""Canonical manifest-driven OTF annotation API.""" + +from .unit_specific_otf import ( + ManifestOTFAnnotator, + ManifestOTFRunResult, + build_manifest_peptide_protein_map, +) + +__all__ = [ + "ManifestOTFAnnotator", + "ManifestOTFRunResult", + "build_manifest_peptide_protein_map", +] diff --git a/metax/peptide_annotator/pep_table_to_otf.py b/metax/peptide_annotator/pep_table_to_otf.py index 6ffe45c7..996b927f 100644 --- a/metax/peptide_annotator/pep_table_to_otf.py +++ b/metax/peptide_annotator/pep_table_to_otf.py @@ -34,6 +34,13 @@ _WINDOWS_PROCESS_POOL_MAX_WORKERS = 61 +def _print_console_safe(line: str) -> None: + """Forward UTF-8 subprocess progress even when Windows uses a legacy code page.""" + encoding = getattr(sys.stdout, "encoding", None) or "utf-8" + safe_line = str(line).encode(encoding, errors="replace").decode(encoding, errors="replace") + print(safe_line, flush=True) + + def _resolve_digested_scan_n_jobs( n_jobs: int | None, *, @@ -560,7 +567,7 @@ def _query_peptide_proteins_nested_via_subprocess( repo_root = pathlib.Path(__file__).resolve().parents[2] env = os.environ.copy() env["PYTHONPATH"] = str(repo_root) + os.pathsep + env.get("PYTHONPATH", "") - env.setdefault("PYTHONIOENCODING", "utf-8") + env["PYTHONIOENCODING"] = "utf-8" creationflags = getattr(subprocess, "CREATE_NO_WINDOW", 0) print("[UnitSpecificDigestedScan/Subprocess] Launching isolated process.", flush=True) return_code, last_lines = run_streaming_subprocess( @@ -569,7 +576,7 @@ def _query_peptide_proteins_nested_via_subprocess( env=env, creationflags=creationflags, max_captured_lines=50, - emit_line=lambda line: print(line, flush=True), + emit_line=_print_console_safe, ) if return_code != 0: tail = "".join(last_lines[-20:]) @@ -1407,7 +1414,7 @@ def _query_peptide_proteins_via_subprocess( env=env, creationflags=creationflags, max_captured_lines=50, - emit_line=print, + emit_line=_print_console_safe, ) if rc != 0: tail = "".join(last_lines[-20:]) diff --git a/metax/peptide_annotator/unit_specific_manifest.py b/metax/peptide_annotator/unit_specific_manifest.py index 4481bfc1..65e1d5ce 100644 --- a/metax/peptide_annotator/unit_specific_manifest.py +++ b/metax/peptide_annotator/unit_specific_manifest.py @@ -9,28 +9,32 @@ import pandas as pd -SCHEMA_VERSION = "metaumbra.unit_specific_manifest.v1" -SUPPORTED_SCHEMA_VERSIONS = { - SCHEMA_VERSION, - "metaumbra.unit_aware_manifest.v1", -} +SCHEMA_VERSION = "metaumbra.genome_selection_manifest.v1" @dataclass -class UnitSpecificUnitSpec: +class GenomeSelectionUnitSpec: analysis_unit_id: str - sample_columns: list[str] + sample_ids: list[str] genome_ids: list[str] n_samples: int + @property + def sample_columns(self) -> list[str]: + """Prepared-table mapping uses the manifest's canonical sample IDs.""" + return self.sample_ids + @dataclass -class UnitSpecificManifest: +class GenomeSelectionManifest: schema_version: str generated_by: dict - default_genome_threshold: str - files: dict - units: dict[str, UnitSpecificUnitSpec] + unit_definition: dict + selection: dict + inputs: dict + artifacts: dict + warnings: list[str] + units: dict[str, GenomeSelectionUnitSpec] selected_genome_threshold: str @@ -58,40 +62,51 @@ def _warn_or_raise(message: str, strict: bool) -> None: warnings.warn(message, stacklevel=2) -def load_unit_specific_manifest( +def load_genome_selection_manifest( manifest_path: str | Path, genome_threshold: str | None = None, strict: bool = True, -) -> UnitSpecificManifest: +) -> GenomeSelectionManifest: manifest_path = Path(manifest_path) data = json.loads(manifest_path.read_text(encoding="utf-8")) schema_version = data.get("schema_version") - if schema_version not in SUPPORTED_SCHEMA_VERSIONS: - raise ValueError(f"Unsupported unit-specific manifest schema_version: {schema_version!r}") - - default_threshold = str(data.get("default_genome_threshold", "")).strip() + if schema_version != SCHEMA_VERSION: + raise ValueError(f"Unsupported genome selection manifest schema_version: {schema_version!r}") + + unit_definition = data.get("unit_definition") + if not isinstance(unit_definition, dict): + raise ValueError("genome selection manifest must contain unit_definition") + selection = data.get("selection") + if not isinstance(selection, dict): + raise ValueError("genome selection manifest must contain selection") + default_threshold = str(selection.get("default_genome_threshold", "")).strip() + available_thresholds = selection.get("available_genome_thresholds") + if available_thresholds != ["q0.05", "q0.01"]: + raise ValueError("selection.available_genome_thresholds must be ['q0.05', 'q0.01']") selected_threshold, selected_genome_key = _normalize_threshold_alias(genome_threshold, default_threshold) raw_units = data.get("units") if not isinstance(raw_units, dict) or not raw_units: - raise ValueError("unit-specific manifest must contain at least one unit") + raise ValueError("genome selection manifest must contain at least one unit") + if int(unit_definition.get("n_units", -1)) != len(raw_units): + raise ValueError("unit_definition.n_units does not match units") seen_samples: dict[str, str] = {} - units: dict[str, UnitSpecificUnitSpec] = {} + units: dict[str, GenomeSelectionUnitSpec] = {} for analysis_unit_id, raw_unit in raw_units.items(): if not isinstance(raw_unit, dict): raise ValueError(f"Unit {analysis_unit_id!r} must be an object") - sample_columns = [str(sample) for sample in raw_unit.get("sample_columns", [])] + sample_columns = [str(sample) for sample in raw_unit.get("sample_ids", [])] if not sample_columns: - raise ValueError(f"Unit {analysis_unit_id!r} has no sample_columns") + raise ValueError(f"Unit {analysis_unit_id!r} has no sample_ids") for sample in sample_columns: previous_unit = seen_samples.get(sample) if previous_unit is not None and previous_unit != analysis_unit_id: raise ValueError( - f"Sample column {sample!r} is assigned to multiple units: " + f"Sample {sample!r} is assigned to multiple units: " f"{previous_unit!r} and {analysis_unit_id!r}" ) seen_samples[sample] = str(analysis_unit_id) @@ -99,6 +114,10 @@ def load_unit_specific_manifest( if selected_genome_key not in raw_unit: raise ValueError(f"Unit {analysis_unit_id!r} is missing {selected_genome_key}") genome_ids = [str(genome) for genome in raw_unit.get(selected_genome_key, [])] + if not genome_ids: + raise ValueError( + f"Unit {analysis_unit_id!r} has no genomes at selected threshold {selected_threshold}" + ) n_samples_value = raw_unit.get("n_samples", len(sample_columns)) try: @@ -124,18 +143,24 @@ def load_unit_specific_manifest( strict=strict, ) - units[str(analysis_unit_id)] = UnitSpecificUnitSpec( + units[str(analysis_unit_id)] = GenomeSelectionUnitSpec( analysis_unit_id=str(analysis_unit_id), - sample_columns=sample_columns, + sample_ids=sample_columns, genome_ids=genome_ids, n_samples=n_samples, ) - return UnitSpecificManifest( + generated_by = data.get("generated_by") + if not isinstance(generated_by, dict) or generated_by.get("software") != "MetaUmbra": + raise ValueError("generated_by.software must be 'MetaUmbra'") + return GenomeSelectionManifest( schema_version=schema_version, - generated_by=dict(data.get("generated_by") or {}), - default_genome_threshold=default_threshold, - files=dict(data.get("files") or {}), + generated_by=dict(generated_by), + unit_definition=dict(unit_definition), + selection=dict(selection), + inputs=dict(data.get("inputs") or {}), + artifacts=dict(data.get("artifacts") or {}), + warnings=[str(item) for item in data.get("warnings", [])], units=units, selected_genome_threshold=selected_threshold, ) @@ -260,7 +285,7 @@ def resolve_manifest_sample_columns( def write_unit_sample_column_mapping( - manifest: UnitSpecificManifest, + manifest: GenomeSelectionManifest, sample_column_mapping: dict[str, str], output_path: str | Path, output_sample_col_prefix: str = "Intensity_", diff --git a/metax/peptide_annotator/unit_specific_otf.py b/metax/peptide_annotator/unit_specific_otf.py index 18ac8500..78650fab 100644 --- a/metax/peptide_annotator/unit_specific_otf.py +++ b/metax/peptide_annotator/unit_specific_otf.py @@ -25,9 +25,9 @@ read_parquet_columns, ) from metax.peptide_annotator.peptable_annotator import _prepare_otf_for_output -from metax.peptide_annotator.unit_specific_manifest import ( - UnitSpecificManifest, - load_unit_specific_manifest, +from metax.peptide_annotator.genome_selection_manifest import ( + GenomeSelectionManifest, + load_genome_selection_manifest, resolve_manifest_sample_columns, write_unit_sample_column_mapping, ) @@ -35,7 +35,7 @@ @dataclass(frozen=True) -class UnitSpecificOTFRunResult: +class ManifestOTFRunResult: output_path: str info_path: str summary_path: str @@ -45,6 +45,8 @@ class UnitSpecificOTFRunResult: completed_units: int skipped_units: int selected_genome_threshold: str + manifest_schema_version: str + warnings: list[str] def _safe_filename(value: str) -> str: @@ -68,10 +70,10 @@ def _create_temporary_unit_directory(parent: Path, analysis_unit_id: str) -> Pat return run_dir -def build_global_unit_specific_peptide_protein_map( +def build_manifest_peptide_protein_map( peptide_df: pd.DataFrame, peptide_col: str, - manifest: UnitSpecificManifest, + manifest: GenomeSelectionManifest, digested_genome_folders: str | list[str], protein_genome_separator: str = "_", n_jobs: int | None = None, @@ -104,11 +106,11 @@ def build_global_unit_specific_peptide_protein_map( ) -class UnitSpecificOTFAnnotator: +class ManifestOTFAnnotator: def __init__( self, peptide_table_path: str, - unit_specific_manifest_path: str, + metaumbra_manifest_path: str, taxafunc_anno_db_path: str, output_path: str, db_path: str | None = None, @@ -125,7 +127,6 @@ def __init__( protein_separator: str = ";", protein_genome_separator: str = "_", save_per_unit_outputs: bool = False, - include_unit_specific_sequence: bool = False, duplicate_peptide_handling_mode: str = "sum", on_missing_sample: str = "error", on_empty_unit: str = "warn-skip", @@ -159,7 +160,7 @@ def __init__( ) self.peptide_table_path = Path(peptide_table_path) - self.unit_specific_manifest_path = Path(unit_specific_manifest_path) + self.metaumbra_manifest_path = Path(metaumbra_manifest_path) self.taxafunc_anno_db_path = Path(taxafunc_anno_db_path) self.requested_output_path = Path(output_path) self.output_path = available_output_path(self.requested_output_path) @@ -182,7 +183,6 @@ def __init__( self.protein_separator = protein_separator self.protein_genome_separator = protein_genome_separator self.save_per_unit_outputs = save_per_unit_outputs - self.include_unit_specific_sequence = include_unit_specific_sequence self.duplicate_peptide_handling_mode = duplicate_peptide_handling_mode self.on_missing_sample = on_missing_sample self.on_empty_unit = on_empty_unit @@ -202,7 +202,7 @@ def __init__( for path, label in [ (self.peptide_table_path, "peptide_table_path"), - (self.unit_specific_manifest_path, "unit_specific_manifest_path"), + (self.metaumbra_manifest_path, "metaumbra_manifest_path"), (self.taxafunc_anno_db_path, "taxafunc_anno_db_path"), ]: if not path.is_file(): @@ -235,7 +235,7 @@ def artifacts_dir(self) -> Path: def temporary_unit_otf_dir(self) -> Path: return self.artifacts_dir / "per_unit" / "unit_otf" - def _all_manifest_samples(self, manifest: UnitSpecificManifest) -> list[str]: + def _all_manifest_samples(self, manifest: GenomeSelectionManifest) -> list[str]: samples: list[str] = [] for unit in manifest.units.values(): for sample in unit.sample_columns: @@ -313,9 +313,9 @@ def unit_candidates(peptide: object) -> tuple[str, str]: return result.loc[result["Proteins"].ne("")].copy() def _write_manifest_used(self) -> None: - target = self.artifacts_dir / "unit_specific_manifest_used.json" + target = self.artifacts_dir / "genome_selection_manifest_used.json" target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(self.unit_specific_manifest_path.read_text(encoding="utf-8"), encoding="utf-8") + target.write_text(self.metaumbra_manifest_path.read_text(encoding="utf-8"), encoding="utf-8") def _resolve_sample_mapping( self, @@ -629,7 +629,7 @@ def info_path(self) -> Path: def _write_merged_info( self, *, - manifest: UnitSpecificManifest, + manifest: GenomeSelectionManifest, manifest_samples: list[str], sample_mapping: dict[str, str], summary_df: pd.DataFrame, @@ -650,7 +650,7 @@ def _write_merged_info( with self.info_path.open("w", encoding="utf-8") as handle: handle.write("MetaX PeptideAnnotator Results\n") handle.write("=" * 50 + "\n") - handle.write(f"Software: MetaX (UnitSpecificOTFAnnotator) v{__version__}\n") + handle.write(f"Software: MetaX (ManifestOTFAnnotator) v{__version__}\n") handle.write(f"Run time: {started_at.strftime('%Y-%m-%d %H:%M:%S')}\n") handle.write("-" * 50 + "\n") handle.write("Parameters:\n") @@ -665,14 +665,13 @@ def _write_merged_info( handle.write(f" - Duplicate handling mode: {self.duplicate_peptide_handling_mode}\n") handle.write("-" * 50 + "\n") handle.write("Unit-specific configuration:\n") - handle.write(f" - Manifest: {self.unit_specific_manifest_path}\n") + handle.write(f" - Manifest: {self.metaumbra_manifest_path}\n") handle.write(f" - Selected genome threshold: {manifest.selected_genome_threshold}\n") handle.write(f" - Units: {len(manifest.units)}\n") handle.write(f" - Manifest samples: {len(manifest_samples)}\n") handle.write(f" - Mapped samples: {len(sample_mapping)}\n") handle.write(f" - On missing sample: {self.on_missing_sample}\n") handle.write(f" - On empty unit: {self.on_empty_unit}\n") - handle.write(f" - Include UnitSpecificSequence: {self.include_unit_specific_sequence}\n") handle.write("-" * 50 + "\n") handle.write("Input/Output:\n") handle.write(f" - Input: {self.peptide_table_path}\n") @@ -712,7 +711,7 @@ def _write_merged_info( def run( self, return_dataframe: bool = False, - ) -> UnitSpecificOTFRunResult | pd.DataFrame: + ) -> ManifestOTFRunResult | pd.DataFrame: started_at = datetime.now() if self.output_path != self.requested_output_path: print( @@ -720,8 +719,8 @@ def run( f"{self.output_path}", flush=True, ) - manifest = load_unit_specific_manifest( - self.unit_specific_manifest_path, + manifest = load_genome_selection_manifest( + self.metaumbra_manifest_path, genome_threshold=self.genome_threshold, strict=True, ) @@ -791,7 +790,7 @@ def run( "intensity in mapped manifest samples.", flush=True, ) - global_mapping = build_global_unit_specific_peptide_protein_map( + global_mapping = build_manifest_peptide_protein_map( peptide_df=peptide_df.loc[ globally_nonzero_mask, [self.peptide_col], @@ -958,8 +957,8 @@ def run( genome_list=unit.genome_ids, duplicate_peptide_handling_mode=self.duplicate_peptide_handling_mode, genome_selection_metadata={ - "genome_selection_method": "metaumbra_unit_specific_manifest", - "unit_specific_manifest_path": str(self.unit_specific_manifest_path), + "genome_selection_method": "metaumbra_genome_selection_manifest", + "metaumbra_manifest_path": str(self.metaumbra_manifest_path), "analysis_unit_id": unit.analysis_unit_id, "selected_genome_threshold": manifest.selected_genome_threshold, **self.peptide_table_prepare_metadata, @@ -969,9 +968,6 @@ def run( ) unit_otf_df.insert(0, "analysis_unit_id", unit.analysis_unit_id) - if self.include_unit_specific_sequence: - sequence_values = unit_otf_df["Sequence"].astype(str) - unit_otf_df.insert(1, "UnitSpecificSequence", unit.analysis_unit_id + "||" + sequence_values) final_unit_output_path = self._unit_output_path( unit.analysis_unit_id, tmpdir, @@ -1079,7 +1075,7 @@ def run( ) manifest_summary = { - "unit_specific_manifest_path": str(self.unit_specific_manifest_path), + "metaumbra_manifest_path": str(self.metaumbra_manifest_path), "selected_genome_threshold": manifest.selected_genome_threshold, "n_units": len(manifest.units), "n_manifest_samples": len(manifest_samples), @@ -1093,7 +1089,7 @@ def run( ) if return_dataframe: return pd.read_csv(self.output_path, sep="\t") - return UnitSpecificOTFRunResult( + return ManifestOTFRunResult( output_path=str(self.output_path), info_path=str(self.info_path), summary_path=str(summary_path), @@ -1103,4 +1099,6 @@ def run( completed_units=completed_units, skipped_units=skipped_units, selected_genome_threshold=manifest.selected_genome_threshold, + manifest_schema_version=manifest.schema_version, + warnings=list(manifest.warnings), ) diff --git a/metax/workflow_recorder/__init__.py b/metax/workflow_recorder/__init__.py index 8f092cd6..227a9831 100644 --- a/metax/workflow_recorder/__init__.py +++ b/metax/workflow_recorder/__init__.py @@ -5,13 +5,14 @@ WorkflowRecorder, auto_otf_report_step, deseq2_step, + direct_otf_step, gui_action_step, limma_step, method_call_step, register_current_python_kernel, set_multi_tables_step, taxafunc_analyzer_step, - unit_specific_otf_step, + manifest_otf_step, ) __all__ = [ @@ -21,11 +22,12 @@ "WorkflowRecorder", "auto_otf_report_step", "deseq2_step", + "direct_otf_step", "gui_action_step", "limma_step", "method_call_step", "register_current_python_kernel", "set_multi_tables_step", "taxafunc_analyzer_step", - "unit_specific_otf_step", + "manifest_otf_step", ] diff --git a/metax/workflow_recorder/recorder.py b/metax/workflow_recorder/recorder.py index 2ef83498..96813c3b 100644 --- a/metax/workflow_recorder/recorder.py +++ b/metax/workflow_recorder/recorder.py @@ -333,29 +333,56 @@ def auto_otf_report_step(config_path: str | Path, result: Any | None = None) -> ) -def unit_specific_otf_step(params: dict[str, Any]) -> AnalysisStep: +def manifest_otf_step(params: dict[str, Any]) -> AnalysisStep: parameters = dict(params) code = _join_code( - "from metax.peptide_annotator.unit_specific_otf import UnitSpecificOTFAnnotator", + "from metax.peptide_annotator.manifest_otf import ManifestOTFAnnotator", "", - _assignment("unit_specific_otf_params", parameters), - "unit_specific_otf = UnitSpecificOTFAnnotator(**unit_specific_otf_params)", - "unit_specific_otf_result = unit_specific_otf.run()", - 'print(f"Saved unit-specific OTF: {unit_specific_otf.output_path}")', + _assignment("manifest_otf_params", parameters), + "manifest_otf = ManifestOTFAnnotator(**manifest_otf_params)", + "manifest_otf_result = manifest_otf.run()", + 'print(f"Saved manifest-driven OTF: {manifest_otf.output_path}")', ) return AnalysisStep( - title="Run Unit-specific Peptide Direct to OTF", - step_type="unit_specific_peptide_direct_to_otf", + title="Run Manifest Peptide Direct to OTF", + step_type="manifest_peptide_direct_to_otf", inputs={ "peptide_table_path": parameters.get("peptide_table_path"), "digested_genome_folders": parameters.get("digested_genome_folders"), "taxafunc_anno_db_path": parameters.get("taxafunc_anno_db_path"), - "unit_specific_manifest_path": parameters.get("unit_specific_manifest_path"), + "metaumbra_manifest_path": parameters.get("metaumbra_manifest_path"), }, outputs={"output_path": parameters.get("output_path")}, parameters=parameters, code=code, - notes=["Recorded from the MetaX GUI unit-specific direct-to-OTF workflow."], + notes=["Recorded from the MetaX GUI manifest-driven direct-to-OTF workflow."], + ) + + +def direct_otf_step(params: dict[str, Any]) -> AnalysisStep: + """Record explicit MetaX automatic or custom-list annotation.""" + parameters = dict(params) + code = _join_code( + "from metax.peptide_annotator.annotation_workflow import GlobalOTFAnnotator", + "", + _assignment("direct_otf_params", parameters), + "direct_otf = GlobalOTFAnnotator(**direct_otf_params)", + "direct_otf_result = direct_otf.run()", + 'print(f"Saved direct OTF: {direct_otf_result.output_path}")', + ) + return AnalysisStep( + title="Run MetaX Direct Peptide to OTF", + step_type="metax_direct_peptide_to_otf", + inputs={ + "peptide_table_path": parameters.get("peptide_table_path"), + "digested_genome_folders": parameters.get("digested_genome_folders"), + "taxafunc_anno_db_path": parameters.get("taxafunc_anno_db_path"), + "genome_list_path": parameters.get("genome_list_path"), + }, + outputs={"output_path": parameters.get("output_path")}, + parameters=parameters, + code=code, + notes=["Uses an explicit MetaX automatic or custom genome-list source."], ) diff --git a/tests/fixtures/genome_selection_manifest.v1.json b/tests/fixtures/genome_selection_manifest.v1.json new file mode 100644 index 00000000..774c180c --- /dev/null +++ b/tests/fixtures/genome_selection_manifest.v1.json @@ -0,0 +1,46 @@ +{ + "schema_version": "metaumbra.genome_selection_manifest.v1", + "generated_by": { + "software": "MetaUmbra", + "version": "1.4.0", + "run_id": "contract-fixture", + "generated_at": "2026-07-15T00:00:00+00:00" + }, + "unit_definition": { + "mode": "metadata", + "sample_id_column": "Run", + "analysis_unit_column": "analysis_unit_id", + "n_units": 2 + }, + "selection": { + "default_genome_threshold": "q0.05", + "available_genome_thresholds": ["q0.05", "q0.01"], + "scoring_method": "per-analysis-unit/empirical-background" + }, + "inputs": { + "peptide_table": {"path": "report.parquet", "format": "diann_parquet"}, + "metadata_table": {"path": "metadata.tsv", "format": "tsv"}, + "genome_digest_directories": ["digests"] + }, + "units": { + "u1": { + "sample_ids": ["s1"], + "n_samples": 1, + "genome_ids_q005": ["g1", "g2"], + "genome_ids_q001": ["g1"] + }, + "u2": { + "sample_ids": ["s2"], + "n_samples": 1, + "genome_ids_q005": ["g2"], + "genome_ids_q001": ["g2"] + } + }, + "artifacts": { + "unit_genome_results": "unit_genome_results.tsv", + "sample_unit_mapping": "sample_unit_mapping.tsv", + "cohort_summary": "cohort_genome_summary.tsv", + "diagnostics_directory": "artifacts/diagnostics" + }, + "warnings": [] +} diff --git a/tests/test_annotation_cli.py b/tests/test_annotation_cli.py deleted file mode 100644 index fa4606f6..00000000 --- a/tests/test_annotation_cli.py +++ /dev/null @@ -1,988 +0,0 @@ -import json -import ast -import shutil -import sqlite3 -import subprocess -import sys -import tomllib -from pathlib import Path -from types import SimpleNamespace - -import pandas as pd -import pytest -from packaging.requirements import Requirement - -import metax.cli.annotate as annotate_cli -import metax.cli.report as report_launcher -import metax.peptide_annotator.annotation_workflow as annotation_workflow -from metax.cli.annotate import ( - ANNOTATION_RESULT_SCHEMA_VERSION, - ExitCode, - build_parser, - load_config_file, - main, -) -from metax.peptide_annotator.annotation_workflow import ( - ANNOTATION_WORKFLOW_API_VERSION, - AnnotationConfigurationError, - GlobalOTFAnnotator, -) -from metax.gui import launcher as gui_launcher - - -def _write_peptide_db(path: Path) -> None: - with sqlite3.connect(path) as connection: - connection.execute( - "CREATE TABLE peptide_proteins (peptide TEXT PRIMARY KEY, proteins TEXT)" - ) - connection.executemany( - "INSERT INTO peptide_proteins VALUES (?, ?)", - [ - ("PEPA", json.dumps(["g1_p1", "g2_p2"])), - ("PEPB", json.dumps(["g2_p3"])), - ], - ) - - -def _write_taxafunc_db(path: Path) -> None: - with sqlite3.connect(path) as connection: - connection.execute("CREATE TABLE id2taxa (ID TEXT PRIMARY KEY, Taxa TEXT)") - connection.execute( - "CREATE TABLE id2annotation (ID TEXT PRIMARY KEY, KEGG_ko TEXT)" - ) - connection.executemany( - "INSERT INTO id2taxa VALUES (?, ?)", - [ - ("g1", "d__Bacteria;p__P1;c__C1;o__O1;f__F1;g__G1;s__S1"), - ("g2", "d__Bacteria;p__P2;c__C2;o__O2;f__F2;g__G2;s__S2"), - ], - ) - connection.executemany( - "INSERT INTO id2annotation VALUES (?, ?)", - [("g1_p1", "K00001"), ("g2_p2", "K00002"), ("g2_p3", "K00003")], - ) - - -@pytest.fixture -def annotation_inputs(tmp_path): - peptide_table = tmp_path / "peptides.tsv" - pd.DataFrame( - { - "Sequence": ["PEPA", "PEPB"], - "Intensity_s1": [10.0, 0.0], - "Intensity_s2": [20.0, 5.0], - } - ).to_csv(peptide_table, sep="\t", index=False) - peptide_db = tmp_path / "peptide.db" - taxafunc_db = tmp_path / "taxafunc.db" - _write_peptide_db(peptide_db) - _write_taxafunc_db(taxafunc_db) - return peptide_table, peptide_db, taxafunc_db - - -def test_global_annotation_cli_writes_result_json(annotation_inputs, tmp_path): - peptide_table, peptide_db, taxafunc_db = annotation_inputs - output = tmp_path / "global_otf.tsv" - result_json = tmp_path / "global_result.json" - - exit_code = main( - [ - "--mode", - "global", - "--peptide-table", - str(peptide_table), - "--peptide-db", - str(peptide_db), - "--taxafunc-db", - str(taxafunc_db), - "--output", - str(output), - "--selection-mode", - "provided", - "--selected-genomes", - "g1", - "--result-json", - str(result_json), - ] - ) - - assert exit_code == ExitCode.SUCCESS - assert output.is_file() - result = json.loads(result_json.read_text(encoding="utf-8")) - assert result["schema_version"] == ANNOTATION_RESULT_SCHEMA_VERSION - assert set(result) == { - "schema_version", - "run", - "inputs", - "parameters", - "stages", - "genome_selection", - "metrics", - "outputs", - "diagnostics", - } - assert result["run"]["status"] == "success" - assert result["run"]["mode"] == "global" - assert result["run"]["software"]["workflow_api_version"] == ANNOTATION_WORKFLOW_API_VERSION - assert result["run"]["duration_seconds"] >= 0 - assert result["parameters"]["selection_mode"] == "provided" - assert result["genome_selection"]["genomes_selected"] == 1 - assert result["metrics"]["mapping"]["peptides_before_mapping"] == 2 - assert result["metrics"]["mapping"]["peptides_after_mapping"] == 1 - assert result["metrics"]["output"]["rows"] == 1 - assert result["outputs"]["otf"]["path"] == str(output) - assert Path(result["outputs"]["annotation_summary"]["path"]).is_file() - assert result["diagnostics"] == {"warnings": [], "error": None} - assert "artifacts" not in result - - -def test_global_annotation_cli_reports_renamed_output_when_target_exists( - annotation_inputs, - tmp_path, - capsys, -): - peptide_table, peptide_db, taxafunc_db = annotation_inputs - requested_output = tmp_path / "global_otf.tsv" - requested_output.write_text("stale output\n", encoding="utf-8") - result_json = tmp_path / "global_result.json" - result_json.write_text("stale result\n", encoding="utf-8") - - exit_code = main( - [ - "--mode", - "global", - "--peptide-table", - str(peptide_table), - "--peptide-db", - str(peptide_db), - "--taxafunc-db", - str(taxafunc_db), - "--output", - str(requested_output), - "--selection-mode", - "provided", - "--selected-genomes", - "g1", - "--result-json", - str(result_json), - ] - ) - - assert exit_code == ExitCode.SUCCESS - assert requested_output.read_text(encoding="utf-8") == "stale output\n" - result = json.loads(result_json.read_text(encoding="utf-8")) - actual_output = Path(result["outputs"]["otf"]["path"]) - actual_info = Path(result["outputs"]["annotation_summary"]["path"]) - assert actual_output != requested_output - assert actual_output.is_file() - assert actual_info == actual_output.with_name(f"{actual_output.stem}_info.txt") - assert actual_info.is_file() - assert f"MetaX annotation completed: {actual_output}" in capsys.readouterr().out - - -def test_global_cli_accepts_long_diann_parquet(annotation_inputs, tmp_path): - _peptide_table, peptide_db, taxafunc_db = annotation_inputs - parquet = tmp_path / "report.parquet" - pd.DataFrame( - { - "Run": ["s1.raw", "s2.raw"], - "Stripped.Sequence": ["PEPA", "PEPA"], - "Precursor.Normalised": [10.0, 20.0], - } - ).to_parquet(parquet, index=False) - output = tmp_path / "diann_otf.tsv" - result_json = tmp_path / "diann_result.json" - - exit_code = main( - [ - "--mode", - "global", - "--peptide-table", - str(parquet), - "--peptide-db", - str(peptide_db), - "--taxafunc-db", - str(taxafunc_db), - "--output", - str(output), - "--selection-mode", - "provided", - "--selected-genomes", - "g1", - "--diann-intensity-col", - "Precursor.Normalised", - "--result-json", - str(result_json), - ] - ) - - assert exit_code == ExitCode.SUCCESS - result = json.loads(result_json.read_text(encoding="utf-8")) - peptide_input = result["inputs"]["peptide_table"] - assert peptide_input["format"] == "diann_parquet" - assert peptide_input["diann"]["intensity_column"] == "Precursor.Normalised" - assert result["metrics"]["input"] == { - "rows": 2, - "columns": 3, - "rows_with_required_values": 2, - "runs": 2, - "unique_peptides": 1, - "aggregated_run_peptide_rows": 2, - "prepared_peptides": 1, - "sample_columns": 2, - } - assert set(result["metrics"]["output"]["samples"]) == { - "Intensity_s1", - "Intensity_s2", - } - output_columns = pd.read_csv(output, sep="\t", nrows=0).columns.tolist() - assert "Intensity_s1" in output_columns - assert "Intensity_s2" in output_columns - - -def test_global_backend_preserves_metaumbra_provenance(monkeypatch, tmp_path): - peptide_table = tmp_path / "peptides.tsv" - pd.DataFrame( - { - "Sequence": ["PEPA"], - "Evidence": [0.9], - "Q.Value": [0.001], - "Intensity_s1": [10.0], - } - ).to_csv(peptide_table, sep="\t", index=False) - digests = tmp_path / "digests" - digests.mkdir() - taxafunc_db = tmp_path / "taxafunc.db" - taxafunc_db.touch() - output = tmp_path / "otf.tsv" - captured = {} - - def fake_scoring(**kwargs): - pd.DataFrame( - {"genome_id": ["g1", "g2"], "qvalue": [0.01, 0.2]} - ).to_csv(kwargs["output_path"], sep="\t", index=False) - return {"output": kwargs["output_path"], "metaumbra_version": "1.3.7"} - - class FakeMapper: - original_peptides_before_mapping = 1 - peptides_after_mapping = 1 - removed_peptides_no_matched = 0 - selected_proteins_num = 1 - selected_genomes_num = 1 - protein_ranked_table = ["g1_p1"] - annotation_run_stats = {} - annotation_output_metrics = { - "rows": 1, - "columns": 3, - "sample_columns": ["Intensity_s1"], - } - - def __init__(self, **kwargs): - self.output_path = Path(kwargs["output_path"]) - - def all_in_one(self, **kwargs): - captured.update(kwargs["genome_selection_metadata"]) - dataframe = pd.DataFrame( - { - "Sequence": ["PEPA"], - "Proteins": ["g1_p1"], - "Intensity_s1": [10.0], - } - ) - dataframe.to_csv(self.output_path, sep="\t", index=False) - return dataframe - - monkeypatch.setattr(annotation_workflow, "run_metaumbra_scoring", fake_scoring) - monkeypatch.setattr(annotation_workflow, "peptideProteinsMapper", FakeMapper) - - run_result = GlobalOTFAnnotator( - peptide_table_path=str(peptide_table), - output_path=str(output), - taxafunc_anno_db_path=str(taxafunc_db), - digested_genome_folders=str(digests), - selection_mode="metaumbra", - metaumbra_peptide_score_col="Evidence", - metaumbra_peptide_error_col="Q.Value", - metaumbra_single_peptide_error_rate_upper_bound=0.01, - metaumbra_genome_qvalue_cutoff=0.05, - ).run() - - assert captured["metaumbra_genome_presence_path"].endswith( - "_metaumbra_genome_presence.tsv" - ) - assert captured["metaumbra_version"] == "1.3.7" - assert captured["metaumbra_peptide_score_col"] == "Evidence" - assert captured["metaumbra_peptide_error_col"] == "Q.Value" - assert captured["metaumbra_single_peptide_error_rate_upper_bound"] == 0.01 - assert captured["metaumbra_genome_qvalue_cutoff"] == 0.05 - assert captured["selected_genomes_from_metaumbra_count"] == 1 - assert captured["selected_genomes_input_count"] == 0 - assert run_result.genome_selection["method"] == "metaumbra" - assert run_result.genome_selection["source"].endswith( - "_metaumbra_genome_presence.tsv" - ) - - -def test_unit_specific_annotation_cli_writes_summary_contract( - annotation_inputs, - tmp_path, -): - peptide_table, peptide_db, taxafunc_db = annotation_inputs - manifest = tmp_path / "unit_specific_manifest.json" - manifest.write_text( - json.dumps( - { - "schema_version": "metaumbra.unit_specific_manifest.v1", - "default_genome_threshold": "q0.05", - "files": {}, - "units": { - "u1": { - "sample_columns": ["s1"], - "n_samples": 1, - "genome_ids_q005": ["g1"], - "genome_ids_q001": ["g1"], - }, - "u2": { - "sample_columns": ["s2"], - "n_samples": 1, - "genome_ids_q005": ["g2"], - "genome_ids_q001": ["g2"], - }, - }, - } - ), - encoding="utf-8", - ) - output = tmp_path / "unit_otf.tsv" - output.write_text("stale unit output\n", encoding="utf-8") - result_json = tmp_path / "unit_result.json" - - exit_code = main( - [ - "--mode", - "unit-specific", - "--peptide-table", - str(peptide_table), - "--unit-specific-manifest", - str(manifest), - "--peptide-db", - str(peptide_db), - "--taxafunc-db", - str(taxafunc_db), - "--output", - str(output), - "--result-json", - str(result_json), - ] - ) - - assert exit_code == ExitCode.SUCCESS - assert output.read_text(encoding="utf-8") == "stale unit output\n" - result = json.loads(result_json.read_text(encoding="utf-8")) - assert result["run"]["mode"] == "unit-specific" - actual_output = Path(result["outputs"]["otf"]["path"]) - assert actual_output != output - assert actual_output.is_file() - assert Path(result["outputs"]["annotation_summary"]["path"]).is_file() - unit_summary = Path(result["outputs"]["unit_summary"]["path"]) - assert unit_summary.is_file() - assert unit_summary.parent == actual_output.with_name( - f"{actual_output.stem}_artifacts" - ) - assert "per_unit_directory" not in result["outputs"] - assert result["metrics"]["units"] == {"completed": 2, "skipped": 0} - assert result["genome_selection"]["method"] == "unit_specific_manifest" - assert result["genome_selection"]["threshold"] == "q0.05" - - -def test_removed_unit_specific_alias_is_rejected(): - with pytest.raises(AnnotationConfigurationError, match="unrecognized arguments"): - build_parser().parse_args(["--unit-specific"]) - - -def test_global_cli_preserves_gui_metaumbra_scoring_default(): - args = build_parser().parse_args([]) - - assert args.metaumbra_single_peptide_error_rate_upper_bound == 0.3 - assert GlobalOTFAnnotator( - peptide_table_path="input.tsv", - output_path="output.tsv", - ).metaumbra_single_peptide_error_rate_upper_bound == 0.3 - - -@pytest.mark.parametrize( - ("parameter", "value", "message"), - [ - ("protein_peptide_coverage_cutoff", 0, "protein_peptide_coverage_cutoff"), - ("protein_peptide_coverage_cutoff", 1.1, "protein_peptide_coverage_cutoff"), - ("metaumbra_genome_qvalue_cutoff", -0.1, "metaumbra_genome_qvalue_cutoff"), - ( - "metaumbra_single_peptide_error_rate_upper_bound", - 1.1, - "metaumbra_single_peptide_error_rate_upper_bound", - ), - ("distinct_genome_threshold", -1, "distinct_genome_threshold"), - ("n_jobs", 0, "n_jobs"), - ], -) -def test_global_backend_rejects_invalid_scientific_parameters( - annotation_inputs, - tmp_path, - parameter, - value, - message, -): - peptide_table, peptide_db, taxafunc_db = annotation_inputs - kwargs = { - "peptide_table_path": str(peptide_table), - "output_path": str(tmp_path / "invalid.tsv"), - "taxafunc_anno_db_path": str(taxafunc_db), - "db_path": str(peptide_db), - "selection_mode": "automatic", - parameter: value, - } - - with pytest.raises(AnnotationConfigurationError, match=message): - GlobalOTFAnnotator(**kwargs)._validate() - - -def test_yaml_config_is_parsed_and_cli_values_can_override(tmp_path): - config_path = tmp_path / "annotation.yaml" - config_path.write_text( - """ -api_version: "1.0" -mode: global -inputs: - peptide_table: input.tsv - peptide_db: peptides.db - taxafunc_db: taxafunc.db -output: - otf: configured.tsv - result_json: result.json -options: - selection_mode: provided - selected_genomes: [g1, g2] -""".strip(), - encoding="utf-8", - ) - - config = load_config_file(config_path) - - assert config["mode"] == "global" - assert config["output"] == str(tmp_path / "configured.tsv") - assert config["result_json"] == str(tmp_path / "result.json") - assert config["peptide_table"] == str(tmp_path / "input.tsv") - assert config["peptide_db"] == str(tmp_path / "peptides.db") - assert config["taxafunc_db"] == str(tmp_path / "taxafunc.db") - assert config["selected_genomes"] == ["g1", "g2"] - parsed = build_parser(config).parse_args(["--output", "overridden.tsv"]) - assert parsed.output == "overridden.tsv" - - -@pytest.mark.parametrize( - ("selection_mode", "selected_genomes", "genome_list", "digest_dirs", "expected"), - [ - (None, ["g1"], None, [], "provided"), - (None, [], "genomes.txt", [], "provided"), - (None, [], None, ["digests"], "metaumbra"), - (None, [], None, [], "automatic"), - ("automatic", [], None, ["digests"], "automatic"), - ], -) -def test_global_selection_mode_resolution( - selection_mode, - selected_genomes, - genome_list, - digest_dirs, - expected, -): - assert annotation_workflow.resolve_global_selection_mode( - selection_mode, - selected_genomes=selected_genomes, - genome_list_path=genome_list, - digested_genome_folders=digest_dirs, - ) == expected - - -@pytest.mark.parametrize( - ("selection_mode", "selected_genomes", "genome_list", "digest_dirs"), - [ - ("provided", [], None, []), - ("automatic", ["g1"], None, []), - ("metaumbra", ["g1"], None, ["digests"]), - ("metaumbra-only", [], "genomes.txt", ["digests"]), - ("provided", ["g1"], "genomes.txt", []), - ], -) -def test_global_selection_mode_rejects_contradictory_inputs( - selection_mode, - selected_genomes, - genome_list, - digest_dirs, -): - with pytest.raises(AnnotationConfigurationError): - annotation_workflow.resolve_global_selection_mode( - selection_mode, - selected_genomes=selected_genomes, - genome_list_path=genome_list, - digested_genome_folders=digest_dirs, - ) - - -def test_metaumbra_mode_requires_digest_directories_when_run(tmp_path): - peptide_table = tmp_path / "peptides.tsv" - peptide_table.write_text("Sequence\tEvidence\tQ.Value\nPEPA\t1\t0.01\n", encoding="utf-8") - - annotator = GlobalOTFAnnotator( - peptide_table_path=str(peptide_table), - output_path=str(tmp_path / "genome_presence.tsv"), - selection_mode="metaumbra-only", - ) - - with pytest.raises( - AnnotationConfigurationError, - match="requires digested_genome_folders", - ): - annotator._validate() - - -@pytest.mark.parametrize("folder_count", [1, 2]) -def test_metaumbra_command_encodes_digest_directories_as_one_argument( - tmp_path, - folder_count, -): - digest_dirs = [ - tmp_path / f"digest folder {index}" for index in range(1, folder_count + 1) - ] - command = annotation_workflow._build_metaumbra_scoring_command( - peptide_table_path=str(tmp_path / "peptides with spaces.tsv"), - digested_genome_folders=[str(path) for path in digest_dirs], - output_path=str(tmp_path / "output with spaces.tsv"), - peptide_col="Sequence", - peptide_score_col="Evidence", - peptide_error_col="Q.Value", - single_peptide_error_rate_upper_bound=0.3, - ) - - option_index = command.index("--genome-digest-dirs") - assert command[option_index + 1] == ";".join(str(path) for path in digest_dirs) - assert command[option_index + 2] == "--output" - - -def test_config_paths_follow_config_directory_and_cli_overrides_follow_cwd( - annotation_inputs, - tmp_path, - monkeypatch, -): - peptide_table, peptide_db, taxafunc_db = annotation_inputs - config_dir = tmp_path / "portable config" - config_dir.mkdir() - results_dir = config_dir / "results" - results_dir.mkdir() - for source in (peptide_table, peptide_db, taxafunc_db): - shutil.copy2(source, config_dir / source.name) - - config_path = config_dir / "annotation.json" - config_path.write_text( - json.dumps( - { - "mode": "global", - "inputs": { - "peptide_table": peptide_table.name, - "peptide_db": peptide_db.name, - "taxafunc_db": taxafunc_db.name, - }, - "output": { - "otf": "results/configured.tsv", - "result_json": "results/configured.json", - }, - "options": {"selected_genomes": ["g1"]}, - } - ), - encoding="utf-8", - ) - launch_dir = tmp_path / "different working directory" - launch_dir.mkdir() - monkeypatch.chdir(launch_dir) - - assert main(["--config", str(config_path)]) == ExitCode.SUCCESS - configured_output = results_dir / "configured.tsv" - configured_result = results_dir / "configured.json" - assert configured_output.is_file() - result = json.loads(configured_result.read_text(encoding="utf-8")) - assert result["parameters"]["selection_mode"] == "provided" - assert result["genome_selection"]["source"] == "selected_genomes" - assert result["outputs"]["otf"]["path"] == str(configured_output) - assert result["inputs"]["config"]["path"] == str(config_path) - - assert main( - [ - "--config", - str(config_path), - "--output", - "cli-override.tsv", - "--result-json", - "cli-override.json", - ] - ) == ExitCode.SUCCESS - assert (launch_dir / "cli-override.tsv").is_file() - override_result = json.loads( - (launch_dir / "cli-override.json").read_text(encoding="utf-8") - ) - assert override_result["outputs"]["otf"]["path"] == "cli-override.tsv" - - -def test_config_path_resolution_preserves_absolute_paths_and_resolves_lists( - tmp_path, -): - config_dir = tmp_path / "config" - config_dir.mkdir() - absolute_digest = tmp_path / "absolute digest" - absolute_digest.mkdir() - absolute_table = tmp_path / "absolute.tsv" - config_path = config_dir / "annotation.json" - config_path.write_text( - json.dumps( - { - "inputs": { - "peptide_table": str(absolute_table), - "digested_genome_folders": [ - "relative digest", - str(absolute_digest), - ], - } - } - ), - encoding="utf-8", - ) - - config = load_config_file(config_path) - - assert config["peptide_table"] == str(absolute_table) - assert config["digested_genome_folders"] == [ - str(config_dir / "relative digest"), - str(absolute_digest), - ] - - -def test_metaumbra_only_preserves_existing_output_and_reports_actual_path( - monkeypatch, - tmp_path, -): - peptide_table = tmp_path / "peptides.tsv" - peptide_table.write_text( - "Sequence\tEvidence\tQ.Value\nPEPA\t1\t0.01\n", - encoding="utf-8", - ) - digest_dir = tmp_path / "digests" - digest_dir.mkdir() - requested_output = tmp_path / "genome_presence.tsv" - requested_output.write_text("stale genome presence\n", encoding="utf-8") - - def fake_scoring(**kwargs): - pd.DataFrame( - {"genome_id": ["g1"], "qvalue": [0.01]} - ).to_csv(kwargs["output_path"], sep="\t", index=False) - return {"output": kwargs["output_path"], "metaumbra_version": "1.3.7"} - - monkeypatch.setattr(annotation_workflow, "run_metaumbra_scoring", fake_scoring) - result = GlobalOTFAnnotator( - peptide_table_path=str(peptide_table), - output_path=str(requested_output), - digested_genome_folders=str(digest_dir), - selection_mode="metaumbra-only", - ).run() - - actual_output = Path(result.output_path) - assert requested_output.read_text(encoding="utf-8") == "stale genome presence\n" - assert actual_output != requested_output - assert actual_output.is_file() - assert result.outputs["genome_presence"]["path"] == str(actual_output) - assert result.genome_selection["source"] == str(actual_output) - - -def test_config_rejects_incompatible_api_version(tmp_path): - config_path = tmp_path / "annotation.json" - config_path.write_text( - json.dumps({"api_version": "2.0", "mode": "global"}), - encoding="utf-8", - ) - - with pytest.raises(AnnotationConfigurationError, match="Unsupported workflow API"): - load_config_file(config_path) - - -def test_missing_input_uses_missing_file_exit_code_and_result_json(tmp_path): - result_json = tmp_path / "failed.json" - exit_code = main( - [ - "--mode", - "global", - "--peptide-table", - str(tmp_path / "missing.tsv"), - "--peptide-db", - str(tmp_path / "missing.db"), - "--taxafunc-db", - str(tmp_path / "missing_taxafunc.db"), - "--output", - str(tmp_path / "out.tsv"), - "--selection-mode", - "automatic", - "--result-json", - str(result_json), - ] - ) - - assert exit_code == ExitCode.MISSING_FILE - result = json.loads(result_json.read_text(encoding="utf-8")) - assert result["run"]["status"] == "error" - assert result["run"]["exit_code"] == ExitCode.MISSING_FILE - assert result["diagnostics"]["error"] == { - "stage": "annotation", - "category": "input", - "type": "FileNotFoundError", - "message": f"Peptide table not found: {tmp_path / 'missing.tsv'}", - } - assert result["stages"]["annotation"]["status"] == "error" - - -def test_invalid_configuration_uses_exit_code_two_and_result_json(tmp_path): - result_json = tmp_path / "invalid.json" - - exit_code = main(["--result-json", str(result_json)]) - - assert exit_code == ExitCode.INVALID_CONFIGURATION - result = json.loads(result_json.read_text(encoding="utf-8")) - assert result["run"]["exit_code"] == ExitCode.INVALID_CONFIGURATION - assert result["diagnostics"]["error"]["type"] == "AnnotationConfigurationError" - assert result["diagnostics"]["error"]["category"] == "configuration" - - -def test_invalid_scientific_parameter_uses_configuration_exit_code(tmp_path): - result_json = tmp_path / "invalid_parameter.json" - - exit_code = main( - [ - "--mode", - "unit-specific", - "--n-jobs", - "0", - "--result-json", - str(result_json), - ] - ) - - assert exit_code == ExitCode.INVALID_CONFIGURATION - result = json.loads(result_json.read_text(encoding="utf-8")) - assert result["diagnostics"]["error"]["category"] == "configuration" - assert "--n-jobs" in result["diagnostics"]["error"]["message"] - - -@pytest.mark.parametrize( - ("exception", "expected_code", "expected_status"), - [ - ( - annotate_cli.OptionalDependencyUnavailable("missing optional runtime"), - ExitCode.OPTIONAL_DEPENDENCY_UNAVAILABLE, - "error", - ), - (RuntimeError("annotation failed"), ExitCode.ANNOTATION_FAILED, "error"), - (KeyboardInterrupt(), ExitCode.CANCELLED, "cancelled"), - ], -) -def test_runtime_failures_use_stable_exit_codes_and_json( - monkeypatch, - tmp_path, - exception, - expected_code, - expected_status, -): - class FakeGlobalAnnotator: - def __init__(self, **_kwargs): - pass - - def run(self): - raise exception - - monkeypatch.setattr(annotate_cli, "GlobalOTFAnnotator", FakeGlobalAnnotator) - result_json = tmp_path / f"result_{int(expected_code)}.json" - - exit_code = main( - [ - "--mode", - "global", - "--peptide-table", - "input.tsv", - "--output", - "output.tsv", - "--result-json", - str(result_json), - ] - ) - - assert exit_code == expected_code - result = json.loads(result_json.read_text(encoding="utf-8")) - assert result["run"]["status"] == expected_status - assert result["run"]["exit_code"] == expected_code - assert result["diagnostics"]["error"]["stage"] == "annotation" - - -def test_annotation_cli_import_does_not_import_pyqt(): - code = ( - "import sys; import metax; import metax.cli.annotate; " - "assert not any(name.startswith('PyQt') for name in sys.modules)" - ) - completed = subprocess.run( - [sys.executable, "-c", code], - cwd=Path(__file__).resolve().parents[1], - capture_output=True, - text=True, - check=False, - ) - - assert completed.returncode == 0, completed.stderr - - -def test_packaging_defines_headless_analyzer_report_and_desktop_profiles(): - pyproject_path = Path(__file__).resolve().parents[1] / "pyproject.toml" - project = tomllib.loads(pyproject_path.read_text(encoding="utf-8"))["project"] - base_names = {Requirement(requirement).name.lower() for requirement in project["dependencies"]} - gui_names = { - Requirement(requirement).name.lower() - for requirement in project["optional-dependencies"]["gui"] - } - full_names = { - Requirement(requirement).name.lower() - for requirement in project["optional-dependencies"]["full"] - } - gui_requirements = "\n".join(project["optional-dependencies"]["gui"]).lower() - analyzer_names = { - Requirement(requirement).name.lower() - for requirement in project["optional-dependencies"]["analyzer"] - } - report_names = { - Requirement(requirement).name.lower() - for requirement in project["optional-dependencies"]["report"] - } - - qt_names = {"pyqt5", "pyqtwebengine", "qt-material", "qtawesome"} - assert base_names.isdisjoint(qt_names) - assert analyzer_names.isdisjoint(qt_names) - assert report_names.isdisjoint(qt_names) - assert { - "scipy", - "scikit-learn", - "joblib", - "statsmodels", - "numba", - "inmoose", - } <= analyzer_names - assert analyzer_names <= report_names - assert { - "matplotlib", - "seaborn", - "adjusttext", - "upsetplot", - "pyecharts", - "distinctipy", - "jinja2", - } <= report_names - assert report_names <= gui_names - assert "pyqt5" in gui_names - assert "metaumbra[gui-pyqt5]" in gui_requirements - assert gui_names == full_names - assert "packaging" in gui_names - assert "python-dateutil" not in gui_names - assert "pyproject-toml" not in gui_names - assert project["scripts"]["metax"] == "metax.gui.launcher:main" - assert project["scripts"]["metax-report"] == "metax.cli.report:main" - - -def test_gui_launcher_explains_how_to_install_missing_gui(monkeypatch, capsys): - def fail_import(_name): - raise ModuleNotFoundError("No module named 'PyQt5'", name="PyQt5") - - monkeypatch.setattr(gui_launcher.importlib, "import_module", fail_import) - - assert gui_launcher.main() == ExitCode.OPTIONAL_DEPENDENCY_UNAVAILABLE - error = capsys.readouterr().err - assert 'pip install "MetaXTools[gui]"' in error - assert "PyQt5" in error - - -def test_gui_launcher_delegates_to_existing_desktop_entrypoint(monkeypatch): - fake_gui = SimpleNamespace(runGUI=lambda: 17) - monkeypatch.setattr(gui_launcher.importlib, "import_module", lambda _name: fake_gui) - - assert gui_launcher.main() == 17 - - -def test_report_launcher_explains_how_to_install_missing_report_stack( - monkeypatch, - capsys, -): - def fail_import(_name): - raise ModuleNotFoundError("No module named 'jinja2'", name="jinja2") - - monkeypatch.setattr(report_launcher.importlib, "import_module", fail_import) - - assert report_launcher.main([]) == ExitCode.OPTIONAL_DEPENDENCY_UNAVAILABLE - error = capsys.readouterr().err - assert 'pip install "MetaXTools[report]"' in error - assert "jinja2" in error - - -def test_report_launcher_delegates_to_report_cli(monkeypatch): - fake_cli = SimpleNamespace(main=lambda argv=None: 23 if argv == ["--help"] else 0) - monkeypatch.setattr( - report_launcher.importlib, - "import_module", - lambda _name: fake_cli, - ) - - assert report_launcher.main(["--help"]) == 23 - - -def test_report_launcher_import_does_not_import_report_stack(): - code = ( - "import sys; import metax.cli.report; " - "assert 'metax.report' not in sys.modules" - ) - completed = subprocess.run( - [sys.executable, "-c", code], - cwd=Path(__file__).resolve().parents[1], - capture_output=True, - text=True, - check=False, - ) - - assert completed.returncode == 0, completed.stderr - - -def test_gui_annotation_handlers_use_shared_backend_classes(): - gui_path = Path(__file__).resolve().parents[1] / "metax" / "gui" / "main_gui.py" - tree = ast.parse(gui_path.read_text(encoding="utf-8")) - methods = { - node.name: node - for node in ast.walk(tree) - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) - } - - global_calls = { - node.func.id - for node in ast.walk(methods["run_pep_dircet_to_otf"]) - if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) - } - unit_calls = { - node.func.id - for node in ast.walk(methods["run_pep_direct_to_otf_unit_specific"]) - if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) - } - assert "GlobalOTFAnnotator" in global_calls - assert "UnitSpecificOTFAnnotator" in unit_calls - assert "peptideProteinsMapper" not in global_calls - assert { - "_prepare_diann_parquet_for_pep_direct_to_otf", - "_run_pep_direct_to_otf_metaumbra_scoring", - "_run_pep_direct_to_otf_with_genome_list", - }.isdisjoint(methods) diff --git a/tests/test_genome_selection_manifest.py b/tests/test_genome_selection_manifest.py new file mode 100644 index 00000000..604e912e --- /dev/null +++ b/tests/test_genome_selection_manifest.py @@ -0,0 +1,68 @@ +import json +from pathlib import Path + +import pytest + +from metax.peptide_annotator.genome_selection_manifest import ( + SCHEMA_VERSION, + load_genome_selection_manifest, + resolve_manifest_sample_columns, +) + + +FIXTURE = Path(__file__).parent / "fixtures" / "genome_selection_manifest.v1.json" + + +def test_load_unified_manifest_and_thresholds(): + manifest = load_genome_selection_manifest(FIXTURE) + assert manifest.schema_version == SCHEMA_VERSION + assert manifest.unit_definition["mode"] == "metadata" + assert manifest.selected_genome_threshold == "q0.05" + assert manifest.units["u1"].sample_ids == ["s1"] + assert manifest.units["u1"].genome_ids == ["g1", "g2"] + strict = load_genome_selection_manifest(FIXTURE, genome_threshold="q0.01") + assert strict.units["u1"].genome_ids == ["g1"] + + +def test_manifest_rejects_old_schema(tmp_path): + data = json.loads(FIXTURE.read_text(encoding="utf-8")) + data["schema_version"] = "metaumbra.unit_specific_manifest.v1" + path = tmp_path / "manifest.json" + path.write_text(json.dumps(data), encoding="utf-8") + with pytest.raises(ValueError, match="Unsupported genome selection"): + load_genome_selection_manifest(path) + + +def test_manifest_rejects_duplicate_sample_assignments(tmp_path): + data = json.loads(FIXTURE.read_text(encoding="utf-8")) + data["units"]["u2"]["sample_ids"] = ["s1"] + path = tmp_path / "manifest.json" + path.write_text(json.dumps(data), encoding="utf-8") + with pytest.raises(ValueError, match="multiple units"): + load_genome_selection_manifest(path) + + +def test_manifest_rejects_empty_unit_and_genome_threshold_contract(tmp_path): + data = json.loads(FIXTURE.read_text(encoding="utf-8")) + data["units"]["u1"]["sample_ids"] = [] + data["units"]["u1"]["n_samples"] = 0 + path = tmp_path / "manifest.json" + path.write_text(json.dumps(data), encoding="utf-8") + with pytest.raises(ValueError, match="no sample_ids"): + load_genome_selection_manifest(path) + + data = json.loads(FIXTURE.read_text(encoding="utf-8")) + data["units"]["u1"]["genome_ids_q005"] = [] + data["units"]["u1"]["genome_ids_q001"] = [] + path.write_text(json.dumps(data), encoding="utf-8") + with pytest.raises(ValueError, match="no genomes at selected threshold q0.05"): + load_genome_selection_manifest(path, genome_threshold="q0.05", strict=True) + + +def test_sample_mapping_is_one_to_one(): + mapping = resolve_manifest_sample_columns( + ["Intensity_s1", "Intensity_s2"], ["s1", "s2"] + ) + assert mapping == {"s1": "Intensity_s1", "s2": "Intensity_s2"} + with pytest.raises(ValueError, match="distinct"): + resolve_manifest_sample_columns(["s1"], ["s1", "s1.raw"]) diff --git a/tests/test_manifest_annotation_cli.py b/tests/test_manifest_annotation_cli.py new file mode 100644 index 00000000..16587881 --- /dev/null +++ b/tests/test_manifest_annotation_cli.py @@ -0,0 +1,127 @@ +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from metax.cli import annotate +from metax.peptide_annotator.manifest_otf import ManifestOTFRunResult +from metax.peptide_annotator.annotation_workflow import read_plain_genome_list_file + + +def test_cli_has_explicit_input_sources_without_legacy_mode_inference(): + parser = annotate.build_parser() + destinations = {action.dest for action in parser._actions} + assert "metaumbra_manifest" in destinations + assert "input_source" in destinations + assert "mode" not in destinations + assert "unit_specific_manifest" not in destinations + assert "genome_list_file" in destinations + assert "intensity_col_prefix" in destinations + source_action = next(action for action in parser._actions if action.dest == "input_source") + assert source_action.choices == ["metaumbra-manifest", "metax-automatic", "genome-list"] + + +def test_custom_genome_list_is_not_interpreted_as_metaumbra_qvalue_output(tmp_path): + genome_list = tmp_path / "genomes.tsv" + genome_list.write_text("genome_id\tqvalue\ng1\t0.9\ng2\t0.01\n", encoding="utf-8") + assert read_plain_genome_list_file(genome_list) == ["g1", "g2"] + + +def test_cli_result_json_contract(monkeypatch, tmp_path): + peptide = tmp_path / "report.parquet" + manifest = tmp_path / "genome_selection_manifest.json" + database = tmp_path / "taxafunc.db" + digest = tmp_path / "digests" + output = tmp_path / "OTF.tsv" + result_json = tmp_path / "result.json" + for path in (peptide, manifest, database): + path.write_text("x", encoding="utf-8") + digest.mkdir() + + class FakeAnnotator: + def __init__(self, **kwargs): + self.kwargs = kwargs + + def run(self): + output.write_text("analysis_unit_id\tSequence\n__global__\tPEP\n", encoding="utf-8") + return ManifestOTFRunResult( + output_path=str(output), info_path="", summary_path="", rows=1, + column_count=2, column_names=["analysis_unit_id", "Sequence"], + completed_units=1, skipped_units=0, selected_genome_threshold="q0.05", + manifest_schema_version="metaumbra.genome_selection_manifest.v1", warnings=[], + ) + + monkeypatch.setattr(annotate, "ManifestOTFAnnotator", FakeAnnotator) + code = annotate.main([ + "--peptide-table", str(peptide), + "--metaumbra-manifest", str(manifest), + "--taxafunc-db", str(database), + "--digested-genome-folders", str(digest), + "--output", str(output), + "--result-json", str(result_json), + ]) + assert code == 0 + result = json.loads(result_json.read_text(encoding="utf-8")) + assert result["manifest"]["path"] == str(manifest) + assert result["number_of_units"] == 1 + assert result["selected_threshold"] == "q0.05" + assert result["outputs"]["otf"]["path"] == str(output) + assert result["input_source"] == "metaumbra-manifest" + assert result["schema_version"] == "metax.annotation_result.v2" + + +@pytest.mark.parametrize( + ("source", "extra_args", "expected_mode"), + [ + ("metax-automatic", [], "automatic"), + ("genome-list", ["--selected-genomes", "g1", "g2"], "provided"), + ], +) +def test_cli_retains_non_metaumbra_and_custom_list_annotation( + monkeypatch, tmp_path, source, extra_args, expected_mode +): + peptide = tmp_path / "report.tsv" + database = tmp_path / "taxafunc.db" + digest = tmp_path / "digests" + output = tmp_path / f"{source}.tsv" + result_json = tmp_path / f"{source}.json" + peptide.write_text("Sequence\tIntensity_s1\nPEP\t1\n", encoding="utf-8") + database.write_text("x", encoding="utf-8") + digest.mkdir() + captured = {} + + class FakeGlobalAnnotator: + def __init__(self, **kwargs): + captured.update(kwargs) + + def run(self): + output.write_text("Sequence\nPEP\n", encoding="utf-8") + return SimpleNamespace( + output_path=str(output), + inputs={}, parameters={}, stages={}, + genome_selection={"method": expected_mode}, metrics={}, + outputs={"otf": {"path": str(output)}}, software={}, warnings=[], + ) + + monkeypatch.setattr(annotate, "GlobalOTFAnnotator", FakeGlobalAnnotator) + code = annotate.main([ + "--input-source", source, + "--peptide-table", str(peptide), + "--taxafunc-db", str(database), + "--digested-genome-folders", str(digest), + "--output", str(output), + "--result-json", str(result_json), + "--intensity-col-prefix", "Abundance", + *extra_args, + ]) + assert code == 0 + assert captured["selection_mode"] == expected_mode + assert captured["intensity_col_prefix"] == "Abundance" + if expected_mode == "automatic": + assert captured["selected_genome_source"] is None + else: + assert captured["selected_genome_source"] == "CLI --selected-genomes" + result = json.loads(result_json.read_text(encoding="utf-8")) + assert result["input_source"] == source + assert result["manifest"] is None diff --git a/tests/test_manifest_otf_backend.py b/tests/test_manifest_otf_backend.py new file mode 100644 index 00000000..0717a104 --- /dev/null +++ b/tests/test_manifest_otf_backend.py @@ -0,0 +1,43 @@ +from pathlib import Path + +import pandas as pd + +from metax.peptide_annotator.genome_selection_manifest import load_genome_selection_manifest +from metax.peptide_annotator import unit_specific_otf as backend + + +FIXTURE = Path(__file__).parent / "fixtures" / "genome_selection_manifest.v1.json" + + +def test_digest_union_is_scanned_once(monkeypatch): + manifest = load_genome_selection_manifest(FIXTURE) + calls = [] + + def fake_scan(**kwargs): + calls.append(kwargs) + return {} + + monkeypatch.setattr( + backend, "query_peptide_proteins_from_digested_genome_folders_nested", fake_scan + ) + backend.build_manifest_peptide_protein_map( + pd.DataFrame({"Sequence": ["PEP1", "PEP2"]}), + "Sequence", + manifest, + "digests", + ) + assert len(calls) == 1 + assert calls[0]["selected_genomes_set"] == {"g1", "g2"} + + +def test_merged_column_order_always_contains_analysis_unit_id(tmp_path): + # This invariant is independent of unit count and is enforced before streaming merge. + method = backend.ManifestOTFAnnotator._merged_column_order + instance = object.__new__(backend.ManifestOTFAnnotator) + instance.output_sample_col_prefix = "Intensity_" + columns = method( + instance, + [{"columns": ["analysis_unit_id", "Sequence", "Taxon", "Intensity_s1"]}], + ["Intensity_s1"], + ) + assert columns[0] == "analysis_unit_id" diff --git a/tests/test_metaumbra_contract_integration.py b/tests/test_metaumbra_contract_integration.py new file mode 100644 index 00000000..0389311d --- /dev/null +++ b/tests/test_metaumbra_contract_integration.py @@ -0,0 +1,59 @@ +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from metax.peptide_annotator.genome_selection_manifest import ( + SCHEMA_VERSION, + load_genome_selection_manifest, +) + + +def test_metaumbra_generated_manifest_is_loaded_directly_by_metax(tmp_path): + workspace = Path(__file__).resolve().parents[2] + schema_path = workspace / "MetaUmbra" / "docs" / "genome_selection_manifest.v1.schema.json" + if not schema_path.is_file(): + pytest.skip("Cross-repository MetaUmbra checkout is not available") + schema = json.loads(schema_path.read_text(encoding="utf-8")) + output = tmp_path / "genome_selection_manifest.json" + script = r''' +import sys +from pathlib import Path +import pandas as pd +from metaumbra.genome_selection_manifest import build_genome_selection_manifest, write_genome_selection_manifest + +output = Path(sys.argv[1]) +mapping = pd.DataFrame({"sample_id": ["s1", "s2"], "analysis_unit_id": ["u1", "u2"]}) +results = pd.DataFrame({ + "analysis_unit_id": ["u1", "u1", "u2", "u2"], + "genome_id": ["g1", "g2", "g1", "g2"], + "pass_q_0_01": [True, False, False, True], + "pass_q_0_05": [True, True, True, True], +}) +manifest = build_genome_selection_manifest( + mapping_df=mapping, + unit_genome_results=results, + unit_mode="per-sample", + sample_id_column="Run", + analysis_unit_column=None, + peptide_table_path=str(output.parent / "report.parquet"), + metadata_table_path=None, + genome_digest_directories=[str(output.parent / "digests")], + artifacts={"unit_genome_results": "unit_genome_results.tsv"}, + scoring_method="per-analysis-unit/empirical-background", + run_id="cross-repo-test", +) +write_genome_selection_manifest(output, manifest) +''' + env = dict(os.environ) + env["PYTHONPATH"] = str(workspace / "MetaUmbra" / "src") + os.pathsep + env.get("PYTHONPATH", "") + subprocess.run([sys.executable, "-c", script, str(output)], check=True, env=env) + manifest = load_genome_selection_manifest(output) + assert schema["$id"] == SCHEMA_VERSION == manifest.schema_version + assert schema["properties"]["schema_version"]["const"] == SCHEMA_VERSION + assert manifest.unit_definition["mode"] == "per-sample" + assert manifest.units["u1"].sample_ids == ["s1"] + assert manifest.units["u2"].genome_ids == ["g1", "g2"] diff --git a/tests/test_pep_table_to_otf_dataframe.py b/tests/test_pep_table_to_otf_dataframe.py index 8082e3f6..f4c97183 100644 --- a/tests/test_pep_table_to_otf_dataframe.py +++ b/tests/test_pep_table_to_otf_dataframe.py @@ -1,11 +1,22 @@ +import io import json import sqlite3 import pandas as pd +from metax.peptide_annotator import pep_table_to_otf from metax.peptide_annotator.pep_table_to_otf import peptideProteinsMapper +def test_subprocess_progress_is_safe_on_legacy_windows_console(monkeypatch): + raw = io.BytesIO() + legacy_stdout = io.TextIOWrapper(raw, encoding="cp1252") + monkeypatch.setattr(pep_table_to_otf.sys, "stdout", legacy_stdout) + pep_table_to_otf._print_console_safe("Scanning ██") + legacy_stdout.flush() + assert raw.getvalue().decode("cp1252").strip() == "Scanning ??" + + def _make_peptide_db(path): with sqlite3.connect(path) as conn: conn.execute("CREATE TABLE peptide_proteins (peptide TEXT PRIMARY KEY, proteins TEXT)") diff --git a/tests/test_peptide_annotator_layout.py b/tests/test_peptide_annotator_layout.py new file mode 100644 index 00000000..c41bb08f --- /dev/null +++ b/tests/test_peptide_annotator_layout.py @@ -0,0 +1,116 @@ +import os +from types import SimpleNamespace + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + +from PyQt5 import QtWidgets + +from metax.gui.main_gui import MetaXGUI + + +def _build_layout_harness(): + window = QtWidgets.QMainWindow() + ui = MetaXGUI.__new__(MetaXGUI) + ui.setupUi(window) + ui.label_pep_direct_to_otf_input_source = QtWidgets.QLabel("Genome selection source") + ui.comboBox_pep_direct_to_otf_input_source = QtWidgets.QComboBox() + ui.comboBox_pep_direct_to_otf_input_source.addItem("Manifest", "metaumbra-manifest") + ui.comboBox_pep_direct_to_otf_input_source.addItem("Automatic", "metax-automatic") + ui.comboBox_pep_direct_to_otf_input_source.addItem("Genome list", "genome-list") + ui.label_pep_direct_to_otf_manifest_summary = QtWidgets.QLabel("summary") + ui.gridLayout_74.addWidget(ui.label_pep_direct_to_otf_input_source, ui.gridLayout_74.rowCount(), 0) + ui.gridLayout_74.addWidget(ui.comboBox_pep_direct_to_otf_input_source, ui.gridLayout_74.rowCount(), 2) + ui._arrange_pep_direct_to_otf_layout() + return window, ui + + +def _position(layout, target): + for index in range(layout.count()): + item = layout.itemAt(index) + if item.widget() is target or item.layout() is target: + return layout.getItemPosition(index) + raise AssertionError(f"Layout item not found: {target}") + + +def test_annotation_layout_follows_source_input_output_order(): + window, ui = _build_layout_harness() + try: + assert _position(ui.gridLayout_74, ui.label_pep_direct_to_otf_input_source) == (0, 0, 1, 2) + assert _position(ui.gridLayout_74, ui.comboBox_pep_direct_to_otf_input_source) == (0, 2, 1, 2) + assert _position(ui.gridLayout_74, ui.lineEdit_pep_direct_to_otf_peptide_path)[0] == 1 + assert _position(ui.gridLayout_74, ui.lineEdit_pep_direct_to_otf_unit_specific_manifest_path)[0] == 2 + assert _position(ui.gridLayout_74, ui.label_pep_direct_to_otf_custom_genome_list)[0] == 2 + assert _position(ui.gridLayout_74, ui.label_pep_direct_to_otf_manifest_summary) == (4, 0, 1, 4) + assert _position(ui.gridLayout_74, ui.lineEdit_pep_direct_to_otf_digestied_genome_pep_path)[0] == 5 + assert _position(ui.gridLayout_74, ui.lineEdit_pep_direct_to_otf_pro2taxafunc_db_path)[0] == 6 + assert _position(ui.gridLayout_74, ui.lineEdit_pep_direct_to_otf_output_path)[0] == 7 + assert _position(ui.gridLayout_74, ui.pushButton_run_pep_direct_to_otf)[0] == 11 + finally: + window.close() + + +def test_source_choice_hides_irrelevant_manifest_or_list_controls(): + window, ui = _build_layout_harness() + try: + ui.comboBox_pep_direct_to_otf_input_source.setCurrentIndex(0) + ui.update_pep_direct_to_otf_mode_state() + assert not ui.lineEdit_pep_direct_to_otf_unit_specific_manifest_path.isHidden() + assert ui.pushButton_pep_direct_to_otf_open_genome_list_file.isHidden() + + ui.comboBox_pep_direct_to_otf_input_source.setCurrentIndex(1) + ui.update_pep_direct_to_otf_mode_state() + assert ui.lineEdit_pep_direct_to_otf_unit_specific_manifest_path.isHidden() + assert ui.pushButton_pep_direct_to_otf_open_genome_list_file.isHidden() + + ui.comboBox_pep_direct_to_otf_input_source.setCurrentIndex(2) + ui.update_pep_direct_to_otf_mode_state() + assert ui.lineEdit_pep_direct_to_otf_unit_specific_manifest_path.isHidden() + assert not ui.pushButton_pep_direct_to_otf_open_genome_list_file.isHidden() + assert not ui.label_pep_direct_to_otf_custom_genome_list.isHidden() + assert ui.pushButton_pep_direct_to_otf_reset_selected_genome_list.text() == "Reset" + finally: + window.close() + + +def test_non_manifest_gui_forwards_custom_intensity_prefix(monkeypatch, tmp_path): + window, ui = _build_layout_harness() + try: + peptide = tmp_path / "peptides.tsv" + database = tmp_path / "taxafunc.db" + digests = tmp_path / "digests" + output = tmp_path / "OTF.tsv" + peptide.write_text("Sequence\tAbundance_s1\nPEP\t1\n", encoding="utf-8") + database.write_text("x", encoding="utf-8") + digests.mkdir() + + ui.MainWindow = window + ui.logger = SimpleNamespace(write_log=lambda *args, **kwargs: None) + ui.pep_direct_to_otf_selected_genomes = [] + ui.pep_direct_to_otf_selected_genome_source = "" + ui.lineEdit_pep_direct_to_otf_peptide_path.setText(str(peptide)) + ui.lineEdit_pep_direct_to_otf_digestied_genome_pep_path.setText(str(digests)) + ui.lineEdit_pep_direct_to_otf_pro2taxafunc_db_path.setText(str(database)) + ui.lineEdit_pep_direct_to_otf_output_path.setText(str(output)) + ui.comboBox_pep_direct_to_otf_peptide_col_name.clear() + ui.comboBox_pep_direct_to_otf_peptide_col_name.addItem("Sequence") + ui.comboBox_pep_direct_to_otf_intensity_column.setEditable(True) + ui.comboBox_pep_direct_to_otf_intensity_column.setEditText("Abundance") + + captured = {} + + class FakeAnnotator: + def __init__(self, **kwargs): + captured.update(kwargs) + + def run(self): + return None + + monkeypatch.setattr("metax.gui.main_gui.GlobalOTFAnnotator", FakeAnnotator) + ui.run_in_new_window = lambda function, **kwargs: function() + ui.run_pep_direct_to_otf_non_metaumbra("metax-automatic") + + assert captured["intensity_col_prefix"] == "Abundance" + assert captured["selection_mode"] == "automatic" + assert captured["selected_genome_source"] is None + finally: + window.close() diff --git a/tests/test_unit_specific_gui_settings_dialog.py b/tests/test_unit_specific_gui_settings_dialog.py deleted file mode 100644 index 7f77ae98..00000000 --- a/tests/test_unit_specific_gui_settings_dialog.py +++ /dev/null @@ -1,542 +0,0 @@ -from dataclasses import asdict -import json -import os - -os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") - -import metax.gui.unit_specific_settings_dialog as unit_specific_settings_dialog -import pandas as pd -import pytest -from PyQt5 import QtWidgets -from metax.gui.main_gui import MetaXGUI -from metax.peptide_annotator.annotation_workflow import GlobalOTFAnnotator -from metax.gui.unit_specific_settings_dialog import ( - UnitSpecificGuiConfig, - UnitSpecificSettingsDialog, - UnitSpecificValidationResultDialog, - validate_unit_specific_manifest_for_gui, -) - - -def _write_manifest(tmp_path): - path = tmp_path / "unit_specific_manifest.json" - path.write_text( - json.dumps( - { - "schema_version": "metaumbra.unit_specific_manifest.v1", - "generated_by": {"tool": "MetaUmbra", "version": "1.3.5"}, - "default_genome_threshold": "q0.05", - "files": {}, - "units": { - "u1": { - "sample_columns": ["s1", "s2"], - "n_samples": 2, - "genome_ids_q005": ["g1", "g2"], - "genome_ids_q001": ["g1"], - }, - "u2": { - "sample_columns": ["s3"], - "n_samples": 1, - "genome_ids_q005": ["g3"], - "genome_ids_q001": ["g3"], - }, - }, - } - ), - encoding="utf-8", - ) - return path - - -def _write_peptide_table(tmp_path, columns): - path = tmp_path / "peptides.tsv" - path.write_text("\t".join(columns) + "\n", encoding="utf-8") - return path - - -def test_unit_specific_gui_config_defaults_and_override(): - default = UnitSpecificGuiConfig() - assert asdict(default) == { - "manifest_path": "", - "genome_threshold": "auto", - "input_sample_col_prefix": "", - "on_missing_sample": "error", - "on_empty_unit": "warn-skip", - "save_per_unit_outputs": False, - "n_jobs": None, - } - - config = UnitSpecificGuiConfig( - manifest_path="unit_specific_manifest.json", - genome_threshold="q0.01", - input_sample_col_prefix="LFQ intensity ", - on_missing_sample="warn-skip", - on_empty_unit="error", - save_per_unit_outputs=True, - n_jobs=4, - ) - - assert config.manifest_path == "unit_specific_manifest.json" - assert config.genome_threshold == "q0.01" - assert config.input_sample_col_prefix == "LFQ intensity " - assert config.on_missing_sample == "warn-skip" - assert config.on_empty_unit == "error" - assert config.save_per_unit_outputs is True - assert config.n_jobs == 4 - - -def test_unit_specific_gui_validation_missing_manifest_points_to_main_window(): - result = validate_unit_specific_manifest_for_gui(manifest_path="") - - assert result.ok is False - assert "main Peptide Direct to OTF window" in result.message - - -def test_unit_specific_settings_dialog_uses_main_window_owned_values(): - app = QtWidgets.QApplication.instance() or QtWidgets.QApplication([]) - dialog = UnitSpecificSettingsDialog( - input_intensity_prefix="Intensity_", - diann_intensity_col="Precursor.Quantity", - current_config=UnitSpecificGuiConfig( - manifest_path="unit_specific_manifest.json", - genome_threshold="q0.01", - ) - ) - - assert dialog.lineEdit_current_manifest_path.text() == "unit_specific_manifest.json" - assert dialog.lineEdit_current_manifest_path.isReadOnly() is True - assert not hasattr(dialog, "pushButton_browse_manifest") - assert not hasattr(dialog, "comboBox_genome_threshold") - assert dialog.get_config().genome_threshold == "q0.01" - assert dialog.tabs.count() == 1 - assert dialog.spinBox_n_jobs.specialValueText() == "Auto" - assert dialog.input_intensity_prefix == "Intensity_" - assert dialog.diann_intensity_col == "Precursor.Quantity" - - dialog.close() - app.processEvents() - - -def test_main_window_controls_own_genome_threshold(): - app = QtWidgets.QApplication.instance() or QtWidgets.QApplication([]) - gui = object.__new__(MetaXGUI) - gui.unit_specific_gui_config = UnitSpecificGuiConfig( - manifest_path="old.json", - genome_threshold="q0.05", - on_missing_sample="warn-skip", - ) - gui.lineEdit_pep_direct_to_otf_unit_specific_manifest_path = QtWidgets.QLineEdit() - gui.lineEdit_pep_direct_to_otf_unit_specific_manifest_path.setText("current.json") - gui.comboBox_pep_direct_to_otf_use_unit_specific_genome_threshold = QtWidgets.QComboBox() - gui.comboBox_pep_direct_to_otf_use_unit_specific_genome_threshold.addItems(["q0.05", "q0.01"]) - gui.comboBox_pep_direct_to_otf_use_unit_specific_genome_threshold.setCurrentText("q0.01") - - config = gui._get_unit_specific_gui_config_from_controls() - - assert config.manifest_path == "current.json" - assert config.genome_threshold == "q0.01" - assert config.on_missing_sample == "warn-skip" - app.processEvents() - - -def test_unit_specific_gui_validation_passes_with_matching_peptide_header(tmp_path): - manifest_path = _write_manifest(tmp_path) - peptide_path = _write_peptide_table( - tmp_path, - ["Sequence", "Intensity_s1", "Intensity_s2", "Intensity_s3"], - ) - - result = validate_unit_specific_manifest_for_gui( - manifest_path=str(manifest_path), - peptide_table_path=str(peptide_path), - peptide_col="Sequence", - peptide_table_separator="\t", - genome_threshold="q0.01", - ) - - assert result.ok is True - assert result.manifest_samples == ["s1", "s2", "s3"] - assert result.mapped_samples == { - "s1": "Intensity_s1", - "s2": "Intensity_s2", - "s3": "Intensity_s3", - } - assert result.missing_samples == [] - assert "Selected genome threshold: q0.01" in result.message - assert "u1: samples=2, genomes=1" in result.message - - -def test_unit_specific_gui_validation_fails_for_missing_sample_by_default(tmp_path): - manifest_path = _write_manifest(tmp_path) - peptide_path = _write_peptide_table(tmp_path, ["Sequence", "Intensity_s1", "Intensity_s2"]) - - result = validate_unit_specific_manifest_for_gui( - manifest_path=str(manifest_path), - peptide_table_path=str(peptide_path), - peptide_col="Sequence", - peptide_table_separator="\t", - on_missing_sample="error", - ) - - assert result.ok is False - assert result.missing_samples == ["s3"] - assert "Manifest sample-column validation failed" in result.message - assert "Missing samples: 1" in result.message - assert "s3" in result.message - assert result.details == "Missing manifest samples:\ns3" - - -@pytest.mark.parametrize("intensity_col", ["Precursor.Normalised", "Precursor.Quantity"]) -def test_unit_specific_gui_validation_supports_long_format_parquet( - tmp_path, - monkeypatch, - intensity_col, -): - manifest_path = _write_manifest(tmp_path) - peptide_path = tmp_path / "report.parquet" - peptide_path.write_bytes(b"parquet placeholder") - monkeypatch.setattr( - unit_specific_settings_dialog, - "_read_peptide_table_header_columns", - lambda peptide_table_path, separator: ["Stripped.Sequence", "Run", intensity_col], - ) - monkeypatch.setattr( - unit_specific_settings_dialog, - "_read_long_format_run_columns", - lambda peptide_table_path: ["s1.raw", "s2.raw", "s3.raw"], - ) - - result = validate_unit_specific_manifest_for_gui( - manifest_path=str(manifest_path), - peptide_table_path=str(peptide_path), - peptide_col="Stripped.Sequence", - ) - - assert result.ok is True - assert result.mapped_samples == {"s1": "s1", "s2": "s2", "s3": "s3"} - assert result.missing_samples == [] - assert "long-format DIA-NN parquet" in result.message - assert "Run-derived sample columns validated" in result.message - assert f"intensity={intensity_col}" in result.message - - -def test_unit_specific_gui_validation_accepts_diann_prepared_sample_columns( - tmp_path, - monkeypatch, -): - manifest_path = tmp_path / "unit_specific_manifest.json" - manifest_path.write_text( - json.dumps( - { - "schema_version": "metaumbra.unit_specific_manifest.v1", - "generated_by": {"tool": "MetaUmbra", "version": "1.3.7"}, - "default_genome_threshold": "q0.05", - "files": {}, - "units": { - "u1": { - "sample_columns": ["s1"], - "n_samples": 1, - "genome_ids_q005": ["g1"], - "genome_ids_q001": ["g1"], - }, - "u2": { - "sample_columns": ["s1_2"], - "n_samples": 1, - "genome_ids_q005": ["g2"], - "genome_ids_q001": ["g2"], - }, - }, - } - ), - encoding="utf-8", - ) - peptide_path = tmp_path / "report.parquet" - peptide_path.write_bytes(b"parquet placeholder") - monkeypatch.setattr( - unit_specific_settings_dialog, - "_read_peptide_table_header_columns", - lambda peptide_table_path, separator: [ - "Stripped.Sequence", - "Run", - "Precursor.Quantity", - ], - ) - monkeypatch.setattr( - unit_specific_settings_dialog, - "_read_long_format_run_columns", - lambda peptide_table_path: [r"C:\batch1\s1.raw", r"C:\batch2\s1.raw"], - ) - - result = validate_unit_specific_manifest_for_gui( - manifest_path=str(manifest_path), - peptide_table_path=str(peptide_path), - peptide_col="Stripped.Sequence", - diann_intensity_col="Precursor.Quantity", - ) - - assert result.ok is True - assert result.mapped_samples == {"s1": "s1", "s1_2": "s1_2"} - assert result.missing_samples == [] - assert "Run-derived sample columns validated" in result.message - - -def test_shared_global_backend_prepares_direct_otf_parquet(tmp_path): - parquet_path = tmp_path / "report.parquet" - pd.DataFrame( - { - "Run": ["s1.raw"], - "Stripped.Sequence": ["PEPA"], - "Evidence": [2.0], - "Q.Value": [0.01], - "Precursor.Quantity": [10.0], - } - ).to_parquet(parquet_path) - prepared_path, separator, peptide_col, intensity_prefix, metadata = ( - GlobalOTFAnnotator( - peptide_table_path=str(parquet_path), - output_path=str(tmp_path / "OTF.tsv"), - selection_mode="metaumbra", - diann_intensity_col="Precursor.Quantity", - )._prepare_input() - ) - - prepared_df = pd.read_csv(prepared_path, sep="\t") - assert separator == "\t" - assert peptide_col == "Stripped.Sequence" - assert intensity_prefix == "Intensity_" - assert prepared_df.loc[0, "Intensity_s1"] == 10.0 - assert metadata["diann_intensity_column"] == "Precursor.Quantity" - - -def test_direct_otf_gui_lists_diann_intensity_columns_and_defaults_to_normalised( - tmp_path, -): - parquet_path = tmp_path / "report.parquet" - pd.DataFrame( - { - "Run": ["s1.raw"], - "Stripped.Sequence": ["PEPA"], - "Evidence": [2.0], - "Q.Value": [0.01], - "Precursor.Normalised": [10.0], - "Precursor.Quantity": [99.0], - } - ).to_parquet(parquet_path) - app = QtWidgets.QApplication.instance() or QtWidgets.QApplication([]) - gui = object.__new__(MetaXGUI) - gui.lineEdit_pep_direct_to_otf_peptide_path = QtWidgets.QLineEdit( - str(parquet_path) - ) - gui.lineEdit_pep_direct_to_otf_pep_table_sep = QtWidgets.QLineEdit("\\t") - gui.comboBox_pep_direct_to_otf_peptide_col_name = QtWidgets.QComboBox() - gui.comboBox_pep_direct_to_otf_intensity_column = QtWidgets.QComboBox() - gui.label_227 = QtWidgets.QLabel() - gui._last_pep_direct_to_otf_peptide_table_signature = "" - gui._apply_metaumbra_columns = lambda schema: None - gui.logger = type( - "Logger", - (), - {"write_log": lambda self, message, level: None}, - )() - - gui.update_pep_direct_to_otf_peptide_table_columns() - - assert gui.comboBox_pep_direct_to_otf_intensity_column.isEditable() is False - assert [ - gui.comboBox_pep_direct_to_otf_intensity_column.itemText(index) - for index in range(gui.comboBox_pep_direct_to_otf_intensity_column.count()) - ] == ["Precursor.Normalised", "Precursor.Quantity"] - assert ( - gui.comboBox_pep_direct_to_otf_intensity_column.currentText() - == "Precursor.Normalised" - ) - assert gui.label_227.text() == "DIA-NN Intensity Column" - assert app is not None - - -def test_direct_otf_gui_keeps_editable_prefix_for_wide_tables(tmp_path): - peptide_path = tmp_path / "peptides.tsv" - pd.DataFrame( - { - "Sequence": ["PEPA"], - "Intensity_s1": [10.0], - "Intensity_s2": [20.0], - } - ).to_csv(peptide_path, sep="\t", index=False) - app = QtWidgets.QApplication.instance() or QtWidgets.QApplication([]) - gui = object.__new__(MetaXGUI) - gui.lineEdit_pep_direct_to_otf_peptide_path = QtWidgets.QLineEdit( - str(peptide_path) - ) - gui.lineEdit_pep_direct_to_otf_pep_table_sep = QtWidgets.QLineEdit("\\t") - gui.comboBox_pep_direct_to_otf_peptide_col_name = QtWidgets.QComboBox() - gui.comboBox_pep_direct_to_otf_intensity_column = QtWidgets.QComboBox() - gui.label_227 = QtWidgets.QLabel() - gui._last_pep_direct_to_otf_peptide_table_signature = "" - gui.logger = type( - "Logger", - (), - {"write_log": lambda self, message, level: None}, - )() - - gui.update_pep_direct_to_otf_peptide_table_columns() - - assert gui.comboBox_pep_direct_to_otf_intensity_column.isEditable() is True - assert ( - gui.comboBox_pep_direct_to_otf_intensity_column.currentText() - == "Intensity" - ) - assert gui.label_227.text() == "Prefix of Intensity Column" - assert app is not None - - -def test_direct_otf_gui_does_not_label_generic_parquet_as_diann(tmp_path): - peptide_path = tmp_path / "peptides.parquet" - pd.DataFrame( - { - "Sequence": ["PEPA"], - "Intensity_s1": [10.0], - "Intensity_s2": [20.0], - } - ).to_parquet(peptide_path) - app = QtWidgets.QApplication.instance() or QtWidgets.QApplication([]) - gui = object.__new__(MetaXGUI) - gui.lineEdit_pep_direct_to_otf_peptide_path = QtWidgets.QLineEdit( - str(peptide_path) - ) - gui.lineEdit_pep_direct_to_otf_pep_table_sep = QtWidgets.QLineEdit("\\t") - gui.comboBox_pep_direct_to_otf_peptide_col_name = QtWidgets.QComboBox() - gui.comboBox_pep_direct_to_otf_intensity_column = QtWidgets.QComboBox() - gui.label_227 = QtWidgets.QLabel() - gui._last_pep_direct_to_otf_peptide_table_signature = "" - gui.logger = type( - "Logger", - (), - {"write_log": lambda self, message, level: None}, - )() - - gui.update_pep_direct_to_otf_peptide_table_columns() - - assert gui.comboBox_pep_direct_to_otf_intensity_column.isEditable() is True - assert ( - gui.comboBox_pep_direct_to_otf_intensity_column.currentText() - == "Intensity" - ) - assert gui.label_227.text() == "Prefix of Intensity Column" - assert app is not None - - -def test_unit_specific_gui_validation_reports_selected_diann_intensity( - tmp_path, - monkeypatch, -): - manifest_path = _write_manifest(tmp_path) - peptide_path = tmp_path / "report.parquet" - peptide_path.write_bytes(b"parquet placeholder") - monkeypatch.setattr( - unit_specific_settings_dialog, - "_read_peptide_table_header_columns", - lambda peptide_table_path, separator: [ - "Stripped.Sequence", - "Run", - "Precursor.Normalised", - "Precursor.Quantity", - ], - ) - monkeypatch.setattr( - unit_specific_settings_dialog, - "_read_long_format_run_columns", - lambda peptide_table_path: ["s1.raw", "s2.raw", "s3.raw"], - ) - - result = validate_unit_specific_manifest_for_gui( - manifest_path=str(manifest_path), - peptide_table_path=str(peptide_path), - peptide_col="Stripped.Sequence", - diann_intensity_col="Precursor.Quantity", - ) - - assert result.ok is True - assert "intensity=Precursor.Quantity" in result.message - - -def test_validation_result_dialog_is_scrollable_and_screen_bounded(): - app = QtWidgets.QApplication.instance() or QtWidgets.QApplication([]) - result = unit_specific_settings_dialog.UnitSpecificManifestValidationResult( - ok=False, - message="Missing samples: 1000", - manifest_samples=[], - mapped_samples={}, - missing_samples=[], - details="\n".join(f"sample_{index}" for index in range(1000)), - ) - dialog = UnitSpecificValidationResultDialog(result) - - assert dialog.result_text.isReadOnly() - assert dialog.result_text.lineWrapMode() == QtWidgets.QPlainTextEdit.NoWrap - assert dialog.result_text.toPlainText().endswith("sample_999") - available = app.primaryScreen().availableGeometry() - assert dialog.width() <= available.width() - assert dialog.height() <= available.height() - - dialog.close() - app.processEvents() - - -def test_unit_specific_gui_validation_warn_skip_passes_with_missing_sample(tmp_path): - manifest_path = _write_manifest(tmp_path) - peptide_path = _write_peptide_table(tmp_path, ["Sequence", "Intensity_s1", "Intensity_s2"]) - - result = validate_unit_specific_manifest_for_gui( - manifest_path=str(manifest_path), - peptide_table_path=str(peptide_path), - peptide_col="Sequence", - peptide_table_separator="\t", - on_missing_sample="warn-skip", - ) - - assert result.ok is True - assert result.mapped_samples == {"s1": "Intensity_s1", "s2": "Intensity_s2"} - assert result.missing_samples == ["s3"] - assert "Missing samples: s3" in result.message - - -def test_unit_specific_gui_validation_fails_for_missing_peptide_column(tmp_path): - manifest_path = _write_manifest(tmp_path) - peptide_path = _write_peptide_table(tmp_path, ["Modified sequence", "Intensity_s1"]) - - result = validate_unit_specific_manifest_for_gui( - manifest_path=str(manifest_path), - peptide_table_path=str(peptide_path), - peptide_col="Sequence", - peptide_table_separator="\t", - ) - - assert result.ok is False - assert "Peptide column 'Sequence' was not found" in result.message - - -def test_unit_specific_gui_validation_marks_parquet_header_skip_as_not_checked(tmp_path, monkeypatch): - manifest_path = _write_manifest(tmp_path) - peptide_path = tmp_path / "peptides.parquet" - peptide_path.write_bytes(b"not a real parquet file") - monkeypatch.setattr( - unit_specific_settings_dialog, - "_read_peptide_table_header_columns", - lambda peptide_table_path, separator: None, - ) - - result = validate_unit_specific_manifest_for_gui( - manifest_path=str(manifest_path), - peptide_table_path=str(peptide_path), - peptide_col="Sequence", - peptide_table_separator="\t", - ) - - assert result.ok is True - assert result.mapped_samples == {} - assert result.missing_samples == [] - assert "Peptide table header validation: skipped" in result.message - assert "Mapped peptide table samples: not checked" in result.message - assert "Missing samples: not checked" in result.message - assert "Reason: parquet header metadata could not be read without pyarrow" in result.message diff --git a/tests/test_unit_specific_manifest.py b/tests/test_unit_specific_manifest.py deleted file mode 100644 index e3714e8f..00000000 --- a/tests/test_unit_specific_manifest.py +++ /dev/null @@ -1,148 +0,0 @@ -import json - -import pytest - -from metax.peptide_annotator.unit_specific_manifest import ( - load_unit_specific_manifest, - resolve_manifest_sample_columns, -) - - -def _write_manifest(tmp_path, units): - path = tmp_path / "unit_specific_manifest.json" - path.write_text( - json.dumps( - { - "schema_version": "metaumbra.unit_specific_manifest.v1", - "generated_by": {"tool": "MetaUmbra", "version": "1.3.5"}, - "default_genome_threshold": "q0.05", - "files": { - "sample_unit_mapping": "sample_unit_mapping.tsv", - "unit_call_counts": "unit_call_counts.tsv", - "unit_specific_genome_list_q005": "unit_specific_genome_list_q005.tsv", - "unit_specific_genome_list_q001": "unit_specific_genome_list_q001.tsv", - }, - "units": units, - } - ), - encoding="utf-8", - ) - return path - - -def test_load_manifest_default_and_switch_thresholds(tmp_path): - path = _write_manifest( - tmp_path, - { - "u1": { - "sample_columns": ["s1"], - "n_samples": 1, - "genome_ids_q005": ["MGYG000001456.1", "g2"], - "genome_ids_q001": ["MGYG000001456.1"], - } - }, - ) - - manifest = load_unit_specific_manifest(path) - assert manifest.selected_genome_threshold == "q0.05" - assert manifest.units["u1"].genome_ids == ["MGYG000001456.1", "g2"] - - manifest = load_unit_specific_manifest(path, genome_threshold="q001") - assert manifest.selected_genome_threshold == "q0.01" - assert manifest.units["u1"].genome_ids == ["MGYG000001456.1"] - - -def test_load_manifest_accepts_unit_aware_schema_alias(tmp_path): - path = _write_manifest( - tmp_path, - { - "u1": { - "sample_columns": ["s1"], - "n_samples": 1, - "genome_ids_q005": ["g1", "g2"], - "genome_ids_q001": ["g1"], - } - }, - ) - data = json.loads(path.read_text(encoding="utf-8")) - data["schema_version"] = "metaumbra.unit_aware_manifest.v1" - path.write_text(json.dumps(data), encoding="utf-8") - - manifest = load_unit_specific_manifest(path) - - assert manifest.schema_version == "metaumbra.unit_aware_manifest.v1" - assert manifest.units["u1"].genome_ids == ["g1", "g2"] - - -def test_manifest_rejects_duplicate_samples(tmp_path): - path = _write_manifest( - tmp_path, - { - "u1": {"sample_columns": ["s1"], "n_samples": 1, "genome_ids_q005": ["g1"], "genome_ids_q001": ["g1"]}, - "u2": {"sample_columns": ["s1"], "n_samples": 1, "genome_ids_q005": ["g2"], "genome_ids_q001": ["g2"]}, - }, - ) - with pytest.raises(ValueError, match="multiple units"): - load_unit_specific_manifest(path) - - -def test_manifest_subset_warning_or_error(tmp_path): - path = _write_manifest( - tmp_path, - { - "u1": { - "sample_columns": ["s1"], - "n_samples": 1, - "genome_ids_q005": ["g1"], - "genome_ids_q001": ["g1", "g2"], - } - }, - ) - with pytest.raises(ValueError, match="not present"): - load_unit_specific_manifest(path, strict=True) - with pytest.warns(UserWarning, match="not present"): - load_unit_specific_manifest(path, strict=False) - - -def test_resolve_manifest_sample_columns_auto_modes(): - peptide_columns = [ - "s_exact", - "Intensity_s_prefixed", - "Intensity_strip", - "_leading", - r"C:\data\basename.raw", - ] - mapping = resolve_manifest_sample_columns( - peptide_columns=peptide_columns, - manifest_sample_columns=["s_exact", "s_prefixed", "strip", "leading", "basename"], - ) - assert mapping["s_exact"] == "s_exact" - assert mapping["s_prefixed"] == "Intensity_s_prefixed" - assert mapping["strip"] == "Intensity_strip" - assert mapping["leading"] == "_leading" - assert mapping["basename"] == r"C:\data\basename.raw" - - -def test_resolve_manifest_sample_columns_input_sample_prefix(): - mapping = resolve_manifest_sample_columns( - peptide_columns=["LFQ intensity s1", "LFQ intensity s2"], - manifest_sample_columns=["s1", "s2"], - input_sample_col_prefix="LFQ intensity ", - ) - assert mapping == { - "s1": "LFQ intensity s1", - "s2": "LFQ intensity s2", - } - - -def test_resolve_manifest_sample_columns_missing_and_ambiguous(): - with pytest.raises(ValueError, match="not found"): - resolve_manifest_sample_columns(["Intensity_a"], ["missing"]) - - with pytest.raises(ValueError, match="ambiguous"): - resolve_manifest_sample_columns([r"C:\a\s.raw", r"D:\b\s.mzML"], ["s"]) - - -def test_resolve_manifest_sample_columns_rejects_duplicate_column_matches(): - with pytest.raises(ValueError, match="distinct peptide table columns"): - resolve_manifest_sample_columns(["s1"], ["s1", "s1.raw"]) diff --git a/tests/test_unit_specific_otf.py b/tests/test_unit_specific_otf.py deleted file mode 100644 index 913ce001..00000000 --- a/tests/test_unit_specific_otf.py +++ /dev/null @@ -1,1800 +0,0 @@ -import json -import sqlite3 -import inspect -from pathlib import Path - -import pandas as pd -import pytest - -import metax.cli.annotate as annotate_cli -from metax.cli.annotate import build_parser -from metax.peptide_annotator.unit_specific_otf import ( - UnitSpecificOTFAnnotator, - UnitSpecificOTFRunResult, - _create_temporary_unit_directory, - build_global_unit_specific_peptide_protein_map, -) -from metax.peptide_annotator.pep_table_to_otf import ( - query_peptide_proteins_from_digested_genome_folders, - query_peptide_proteins_from_digested_genome_folders_nested, -) - - -def _write_manifest(path): - path.write_text( - json.dumps( - { - "schema_version": "metaumbra.unit_specific_manifest.v1", - "generated_by": {"tool": "MetaUmbra", "version": "1.3.5"}, - "default_genome_threshold": "q0.05", - "files": {}, - "units": { - "u1": { - "sample_columns": ["s1"], - "n_samples": 1, - "genome_ids_q005": ["g1"], - "genome_ids_q001": ["g1"], - }, - "u2": { - "sample_columns": ["s2"], - "n_samples": 1, - "genome_ids_q005": ["g2"], - "genome_ids_q001": ["g2"], - }, - }, - } - ), - encoding="utf-8", - ) - - -def _write_peptide_protein_db(path): - with sqlite3.connect(path) as conn: - conn.execute("CREATE TABLE peptide_proteins (peptide TEXT PRIMARY KEY, proteins TEXT)") - conn.execute("INSERT INTO peptide_proteins VALUES (?, ?)", ("PEPA", json.dumps(["g1_p1", "g2_p2"]))) - conn.execute("INSERT INTO peptide_proteins VALUES (?, ?)", ("PEPB", json.dumps(["g2_p3"]))) - conn.commit() - - -def _write_taxafunc_db(path): - with sqlite3.connect(path) as conn: - conn.execute("CREATE TABLE id2taxa (ID TEXT PRIMARY KEY, Taxa TEXT)") - conn.execute("CREATE TABLE id2annotation (ID TEXT PRIMARY KEY, KEGG_ko TEXT)") - conn.executemany( - "INSERT INTO id2taxa VALUES (?, ?)", - [ - ("g1", "d__Bacteria;p__P1;c__C1;o__O1;f__F1;g__G1;s__S1"), - ("g2", "d__Bacteria;p__P2;c__C2;o__O2;f__F2;g__G2;s__S2"), - ], - ) - conn.executemany( - "INSERT INTO id2annotation VALUES (?, ?)", - [ - ("g1_p1", "K00001"), - ("g2_p2", "K00002"), - ("g2_p3", "K00003"), - ], - ) - conn.commit() - - -def test_unit_specific_otf_builds_units_and_artifacts(monkeypatch, tmp_path, capsys): - calls = [] - numeric_apply_calls = 0 - original_dataframe_apply = pd.DataFrame.apply - - def tracking_dataframe_apply(self, func, *args, **kwargs): - nonlocal numeric_apply_calls - if func is pd.to_numeric: - numeric_apply_calls += 1 - return original_dataframe_apply(self, func, *args, **kwargs) - - monkeypatch.setattr(pd.DataFrame, "apply", tracking_dataframe_apply) - - class FakeMapper: - def __init__(self, **kwargs): - self.kwargs = kwargs - self.peptide_df = kwargs["peptide_df"] - self.final_peptide_table = self.peptide_df.copy() - self.peptides_after_mapping = len(self.peptide_df) - self.selected_genomes_num = len(kwargs["genome_list"]) - assert all( - Path(previous_call["output_path"]).parent.is_dir() - for previous_call in calls - ) - assert all( - Path(previous_call["output_path"]).is_file() - for previous_call in calls - ) - calls.append(kwargs) - - def all_in_one(self, **kwargs): - assert kwargs["save_output"] is False - genome = self.kwargs["genome_list"][0] - sample_cols = [c for c in self.peptide_df.columns if c.startswith("Intensity_")] - return pd.DataFrame( - { - "Sequence": self.peptide_df["Sequence"], - "Proteins": [f"{genome}_p1"] * len(self.peptide_df), - "LCA_level": ["genome"] * len(self.peptide_df), - "Taxon": [f"d__Bacteria|m__{genome}"] * len(self.peptide_df), - "Taxon_prop": [1.0] * len(self.peptide_df), - "None_func": ["none_func"] * len(self.peptide_df), - "None_func_prop": [1.0] * len(self.peptide_df), - **{col: self.peptide_df[col].tolist() for col in sample_cols}, - } - ) - - monkeypatch.setattr("metax.peptide_annotator.unit_specific_otf.peptideProteinsMapper", FakeMapper) - - peptide_table = tmp_path / "peptides.tsv" - pd.DataFrame( - { - "Sequence": ["PEPA", "PEPB"], - "s1": [10, 0], - "s2": [20, 5], - } - ).to_csv(peptide_table, sep="\t", index=False) - manifest = tmp_path / "unit_specific_manifest.json" - _write_manifest(manifest) - taxafunc_db = tmp_path / "taxafunc.db" - taxafunc_db.write_text("", encoding="utf-8") - peptide_db = tmp_path / "peptide.db" - peptide_db.write_text("", encoding="utf-8") - output = tmp_path / "OTF_unit_specific.tsv" - - result = UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(output), - db_path=str(peptide_db), - peptide_col="Sequence", - duplicate_peptide_handling_mode="max", - ).run(return_dataframe=True) - - assert [call["genome_list"] for call in calls] == [["g1"], ["g2"]] - assert calls[0]["selected_genomes_set"] == {"g1"} - assert calls[1]["selected_genomes_set"] == {"g2"} - assert [call["duplicate_peptide_handling_mode"] for call in calls] == ["max", "max"] - assert numeric_apply_calls == 1 - expected_temp_root = ( - tmp_path / "OTF_unit_specific_artifacts" / "per_unit" / "unit_otf" - ) - assert all( - Path(call["output_path"]).parent.parent == expected_temp_root - for call in calls - ) - assert [Path(call["output_path"]).parent.name for call in calls] == [ - "run_u1", - "run_u2", - ] - assert expected_temp_root.is_dir() - assert list(expected_temp_root.iterdir()) == [] - assert "UnitSpecificSequence" not in result.columns - assert result[["analysis_unit_id", "Sequence"]].drop_duplicates().values.tolist() == [ - ["u1", "PEPA"], - ["u2", "PEPA"], - ["u2", "PEPB"], - ] - assert result.loc[result["analysis_unit_id"] == "u1", "Intensity_s2"].isna().all() - assert result.loc[result["analysis_unit_id"] == "u2", "Intensity_s1"].isna().all() - assert output.is_file() - info_path = tmp_path / "OTF_unit_specific_info.txt" - assert info_path.is_file() - info_text = info_path.read_text(encoding="utf-8") - assert "MetaX PeptideAnnotator Results" in info_text - assert "Software: MetaX (UnitSpecificOTFAnnotator)" in info_text - assert "Completed units: 2" in info_text - assert "Shape: 3 rows" in info_text - assert (tmp_path / "OTF_unit_specific_artifacts" / "unit_sample_column_mapping.tsv").is_file() - assert (tmp_path / "OTF_unit_specific_artifacts" / "unit_annotation_summary.tsv").is_file() - progress_log = capsys.readouterr().out - assert "[Unit-specific] Preparing annotation for 2 units" in progress_log - assert "[Unit-specific] Unit 1 of 2: u1 started" in progress_log - assert "[Unit-specific] Unit 1 of 2: u1 completed" in progress_log - assert "[Unit-specific] Unit 2 of 2: u2 started" in progress_log - assert "[Unit-specific] Unit 2 of 2: u2 completed" in progress_log - assert "2 units total, 2 completed, 0 skipped" in progress_log - - -def test_temporary_unit_directory_adds_timestamp_only_on_conflict(tmp_path): - parent = tmp_path / "unit_otf" - parent.mkdir() - - run_dir = _create_temporary_unit_directory(parent, "u1") - assert run_dir.name == "run_u1" - assert run_dir.is_dir() - - conflicting_run_dir = _create_temporary_unit_directory(parent, "u1") - assert conflicting_run_dir.name.startswith("run_u1_") - assert conflicting_run_dir.name != "run_u1" - assert conflicting_run_dir.is_dir() - - -def test_default_run_does_not_read_merged_output(monkeypatch, tmp_path): - peptide_table = tmp_path / "peptides.tsv" - pd.DataFrame( - { - "Sequence": ["PEPA", "PEPB"], - "Intensity_s1": [10, 0], - "Intensity_s2": [20, 5], - } - ).to_csv(peptide_table, sep="\t", index=False) - manifest = tmp_path / "unit_specific_manifest.json" - _write_manifest(manifest) - peptide_db = tmp_path / "peptide.db" - _write_peptide_protein_db(peptide_db) - taxafunc_db = tmp_path / "taxafunc.db" - _write_taxafunc_db(taxafunc_db) - output = tmp_path / "OTF_unit_specific.tsv" - - original_read_csv = pd.read_csv - - def guarded_read_csv(path, *args, **kwargs): - if Path(path) == output: - raise AssertionError("default run must not read the merged output") - return original_read_csv(path, *args, **kwargs) - - monkeypatch.setattr( - "metax.peptide_annotator.unit_specific_otf.pd.read_csv", - guarded_read_csv, - ) - - result = UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(output), - db_path=str(peptide_db), - ).run() - - assert isinstance(result, UnitSpecificOTFRunResult) - assert result.output_path == str(output) - assert result.rows == 3 - assert result.column_count == len(result.column_names) - assert result.column_names == original_read_csv( - output, - sep="\t", - nrows=0, - ).columns.tolist() - assert result.completed_units == 2 - assert result.skipped_units == 0 - assert result.selected_genome_threshold == "q0.05" - assert "KEGG_ko" in result.column_names - assert "KEGG_ko_prop" in result.column_names - info_text = Path(result.info_path).read_text(encoding="utf-8") - assert "Unique sequences: NA" in info_text - assert "Unique protein groups: NA" in info_text - - -def test_return_dataframe_explicitly_reads_merged_output(monkeypatch, tmp_path): - peptide_table = tmp_path / "peptides.tsv" - pd.DataFrame( - { - "Sequence": ["PEPA"], - "Intensity_s1": [10], - "Intensity_s2": [5], - } - ).to_csv(peptide_table, sep="\t", index=False) - manifest = tmp_path / "unit_specific_manifest.json" - _write_manifest(manifest) - peptide_db = tmp_path / "peptide.db" - _write_peptide_protein_db(peptide_db) - taxafunc_db = tmp_path / "taxafunc.db" - _write_taxafunc_db(taxafunc_db) - output = tmp_path / "OTF_unit_specific.tsv" - - output_reads = 0 - original_read_csv = pd.read_csv - - def tracking_read_csv(path, *args, **kwargs): - nonlocal output_reads - if Path(path) == output: - output_reads += 1 - return original_read_csv(path, *args, **kwargs) - - monkeypatch.setattr( - "metax.peptide_annotator.unit_specific_otf.pd.read_csv", - tracking_read_csv, - ) - - result = UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(output), - db_path=str(peptide_db), - ).run(return_dataframe=True) - - assert isinstance(result, pd.DataFrame) - assert output_reads == 1 - assert result["analysis_unit_id"].tolist() == ["u1", "u2"] - - -def test_stream_merge_unit_outputs_is_chunked_and_fills_columns(tmp_path): - peptide_table = tmp_path / "peptides.tsv" - manifest = tmp_path / "manifest.json" - taxafunc_db = tmp_path / "taxafunc.db" - peptide_db = tmp_path / "peptide.db" - for path in [peptide_table, manifest, taxafunc_db, peptide_db]: - path.write_text("", encoding="utf-8") - - annotator = UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(tmp_path / "merged.tsv"), - db_path=str(peptide_db), - ) - unit1 = tmp_path / "u1.tsv" - unit2 = tmp_path / "u2.tsv" - pd.DataFrame( - { - "analysis_unit_id": ["u1", "u1"], - "Sequence": ["PEPA", "PEPB"], - "Proteins": ["g1_p1", "g1_p2"], - "Intensity_s1": [10, 20], - } - ).to_csv(unit1, sep="\t", index=False) - pd.DataFrame( - { - "analysis_unit_id": ["u2"], - "Sequence": ["PEPC"], - "Proteins": ["g2_p1"], - "Mock_func": ["K00001"], - "Intensity_s2": [30], - } - ).to_csv(unit2, sep="\t", index=False) - records = [ - { - "path": str(unit1), - "columns": pd.read_csv(unit1, sep="\t", nrows=0).columns.tolist(), - "temporary": False, - }, - { - "path": str(unit2), - "columns": pd.read_csv(unit2, sep="\t", nrows=0).columns.tolist(), - "temporary": True, - }, - ] - - columns, rows = annotator._stream_merge_unit_outputs( - records, - ["Intensity_s1", "Intensity_s2"], - merge_chunksize=1, - collect_unique_stats=True, - ) - - merged = pd.read_csv(annotator.output_path, sep="\t") - assert rows == 3 - assert annotator._last_unique_sequences == 3 - assert annotator._last_unique_protein_groups == 3 - assert merged.columns.tolist() == columns - assert columns == [ - "analysis_unit_id", - "Sequence", - "Proteins", - "Mock_func", - "Intensity_s1", - "Intensity_s2", - ] - assert merged.loc[merged["analysis_unit_id"] == "u1", "Intensity_s2"].isna().all() - assert merged.loc[merged["analysis_unit_id"] == "u2", "Intensity_s1"].isna().all() - assert merged.loc[merged["analysis_unit_id"] == "u1", "Mock_func"].isna().all() - raw_rows = annotator.output_path.read_text(encoding="utf-8").splitlines() - assert raw_rows[1].endswith("\t10\t") - assert raw_rows[3].endswith("\t\t30") - assert ( - annotator.output_path.read_text(encoding="utf-8").count("analysis_unit_id") - == 1 - ) - assert unit1.is_file() - assert not unit2.exists() - - -def test_stream_merge_unit_outputs_supports_temporary_pickle(tmp_path): - peptide_table = tmp_path / "peptides.tsv" - manifest = tmp_path / "manifest.json" - taxafunc_db = tmp_path / "taxafunc.db" - peptide_db = tmp_path / "peptide.db" - for path in [peptide_table, manifest, taxafunc_db, peptide_db]: - path.write_text("", encoding="utf-8") - - annotator = UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(tmp_path / "merged.tsv"), - db_path=str(peptide_db), - ) - unit_path = tmp_path / "u1.pkl" - unit_frame = pd.DataFrame( - { - "analysis_unit_id": ["u1", "u1"], - "Sequence": ["PEPA", "PEPB"], - "Proteins": ["g1_p1", "g1_p2"], - "Intensity_s1": [10, 20], - } - ) - unit_frame.to_pickle(unit_path) - - columns, rows = annotator._stream_merge_unit_outputs( - [ - { - "path": str(unit_path), - "columns": unit_frame.columns.tolist(), - "temporary": True, - "format": "pickle", - } - ], - ["Intensity_s1"], - merge_chunksize=1, - ) - - merged = pd.read_csv(annotator.output_path, sep="\t") - assert rows == 2 - assert merged.columns.tolist() == columns - assert merged["Sequence"].tolist() == ["PEPA", "PEPB"] - assert not unit_path.exists() - - -def test_add_unit_protein_mapping_preserves_manifest_genome_order(tmp_path): - peptide_table = tmp_path / "peptides.tsv" - manifest = tmp_path / "manifest.json" - taxafunc_db = tmp_path / "taxafunc.db" - peptide_db = tmp_path / "peptide.db" - for path in [peptide_table, manifest, taxafunc_db, peptide_db]: - path.write_text("", encoding="utf-8") - annotator = UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(tmp_path / "out.tsv"), - db_path=str(peptide_db), - ) - - result = annotator._add_unit_protein_mapping( - pd.DataFrame({"Sequence": ["PEPA"]}), - { - "PEPA": { - "g3": {"g3_p3"}, - "g1": {"g1_p1"}, - "g2": {"g2_p2"}, - } - }, - ["g2", "g1"], - ) - - assert result["Proteins"].tolist() == ["g1_p1;g2_p2"] - assert result["Genomes"].tolist() == ["g2;g1"] - - -def test_saved_per_unit_output_is_final_unit_specific_table(tmp_path): - peptide_table = tmp_path / "peptides.tsv" - pd.DataFrame( - { - "Sequence": ["PEPA"], - "Intensity_s1": [10], - "Intensity_s2": [0], - } - ).to_csv(peptide_table, sep="\t", index=False) - manifest = tmp_path / "unit_specific_manifest.json" - _write_manifest(manifest) - peptide_db = tmp_path / "peptide.db" - _write_peptide_protein_db(peptide_db) - taxafunc_db = tmp_path / "taxafunc.db" - _write_taxafunc_db(taxafunc_db) - - UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(tmp_path / "out.tsv"), - db_path=str(peptide_db), - save_per_unit_outputs=True, - include_unit_specific_sequence=True, - ).run() - - per_unit_paths = list((tmp_path / "out_artifacts" / "per_unit").glob("u1_*_OTF.tsv")) - assert len(per_unit_paths) == 1 - per_unit_path = per_unit_paths[0] - per_unit = pd.read_csv(per_unit_path, sep="\t") - assert per_unit["analysis_unit_id"].tolist() == ["u1"] - assert per_unit["UnitSpecificSequence"].tolist() == ["u1||PEPA"] - temporary_unit_root = ( - tmp_path / "out_artifacts" / "per_unit" / "unit_otf" - ) - assert temporary_unit_root.is_dir() - assert list(temporary_unit_root.iterdir()) == [] - - -@pytest.mark.parametrize("intensity_col", ["Precursor.Normalised", "Precursor.Quantity"]) -def test_unit_specific_otf_prepares_diann_parquet_with_shared_alias_rules( - tmp_path, - intensity_col, -): - peptide_table = tmp_path / "report.parquet" - pd.DataFrame( - { - "Run": ["s1.raw", "s2.raw"], - "Stripped.Sequence": ["PEPA", "PEPA"], - intensity_col: [10.0, 20.0], - } - ).to_parquet(peptide_table) - manifest = tmp_path / "unit_specific_manifest.json" - _write_manifest(manifest) - taxafunc_db = tmp_path / "taxafunc.db" - taxafunc_db.write_text("", encoding="utf-8") - digested_dir = tmp_path / "digested" - digested_dir.mkdir() - - annotator = UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(tmp_path / "OTF.tsv"), - digested_genome_folders=str(digested_dir), - peptide_col="Stripped.Sequence", - ) - - prepared = annotator._read_peptide_table(["s1", "s2"]) - - assert annotator.peptide_col == "Stripped.Sequence" - assert prepared.loc[0, "s1"] == 10.0 - assert prepared.loc[0, "s2"] == 20.0 - assert annotator._last_sample_mapping == {"s1": "s1", "s2": "s2"} - assert annotator.peptide_table_prepare_metadata["diann_intensity_column"] == intensity_col - - -def test_unit_specific_otf_uses_selected_diann_intensity_column(tmp_path): - peptide_table = tmp_path / "report.parquet" - pd.DataFrame( - { - "Run": ["s1.raw", "s2.raw"], - "Stripped.Sequence": ["PEPA", "PEPA"], - "Precursor.Normalised": [10.0, 20.0], - "Precursor.Quantity": [100.0, 200.0], - } - ).to_parquet(peptide_table) - manifest = tmp_path / "unit_specific_manifest.json" - _write_manifest(manifest) - taxafunc_db = tmp_path / "taxafunc.db" - taxafunc_db.write_text("", encoding="utf-8") - digested_dir = tmp_path / "digested" - digested_dir.mkdir() - - annotator = UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(tmp_path / "OTF.tsv"), - digested_genome_folders=str(digested_dir), - peptide_col="Stripped.Sequence", - diann_intensity_col="Precursor.Quantity", - ) - - prepared = annotator._read_peptide_table(["s1", "s2"]) - - assert prepared.loc[0, "s1"] == 100.0 - assert prepared.loc[0, "s2"] == 200.0 - assert ( - annotator.peptide_table_prepare_metadata["diann_intensity_column"] - == "Precursor.Quantity" - ) - - -def test_unit_specific_otf_diann_mapping_preserves_duplicate_run_basenames(tmp_path): - peptide_table = tmp_path / "report.parquet" - pd.DataFrame( - { - "Run": [r"C:\batch1\s1.raw", r"C:\batch2\s1.raw"], - "Stripped.Sequence": ["PEPA", "PEPA"], - "Precursor.Quantity": [10.0, 20.0], - } - ).to_parquet(peptide_table) - manifest = tmp_path / "unit_specific_manifest.json" - manifest.write_text( - json.dumps( - { - "schema_version": "metaumbra.unit_specific_manifest.v1", - "generated_by": {"tool": "MetaUmbra", "version": "1.3.7"}, - "default_genome_threshold": "q0.05", - "files": {}, - "units": { - "u1": { - "sample_columns": [r"C:\batch1\s1.raw"], - "n_samples": 1, - "genome_ids_q005": ["g1"], - "genome_ids_q001": ["g1"], - }, - "u2": { - "sample_columns": [r"C:\batch2\s1.raw"], - "n_samples": 1, - "genome_ids_q005": ["g2"], - "genome_ids_q001": ["g2"], - }, - }, - } - ), - encoding="utf-8", - ) - taxafunc_db = tmp_path / "taxafunc.db" - taxafunc_db.write_text("", encoding="utf-8") - digested_dir = tmp_path / "digested" - digested_dir.mkdir() - - annotator = UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(tmp_path / "OTF.tsv"), - digested_genome_folders=str(digested_dir), - peptide_col="Stripped.Sequence", - diann_intensity_col="Precursor.Quantity", - ) - - prepared = annotator._read_peptide_table([r"C:\batch1\s1.raw", r"C:\batch2\s1.raw"]) - - assert prepared.loc[0, "s1"] == 10.0 - assert prepared.loc[0, "s1_2"] == 20.0 - assert annotator._last_sample_mapping == { - r"C:\batch1\s1.raw": "s1", - r"C:\batch2\s1.raw": "s1_2", - } - - -def test_unit_specific_otf_diann_mapping_prefers_prepared_columns(tmp_path): - peptide_table = tmp_path / "report.parquet" - pd.DataFrame( - { - "Run": [r"C:\batch1\s1.raw", r"C:\batch2\s1.raw"], - "Stripped.Sequence": ["PEPA", "PEPA"], - "Precursor.Quantity": [10.0, 20.0], - } - ).to_parquet(peptide_table) - manifest = tmp_path / "unit_specific_manifest.json" - manifest.write_text( - json.dumps( - { - "schema_version": "metaumbra.unit_specific_manifest.v1", - "generated_by": {"tool": "MetaUmbra", "version": "1.3.7"}, - "default_genome_threshold": "q0.05", - "files": {}, - "units": { - "u1": { - "sample_columns": ["s1"], - "n_samples": 1, - "genome_ids_q005": ["g1"], - "genome_ids_q001": ["g1"], - }, - "u2": { - "sample_columns": ["s1_2"], - "n_samples": 1, - "genome_ids_q005": ["g2"], - "genome_ids_q001": ["g2"], - }, - }, - } - ), - encoding="utf-8", - ) - taxafunc_db = tmp_path / "taxafunc.db" - taxafunc_db.write_text("", encoding="utf-8") - digested_dir = tmp_path / "digested" - digested_dir.mkdir() - - annotator = UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(tmp_path / "OTF.tsv"), - digested_genome_folders=str(digested_dir), - peptide_col="Stripped.Sequence", - diann_intensity_col="Precursor.Quantity", - ) - - prepared = annotator._read_peptide_table(["s1", "s1_2"]) - - assert prepared.loc[0, "s1"] == 10.0 - assert prepared.loc[0, "s1_2"] == 20.0 - assert annotator._last_sample_mapping == { - "s1": "s1", - "s1_2": "s1_2", - } - - -def test_unit_specific_otf_keeps_wide_parquet_compatible(tmp_path): - peptide_table = tmp_path / "wide.parquet" - pd.DataFrame( - { - "Sequence": ["PEPA"], - "Intensity_s1": [10.0], - "Intensity_s2": [20.0], - } - ).to_parquet(peptide_table) - manifest = tmp_path / "unit_specific_manifest.json" - _write_manifest(manifest) - taxafunc_db = tmp_path / "taxafunc.db" - taxafunc_db.write_text("", encoding="utf-8") - digested_dir = tmp_path / "digested" - digested_dir.mkdir() - - annotator = UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(tmp_path / "OTF.tsv"), - digested_genome_folders=str(digested_dir), - ) - - prepared = annotator._read_peptide_table() - - assert prepared.columns.tolist() == ["Sequence", "Intensity_s1", "Intensity_s2"] - assert annotator.peptide_table_prepare_metadata["input_peptide_table_format"] == "parquet" - - -def test_unit_specific_otf_reads_only_required_tsv_columns_with_prefix_aliases( - monkeypatch, - tmp_path, -): - peptide_table = tmp_path / "wide.tsv" - pd.DataFrame( - { - "Sequence": ["PEPA"], - "Input_s1": [10.0], - "Input_s2": [20.0], - "Irrelevant": ["unused"], - } - ).to_csv(peptide_table, sep="\t", index=False) - manifest = tmp_path / "unit_specific_manifest.json" - _write_manifest(manifest) - taxafunc_db = tmp_path / "taxafunc.db" - taxafunc_db.write_text("", encoding="utf-8") - digested_dir = tmp_path / "digested" - digested_dir.mkdir() - read_calls = [] - original_read_csv = pd.read_csv - - def tracking_read_csv(path, *args, **kwargs): - if Path(path) == peptide_table: - read_calls.append(dict(kwargs)) - return original_read_csv(path, *args, **kwargs) - - monkeypatch.setattr( - "metax.peptide_annotator.unit_specific_otf.pd.read_csv", - tracking_read_csv, - ) - annotator = UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(tmp_path / "OTF.tsv"), - digested_genome_folders=str(digested_dir), - input_sample_col_prefix="Input_", - ) - - prepared = annotator._read_peptide_table(["s1", "s2"]) - - assert prepared.columns.tolist() == ["Sequence", "Input_s1", "Input_s2"] - assert annotator._last_sample_mapping == { - "s1": "Input_s1", - "s2": "Input_s2", - } - assert read_calls[0]["nrows"] == 0 - assert read_calls[1]["usecols"] == ["Sequence", "Input_s1", "Input_s2"] - - -def test_unit_specific_otf_reads_only_required_wide_parquet_columns( - monkeypatch, - tmp_path, -): - peptide_table = tmp_path / "wide.parquet" - pd.DataFrame( - { - "Sequence": ["PEPA"], - "Intensity_s1": [10.0], - "Intensity_s2": [20.0], - "Irrelevant": ["unused"], - } - ).to_parquet(peptide_table) - manifest = tmp_path / "unit_specific_manifest.json" - _write_manifest(manifest) - taxafunc_db = tmp_path / "taxafunc.db" - taxafunc_db.write_text("", encoding="utf-8") - digested_dir = tmp_path / "digested" - digested_dir.mkdir() - read_calls = [] - original_read_parquet = pd.read_parquet - - def tracking_read_parquet(path, *args, **kwargs): - if Path(path) == peptide_table: - read_calls.append(dict(kwargs)) - return original_read_parquet(path, *args, **kwargs) - - monkeypatch.setattr( - "metax.peptide_annotator.unit_specific_otf.pd.read_parquet", - tracking_read_parquet, - ) - annotator = UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(tmp_path / "OTF.tsv"), - digested_genome_folders=str(digested_dir), - ) - - prepared = annotator._read_peptide_table(["s1", "s2"]) - - assert prepared.columns.tolist() == ["Sequence", "Intensity_s1", "Intensity_s2"] - assert read_calls == [ - {"columns": ["Sequence", "Intensity_s1", "Intensity_s2"]} - ] - - -def test_build_unit_dataframe_filters_before_copy_and_preserves_column_order(tmp_path): - peptide_table = tmp_path / "peptides.tsv" - manifest = tmp_path / "manifest.json" - taxafunc_db = tmp_path / "taxafunc.db" - peptide_db = tmp_path / "peptide.db" - for path in [peptide_table, manifest, taxafunc_db, peptide_db]: - path.write_text("", encoding="utf-8") - annotator = UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(tmp_path / "out.tsv"), - db_path=str(peptide_db), - ) - peptide_df = pd.DataFrame( - { - "Sequence": ["ZERO", "S2_ONLY", "BOTH"], - "input_s1": [0, 0, 2], - "input_s2": [0, 3, 4], - "unused": [1, 1, 1], - } - ) - - unit_df, canonical_cols = annotator._build_unit_dataframe( - peptide_df, - ["s2", "s1"], - {"s1": "input_s1", "s2": "input_s2"}, - ) - - assert canonical_cols == ["Intensity_s2", "Intensity_s1"] - assert unit_df.columns.tolist() == [ - "Sequence", - "Intensity_s2", - "Intensity_s1", - ] - assert unit_df["Sequence"].tolist() == ["S2_ONLY", "BOTH"] - assert unit_df[["Intensity_s2", "Intensity_s1"]].values.tolist() == [ - [3, 0], - [4, 2], - ] - - -def test_warn_skip_missing_sample_writes_sparse_canonical_column(tmp_path): - peptide_table = tmp_path / "peptides.tsv" - pd.DataFrame( - { - "Sequence": ["PEPA"], - "Intensity_s1": [10], - "Irrelevant": ["unused"], - } - ).to_csv(peptide_table, sep="\t", index=False) - manifest = tmp_path / "unit_specific_manifest.json" - _write_manifest(manifest) - peptide_db = tmp_path / "peptide.db" - _write_peptide_protein_db(peptide_db) - taxafunc_db = tmp_path / "taxafunc.db" - _write_taxafunc_db(taxafunc_db) - - with pytest.warns(UserWarning) as warning_records: - result = UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(tmp_path / "out.tsv"), - db_path=str(peptide_db), - on_missing_sample="warn-skip", - ).run(return_dataframe=True) - - assert any( - "Manifest sample 's2' was not found" in str(record.message) - for record in warning_records - ) - assert result["analysis_unit_id"].tolist() == ["u1"] - assert result.columns[-2:].tolist() == ["Intensity_s1", "Intensity_s2"] - assert result["Intensity_s2"].isna().all() - - -def test_unit_specific_otf_defaults_and_path_validation(tmp_path): - peptide_table = tmp_path / "peptides.tsv" - pd.DataFrame({"Sequence": ["PEPA"], "Intensity_s1": [10]}).to_csv(peptide_table, sep="\t", index=False) - manifest = tmp_path / "unit_specific_manifest.json" - _write_manifest(manifest) - taxafunc_db = tmp_path / "taxafunc.db" - taxafunc_db.write_text("", encoding="utf-8") - peptide_db = tmp_path / "peptide.db" - peptide_db.write_text("", encoding="utf-8") - - annotator = UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(tmp_path / "out.tsv"), - db_path=str(peptide_db), - ) - assert annotator.distinct_genome_threshold == 0 - - parser = build_parser() - args = parser.parse_args(["--mode", "unit-specific"]) - assert args.distinct_genome_threshold == 0 - assert args.duplicate_peptide_handling_mode == "sum" - assert args.include_unit_specific_sequence is False - assert args.output_sample_col_prefix == "Intensity_" - - with pytest.raises(ValueError, match="duplicate_peptide_handling_mode"): - UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(tmp_path / "out.tsv"), - db_path=str(peptide_db), - duplicate_peptide_handling_mode="bad-mode", - ) - - with pytest.raises(ValueError, match="n_jobs"): - UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(tmp_path / "out.tsv"), - db_path=str(peptide_db), - n_jobs=0, - ) - - with pytest.raises(FileNotFoundError, match="db_path"): - UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(tmp_path / "out.tsv"), - db_path=str(tmp_path / "missing.db"), - ) - - not_a_dir = tmp_path / "not_a_dir.tsv" - not_a_dir.write_text("", encoding="utf-8") - with pytest.raises(FileNotFoundError, match="digested_genome_folders"): - UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(tmp_path / "out.tsv"), - digested_genome_folders=[str(not_a_dir)], - ) - - -def test_unit_specific_otf_real_sqlite_integration(tmp_path): - peptide_table = tmp_path / "peptides.tsv" - pd.DataFrame( - { - "Sequence": ["PEPA", "PEPB"], - "Intensity_s1": [10, 0], - "Intensity_s2": [20, 5], - } - ).to_csv(peptide_table, sep="\t", index=False) - manifest = tmp_path / "unit_specific_manifest.json" - _write_manifest(manifest) - peptide_db = tmp_path / "peptide.db" - _write_peptide_protein_db(peptide_db) - taxafunc_db = tmp_path / "taxafunc.db" - _write_taxafunc_db(taxafunc_db) - output = tmp_path / "OTF_unit_specific.tsv" - - result = UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(output), - db_path=str(peptide_db), - peptide_col="Sequence", - ).run(return_dataframe=True) - - assert "UnitSpecificSequence" not in result.columns - assert result["Sequence"].tolist() == ["PEPA", "PEPA", "PEPB"] - assert result.loc[result["analysis_unit_id"] == "u1", "Proteins"].tolist() == ["g1_p1"] - assert result.loc[result["analysis_unit_id"] == "u2", "Proteins"].tolist() == ["g2_p2", "g2_p3"] - assert output.is_file() - - -def test_unit_specific_otf_can_include_unit_specific_sequence_for_debug(tmp_path, capsys): - peptide_table = tmp_path / "peptides.tsv" - pd.DataFrame({"Sequence": ["PEPA"], "Intensity_s1": [10], "Intensity_s2": [0]}).to_csv( - peptide_table, - sep="\t", - index=False, - ) - manifest = tmp_path / "unit_specific_manifest.json" - _write_manifest(manifest) - peptide_db = tmp_path / "peptide.db" - _write_peptide_protein_db(peptide_db) - taxafunc_db = tmp_path / "taxafunc.db" - _write_taxafunc_db(taxafunc_db) - - result = UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(tmp_path / "OTF_unit_specific.tsv"), - db_path=str(peptide_db), - include_unit_specific_sequence=True, - ).run(return_dataframe=True) - - assert "UnitSpecificSequence" in result.columns - assert result["UnitSpecificSequence"].tolist() == ["u1||PEPA"] - progress_log = capsys.readouterr().out - assert "[Unit-specific] Unit 2 of 2: u2 skipped" in progress_log - assert "2 units total, 1 completed, 1 skipped" in progress_log - - -def test_unit_specific_run_summary_records_sparse_zero_output(tmp_path): - peptide_table = tmp_path / "peptides.tsv" - pd.DataFrame( - { - "Sequence": ["PEPA", "PEPB"], - "Intensity_s1": [10, 0], - "Intensity_s2": [0, 20], - } - ).to_csv(peptide_table, sep="\t", index=False) - manifest = tmp_path / "unit_specific_manifest.json" - _write_manifest(manifest) - peptide_db = tmp_path / "peptide.db" - _write_peptide_protein_db(peptide_db) - taxafunc_db = tmp_path / "taxafunc.db" - _write_taxafunc_db(taxafunc_db) - output = tmp_path / "OTF_unit_specific.tsv" - - result = UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(output), - db_path=str(peptide_db), - ).run(return_dataframe=True) - - assert result.loc[result["analysis_unit_id"] == "u1", "Intensity_s2"].isna().all() - assert result.loc[result["analysis_unit_id"] == "u2", "Intensity_s1"].isna().all() - summary = json.loads( - (tmp_path / "OTF_unit_specific_artifacts" / "run_summary.json").read_text( - encoding="utf-8" - ) - ) - assert summary["sparse_zero_intensity_output"] is True - info_text = (tmp_path / "OTF_unit_specific_info.txt").read_text(encoding="utf-8") - assert "sparse_zero_intensity_output: True" in info_text - - -def _write_shared_genome_manifest(path): - path.write_text( - json.dumps( - { - "schema_version": "metaumbra.unit_specific_manifest.v1", - "generated_by": {"tool": "MetaUmbra", "version": "1.3.5"}, - "default_genome_threshold": "q0.05", - "files": {}, - "units": { - "u1": { - "sample_columns": ["s1"], - "n_samples": 1, - "genome_ids_q005": ["g1", "g2"], - "genome_ids_q001": ["g1", "g2"], - }, - "u2": { - "sample_columns": ["s2"], - "n_samples": 1, - "genome_ids_q005": ["g2", "g3"], - "genome_ids_q001": ["g2", "g3"], - }, - }, - } - ), - encoding="utf-8", - ) - - -def test_global_unit_specific_mapping_scans_shared_genomes_once(monkeypatch, tmp_path): - manifest_path = tmp_path / "manifest.json" - _write_shared_genome_manifest(manifest_path) - from metax.peptide_annotator.unit_specific_manifest import load_unit_specific_manifest - - manifest = load_unit_specific_manifest(manifest_path) - calls = [] - - def fake_query(**kwargs): - calls.append(kwargs) - return { - "PEPA": { - "g1": {"g1_p1"}, - "g2": {"g2_p2"}, - "g3": {"g3_p3"}, - }, - "PEPB": {"g2": {"g2_p4"}}, - } - - monkeypatch.setattr( - "metax.peptide_annotator.unit_specific_otf.query_peptide_proteins_from_digested_genome_folders_nested", - fake_query, - ) - result = build_global_unit_specific_peptide_protein_map( - peptide_df=pd.DataFrame( - { - "Sequence": ["PEPA", "PEPB", "PEPC"], - "s1": [1, 0, 0], - "s2": [0, 1, 0], - } - ), - peptide_col="Sequence", - manifest=manifest, - digested_genome_folders=str(tmp_path), - ) - - assert len(calls) == 1 - assert calls[0]["selected_genomes_set"] == {"g1", "g2", "g3"} - assert calls[0]["peptide_list"] == ["PEPA", "PEPB", "PEPC"] - assert result == { - "PEPA": { - "g1": {"g1_p1"}, - "g2": {"g2_p2"}, - "g3": {"g3_p3"}, - }, - "PEPB": {"g2": {"g2_p4"}}, - } - - -def test_nested_digested_scanner_reads_each_union_genome_once(monkeypatch, tmp_path): - digested_dir = tmp_path / "digested" - digested_dir.mkdir() - for genome_id, rows in { - "g1": [("p1", "PEPA")], - "g2": [("p2;p3", "PEPA"), ("p4", "PEPB")], - "g3": [("g3_p5", "PEPB")], - "unused": [("p6", "PEPA")], - }.items(): - pd.DataFrame(rows, columns=["Protein", "Peptide"]).to_csv( - digested_dir / f"{genome_id}.tsv", - sep="\t", - index=False, - ) - - import metax.peptide_annotator.pep_table_to_otf as mapping_module - - original_processor = mapping_module._process_digested_genome_batch_for_nested_mapping - scanned_files = [] - - def recording_processor(file_paths, *args, **kwargs): - scanned_files.extend(file_paths) - return original_processor(file_paths, *args, **kwargs) - - monkeypatch.setattr( - mapping_module, - "_process_digested_genome_batch_for_nested_mapping", - recording_processor, - ) - result = query_peptide_proteins_from_digested_genome_folders_nested( - digested_genome_folders=str(digested_dir), - peptide_list=["PEPA", "PEPB", "ABSENT"], - selected_genomes_set={"g1", "g2", "g3"}, - n_jobs=2, - parallel_backend="thread", - ) - - assert sorted(Path(path).stem for path in scanned_files) == ["g1", "g2", "g3"] - assert result == { - "PEPA": { - "g1": {"g1_p1"}, - "g2": {"g2_p2", "g2_p3"}, - }, - "PEPB": { - "g2": {"g2_p4"}, - "g3": {"g3_p5"}, - }, - } - - -def test_nested_scanner_defaults_to_subprocess(monkeypatch, tmp_path): - calls = [] - - def fake_subprocess(**kwargs): - calls.append(kwargs) - return {"PEPA": {"g1": {"g1_p1"}}} - - monkeypatch.setattr( - "metax.peptide_annotator.pep_table_to_otf._query_peptide_proteins_nested_via_subprocess", - fake_subprocess, - ) - result = query_peptide_proteins_from_digested_genome_folders_nested( - digested_genome_folders=str(tmp_path), - peptide_list=["PEPA"], - selected_genomes_set={"g1"}, - ) - - assert inspect.signature( - query_peptide_proteins_from_digested_genome_folders_nested - ).parameters["parallel_backend"].default == "subprocess" - assert len(calls) == 1 - assert result == {"PEPA": {"g1": {"g1_p1"}}} - - -def test_digested_scan_caps_windows_process_workers(monkeypatch): - import metax.peptide_annotator.pep_table_to_otf as mapping_module - - monkeypatch.setattr(mapping_module.os, "name", "nt") - monkeypatch.setattr(mapping_module.os, "cpu_count", lambda: 96) - - assert mapping_module._resolve_digested_scan_n_jobs( - None, - parallel_backend="process", - ) == 61 - assert mapping_module._resolve_digested_scan_n_jobs( - 95, - parallel_backend="process", - ) == 61 - assert mapping_module._resolve_digested_scan_n_jobs( - 95, - parallel_backend="thread", - ) == 95 - - -def test_nested_scanner_subprocess_matches_direct_backends(tmp_path): - digested_dir = tmp_path / "digested" - digested_dir.mkdir() - pd.DataFrame( - [("p1;p2", "PEPA"), ("p3", "PEPB")], - columns=["Protein", "Peptide"], - ).to_csv(digested_dir / "g1.tsv", sep="\t", index=False) - expected = { - "PEPA": {"g1": {"g1_p1", "g1_p2"}}, - "PEPB": {"g1": {"g1_p3"}}, - } - - results = { - backend: query_peptide_proteins_from_digested_genome_folders_nested( - digested_genome_folders=str(digested_dir), - peptide_list=["PEPA", "PEPB"], - selected_genomes_set={"g1"}, - n_jobs=1, - parallel_backend=backend, - ) - for backend in ("thread", "process", "subprocess") - } - - assert results == {backend: expected for backend in results} - - -def test_nested_scanner_warns_for_malformed_genome(tmp_path, capsys): - digested_dir = tmp_path / "digested" - digested_dir.mkdir() - pd.DataFrame({"Wrong": ["x"]}).to_csv( - digested_dir / "z_bad.tsv", - sep="\t", - index=False, - ) - pd.DataFrame({"Protein": ["p1"], "Peptide": ["PEPA"]}).to_csv( - digested_dir / "a_good.tsv", - sep="\t", - index=False, - ) - - result = query_peptide_proteins_from_digested_genome_folders_nested( - digested_genome_folders=str(digested_dir), - peptide_list=["PEPA"], - selected_genomes_set={"z_bad", "a_good"}, - digested_peptide_col="Peptide", - digested_protein_col="Protein", - n_jobs=1, - parallel_backend="thread", - ) - - assert result == {"PEPA": {"a_good": {"a_good_p1"}}} - output = capsys.readouterr().out - assert "Warning: skipped malformed genome TSV" in output - assert "z_bad.tsv" in output - assert "ValueError" in output - - -def test_digested_scanner_skips_malformed_first_file_without_losing_later_hits( - tmp_path, - capsys, -): - digested_dir = tmp_path / "digested" - digested_dir.mkdir() - pd.DataFrame({"Unrelated": ["x"]}).to_csv( - digested_dir / "0_bad.tsv", - sep="\t", - index=False, - ) - pd.DataFrame({"Protein": ["p1"], "Peptide": ["PEPA"]}).to_csv( - digested_dir / "1_good.tsv", - sep="\t", - index=False, - ) - - mapping, peptide_col, protein_col = query_peptide_proteins_from_digested_genome_folders( - digested_genome_folders=str(digested_dir), - peptide_list=["PEPA"], - n_jobs=1, - parallel_backend="thread", - ) - - assert mapping == {"PEPA": "1_good_p1"} - assert (peptide_col, protein_col) == ("", "") - output = capsys.readouterr().out - assert "0_bad.tsv" in output - assert "ValueError" in output - - -def test_nested_digested_scanner_resolves_columns_for_each_header(tmp_path): - digested_dir = tmp_path / "digested" - digested_dir.mkdir() - pd.DataFrame({"Protein": ["p1"], "Peptide": ["PEPA"]}).to_csv( - digested_dir / "g1.tsv", - sep="\t", - index=False, - ) - pd.DataFrame({"protein_id": ["p2"], "Sequence": ["PEPB"]}).to_csv( - digested_dir / "g2.tsv", - sep="\t", - index=False, - ) - - result = query_peptide_proteins_from_digested_genome_folders_nested( - digested_genome_folders=str(digested_dir), - peptide_list=["PEPA", "PEPB"], - selected_genomes_set={"g1", "g2"}, - n_jobs=1, - parallel_backend="thread", - ) - - assert result == { - "PEPA": {"g1": {"g1_p1"}}, - "PEPB": {"g2": {"g2_p2"}}, - } - - -def test_digested_scanner_respects_explicit_columns_per_file(tmp_path): - digested_dir = tmp_path / "digested" - digested_dir.mkdir() - pd.DataFrame({"CustomPeptide": ["PEPA"], "CustomProtein": ["p1"]}).to_csv( - digested_dir / "g1.tsv", - sep="\t", - index=False, - ) - - mapping, peptide_col, protein_col = query_peptide_proteins_from_digested_genome_folders( - digested_genome_folders=str(digested_dir), - peptide_list=["PEPA"], - digested_peptide_col="CustomPeptide", - digested_protein_col="CustomProtein", - n_jobs=1, - parallel_backend="thread", - ) - - assert mapping == {"PEPA": "g1_p1"} - assert (peptide_col, protein_col) == ("CustomPeptide", "CustomProtein") - - -def test_digested_scanner_falls_back_to_inference_when_explicit_columns_are_missing( - tmp_path, -): - digested_dir = tmp_path / "digested" - digested_dir.mkdir() - pd.DataFrame({"protein_id": ["p1"], "Sequence": ["PEPA"]}).to_csv( - digested_dir / "g1.tsv", - sep="\t", - index=False, - ) - - mapping, _, _ = query_peptide_proteins_from_digested_genome_folders( - digested_genome_folders=str(digested_dir), - peptide_list=["PEPA"], - digested_peptide_col="Peptide", - digested_protein_col="Protein", - n_jobs=1, - parallel_backend="thread", - ) - - assert mapping == {"PEPA": "g1_p1"} - - -def test_unit_specific_cli_passes_extended_options(monkeypatch, tmp_path): - captured = {} - defaults = build_parser().parse_args([]) - assert defaults.merge_chunksize == 100_000 - assert defaults.collect_unique_stats is False - - class FakeAnnotator: - def __init__(self, **kwargs): - captured.update(kwargs) - - def run(self): - return pd.DataFrame() - - monkeypatch.setattr(annotate_cli, "UnitSpecificOTFAnnotator", FakeAnnotator) - result = annotate_cli.main( - [ - "--mode", - "unit-specific", - "--peptide-table", - str(tmp_path / "peptides.tsv"), - "--unit-specific-manifest", - str(tmp_path / "manifest.json"), - "--taxafunc-db", - str(tmp_path / "taxafunc.db"), - "--output", - str(tmp_path / "out.tsv"), - "--peptide-db", - str(tmp_path / "peptide.db"), - "--duplicate-peptide-handling-mode", - "max", - "--include-unit-specific-sequence", - "--merge-chunksize", - "7", - "--collect-unique-stats", - ] - ) - - assert result == 0 - assert captured["duplicate_peptide_handling_mode"] == "max" - assert captured["include_unit_specific_sequence"] is True - assert captured["output_sample_col_prefix"] == "Intensity_" - assert captured["merge_chunksize"] == 7 - assert captured["collect_unique_stats"] is True - - -def test_run_prefilters_globally_zero_peptides_before_global_mapping( - monkeypatch, - tmp_path, - capsys, -): - manifest = tmp_path / "manifest.json" - _write_manifest(manifest) - peptide_table = tmp_path / "peptides.tsv" - pd.DataFrame( - { - "Sequence": ["PEPA", "PEPB", "PEPC"], - "s1": [10, 0, 0], - "s2": [0, 20, 0], - } - ).to_csv(peptide_table, sep="\t", index=False) - taxafunc_db = tmp_path / "taxafunc.db" - taxafunc_db.write_text("", encoding="utf-8") - digested_dir = tmp_path / "digested" - digested_dir.mkdir() - captured_peptides = [] - - def fake_build_global_map(**kwargs): - captured_peptides.extend(kwargs["peptide_df"]["Sequence"].tolist()) - return { - "PEPA": {"g1": {"g1_p1"}}, - "PEPB": {"g2": {"g2_p1"}}, - } - - class FakeMapper: - def __init__(self, **kwargs): - self.peptide_table = kwargs["peptide_df"].copy() - self.final_peptide_table = self.peptide_table.copy() - self.peptides_after_mapping = len(self.peptide_table) - self.selected_genomes_num = len(kwargs["genome_list"]) - - def all_in_one(self, **kwargs): - sample_cols = [ - column - for column in self.peptide_table - if column.startswith("Intensity_") - ] - return pd.DataFrame( - { - "Sequence": self.peptide_table["Sequence"], - "Proteins": self.peptide_table["Proteins"], - "LCA_level": ["genome"] * len(self.peptide_table), - "Taxon": ["d__Bacteria"] * len(self.peptide_table), - "Taxon_prop": [1.0] * len(self.peptide_table), - **{ - column: self.peptide_table[column].tolist() - for column in sample_cols - }, - } - ) - - monkeypatch.setattr( - "metax.peptide_annotator.unit_specific_otf.build_global_unit_specific_peptide_protein_map", - fake_build_global_map, - ) - monkeypatch.setattr( - "metax.peptide_annotator.unit_specific_otf.peptideProteinsMapper", - FakeMapper, - ) - - result = UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(tmp_path / "out.tsv"), - digested_genome_folders=str(digested_dir), - ).run(return_dataframe=True) - - assert captured_peptides == ["PEPA", "PEPB"] - assert result["Sequence"].tolist() == ["PEPA", "PEPB"] - assert "PEPC" not in result["Sequence"].tolist() - assert ( - "[Unit-specific] Digested scan candidates: 2 unique peptides with nonzero " - "intensity in mapped manifest samples." - in capsys.readouterr().out - ) - - -def test_unit_specific_folder_mode_reuses_global_map_and_filters_by_unit(monkeypatch, tmp_path): - manifest = tmp_path / "manifest.json" - _write_shared_genome_manifest(manifest) - peptide_table = tmp_path / "peptides.tsv" - pd.DataFrame( - { - "Sequence": ["PEPA", "PEPB", "PEPZERO"], - "s1": [10, 0, 0], - "s2": [20, 5, 0], - } - ).to_csv(peptide_table, sep="\t", index=False) - taxafunc_db = tmp_path / "taxafunc.db" - taxafunc_db.write_text("", encoding="utf-8") - digested_dir = tmp_path / "digested" - digested_dir.mkdir() - calls = [] - mapper_inputs = [] - - def fake_query(**kwargs): - calls.append(kwargs) - return { - "PEPA": { - "g1": {"g1_p1"}, - "g2": {"g2_p2"}, - "g3": {"g3_p3"}, - }, - "PEPB": {"g3": {"g3_p4"}}, - } - - class FakeMapper: - def __init__(self, **kwargs): - assert kwargs["continue_base_on_annotaied_peptide_table"] is True - self.peptide_table = kwargs["peptide_df"].copy() - self.final_peptide_table = self.peptide_table.copy() - self.peptides_after_mapping = len(self.peptide_table) - self.selected_genomes_num = len(kwargs["genome_list"]) - self.kwargs = kwargs - mapper_inputs.append(self.peptide_table.copy()) - - def all_in_one(self, **kwargs): - sample_cols = [c for c in self.peptide_table if c.startswith("Intensity_")] - unit_genomes = self.kwargs["genome_list"] - unit_label = "+".join(unit_genomes) - return pd.DataFrame( - { - "Sequence": self.peptide_table["Sequence"], - "Proteins": self.peptide_table["Proteins"], - "LCA_level": ["genome"] * len(self.peptide_table), - "Taxon": [f"d__Bacteria|m__{unit_label}"] * len(self.peptide_table), - "Taxon_prop": [1.0] * len(self.peptide_table), - "Mock_func": [f"func_{unit_label}"] * len(self.peptide_table), - **{col: self.peptide_table[col].tolist() for col in sample_cols}, - } - ) - - monkeypatch.setattr( - "metax.peptide_annotator.unit_specific_otf.query_peptide_proteins_from_digested_genome_folders_nested", - fake_query, - ) - monkeypatch.setattr("metax.peptide_annotator.unit_specific_otf.peptideProteinsMapper", FakeMapper) - - result = UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(tmp_path / "out.tsv"), - digested_genome_folders=str(digested_dir), - ).run(return_dataframe=True) - - assert len(calls) == 1 - assert calls[0]["peptide_list"] == ["PEPA", "PEPB"] - assert mapper_inputs[0]["Proteins"].tolist() == ["g1_p1;g2_p2"] - assert mapper_inputs[1]["Proteins"].tolist() == ["g2_p2;g3_p3", "g3_p4"] - assert "PEPZERO" not in result["Sequence"].tolist() - assert result.loc[result["analysis_unit_id"] == "u1", "Proteins"].tolist() == ["g1_p1;g2_p2"] - assert result.loc[result["analysis_unit_id"] == "u2", "Proteins"].tolist() == [ - "g2_p2;g3_p3", - "g3_p4", - ] - pepa = result.loc[result["Sequence"] == "PEPA"].set_index("analysis_unit_id") - assert pepa.loc["u1", "Taxon"] != pepa.loc["u2", "Taxon"] - assert pepa.loc["u1", "Mock_func"] != pepa.loc["u2", "Mock_func"] - artifacts_dir = tmp_path / "out_artifacts" - assert not list(artifacts_dir.rglob("*.db")) - assert not list(artifacts_dir.rglob("*.parquet")) - assert not list(artifacts_dir.rglob("*global*map*")) - - -@pytest.mark.parametrize("on_empty_unit", ["warn-skip", "error"]) -def test_unit_specific_folder_mode_handles_unit_without_mapped_proteins( - monkeypatch, - tmp_path, - on_empty_unit, -): - manifest = tmp_path / "manifest.json" - _write_manifest(manifest) - peptide_table = tmp_path / "peptides.tsv" - pd.DataFrame( - { - "Sequence": ["PEPA"], - "s1": [10], - "s2": [10], - } - ).to_csv(peptide_table, sep="\t", index=False) - taxafunc_db = tmp_path / "taxafunc.db" - taxafunc_db.write_text("", encoding="utf-8") - digested_dir = tmp_path / "digested" - digested_dir.mkdir() - - monkeypatch.setattr( - "metax.peptide_annotator.unit_specific_otf.query_peptide_proteins_from_digested_genome_folders_nested", - lambda **kwargs: {"PEPA": {"g1": {"g1_p1"}}}, - ) - - class FakeMapper: - def __init__(self, **kwargs): - self.peptide_table = kwargs["peptide_df"].copy() - self.final_peptide_table = self.peptide_table.copy() - self.peptides_after_mapping = len(self.peptide_table) - self.selected_genomes_num = len(kwargs["genome_list"]) - - def all_in_one(self, **kwargs): - return pd.DataFrame( - { - "Sequence": self.peptide_table["Sequence"], - "Proteins": self.peptide_table["Proteins"], - "LCA_level": ["genome"], - "Taxon": ["d__Bacteria"], - "Taxon_prop": [1.0], - "Intensity_s1": self.peptide_table.get("Intensity_s1", 0), - } - ) - - monkeypatch.setattr("metax.peptide_annotator.unit_specific_otf.peptideProteinsMapper", FakeMapper) - annotator = UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(tmp_path / "out.tsv"), - digested_genome_folders=str(digested_dir), - on_empty_unit=on_empty_unit, - ) - - if on_empty_unit == "error": - with pytest.raises(ValueError, match="u2: unit has no peptides mapped"): - annotator.run() - else: - with pytest.warns(UserWarning, match="u2: unit has no peptides mapped"): - result = annotator.run(return_dataframe=True) - assert result["analysis_unit_id"].tolist() == ["u1"] - - -def test_unit_specific_otf_raises_on_invalid_output_prefix(tmp_path): - peptide_table = tmp_path / "peptides.tsv" - peptide_table.write_text("Sequence\ts1\nPEPA\t10\n", encoding="utf-8") - manifest = tmp_path / "unit_specific_manifest.json" - _write_manifest(manifest) - taxafunc_db = tmp_path / "taxafunc.db" - _write_taxafunc_db(taxafunc_db) - digested_dir = tmp_path / "digested" - digested_dir.mkdir() - - with pytest.raises(ValueError, match="Unit-specific OTF output_sample_col_prefix must be 'Intensity_'"): - UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(tmp_path / "OTF.tsv"), - digested_genome_folders=str(digested_dir), - output_sample_col_prefix="LFQ intensity ", - ) - - -def test_unit_specific_otf_per_unit_output_filename_collisions(monkeypatch, tmp_path, capsys): - peptide_table = tmp_path / "peptides.tsv" - peptide_table.write_text("Sequence\ts1\ts2\nPEPA\t10\t20\n", encoding="utf-8") - - # Create two units whose IDs will collide when sanitized to a safe filename - manifest_data = { - "schema_version": "metaumbra.unit_specific_manifest.v1", - "generated_by": {"tool": "test"}, - "default_genome_threshold": "q0.05", - "files": {}, - "units": { - "a/b": { - "sample_columns": ["s1"], - "n_samples": 1, - "genome_ids_q005": ["g1"], - "genome_ids_q001": ["g1"], - }, - "a?b": { - "sample_columns": ["s2"], - "n_samples": 1, - "genome_ids_q005": ["g2"], - "genome_ids_q001": ["g2"], - } - } - } - manifest = tmp_path / "unit_specific_manifest.json" - manifest.write_text(json.dumps(manifest_data), encoding="utf-8") - - taxafunc_db = tmp_path / "taxafunc.db" - _write_taxafunc_db(taxafunc_db) - - peptide_db = tmp_path / "peptide.db" - peptide_db.write_text("", encoding="utf-8") - - class FakeMapper: - def __init__(self, *args, **kwargs): - self.peptide_table = kwargs["peptide_df"] - self.final_peptide_table = self.peptide_table.copy() - self.peptides_before_mapping = len(self.peptide_table) - self.peptides_after_mapping = len(self.peptide_table) - self.selected_genomes_num = len(kwargs["genome_list"]) - - def all_in_one(self, **kwargs): - # Return intensity columns exactly as they exist in the chunk - df_dict = { - "Sequence": self.peptide_table["Sequence"], - "Proteins": ["prot1"] * len(self.peptide_table), - "LCA_level": ["genome"] * len(self.peptide_table), - "Taxon": ["d__Bacteria"] * len(self.peptide_table), - "Taxon_prop": [1.0] * len(self.peptide_table), - } - # Add whichever Intensity_ columns are present - for col in self.peptide_table.columns: - if col.startswith("Intensity_"): - df_dict[col] = self.peptide_table[col] - - return pd.DataFrame(df_dict) - - monkeypatch.setattr("metax.peptide_annotator.unit_specific_otf.peptideProteinsMapper", FakeMapper) - - annotator = UnitSpecificOTFAnnotator( - peptide_table_path=str(peptide_table), - unit_specific_manifest_path=str(manifest), - taxafunc_anno_db_path=str(taxafunc_db), - output_path=str(tmp_path / "OTF.tsv"), - db_path=str(peptide_db), - save_per_unit_outputs=True, - ) - result = annotator.run() - - # Check that they both succeeded and have different output paths - per_unit_dir = tmp_path / "OTF_artifacts" / "per_unit" - saved_files = list(per_unit_dir.glob("*_OTF.tsv")) - assert len(saved_files) == 2 - assert saved_files[0].name != saved_files[1].name - - # Assert unit_output_records / summary preserve the original analysis_unit_id - summary_text = (tmp_path / "OTF_artifacts" / "unit_annotation_summary.tsv").read_text(encoding="utf-8") - assert "a/b" in summary_text - assert "a?b" in summary_text - - # Assert merged output contains rows from both units, not duplicated rows from the overwritten file - merged_text = (tmp_path / "OTF.tsv").read_text(encoding="utf-8") - assert "Intensity_s1" in merged_text - assert "Intensity_s2" in merged_text - - # Ensure they don't have duplicated contents, since s1 and s2 are different columns - df = pd.read_csv(tmp_path / "OTF.tsv", sep="\t", skipinitialspace=True) - assert len(df) == 2 - assert set(df["analysis_unit_id"].tolist()) == {"a/b", "a?b"} diff --git a/tests/test_workflow_recorder.py b/tests/test_workflow_recorder.py index 32b9e9b4..ce82ec32 100644 --- a/tests/test_workflow_recorder.py +++ b/tests/test_workflow_recorder.py @@ -19,8 +19,9 @@ set_multi_tables_step, taxafunc_analyzer_step, deseq2_step, + direct_otf_step, limma_step, - unit_specific_otf_step, + manifest_otf_step, ) @@ -150,10 +151,10 @@ def test_auto_otf_report_step_uses_saved_config(tmp_path: Path): assert step.outputs["index_html_path"].endswith("index.html") -def test_unit_specific_otf_step_records_replayable_parameters(): +def test_manifest_otf_step_records_replayable_parameters(): params = { "peptide_table_path": "peptides.tsv", - "unit_specific_manifest_path": "manifest.json", + "metaumbra_manifest_path": "manifest.json", "taxafunc_anno_db_path": "taxafunc.db", "output_path": "out.tsv", "digested_genome_folders": "digested", @@ -170,14 +171,29 @@ def test_unit_specific_otf_step_records_replayable_parameters(): "n_jobs": None, } - step = unit_specific_otf_step(params) + step = manifest_otf_step(params) - assert step.step_type == "unit_specific_peptide_direct_to_otf" + assert step.step_type == "manifest_peptide_direct_to_otf" assert step.parameters == params - assert step.inputs["unit_specific_manifest_path"] == "manifest.json" + assert step.inputs["metaumbra_manifest_path"] == "manifest.json" assert step.outputs["output_path"] == "out.tsv" - assert "UnitSpecificOTFAnnotator(**unit_specific_otf_params)" in step.code - compile(step.code, "", "exec") + assert "ManifestOTFAnnotator(**manifest_otf_params)" in step.code + compile(step.code, "", "exec") + + +def test_direct_otf_step_records_automatic_or_list_source(): + params = { + "peptide_table_path": "peptides.tsv", + "taxafunc_anno_db_path": "taxafunc.db", + "output_path": "out.tsv", + "digested_genome_folders": "digested", + "selection_mode": "provided", + "selected_genomes": ["g1", "g2"], + } + step = direct_otf_step(params) + assert step.step_type == "metax_direct_peptide_to_otf" + assert "GlobalOTFAnnotator(**direct_otf_params)" in step.code + compile(step.code, "", "exec") def test_taxafunc_and_multi_table_steps_generate_runnable_cells(): From 3f46ad6de83a67020b4c4bb635061af62cbc3512 Mon Sep 17 00:00:00 2001 From: Qing <44231502+byemaxx@users.noreply.github.com> Date: Wed, 15 Jul 2026 16:16:34 -0400 Subject: [PATCH 2/5] Update CI for unified annotation tests --- .github/workflows/installation-profiles.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/installation-profiles.yml b/.github/workflows/installation-profiles.yml index 359fab02..437b7b52 100644 --- a/.github/workflows/installation-profiles.yml +++ b/.github/workflows/installation-profiles.yml @@ -18,7 +18,7 @@ jobs: run: | python -c "import sys, metax, metax.cli.annotate, metax.cli.report; assert not any(name.startswith('PyQt') for name in sys.modules); assert 'metax.report' not in sys.modules" python -m metax.cli.annotate --help - python -m pytest tests/test_annotation_cli.py tests/test_annotation_subprocess_cleanup.py tests/test_pep_table_to_otf_dataframe.py tests/test_unit_specific_manifest.py -q + python -m pytest tests/test_manifest_annotation_cli.py tests/test_annotation_subprocess_cleanup.py tests/test_pep_table_to_otf_dataframe.py tests/test_genome_selection_manifest.py tests/test_manifest_otf_backend.py -q headless-analysis-profiles: runs-on: ubuntu-latest @@ -36,13 +36,13 @@ jobs: if: matrix.extra == 'analyzer' run: | python -c "from metax.taxafunc_analyzer.analyzer import TaxaFuncAnalyzer; import sys; assert not any(name.startswith('PyQt') for name in sys.modules)" - python -m pytest tests/test_analyzer.py tests/test_annotation_cli.py -q + python -m pytest tests/test_analyzer.py tests/test_manifest_annotation_cli.py -q - name: Verify Report profile if: matrix.extra == 'report' run: | python -c "import metax.report; import sys; assert not any(name.startswith('PyQt') for name in sys.modules)" python -m metax.cli.report --help - python -m pytest tests/test_auto_report_config.py tests/test_auto_report_workflow.py tests/test_annotation_cli.py -q + python -m pytest tests/test_auto_report_config.py tests/test_auto_report_workflow.py tests/test_manifest_annotation_cli.py -q gui-and-full-metadata: runs-on: ubuntu-latest @@ -63,7 +63,7 @@ jobs: - name: Run shared backend and GUI annotation tests env: QT_QPA_PLATFORM: offscreen - run: python -m pytest tests/test_annotation_cli.py tests/test_unit_specific_gui_settings_dialog.py -q + run: python -m pytest tests/test_manifest_annotation_cli.py tests/test_peptide_annotator_layout.py -q windows-annotation: runs-on: windows-latest @@ -79,4 +79,4 @@ jobs: run: | python -c "import sys, metax, metax.cli.annotate; assert not any(name.startswith('PyQt') for name in sys.modules)" python -m metax.cli.annotate --help - python -m pytest tests/test_annotation_cli.py tests/test_annotation_subprocess_cleanup.py tests/test_pep_table_to_otf_dataframe.py tests/test_unit_specific_manifest.py -q + python -m pytest tests/test_manifest_annotation_cli.py tests/test_annotation_subprocess_cleanup.py tests/test_pep_table_to_otf_dataframe.py tests/test_genome_selection_manifest.py tests/test_manifest_otf_backend.py -q From fb54007fcf10ea10a158719d33da2d869fad9b70 Mon Sep 17 00:00:00 2001 From: Qing <44231502+byemaxx@users.noreply.github.com> Date: Wed, 15 Jul 2026 16:37:12 -0400 Subject: [PATCH 3/5] Bump version to 2.6.0 --- Docs/ChangeLog.md | 13 +++++++++++++ metax/utils/version.py | 4 ++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/Docs/ChangeLog.md b/Docs/ChangeLog.md index 85f8d52d..2ca6af7a 100644 --- a/Docs/ChangeLog.md +++ b/Docs/ChangeLog.md @@ -1,3 +1,16 @@ +# Version: 2.6.0 +## Date: 2026-07-15 +### Changes: +- New: Added unified MetaUmbra genome-selection manifest annotation for one or multiple analysis units. +- New: Added explicit MetaUmbra manifest, MetaX automatic, and custom genome-list annotation sources. +- Change: Unified single-unit and multi-unit OTF annotation around one manifest-driven backend and retained `analysis_unit_id` in canonical OTF output. +- Change: Reorganized the Peptide Annotator GUI around genome source, inputs, manifest validation, output, and advanced settings. +- Change: Unified manifests are produced by MetaUmbra 1.4.0 or newer; updated the MetaX API compatibility version to 11 while retaining MetaX's existing non-manifest installation baseline. +- Fix: Preserved custom intensity prefixes and accurate provenance for automatic and custom-list annotation. +- Fix: Rejected empty selected-threshold genome lists and made Windows console progress output encoding-safe. +- Test: Added unified manifest contract, CLI, backend, GUI layout, and cross-repository integration coverage. + + # Version: 2.5.0 ## Date: 2026-07-14 ### Changes: diff --git a/metax/utils/version.py b/metax/utils/version.py index f0a4108b..e4f6390a 100644 --- a/metax/utils/version.py +++ b/metax/utils/version.py @@ -1,2 +1,2 @@ -__version__ = '2.5.0' -API_version = '10' +__version__ = '2.6.0' +API_version = '11' From 18d5b4e8ab81cd05ba73407c7a9239d1dd5e44c6 Mon Sep 17 00:00:00 2001 From: Qing <44231502+byemaxx@users.noreply.github.com> Date: Sat, 18 Jul 2026 11:13:11 -0400 Subject: [PATCH 4/5] test: initialize QApplication in layout harness --- tests/test_peptide_annotator_layout.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_peptide_annotator_layout.py b/tests/test_peptide_annotator_layout.py index c41bb08f..b85c5381 100644 --- a/tests/test_peptide_annotator_layout.py +++ b/tests/test_peptide_annotator_layout.py @@ -9,6 +9,7 @@ def _build_layout_harness(): + app = QtWidgets.QApplication.instance() or QtWidgets.QApplication([]) window = QtWidgets.QMainWindow() ui = MetaXGUI.__new__(MetaXGUI) ui.setupUi(window) @@ -21,7 +22,7 @@ def _build_layout_harness(): ui.gridLayout_74.addWidget(ui.label_pep_direct_to_otf_input_source, ui.gridLayout_74.rowCount(), 0) ui.gridLayout_74.addWidget(ui.comboBox_pep_direct_to_otf_input_source, ui.gridLayout_74.rowCount(), 2) ui._arrange_pep_direct_to_otf_layout() - return window, ui + return app, window, ui def _position(layout, target): @@ -33,7 +34,7 @@ def _position(layout, target): def test_annotation_layout_follows_source_input_output_order(): - window, ui = _build_layout_harness() + app, window, ui = _build_layout_harness() try: assert _position(ui.gridLayout_74, ui.label_pep_direct_to_otf_input_source) == (0, 0, 1, 2) assert _position(ui.gridLayout_74, ui.comboBox_pep_direct_to_otf_input_source) == (0, 2, 1, 2) @@ -47,10 +48,11 @@ def test_annotation_layout_follows_source_input_output_order(): assert _position(ui.gridLayout_74, ui.pushButton_run_pep_direct_to_otf)[0] == 11 finally: window.close() + app.processEvents() def test_source_choice_hides_irrelevant_manifest_or_list_controls(): - window, ui = _build_layout_harness() + app, window, ui = _build_layout_harness() try: ui.comboBox_pep_direct_to_otf_input_source.setCurrentIndex(0) ui.update_pep_direct_to_otf_mode_state() @@ -70,10 +72,11 @@ def test_source_choice_hides_irrelevant_manifest_or_list_controls(): assert ui.pushButton_pep_direct_to_otf_reset_selected_genome_list.text() == "Reset" finally: window.close() + app.processEvents() def test_non_manifest_gui_forwards_custom_intensity_prefix(monkeypatch, tmp_path): - window, ui = _build_layout_harness() + app, window, ui = _build_layout_harness() try: peptide = tmp_path / "peptides.tsv" database = tmp_path / "taxafunc.db" @@ -114,3 +117,4 @@ def run(self): assert captured["selected_genome_source"] is None finally: window.close() + app.processEvents() From 256af844163105e74c5a46e7ee3919f2839f6153 Mon Sep 17 00:00:00 2001 From: Qing <44231502+byemaxx@users.noreply.github.com> Date: Sun, 19 Jul 2026 11:45:44 -0400 Subject: [PATCH 5/5] Validate manifest contracts before annotation --- .github/workflows/installation-profiles.yml | 10 +- metax/peptide_annotator/pep_table_to_otf.py | 62 +++++- .../unit_specific_manifest.py | 195 ++++++++++++++---- tests/test_genome_selection_manifest.py | 63 +++++- tests/test_manifest_otf_backend.py | 65 ++++++ tests/test_metaumbra_contract_integration.py | 15 +- 6 files changed, 362 insertions(+), 48 deletions(-) diff --git a/.github/workflows/installation-profiles.yml b/.github/workflows/installation-profiles.yml index 437b7b52..7a260a6d 100644 --- a/.github/workflows/installation-profiles.yml +++ b/.github/workflows/installation-profiles.yml @@ -9,16 +9,24 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Check out the pinned MetaUmbra manifest producer + uses: actions/checkout@v4 + with: + repository: byemaxx/MetaUmbra + ref: b3f719b49b8e73cd324e2841df7f686d4f6c909b + path: MetaUmbra - uses: actions/setup-python@v5 with: python-version: "3.11" - run: python -m pip install --upgrade pip - run: python -m pip install . pytest - name: Verify lightweight imports and annotation CLI + env: + METAX_METAUMBRA_SOURCE: ${{ github.workspace }}/MetaUmbra run: | python -c "import sys, metax, metax.cli.annotate, metax.cli.report; assert not any(name.startswith('PyQt') for name in sys.modules); assert 'metax.report' not in sys.modules" python -m metax.cli.annotate --help - python -m pytest tests/test_manifest_annotation_cli.py tests/test_annotation_subprocess_cleanup.py tests/test_pep_table_to_otf_dataframe.py tests/test_genome_selection_manifest.py tests/test_manifest_otf_backend.py -q + python -m pytest tests/test_manifest_annotation_cli.py tests/test_annotation_subprocess_cleanup.py tests/test_pep_table_to_otf_dataframe.py tests/test_genome_selection_manifest.py tests/test_manifest_otf_backend.py tests/test_metaumbra_contract_integration.py -q headless-analysis-profiles: runs-on: ubuntu-latest diff --git a/metax/peptide_annotator/pep_table_to_otf.py b/metax/peptide_annotator/pep_table_to_otf.py index 996b927f..1b4f99bf 100644 --- a/metax/peptide_annotator/pep_table_to_otf.py +++ b/metax/peptide_annotator/pep_table_to_otf.py @@ -591,6 +591,52 @@ def _query_peptide_proteins_nested_via_subprocess( return _read_nested_mapping_tsv(out_mapping) +def _resolve_selected_digest_genome_files( + valid_folders: list[str], + selected_genomes: set[str], +) -> list[str]: + """Resolve one digest TSV per selected genome during the existing directory walk.""" + matched_files: dict[str, dict[str, str]] = {} + for folder in valid_folders: + for path in pathlib.Path(folder).glob("*.tsv"): + genome_id = path.stem + if genome_id not in selected_genomes: + continue + resolved_path = str(path.resolve()) + matched_files.setdefault(genome_id, {})[resolved_path] = str(path) + + missing_genomes = sorted(selected_genomes - set(matched_files)) + if missing_genomes: + preview = ", ".join(missing_genomes[:20]) + suffix = "" if len(missing_genomes) <= 20 else ", ..." + raise FileNotFoundError( + "Missing digested genome TSV files for " + f"{len(missing_genomes)} selected manifest genomes: {preview}{suffix}. " + f"Searched directories: {valid_folders}" + ) + + duplicate_genomes = { + genome_id: sorted(paths.values()) + for genome_id, paths in matched_files.items() + if len(paths) > 1 + } + if duplicate_genomes: + details = "; ".join( + f"{genome_id}: {paths}" + for genome_id, paths in sorted(duplicate_genomes.items())[:10] + ) + suffix = "" if len(duplicate_genomes) <= 10 else "; ..." + raise ValueError( + "Selected manifest genomes must resolve to exactly one digest TSV; " + f"duplicates found for {len(duplicate_genomes)} genomes: {details}{suffix}" + ) + + return [ + next(iter(matched_files[genome_id].values())) + for genome_id in sorted(selected_genomes) + ] + + def query_peptide_proteins_from_digested_genome_folders_nested( digested_genome_folders: str | list[str], peptide_list: list[str], @@ -630,13 +676,15 @@ def query_peptide_proteins_from_digested_genome_folders_nested( if not valid_folders: raise ValueError(f"No valid digested genome folders found: {folders}") - selected_genomes = {str(genome_id) for genome_id in selected_genomes_set} - all_files = [ - str(path) - for folder in valid_folders - for path in pathlib.Path(folder).glob("*.tsv") - if path.stem in selected_genomes - ] + selected_genomes = { + str(genome_id) + for genome_id in selected_genomes_set + if str(genome_id) + } + all_files = _resolve_selected_digest_genome_files( + valid_folders, + selected_genomes, + ) if not all_files: print( "[UnitSpecificDigestedScan] No genome TSV files match the manifest genome union.", diff --git a/metax/peptide_annotator/unit_specific_manifest.py b/metax/peptide_annotator/unit_specific_manifest.py index 65e1d5ce..99df6c4c 100644 --- a/metax/peptide_annotator/unit_specific_manifest.py +++ b/metax/peptide_annotator/unit_specific_manifest.py @@ -10,6 +10,17 @@ SCHEMA_VERSION = "metaumbra.genome_selection_manifest.v1" +_TOP_LEVEL_FIELDS = { + "schema_version", + "generated_by", + "unit_definition", + "selection", + "inputs", + "units", + "artifacts", + "warnings", +} +_GENOME_THRESHOLDS = {"q0.05", "q0.01"} @dataclass @@ -62,34 +73,139 @@ def _warn_or_raise(message: str, strict: bool) -> None: warnings.warn(message, stacklevel=2) +def _require_mapping(value: object, label: str) -> dict: + if not isinstance(value, dict): + raise ValueError(f"{label} must be an object") + return value + + +def _require_string(value: object, label: str) -> str: + if not isinstance(value, str): + raise ValueError(f"{label} must be a string") + return value + + +def _require_integer(value: object, label: str, *, minimum: int | None = None) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{label} must be an integer") + if minimum is not None and value < minimum: + raise ValueError(f"{label} must be greater than or equal to {minimum}") + return value + + +def _require_unique_string_list( + value: object, + label: str, + *, + allow_empty: bool, + unique: bool = True, +) -> list[str]: + if not isinstance(value, list): + raise ValueError(f"{label} must be an array of strings") + if not allow_empty and not value: + raise ValueError(f"{label} must contain at least one item") + if any(not isinstance(item, str) for item in value): + raise ValueError(f"{label} must contain only strings") + if unique and len(set(value)) != len(value): + raise ValueError(f"{label} must contain unique items") + return list(value) + + +def _validate_manifest_structure(data: object) -> dict: + manifest = _require_mapping(data, "genome selection manifest") + missing_fields = sorted(_TOP_LEVEL_FIELDS - set(manifest)) + if missing_fields: + raise ValueError( + "genome selection manifest is missing required fields: " + + ", ".join(missing_fields) + ) + unknown_fields = sorted(set(manifest) - _TOP_LEVEL_FIELDS) + if unknown_fields: + raise ValueError( + "genome selection manifest contains unsupported fields: " + + ", ".join(unknown_fields) + ) + + generated_by = _require_mapping(manifest["generated_by"], "generated_by") + if generated_by.get("software") != "MetaUmbra": + raise ValueError("generated_by.software must be 'MetaUmbra'") + for field in ("version", "run_id", "generated_at"): + _require_string(generated_by.get(field), f"generated_by.{field}") + + unit_definition = _require_mapping( + manifest["unit_definition"], + "unit_definition", + ) + mode = _require_string(unit_definition.get("mode"), "unit_definition.mode") + if mode not in {"all-samples", "per-sample", "metadata"}: + raise ValueError( + "unit_definition.mode must be 'all-samples', 'per-sample', or 'metadata'" + ) + _require_string( + unit_definition.get("sample_id_column"), + "unit_definition.sample_id_column", + ) + analysis_unit_column = unit_definition.get("analysis_unit_column") + if analysis_unit_column is not None: + _require_string( + analysis_unit_column, + "unit_definition.analysis_unit_column", + ) + _require_integer( + unit_definition.get("n_units"), + "unit_definition.n_units", + minimum=1, + ) + + selection = _require_mapping(manifest["selection"], "selection") + default_threshold = _require_string( + selection.get("default_genome_threshold"), + "selection.default_genome_threshold", + ) + if default_threshold not in _GENOME_THRESHOLDS: + raise ValueError( + "selection.default_genome_threshold must be 'q0.05' or 'q0.01'" + ) + if selection.get("available_genome_thresholds") != ["q0.05", "q0.01"]: + raise ValueError( + "selection.available_genome_thresholds must be ['q0.05', 'q0.01']" + ) + _require_string(selection.get("scoring_method"), "selection.scoring_method") + + _require_mapping(manifest["inputs"], "inputs") + _require_mapping(manifest["artifacts"], "artifacts") + _require_unique_string_list( + manifest["warnings"], + "warnings", + allow_empty=True, + unique=False, + ) + return manifest + + def load_genome_selection_manifest( manifest_path: str | Path, genome_threshold: str | None = None, strict: bool = True, ) -> GenomeSelectionManifest: manifest_path = Path(manifest_path) - data = json.loads(manifest_path.read_text(encoding="utf-8")) + data = _validate_manifest_structure( + json.loads(manifest_path.read_text(encoding="utf-8")) + ) schema_version = data.get("schema_version") if schema_version != SCHEMA_VERSION: raise ValueError(f"Unsupported genome selection manifest schema_version: {schema_version!r}") - unit_definition = data.get("unit_definition") - if not isinstance(unit_definition, dict): - raise ValueError("genome selection manifest must contain unit_definition") - selection = data.get("selection") - if not isinstance(selection, dict): - raise ValueError("genome selection manifest must contain selection") + unit_definition = data["unit_definition"] + selection = data["selection"] default_threshold = str(selection.get("default_genome_threshold", "")).strip() - available_thresholds = selection.get("available_genome_thresholds") - if available_thresholds != ["q0.05", "q0.01"]: - raise ValueError("selection.available_genome_thresholds must be ['q0.05', 'q0.01']") selected_threshold, selected_genome_key = _normalize_threshold_alias(genome_threshold, default_threshold) - raw_units = data.get("units") + raw_units = data["units"] if not isinstance(raw_units, dict) or not raw_units: raise ValueError("genome selection manifest must contain at least one unit") - if int(unit_definition.get("n_units", -1)) != len(raw_units): + if unit_definition["n_units"] != len(raw_units): raise ValueError("unit_definition.n_units does not match units") seen_samples: dict[str, str] = {} @@ -98,9 +214,11 @@ def load_genome_selection_manifest( if not isinstance(raw_unit, dict): raise ValueError(f"Unit {analysis_unit_id!r} must be an object") - sample_columns = [str(sample) for sample in raw_unit.get("sample_ids", [])] - if not sample_columns: - raise ValueError(f"Unit {analysis_unit_id!r} has no sample_ids") + sample_columns = _require_unique_string_list( + raw_unit.get("sample_ids"), + f"Unit {analysis_unit_id!r}.sample_ids", + allow_empty=False, + ) for sample in sample_columns: previous_unit = seen_samples.get(sample) @@ -113,35 +231,40 @@ def load_genome_selection_manifest( if selected_genome_key not in raw_unit: raise ValueError(f"Unit {analysis_unit_id!r} is missing {selected_genome_key}") - genome_ids = [str(genome) for genome in raw_unit.get(selected_genome_key, [])] + q005 = _require_unique_string_list( + raw_unit.get("genome_ids_q005"), + f"Unit {analysis_unit_id!r}.genome_ids_q005", + allow_empty=True, + ) + q001 = _require_unique_string_list( + raw_unit.get("genome_ids_q001"), + f"Unit {analysis_unit_id!r}.genome_ids_q001", + allow_empty=True, + ) + genome_ids = q005 if selected_genome_key == "genome_ids_q005" else q001 if not genome_ids: raise ValueError( f"Unit {analysis_unit_id!r} has no genomes at selected threshold {selected_threshold}" ) - n_samples_value = raw_unit.get("n_samples", len(sample_columns)) - try: - n_samples = int(n_samples_value) - except Exception as exc: - raise ValueError(f"Unit {analysis_unit_id!r} has invalid n_samples: {n_samples_value!r}") from exc + n_samples = _require_integer( + raw_unit.get("n_samples"), + f"Unit {analysis_unit_id!r}.n_samples", + minimum=1, + ) if n_samples != len(sample_columns): raise ValueError( f"Unit {analysis_unit_id!r} declares n_samples={n_samples}, " - f"but has {len(sample_columns)} sample_columns" + f"but has {len(sample_columns)} sample_ids" ) - q005 = raw_unit.get("genome_ids_q005") - q001 = raw_unit.get("genome_ids_q001") - if q005 is not None and q001 is not None: - q005_set = {str(genome) for genome in q005} - q001_set = {str(genome) for genome in q001} - missing = q001_set - q005_set - if missing: - preview = ", ".join(sorted(missing)[:10]) - _warn_or_raise( - f"Unit {analysis_unit_id!r} has q0.01 genomes not present in q0.05: {preview}", - strict=strict, - ) + missing = set(q001) - set(q005) + if missing: + preview = ", ".join(sorted(missing)[:10]) + _warn_or_raise( + f"Unit {analysis_unit_id!r} has q0.01 genomes not present in q0.05: {preview}", + strict=strict, + ) units[str(analysis_unit_id)] = GenomeSelectionUnitSpec( analysis_unit_id=str(analysis_unit_id), @@ -150,9 +273,7 @@ def load_genome_selection_manifest( n_samples=n_samples, ) - generated_by = data.get("generated_by") - if not isinstance(generated_by, dict) or generated_by.get("software") != "MetaUmbra": - raise ValueError("generated_by.software must be 'MetaUmbra'") + generated_by = data["generated_by"] return GenomeSelectionManifest( schema_version=schema_version, generated_by=dict(generated_by), diff --git a/tests/test_genome_selection_manifest.py b/tests/test_genome_selection_manifest.py index 604e912e..0d376e17 100644 --- a/tests/test_genome_selection_manifest.py +++ b/tests/test_genome_selection_manifest.py @@ -48,7 +48,7 @@ def test_manifest_rejects_empty_unit_and_genome_threshold_contract(tmp_path): data["units"]["u1"]["n_samples"] = 0 path = tmp_path / "manifest.json" path.write_text(json.dumps(data), encoding="utf-8") - with pytest.raises(ValueError, match="no sample_ids"): + with pytest.raises(ValueError, match="sample_ids must contain at least one item"): load_genome_selection_manifest(path) data = json.loads(FIXTURE.read_text(encoding="utf-8")) @@ -59,6 +59,67 @@ def test_manifest_rejects_empty_unit_and_genome_threshold_contract(tmp_path): load_genome_selection_manifest(path, genome_threshold="q0.05", strict=True) +@pytest.mark.parametrize( + ("field", "value"), + [ + ("sample_ids", "s1"), + ("genome_ids_q005", "g1"), + ("genome_ids_q001", "g1"), + ], +) +def test_manifest_rejects_string_values_for_array_fields(tmp_path, field, value): + data = json.loads(FIXTURE.read_text(encoding="utf-8")) + data["units"]["u1"][field] = value + path = tmp_path / "manifest.json" + path.write_text(json.dumps(data), encoding="utf-8") + + with pytest.raises(ValueError, match="must be an array of strings"): + load_genome_selection_manifest(path) + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("sample_ids", ["s1", "s1"]), + ("genome_ids_q005", ["g1", "g1"]), + ("genome_ids_q001", ["g1", "g1"]), + ], +) +def test_manifest_rejects_duplicate_array_items(tmp_path, field, value): + data = json.loads(FIXTURE.read_text(encoding="utf-8")) + data["units"]["u1"][field] = value + if field == "sample_ids": + data["units"]["u1"]["n_samples"] = len(value) + path = tmp_path / "manifest.json" + path.write_text(json.dumps(data), encoding="utf-8") + + with pytest.raises(ValueError, match="must contain unique items"): + load_genome_selection_manifest(path) + + +def test_manifest_rejects_missing_required_provenance(tmp_path): + data = json.loads(FIXTURE.read_text(encoding="utf-8")) + data["generated_by"].pop("run_id") + path = tmp_path / "manifest.json" + path.write_text(json.dumps(data), encoding="utf-8") + + with pytest.raises(ValueError, match="generated_by.run_id must be a string"): + load_genome_selection_manifest(path) + + +def test_manifest_subset_violation_respects_strict_mode(tmp_path): + data = json.loads(FIXTURE.read_text(encoding="utf-8")) + data["units"]["u1"]["genome_ids_q001"].append("g3") + path = tmp_path / "manifest.json" + path.write_text(json.dumps(data), encoding="utf-8") + + with pytest.raises(ValueError, match="q0.01 genomes not present"): + load_genome_selection_manifest(path, strict=True) + with pytest.warns(UserWarning, match="q0.01 genomes not present"): + manifest = load_genome_selection_manifest(path, strict=False) + assert manifest.units["u1"].genome_ids == ["g1", "g2"] + + def test_sample_mapping_is_one_to_one(): mapping = resolve_manifest_sample_columns( ["Intensity_s1", "Intensity_s2"], ["s1", "s2"] diff --git a/tests/test_manifest_otf_backend.py b/tests/test_manifest_otf_backend.py index 0717a104..93661f73 100644 --- a/tests/test_manifest_otf_backend.py +++ b/tests/test_manifest_otf_backend.py @@ -1,6 +1,7 @@ from pathlib import Path import pandas as pd +import pytest from metax.peptide_annotator.genome_selection_manifest import load_genome_selection_manifest from metax.peptide_annotator import unit_specific_otf as backend @@ -41,3 +42,67 @@ def test_merged_column_order_always_contains_analysis_unit_id(tmp_path): ["Intensity_s1"], ) assert columns[0] == "analysis_unit_id" + + +def test_nested_digest_scan_rejects_missing_manifest_genome(tmp_path): + digests = tmp_path / "digests" + digests.mkdir() + (digests / "g1.tsv").write_text( + "Peptide\tProtein\nPEP\tg1_p1\n", + encoding="utf-8", + ) + + with pytest.raises(FileNotFoundError, match=r"1 selected manifest genomes: g2"): + backend.query_peptide_proteins_from_digested_genome_folders_nested( + digested_genome_folders=str(digests), + peptide_list=["PEP"], + selected_genomes_set={"g1", "g2"}, + digested_peptide_col="Peptide", + digested_protein_col="Protein", + parallel_backend="thread", + n_jobs=1, + ) + + +def test_nested_digest_scan_rejects_duplicate_genome_files(tmp_path): + digest_a = tmp_path / "digest-a" + digest_b = tmp_path / "digest-b" + digest_a.mkdir() + digest_b.mkdir() + for folder in (digest_a, digest_b): + (folder / "g1.tsv").write_text( + "Peptide\tProtein\nPEP\tg1_p1\n", + encoding="utf-8", + ) + + with pytest.raises(ValueError, match=r"duplicates found for 1 genomes: g1"): + backend.query_peptide_proteins_from_digested_genome_folders_nested( + digested_genome_folders=[str(digest_a), str(digest_b)], + peptide_list=["PEP"], + selected_genomes_set={"g1"}, + digested_peptide_col="Peptide", + digested_protein_col="Protein", + parallel_backend="thread", + n_jobs=1, + ) + + +def test_nested_digest_scan_uses_validated_genome_file_once(tmp_path): + digests = tmp_path / "digests" + digests.mkdir() + (digests / "g1.tsv").write_text( + "Peptide\tProtein\nPEP\tg1_p1\n", + encoding="utf-8", + ) + + mapping = backend.query_peptide_proteins_from_digested_genome_folders_nested( + digested_genome_folders=str(digests), + peptide_list=["PEP"], + selected_genomes_set={"g1"}, + digested_peptide_col="Peptide", + digested_protein_col="Protein", + parallel_backend="thread", + n_jobs=1, + ) + + assert mapping == {"PEP": {"g1": {"g1_p1"}}} diff --git a/tests/test_metaumbra_contract_integration.py b/tests/test_metaumbra_contract_integration.py index 0389311d..87455200 100644 --- a/tests/test_metaumbra_contract_integration.py +++ b/tests/test_metaumbra_contract_integration.py @@ -14,8 +14,19 @@ def test_metaumbra_generated_manifest_is_loaded_directly_by_metax(tmp_path): workspace = Path(__file__).resolve().parents[2] - schema_path = workspace / "MetaUmbra" / "docs" / "genome_selection_manifest.v1.schema.json" + configured_source = os.environ.get("METAX_METAUMBRA_SOURCE") + metaumbra_source = ( + Path(configured_source).expanduser().resolve() + if configured_source + else workspace / "MetaUmbra" + ) + schema_path = metaumbra_source / "docs" / "genome_selection_manifest.v1.schema.json" if not schema_path.is_file(): + if os.environ.get("CI"): + pytest.fail( + "MetaUmbra contract checkout is required in CI; " + f"expected schema at {schema_path}" + ) pytest.skip("Cross-repository MetaUmbra checkout is not available") schema = json.loads(schema_path.read_text(encoding="utf-8")) output = tmp_path / "genome_selection_manifest.json" @@ -49,7 +60,7 @@ def test_metaumbra_generated_manifest_is_loaded_directly_by_metax(tmp_path): write_genome_selection_manifest(output, manifest) ''' env = dict(os.environ) - env["PYTHONPATH"] = str(workspace / "MetaUmbra" / "src") + os.pathsep + env.get("PYTHONPATH", "") + env["PYTHONPATH"] = str(metaumbra_source / "src") + os.pathsep + env.get("PYTHONPATH", "") subprocess.run([sys.executable, "-c", script, str(output)], check=True, env=env) manifest = load_genome_selection_manifest(output) assert schema["$id"] == SCHEMA_VERSION == manifest.schema_version