diff --git a/.gitignore b/.gitignore index 05f3e66..1e12ad7 100644 --- a/.gitignore +++ b/.gitignore @@ -107,6 +107,7 @@ celerybeat.pid # Environments .env +.env.* .venv env/ venv/ @@ -114,6 +115,21 @@ ENV/ env.bak/ venv.bak/ +# Root-level ROBERT generated run outputs +/CURATE/ +/GENERATE/ +/VERIFY/ +/PREDICT/ +/REPORT/ +/AQME/ +/EVALUATE/ +/ROBERT_report.pdf +/report.css +/report_debug.txt + +# Local project archive created by the timestamped wrapper +/json-output-for-agent/runs/ + # Spyder project settings .spyderproject .spyproject @@ -131,3 +147,13 @@ dmypy.json # Pyre type checker .pyre/ + +# Local ROBERT comparison data +comparison/original_robert/ +comparison/chatbob_robert/ + +# Locally generated JSON outputs +/JSON/ + +# Databases provided by Juanvi for testing +/databases/ \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..90d9f89 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,6 @@ +{ + "python-envs.defaultEnvManager": "ms-python.python:conda", + "python-envs.defaultPackageManager": "ms-python.python:conda", + "python.defaultInterpreterPath": "/Users/cjcscha/mambaforge/envs/cheminf/bin/python", + "python.terminal.activateEnvironment": true +} \ No newline at end of file diff --git a/MEMORY.md b/MEMORY.md new file mode 100644 index 0000000..000f493 --- /dev/null +++ b/MEMORY.md @@ -0,0 +1,271 @@ +# MEMORY.md — ROBERT / ChatBob Project State + +## Current Project Goal + +Build a lightweight interpretation layer for completed ROBERT runs. + +The near-term technical goal is to make ROBERT generate stable, structured JSON evidence during a run so a future Dash app, ChatBob, can explain ROBERT results to chemists. + +ChatBob is not intended to replace ROBERT or EasyROB. + +- ROBERT = scientific engine +- EasyROB = setup / execution helper +- ChatBob = result interpretation and teaching layer + +## Current Architecture + +ROBERT still runs normally and creates its standard outputs. + +The JSON layer adds extra files only. It must not change, append to, rename, delete, reorder, or otherwise modify standard ROBERT outputs. + +Current JSON-related helper file: + +```text +robert/json_output_for_agent.py +``` + +Current project planning folder: + +```text +json-output-for-agent/ +``` + +This folder may later be renamed to: + +```text +json-output/ +``` + +Future ChatBob code should live separately, probably in: + +```text +chatbob/ +``` + +## Non-Negotiable Rules + +1. Plan before implementation. +2. Wait for explicit user approval before editing code. +3. Keep changes small, local, reversible, and easy to explain. +4. Do not change ROBERT scientific behavior. +5. Do not change ROBERT scoring logic. +6. Do not change model-selection logic. +7. Do not change thresholds. +8. Do not change existing CLI behavior. +9. Do not modify standard ROBERT output files. +10. JSON-layer status, warnings, success records, and failure records must go only into JSON-layer audit files. +11. Do not write JSON-layer messages into `.dat`, `.csv`, `.pdf`, image, model, or report files produced by standard ROBERT. +12. If evidence is unavailable, record it as unavailable. Do not guess. +13. Use plain-English comments for new code. +14. Avoid broad repository scans unless explicitly approved. + +## Current JSON Artifacts + +Current implemented artifacts: + +```text +CURATE/dataset_profile.json +CURATE/curate_audit.json +CURATE/json_output_audit.json +``` + +Purpose: + +```text +dataset_profile.json = what the raw input dataset looked like before ROBERT changed it +curate_audit.json = what CURATE did during the curation step +json_output_audit.json = whether JSON artifacts were attempted and whether they succeeded +``` + +The JSON layer is intended to be fail-soft. If JSON writing fails, ROBERT should continue normally. + +## Current Validation Status + +Completed validation: + +* Regression CURATE validation passed. +* Classification CURATE validation passed. +* `dataset_profile.json` was created and validated. +* `curate_audit.json` was created and validated. +* `json_output_audit.json` records JSON-layer events. +* Standard `CURATE_data.dat` was checked and did not contain JSON-layer text. +* Timestamped wrapper archive behavior has been validated as copy-only. +* Missing module folders are represented in manifests with `module_dir_exists: false` and `file_count: 0`. + +Known partial issue: + +* Full REPORT PDF generation was partially blocked in the local environment by missing WeasyPrint system libraries. ROBERT runs still completed. + +Still needed: + +* Fault-injection validation specifically for `curate_audit.json` write failure. + +## Current Source Data Folder + +Temporary validation datasets currently live in: + +```text +databases/ +``` + +This folder is treated as protected source data. + +Rules: + +* Do not edit CSV files in `databases/`. +* Do not move them. +* Do not overwrite them. +* Do not add generated ROBERT outputs into this folder. +* Do not ignore `databases/` for now. +* Before committing database files, confirm they are safe to share, reasonably small, and not private or unpublished. + +Later TODO: + +* Decide whether `databases/` should remain in the repo, move to `tests/fixtures/`, or be removed after formal JSON unit tests are created. + +## Current Documentation Files + +Important project docs: + +```text +json-output-for-agent/AGENTS.md +json-output-for-agent/PROJECT_RULES.md +json-output-for-agent/TASKS.md +json-output-for-agent/task_tracker_plain_english.md +json-output-for-agent/json_schema_notes.md +json-output-for-agent/output_map.md +json-output-for-agent/README.md +``` + +Update rules: + +* Routine validation updates: update `task_tracker_plain_english.md`. +* Schema changes: update `json_schema_notes.md`. +* Task status changes: update `TASKS.md`. +* Project-direction changes: update `AGENTS.md` and `README.md`. + +Avoid updating many docs for small routine checks. + +## Current Next Task + +Next task: + +```text +Run fault-injection validation specifically for curate_audit.json. +``` + +Goal: + +Confirm that if writing `curate_audit.json` fails: + +1. CURATE still completes normally. +2. Standard CURATE outputs are still produced. +3. `CURATE_data.dat` contains no JSON-layer messages. +4. `json_output_audit.json` records the `curate_audit.json` failure. +5. `dataset_profile.json` behavior is unaffected. + +Do not implement new hooks during this task. + +## Future Module-Native Hook Direction + +Module-native JSON hooks are important. + +Current pattern: + +```text +dataset_profile.json = raw input snapshot +curate_audit.json = CURATE module audit +``` + +Future likely artifacts: + +```text +GENERATE/generate_audit.json +VERIFY/verify_audit.json +PREDICT/predict_audit.json +REPORT/report_audit.json +run_context.json +``` + +Each future hook must: + +* capture information ROBERT already produces, +* write only new JSON files, +* never alter standard ROBERT outputs, +* include `schema_version`, +* be fail-soft, +* record JSON status only in JSON audit files, +* avoid invented explanations or thresholds. + +## Future ChatBob Direction + +ChatBob will likely be a Dash app. + +Possible tabs: + +```text +Home +Run Overview +Ask ChatBob +Teaching Mode +Next Steps +Troubleshooting +Evidence Viewer +``` + +ChatBob should: + +* select a timestamped ROBERT run, +* read JSON evidence from the run, +* answer one-time questions using OpenAI Responses, +* support FAQ-style questions, +* support RAG over ROBERT docs and user-uploaded PDFs, +* explain ROBERT results for chemists, +* show evidence used in answers. + +ChatBob should not: + +* run ROBERT as its primary role, +* replace ROBERT, +* rescore ROBERT, +* modify ROBERT outputs, +* invent scientific conclusions. + +## Files Not To Touch Without Approval + +Do not touch these without explicit approval: + +```text +standard ROBERT output files +standard ROBERT scoring code +standard ROBERT model-selection code +standard ROBERT CLI behavior +files inside databases/ +files inside tests/ +``` + +## Token-Saving Instructions For Agents + +Before inspecting files, ask whether inspection is needed. + +Prefer narrow file lists. + +Do not scan the full repository unless explicitly approved. + +For most tasks, follow this rhythm: + +1. Plan only. +2. Wait for approval. +3. Execute only the approved plan. +4. Validate only the requested behavior. +5. Update only the necessary tracker file. + +When starting a new chat, read only: + +```text +AGENTS.md +MEMORY.md +TASKS.md if needed +``` + +Do not re-read the entire workspace. diff --git a/comparison/compare_robert_outputs.ipynb b/comparison/compare_robert_outputs.ipynb new file mode 100644 index 0000000..bc7d773 --- /dev/null +++ b/comparison/compare_robert_outputs.ipynb @@ -0,0 +1,1233 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "ec3fbaf6", + "metadata": {}, + "source": [ + "# Compare Original ROBERT and ChatBob-Modified ROBERT Outputs\n", + "\n", + "## Purpose\n", + "\n", + "This notebook compares the outputs from two completed ROBERT runs:\n", + "\n", + "1. **Original ROBERT**, an unchanged reference version of ROBERT.\n", + "2. **ChatBob ROBERT**, the modified version that adds structured JSON outputs for later use by the ChatBob interface and companion agent.\n", + "\n", + "The purpose of this comparison is to confirm that the ChatBob-related modifications do not change ROBERT's standard scientific outputs.\n", + "\n", + "ROBERT remains the scientific source of truth. The JSON layer is intended to record evidence that ROBERT already produces without changing its:\n", + "\n", + "- data-curation behavior,\n", + "- descriptor selection,\n", + "- model generation and model selection,\n", + "- verification tests,\n", + "- predictions,\n", + "- scoring,\n", + "- standard `.dat` files,\n", + "- standard `.csv` files,\n", + "- plots, or\n", + "- PDF report.\n", + "\n", + "## Assumptions\n", + "\n", + "This notebook does **not** run ROBERT itself.\n", + "\n", + "Before using the notebook, the user must run the same dataset through both versions of ROBERT and place the completed outputs into two separate folders.\n", + "\n", + "The expected comparison structure is:\n", + "\n", + "```text\n", + "comparison/\n", + "├── original_robert/\n", + "│ ├── CURATE/\n", + "│ ├── GENERATE/\n", + "│ ├── VERIFY/\n", + "│ ├── PREDICT/\n", + "│ └── ROBERT_report.pdf\n", + "│\n", + "├── chatbob_robert/\n", + "│ ├── CURATE/\n", + "│ ├── GENERATE/\n", + "│ ├── VERIFY/\n", + "│ ├── PREDICT/\n", + "│ ├── ROBERT_report.pdf\n", + "│ └── additional JSON files\n", + "│\n", + "└── compare_robert_outputs.ipynb" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "6e99a663", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Comparison folder: /Users/cjcscha/ROBERT/chat_rob_UI/robert/comparison\n", + "Original ROBERT run: /Users/cjcscha/ROBERT/chat_rob_UI/robert/comparison/original_robert\n", + "ChatBob ROBERT run: /Users/cjcscha/ROBERT/chat_rob_UI/robert/comparison/chatbob_robert\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "\n", + "# The notebook should be stored inside the comparison folder:\n", + "#\n", + "# comparison/\n", + "# ├── compare_robert_outputs.ipynb\n", + "# ├── original_robert/\n", + "# └── chatbob_robert/\n", + "#\n", + "# These paths are resolved relative to the directory from which the notebook\n", + "# kernel is running, which should normally be the comparison folder.\n", + "\n", + "COMPARISON_ROOT = Path.cwd().resolve()\n", + "\n", + "ORIGINAL_RUN = COMPARISON_ROOT / \"original_robert\"\n", + "CHATBOB_RUN = COMPARISON_ROOT / \"chatbob_robert\"\n", + "\n", + "print(f\"Comparison folder: {COMPARISON_ROOT}\")\n", + "print(f\"Original ROBERT run: {ORIGINAL_RUN}\")\n", + "print(f\"ChatBob ROBERT run: {CHATBOB_RUN}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "5ad4495b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Notebook working directory: /Users/cjcscha/ROBERT/chat_rob_UI/robert/comparison\n", + "\n", + "PASS: Original ROBERT folder found\n", + " /Users/cjcscha/ROBERT/chat_rob_UI/robert/comparison/original_robert\n", + "PASS: ChatBob ROBERT folder found\n", + " /Users/cjcscha/ROBERT/chat_rob_UI/robert/comparison/chatbob_robert\n", + "\n", + "PASS: Both ROBERT run folders are available.\n" + ] + } + ], + "source": [ + "# Validate the comparison folder structure before attempting any file comparisons.\n", + "\n", + "expected_run_folders = {\n", + " \"Original ROBERT\": ORIGINAL_RUN,\n", + " \"ChatBob ROBERT\": CHATBOB_RUN,\n", + "}\n", + "\n", + "print(f\"Notebook working directory: {COMPARISON_ROOT}\\n\")\n", + "\n", + "all_paths_valid = True\n", + "\n", + "for label, run_path in expected_run_folders.items():\n", + " if run_path.exists() and run_path.is_dir():\n", + " print(f\"PASS: {label} folder found\")\n", + " print(f\" {run_path}\")\n", + " else:\n", + " print(f\"FAIL: {label} folder not found\")\n", + " print(f\" Expected location: {run_path}\")\n", + " all_paths_valid = False\n", + "\n", + "if all_paths_valid:\n", + " print(\"\\nPASS: Both ROBERT run folders are available.\")\n", + "else:\n", + " print(\n", + " \"\\nACTION REQUIRED: Check the notebook working directory and the \"\n", + " \"folder names defined in Cell 2.\"\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "a5ad854f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Original ROBERT\n", + "===============\n", + "PASS Directory: CURATE/\n", + "PASS Directory: GENERATE/\n", + "PASS Directory: VERIFY/\n", + "PASS Directory: PREDICT/\n", + "PASS File: ROBERT_report.pdf\n", + "\n", + "ChatBob ROBERT\n", + "==============\n", + "PASS Directory: CURATE/\n", + "PASS Directory: GENERATE/\n", + "PASS Directory: VERIFY/\n", + "PASS Directory: PREDICT/\n", + "PASS File: ROBERT_report.pdf\n" + ] + } + ], + "source": [ + "# Check that both run folders contain the expected standard ROBERT outputs.\n", + "#\n", + "# A missing item does not automatically mean the run failed. For example,\n", + "# ROBERT_report.pdf may be absent if PDF-generation dependencies were unavailable.\n", + "# This cell records what is present before we begin comparing file contents.\n", + "\n", + "EXPECTED_DIRECTORIES = [\n", + " \"CURATE\",\n", + " \"GENERATE\",\n", + " \"VERIFY\",\n", + " \"PREDICT\",\n", + "]\n", + "\n", + "EXPECTED_ROOT_FILES = [\n", + " \"ROBERT_report.pdf\",\n", + "]\n", + "\n", + "run_structure_results = {}\n", + "\n", + "for run_label, run_path in expected_run_folders.items():\n", + " print(f\"\\n{run_label}\")\n", + " print(\"=\" * len(run_label))\n", + "\n", + " directory_results = {}\n", + " file_results = {}\n", + "\n", + " for directory_name in EXPECTED_DIRECTORIES:\n", + " directory_path = run_path / directory_name\n", + " exists = directory_path.exists() and directory_path.is_dir()\n", + " directory_results[directory_name] = exists\n", + "\n", + " status = \"PASS\" if exists else \"MISSING\"\n", + " print(f\"{status:7} Directory: {directory_name}/\")\n", + "\n", + " for file_name in EXPECTED_ROOT_FILES:\n", + " file_path = run_path / file_name\n", + " exists = file_path.exists() and file_path.is_file()\n", + " file_results[file_name] = exists\n", + "\n", + " status = \"PASS\" if exists else \"MISSING\"\n", + " print(f\"{status:7} File: {file_name}\")\n", + "\n", + " run_structure_results[run_label] = {\n", + " \"directories\": directory_results,\n", + " \"files\": file_results,\n", + " }" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "6544a14d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Matching file paths: 56\n", + "Only in original: 0\n", + "Only in ChatBob: 0\n" + ] + } + ], + "source": [ + "# Collect all standard files from both ROBERT output folders.\n", + "# The notebook assumes the user has already placed the correct files\n", + "# into original_robert/ and chatbob_robert/.\n", + "\n", + "def collect_relative_files(run_path):\n", + " return {\n", + " path.relative_to(run_path)\n", + " for path in run_path.rglob(\"*\")\n", + " if path.is_file()\n", + " }\n", + "\n", + "original_files = collect_relative_files(ORIGINAL_RUN)\n", + "chatbob_files = collect_relative_files(CHATBOB_RUN)\n", + "\n", + "common_files = sorted(original_files & chatbob_files)\n", + "original_only_files = sorted(original_files - chatbob_files)\n", + "chatbob_only_files = sorted(chatbob_files - original_files)\n", + "\n", + "print(f\"Matching file paths: {len(common_files)}\")\n", + "print(f\"Only in original: {len(original_only_files)}\")\n", + "print(f\"Only in ChatBob: {len(chatbob_only_files)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "f596b44d", + "metadata": {}, + "outputs": [], + "source": [ + "if original_only_files:\n", + " print(\"\\nOnly in original ROBERT:\")\n", + " for path in original_only_files:\n", + " print(path)\n", + "\n", + "if chatbob_only_files:\n", + " print(\"\\nOnly in ChatBob ROBERT:\")\n", + " for path in chatbob_only_files:\n", + " print(path)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "71c6788f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Matching .dat files found: 4\n", + "\n", + "DIFFERENT CURATE/CURATE_data.dat\n", + "--- original/CURATE/CURATE_data.dat\n", + "+++ chatbob/CURATE/CURATE_data.dat\n", + "@@ -1,7 +1,7 @@\n", + "-ROBERT v 2.1.2 2026/07/10 14:21:21 \n", + "+ROBERT v 2.1.2 2026/07/13 14:09:50 \n", + " How to cite: Dalmau, D.; Alegre Requena, J. V. WIREs Comput Mol Sci. 2024, 14, e1733.\n", + " \n", + "-Command line used in ROBERT: python -m robert --csv_name \"/Users/cjcscha/ROBERT/robert-1/H_predict_ln-k.csv\" --y \"ln(k)_rate\" --names \"Coupling\" --ignore \"Coupling\"\n", + "+Command line used in ROBERT: python -m robert --csv_name \"/Users/cjcscha/ROBERT/chat_rob_UI/robert/databases/Regression/H_predict_ln-k.csv\" --y \"ln(k)_rate\" --names \"Coupling\" --ignore \"Coupling\"\n", + " \n", + " \n", + " o Starting data curation with the CURATE module\n", + "@@ -38,9 +38,9 @@\n", + " o Model MVL: 4 descriptors remaining:\n", + " IR-freq-C=O_interm_boltz, NBO-C1 C=O_anion_boltz, Sterimol-B1-N1-C4_amine_boltz, Vbur-2.0A_anion_boltz\n", + " \n", + "-o Model-specific curated databases were stored in /Users/cjcscha/ROBERT/robert-1/CURATE\n", + "+o Model-specific curated databases were stored in /Users/cjcscha/ROBERT/chat_rob_UI/robert/CURATE\n", + " \n", + " o The Pearson heatmap was stored in CURATE/Pearson_heatmap.png.\n", + " \n", + "-Time CURATE: 0.45 seconds\n", + "+Time CURATE: 0.46 seconds\n", + " \n", + "\n", + "DIFFERENT GENERATE/GENERATE_data.dat\n", + "--- original/GENERATE/GENERATE_data.dat\n", + "+++ chatbob/GENERATE/GENERATE_data.dat\n", + "@@ -1,7 +1,7 @@\n", + "-ROBERT v 2.1.2 2026/07/10 14:21:21 \n", + "+ROBERT v 2.1.2 2026/07/13 14:09:50 \n", + " How to cite: Dalmau, D.; Alegre Requena, J. V. WIREs Comput Mol Sci. 2024, 14, e1733.\n", + " \n", + "-Command line used in ROBERT: python -m robert --csv_name \"/Users/cjcscha/ROBERT/robert-1/H_predict_ln-k.csv\" --y \"ln(k)_rate\" --names \"Coupling\" --ignore \"Coupling\"\n", + "+Command line used in ROBERT: python -m robert --csv_name \"/Users/cjcscha/ROBERT/chat_rob_UI/robert/databases/Regression/H_predict_ln-k.csv\" --y \"ln(k)_rate\" --names \"Coupling\" --ignore \"Coupling\"\n", + " \n", + " \n", + " o Starting generation of ML models with the GENERATE module\n", + "@@ -19,7 +19,7 @@\n", + " 1. 50% = RMSE from a 10x repeated 5-fold CV (interpoplation)\n", + " \n", + " 2. 50% = RMSE from the bottom or top (worst performing) fold in a sorted 5-fold CV (extrapolation)\n", + "- \n", + "+ \n", + " \n", + " - 1/4 - ML model: RF \n", + " o Using model-specific curated database: H_predict_ln-k_CURATE_RF.csv\n", + "@@ -58,5 +58,5 @@\n", + " \n", + " o Heatmap_ML_models_PFI succesfully created in GENERATE/Raw_data\n", + " \n", + "-Time GENERATE: 58.29 seconds\n", + "+Time GENERATE: 57.82 seconds\n", + " \n", + "\n", + "DIFFERENT PREDICT/PREDICT_data.dat\n", + "--- original/PREDICT/PREDICT_data.dat\n", + "+++ chatbob/PREDICT/PREDICT_data.dat\n", + "@@ -1,7 +1,7 @@\n", + "-ROBERT v 2.1.2 2026/07/10 14:21:21 \n", + "+ROBERT v 2.1.2 2026/07/13 14:09:50 \n", + " How to cite: Dalmau, D.; Alegre Requena, J. V. WIREs Comput Mol Sci. 2024, 14, e1733.\n", + " \n", + "-Command line used in ROBERT: python -m robert --csv_name \"/Users/cjcscha/ROBERT/robert-1/H_predict_ln-k.csv\" --y \"ln(k)_rate\" --names \"Coupling\" --ignore \"Coupling\"\n", + "+Command line used in ROBERT: python -m robert --csv_name \"/Users/cjcscha/ROBERT/chat_rob_UI/robert/databases/Regression/H_predict_ln-k.csv\" --y \"ln(k)_rate\" --names \"Coupling\" --ignore \"Coupling\"\n", + " \n", + " \n", + " o Representation of predictions and analysis of ML models with the PREDICT module\n", + "@@ -129,5 +129,5 @@\n", + " - The number of points in each quartile is Q1: 9, Q2: 5, Q3: 15, Q4: 15\n", + " x WARNING! Your data is slightly not uniform (Q2 has 5 points while Q3 has 15)\n", + " \n", + "-Time PREDICT: 6.62 seconds\n", + "+Time PREDICT: 6.67 seconds\n", + " \n", + "\n", + "DIFFERENT VERIFY/VERIFY_data.dat\n", + "--- original/VERIFY/VERIFY_data.dat\n", + "+++ chatbob/VERIFY/VERIFY_data.dat\n", + "@@ -1,7 +1,7 @@\n", + "-ROBERT v 2.1.2 2026/07/10 14:21:21 \n", + "+ROBERT v 2.1.2 2026/07/13 14:09:50 \n", + " How to cite: Dalmau, D.; Alegre Requena, J. V. WIREs Comput Mol Sci. 2024, 14, e1733.\n", + " \n", + "-Command line used in ROBERT: python -m robert --csv_name \"/Users/cjcscha/ROBERT/robert-1/H_predict_ln-k.csv\" --y \"ln(k)_rate\" --names \"Coupling\" --ignore \"Coupling\"\n", + "+Command line used in ROBERT: python -m robert --csv_name \"/Users/cjcscha/ROBERT/chat_rob_UI/robert/databases/Regression/H_predict_ln-k.csv\" --y \"ln(k)_rate\" --names \"Coupling\" --ignore \"Coupling\"\n", + " \n", + " \n", + " o Starting tests to verify the prediction ability of the ML models with the VERIFY module\n", + "@@ -60,5 +60,5 @@\n", + " o onehot: PASSED, RMSE = 2.0, higher than thresholds\n", + " - Sorted 5-fold CV : R2 = [0.19, 0.0, 0.36, 0.39, 0.2], MAE = [1.95, 0.98, 0.92, 0.71, 1.77], RMSE = [2.08, 1.24, 1.08, 0.95, 1.92]\n", + " \n", + "-Time VERIFY: 1.82 seconds\n", + "+Time VERIFY: 1.83 seconds\n", + " \n", + "\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import difflib\n", + "\n", + "# Compare matching ROBERT .dat files.\n", + "# The first check is exact: are the two files byte-for-byte identical?\n", + "# If not, show a short unified diff so the differences can be inspected.\n", + "\n", + "dat_files = sorted(\n", + " path for path in common_files\n", + " if path.suffix.lower() == \".dat\"\n", + ")\n", + "\n", + "print(f\"Matching .dat files found: {len(dat_files)}\\n\")\n", + "\n", + "dat_comparison_results = {}\n", + "\n", + "for relative_path in dat_files:\n", + " original_path = ORIGINAL_RUN / relative_path\n", + " chatbob_path = CHATBOB_RUN / relative_path\n", + "\n", + " original_bytes = original_path.read_bytes()\n", + " chatbob_bytes = chatbob_path.read_bytes()\n", + "\n", + " identical = original_bytes == chatbob_bytes\n", + " dat_comparison_results[str(relative_path)] = identical\n", + "\n", + " status = \"IDENTICAL\" if identical else \"DIFFERENT\"\n", + " print(f\"{status:10} {relative_path}\")\n", + "\n", + " if not identical:\n", + " original_lines = original_path.read_text(\n", + " errors=\"replace\"\n", + " ).splitlines()\n", + "\n", + " chatbob_lines = chatbob_path.read_text(\n", + " errors=\"replace\"\n", + " ).splitlines()\n", + "\n", + " diff = list(\n", + " difflib.unified_diff(\n", + " original_lines,\n", + " chatbob_lines,\n", + " fromfile=f\"original/{relative_path}\",\n", + " tofile=f\"chatbob/{relative_path}\",\n", + " lineterm=\"\",\n", + " )\n", + " )\n", + "\n", + " print(\"\\n\".join(diff[:40]))\n", + "\n", + " if len(diff) > 40:\n", + " print(f\"... {len(diff) - 40} additional diff lines not shown\")\n", + "\n", + " print()" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "2e539b84", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Matching CSV files found: 28\n", + "\n", + "CURATE/CURATE_options.csv\n", + "-------------------------\n", + "Numeric cells different: 0\n", + "Text cells different: 1\n", + "Maximum absolute numeric difference: 0\n", + "\n", + "CURATE/H_predict_ln-k_CURATE.csv\n", + "--------------------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n", + "CURATE/H_predict_ln-k_CURATE_GB.csv\n", + "-----------------------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n", + "CURATE/H_predict_ln-k_CURATE_MVL.csv\n", + "------------------------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n", + "CURATE/H_predict_ln-k_CURATE_NN.csv\n", + "-----------------------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n", + "CURATE/H_predict_ln-k_CURATE_RF.csv\n", + "-----------------------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n", + "GENERATE/Best_model/No_PFI/NN.csv\n", + "---------------------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n", + "GENERATE/Best_model/No_PFI/NN_db.csv\n", + "------------------------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n", + "GENERATE/Best_model/PFI/MVL_PFI.csv\n", + "-----------------------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n", + "GENERATE/Best_model/PFI/MVL_PFI_db.csv\n", + "--------------------------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n", + "GENERATE/Raw_data/No_PFI/GB.csv\n", + "-------------------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n", + "GENERATE/Raw_data/No_PFI/GB_db.csv\n", + "----------------------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n", + "GENERATE/Raw_data/No_PFI/MVL.csv\n", + "--------------------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n", + "GENERATE/Raw_data/No_PFI/MVL_db.csv\n", + "-----------------------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n", + "GENERATE/Raw_data/No_PFI/NN.csv\n", + "-------------------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n", + "GENERATE/Raw_data/No_PFI/NN_db.csv\n", + "----------------------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n", + "GENERATE/Raw_data/No_PFI/RF.csv\n", + "-------------------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n", + "GENERATE/Raw_data/No_PFI/RF_db.csv\n", + "----------------------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n", + "GENERATE/Raw_data/PFI/GB_PFI.csv\n", + "--------------------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n", + "GENERATE/Raw_data/PFI/GB_PFI_db.csv\n", + "-----------------------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n", + "GENERATE/Raw_data/PFI/MVL_PFI.csv\n", + "---------------------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n", + "GENERATE/Raw_data/PFI/MVL_PFI_db.csv\n", + "------------------------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n", + "GENERATE/Raw_data/PFI/NN_PFI.csv\n", + "--------------------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n", + "GENERATE/Raw_data/PFI/NN_PFI_db.csv\n", + "-----------------------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n", + "GENERATE/Raw_data/PFI/RF_PFI.csv\n", + "--------------------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n", + "GENERATE/Raw_data/PFI/RF_PFI_db.csv\n", + "-----------------------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n", + "PREDICT/MVL_PFI.csv\n", + "-------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n", + "PREDICT/NN_No_PFI.csv\n", + "---------------------\n", + "IDENTICAL within numerical tolerance\n", + "\n" + ] + } + ], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "\n", + "# Compare matching CSV files produced by the two ROBERT runs.\n", + "#\n", + "# Numeric values are compared using a small tolerance because values that are\n", + "# scientifically equivalent may sometimes differ only through floating-point\n", + "# formatting. Text values are compared exactly.\n", + "#\n", + "# This comparison assumes that corresponding rows appear in the same order.\n", + "\n", + "RTOL = 1e-8\n", + "ATOL = 1e-10\n", + "\n", + "csv_files = sorted(\n", + " path for path in common_files\n", + " if path.suffix.lower() == \".csv\"\n", + ")\n", + "\n", + "print(f\"Matching CSV files found: {len(csv_files)}\\n\")\n", + "\n", + "csv_comparison_results = {}\n", + "\n", + "for relative_path in csv_files:\n", + " original_path = ORIGINAL_RUN / relative_path\n", + " chatbob_path = CHATBOB_RUN / relative_path\n", + "\n", + " original_df = pd.read_csv(original_path)\n", + " chatbob_df = pd.read_csv(chatbob_path)\n", + "\n", + " result = {\n", + " \"same_shape\": original_df.shape == chatbob_df.shape,\n", + " \"same_columns\": list(original_df.columns) == list(chatbob_df.columns),\n", + " \"numeric_differences\": None,\n", + " \"text_differences\": None,\n", + " \"maximum_absolute_difference\": None,\n", + " }\n", + "\n", + " print(relative_path)\n", + " print(\"-\" * len(str(relative_path)))\n", + "\n", + " if not result[\"same_shape\"]:\n", + " print(\n", + " f\"DIFFERENT SHAPE: original {original_df.shape}, \"\n", + " f\"ChatBob {chatbob_df.shape}\\n\"\n", + " )\n", + " csv_comparison_results[str(relative_path)] = result\n", + " continue\n", + "\n", + " if not result[\"same_columns\"]:\n", + " print(\"DIFFERENT COLUMNS\")\n", + " print(f\"Original: {list(original_df.columns)}\")\n", + " print(f\"ChatBob: {list(chatbob_df.columns)}\\n\")\n", + " csv_comparison_results[str(relative_path)] = result\n", + " continue\n", + "\n", + " numeric_columns = original_df.select_dtypes(\n", + " include=[np.number]\n", + " ).columns.tolist()\n", + "\n", + " text_columns = [\n", + " column\n", + " for column in original_df.columns\n", + " if column not in numeric_columns\n", + " ]\n", + "\n", + " numeric_difference_count = 0\n", + " maximum_absolute_difference = 0.0\n", + "\n", + " for column in numeric_columns:\n", + " original_values = original_df[column].to_numpy()\n", + " chatbob_values = chatbob_df[column].to_numpy()\n", + "\n", + " values_match = np.isclose(\n", + " original_values,\n", + " chatbob_values,\n", + " rtol=RTOL,\n", + " atol=ATOL,\n", + " equal_nan=True,\n", + " )\n", + "\n", + " numeric_difference_count += int((~values_match).sum())\n", + "\n", + " finite_differences = np.abs(\n", + " original_values.astype(float)\n", + " - chatbob_values.astype(float)\n", + " )\n", + "\n", + " finite_differences = finite_differences[\n", + " np.isfinite(finite_differences)\n", + " ]\n", + "\n", + " if finite_differences.size:\n", + " maximum_absolute_difference = max(\n", + " maximum_absolute_difference,\n", + " float(finite_differences.max()),\n", + " )\n", + "\n", + " text_difference_count = 0\n", + "\n", + " for column in text_columns:\n", + " original_values = original_df[column].fillna(\"\").astype(str)\n", + " chatbob_values = chatbob_df[column].fillna(\"\").astype(str)\n", + "\n", + " text_difference_count += int(\n", + " (original_values != chatbob_values).sum()\n", + " )\n", + "\n", + " result[\"numeric_differences\"] = numeric_difference_count\n", + " result[\"text_differences\"] = text_difference_count\n", + " result[\"maximum_absolute_difference\"] = maximum_absolute_difference\n", + "\n", + " if numeric_difference_count == 0 and text_difference_count == 0:\n", + " print(\"IDENTICAL within numerical tolerance\")\n", + " else:\n", + " print(f\"Numeric cells different: {numeric_difference_count}\")\n", + " print(f\"Text cells different: {text_difference_count}\")\n", + " print(\n", + " \"Maximum absolute numeric difference: \"\n", + " f\"{maximum_absolute_difference:.12g}\"\n", + " )\n", + "\n", + " print()\n", + "\n", + " csv_comparison_results[str(relative_path)] = result" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "e5a8cd26", + "metadata": {}, + "outputs": [], + "source": [ + "# Inspect CSV files that have different shapes or column structures.\n", + "\n", + "for relative_path in csv_files:\n", + " original_path = ORIGINAL_RUN / relative_path\n", + " chatbob_path = CHATBOB_RUN / relative_path\n", + "\n", + " original_df = pd.read_csv(original_path)\n", + " chatbob_df = pd.read_csv(chatbob_path)\n", + "\n", + " if (\n", + " original_df.shape != chatbob_df.shape\n", + " or list(original_df.columns) != list(chatbob_df.columns)\n", + " ):\n", + " original_columns = list(original_df.columns)\n", + " chatbob_columns = list(chatbob_df.columns)\n", + "\n", + " only_in_original = [\n", + " column for column in original_columns\n", + " if column not in chatbob_columns\n", + " ]\n", + "\n", + " only_in_chatbob = [\n", + " column for column in chatbob_columns\n", + " if column not in original_columns\n", + " ]\n", + "\n", + " print(f\"\\n{relative_path}\")\n", + " print(\"=\" * len(str(relative_path)))\n", + "\n", + " print(f\"Original shape: {original_df.shape}\")\n", + " print(f\"ChatBob shape: {chatbob_df.shape}\")\n", + "\n", + " print(\"\\nOriginal columns:\")\n", + " for column in original_columns:\n", + " print(f\" {column}\")\n", + "\n", + " print(\"\\nChatBob columns:\")\n", + " for column in chatbob_columns:\n", + " print(f\" {column}\")\n", + "\n", + " print(\"\\nOnly in original:\")\n", + " if only_in_original:\n", + " for column in only_in_original:\n", + " print(f\" {column}\")\n", + " else:\n", + " print(\" None\")\n", + "\n", + " print(\"\\nOnly in ChatBob:\")\n", + " if only_in_chatbob:\n", + " for column in only_in_chatbob:\n", + " print(f\" {column}\")\n", + " else:\n", + " print(\" None\")" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "167b5244", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "DIFFERENT shared columns: CURATE/CURATE_options.csv\n", + " Numeric cells different: 0\n", + " Text cells different: 1\n", + " Maximum absolute difference: 0\n", + "IDENTICAL shared columns: CURATE/H_predict_ln-k_CURATE.csv\n", + "IDENTICAL shared columns: CURATE/H_predict_ln-k_CURATE_GB.csv\n", + "IDENTICAL shared columns: CURATE/H_predict_ln-k_CURATE_MVL.csv\n", + "IDENTICAL shared columns: CURATE/H_predict_ln-k_CURATE_NN.csv\n", + "IDENTICAL shared columns: CURATE/H_predict_ln-k_CURATE_RF.csv\n", + "IDENTICAL shared columns: GENERATE/Best_model/No_PFI/NN.csv\n", + "IDENTICAL shared columns: GENERATE/Best_model/No_PFI/NN_db.csv\n", + "IDENTICAL shared columns: GENERATE/Best_model/PFI/MVL_PFI.csv\n", + "IDENTICAL shared columns: GENERATE/Best_model/PFI/MVL_PFI_db.csv\n", + "IDENTICAL shared columns: GENERATE/Raw_data/No_PFI/GB.csv\n", + "IDENTICAL shared columns: GENERATE/Raw_data/No_PFI/GB_db.csv\n", + "IDENTICAL shared columns: GENERATE/Raw_data/No_PFI/MVL.csv\n", + "IDENTICAL shared columns: GENERATE/Raw_data/No_PFI/MVL_db.csv\n", + "IDENTICAL shared columns: GENERATE/Raw_data/No_PFI/NN.csv\n", + "IDENTICAL shared columns: GENERATE/Raw_data/No_PFI/NN_db.csv\n", + "IDENTICAL shared columns: GENERATE/Raw_data/No_PFI/RF.csv\n", + "IDENTICAL shared columns: GENERATE/Raw_data/No_PFI/RF_db.csv\n", + "IDENTICAL shared columns: GENERATE/Raw_data/PFI/GB_PFI.csv\n", + "IDENTICAL shared columns: GENERATE/Raw_data/PFI/GB_PFI_db.csv\n", + "IDENTICAL shared columns: GENERATE/Raw_data/PFI/MVL_PFI.csv\n", + "IDENTICAL shared columns: GENERATE/Raw_data/PFI/MVL_PFI_db.csv\n", + "IDENTICAL shared columns: GENERATE/Raw_data/PFI/NN_PFI.csv\n", + "IDENTICAL shared columns: GENERATE/Raw_data/PFI/NN_PFI_db.csv\n", + "IDENTICAL shared columns: GENERATE/Raw_data/PFI/RF_PFI.csv\n", + "IDENTICAL shared columns: GENERATE/Raw_data/PFI/RF_PFI_db.csv\n", + "IDENTICAL shared columns: PREDICT/MVL_PFI.csv\n", + "IDENTICAL shared columns: PREDICT/NN_No_PFI.csv\n" + ] + } + ], + "source": [ + "# Compare only the columns shared by both versions of each CSV.\n", + "#\n", + "# This allows us to ignore additional columns introduced in one ROBERT version,\n", + "# such as ln(k)_rate_pred_conformal_hw, while still checking whether the common\n", + "# scientific outputs are identical.\n", + "\n", + "shared_column_results = {}\n", + "\n", + "for relative_path in csv_files:\n", + " original_path = ORIGINAL_RUN / relative_path\n", + " chatbob_path = CHATBOB_RUN / relative_path\n", + "\n", + " original_df = pd.read_csv(original_path)\n", + " chatbob_df = pd.read_csv(chatbob_path)\n", + "\n", + " shared_columns = [\n", + " column\n", + " for column in original_df.columns\n", + " if column in chatbob_df.columns\n", + " ]\n", + "\n", + " original_shared = original_df[shared_columns]\n", + " chatbob_shared = chatbob_df[shared_columns]\n", + "\n", + " if original_shared.shape != chatbob_shared.shape:\n", + " print(f\"{relative_path}: DIFFERENT ROW COUNTS\")\n", + " continue\n", + "\n", + " numeric_columns = original_shared.select_dtypes(\n", + " include=[np.number]\n", + " ).columns.tolist()\n", + "\n", + " text_columns = [\n", + " column\n", + " for column in shared_columns\n", + " if column not in numeric_columns\n", + " ]\n", + "\n", + " differing_numeric_cells = 0\n", + " differing_text_cells = 0\n", + " maximum_absolute_difference = 0.0\n", + "\n", + " for column in numeric_columns:\n", + " original_values = original_shared[column].to_numpy(dtype=float)\n", + " chatbob_values = chatbob_shared[column].to_numpy(dtype=float)\n", + "\n", + " matches = np.isclose(\n", + " original_values,\n", + " chatbob_values,\n", + " rtol=RTOL,\n", + " atol=ATOL,\n", + " equal_nan=True,\n", + " )\n", + "\n", + " differing_numeric_cells += int((~matches).sum())\n", + "\n", + " differences = np.abs(original_values - chatbob_values)\n", + " finite_differences = differences[np.isfinite(differences)]\n", + "\n", + " if finite_differences.size:\n", + " maximum_absolute_difference = max(\n", + " maximum_absolute_difference,\n", + " float(finite_differences.max()),\n", + " )\n", + "\n", + " for column in text_columns:\n", + " original_values = (\n", + " original_shared[column]\n", + " .fillna(\"\")\n", + " .astype(str)\n", + " )\n", + "\n", + " chatbob_values = (\n", + " chatbob_shared[column]\n", + " .fillna(\"\")\n", + " .astype(str)\n", + " )\n", + "\n", + " differing_text_cells += int(\n", + " (original_values != chatbob_values).sum()\n", + " )\n", + "\n", + " shared_column_results[str(relative_path)] = {\n", + " \"shared_columns\": shared_columns,\n", + " \"numeric_differences\": differing_numeric_cells,\n", + " \"text_differences\": differing_text_cells,\n", + " \"maximum_absolute_difference\": maximum_absolute_difference,\n", + " }\n", + "\n", + " if differing_numeric_cells == 0 and differing_text_cells == 0:\n", + " print(f\"IDENTICAL shared columns: {relative_path}\")\n", + " else:\n", + " print(f\"\\nDIFFERENT shared columns: {relative_path}\")\n", + " print(f\" Numeric cells different: {differing_numeric_cells}\")\n", + " print(f\" Text cells different: {differing_text_cells}\")\n", + " print(\n", + " \" Maximum absolute difference: \"\n", + " f\"{maximum_absolute_difference:.12g}\"\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "e461ca41", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "CURATE/CURATE_options.csv\n", + "=========================\n", + "Column: csv_name\n", + " Type: text\n", + " Differing rows: 1\n" + ] + } + ], + "source": [ + "# Identify which shared columns differ in each CSV.\n", + "#\n", + "# For numeric columns, report:\n", + "# - number of differing rows,\n", + "# - maximum absolute difference,\n", + "# - mean absolute difference.\n", + "#\n", + "# For text columns, report the number of differing rows.\n", + "\n", + "for relative_path in csv_files:\n", + " original_path = ORIGINAL_RUN / relative_path\n", + " chatbob_path = CHATBOB_RUN / relative_path\n", + "\n", + " original_df = pd.read_csv(original_path)\n", + " chatbob_df = pd.read_csv(chatbob_path)\n", + "\n", + " shared_columns = [\n", + " column\n", + " for column in original_df.columns\n", + " if column in chatbob_df.columns\n", + " ]\n", + "\n", + " column_differences = []\n", + "\n", + " for column in shared_columns:\n", + " original_column = original_df[column]\n", + " chatbob_column = chatbob_df[column]\n", + "\n", + " if (\n", + " pd.api.types.is_numeric_dtype(original_column)\n", + " and pd.api.types.is_numeric_dtype(chatbob_column)\n", + " ):\n", + " original_values = original_column.to_numpy(dtype=float)\n", + " chatbob_values = chatbob_column.to_numpy(dtype=float)\n", + "\n", + " matches = np.isclose(\n", + " original_values,\n", + " chatbob_values,\n", + " rtol=RTOL,\n", + " atol=ATOL,\n", + " equal_nan=True,\n", + " )\n", + "\n", + " differing_rows = int((~matches).sum())\n", + "\n", + " if differing_rows > 0:\n", + " absolute_differences = np.abs(\n", + " original_values - chatbob_values\n", + " )\n", + "\n", + " finite_differences = absolute_differences[\n", + " np.isfinite(absolute_differences)\n", + " ]\n", + "\n", + " column_differences.append(\n", + " {\n", + " \"column\": column,\n", + " \"type\": \"numeric\",\n", + " \"differing_rows\": differing_rows,\n", + " \"maximum_absolute_difference\": (\n", + " float(finite_differences.max())\n", + " if finite_differences.size\n", + " else np.nan\n", + " ),\n", + " \"mean_absolute_difference\": (\n", + " float(finite_differences.mean())\n", + " if finite_differences.size\n", + " else np.nan\n", + " ),\n", + " }\n", + " )\n", + "\n", + " else:\n", + " original_values = (\n", + " original_column.fillna(\"\").astype(str)\n", + " )\n", + " chatbob_values = (\n", + " chatbob_column.fillna(\"\").astype(str)\n", + " )\n", + "\n", + " differing_rows = int(\n", + " (original_values != chatbob_values).sum()\n", + " )\n", + "\n", + " if differing_rows > 0:\n", + " column_differences.append(\n", + " {\n", + " \"column\": column,\n", + " \"type\": \"text\",\n", + " \"differing_rows\": differing_rows,\n", + " }\n", + " )\n", + "\n", + " if column_differences:\n", + " print(f\"\\n{relative_path}\")\n", + " print(\"=\" * len(str(relative_path)))\n", + "\n", + " for result in column_differences:\n", + " print(f\"Column: {result['column']}\")\n", + " print(f\" Type: {result['type']}\")\n", + " print(f\" Differing rows: {result['differing_rows']}\")\n", + "\n", + " if result[\"type\"] == \"numeric\":\n", + " print(\n", + " \" Maximum difference: \"\n", + " f\"{result['maximum_absolute_difference']:.12g}\"\n", + " )\n", + " print(\n", + " \" Mean difference: \"\n", + " f\"{result['mean_absolute_difference']:.12g}\"\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "0f111902", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NORMALIZED DAT COMPARISON\n", + "=========================\n", + "IDENTICAL CURATE/CURATE_data.dat\n", + "IDENTICAL GENERATE/GENERATE_data.dat\n", + "IDENTICAL PREDICT/PREDICT_data.dat\n", + "IDENTICAL VERIFY/VERIFY_data.dat\n" + ] + } + ], + "source": [ + "import re\n", + "\n", + "# Compare DAT files after removing expected run-specific differences.\n", + "#\n", + "# The normalization removes:\n", + "# - the run timestamp from the ROBERT version line,\n", + "# - the command-line CSV path,\n", + "# - absolute output-directory paths,\n", + "# - module execution times,\n", + "# - trailing whitespace.\n", + "#\n", + "# It does not remove scientific results.\n", + "\n", + "def normalize_dat_text(path):\n", + " lines = path.read_text(errors=\"replace\").splitlines()\n", + " normalized_lines = []\n", + "\n", + " for line in lines:\n", + " # Remove trailing spaces and tabs.\n", + " line = line.rstrip()\n", + "\n", + " # Normalize the timestamp while preserving the ROBERT version.\n", + " line = re.sub(\n", + " r\"^(ROBERT v\\s+\\S+)\\s+\\d{4}/\\d{2}/\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}\\s*$\",\n", + " r\"\\1 \",\n", + " line,\n", + " )\n", + "\n", + " # Normalize the input CSV path in the recorded command line.\n", + " line = re.sub(\n", + " r'(--csv_name\\s+)\"[^\"]+\"',\n", + " r'\\1\"\"',\n", + " line,\n", + " )\n", + "\n", + " # Normalize absolute paths used in output-location messages.\n", + " if \"were stored in \" in line:\n", + " line = re.sub(\n", + " r\"were stored in .*$\",\n", + " \"were stored in \",\n", + " line,\n", + " )\n", + "\n", + " # Normalize module execution times.\n", + " line = re.sub(\n", + " r\"^Time (CURATE|GENERATE|VERIFY|PREDICT):\\s+\"\n", + " r\"[0-9.]+\\s+seconds$\",\n", + " r\"Time \\1: \",\n", + " line,\n", + " )\n", + "\n", + " normalized_lines.append(line)\n", + "\n", + " return \"\\n\".join(normalized_lines)\n", + "\n", + "\n", + "print(\"NORMALIZED DAT COMPARISON\")\n", + "print(\"=========================\")\n", + "\n", + "normalized_dat_results = {}\n", + "\n", + "for relative_path in dat_files:\n", + " original_path = ORIGINAL_RUN / relative_path\n", + " chatbob_path = CHATBOB_RUN / relative_path\n", + "\n", + " original_normalized = normalize_dat_text(original_path)\n", + " chatbob_normalized = normalize_dat_text(chatbob_path)\n", + "\n", + " identical = original_normalized == chatbob_normalized\n", + " normalized_dat_results[str(relative_path)] = identical\n", + "\n", + " status = \"IDENTICAL\" if identical else \"DIFFERENT\"\n", + " print(f\"{status:10} {relative_path}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "17ad1293", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ROBERT OUTPUT VALIDATION SUMMARY\n", + "================================\n", + "Normalized DAT comparison: PASS\n", + "Scientific CSV comparison: PASS\n", + "Expected metadata difference: CURATE/CURATE_options.csv records a different input CSV path.\n", + "\n", + "Conclusion: The ChatBob-modified ROBERT 2.1.2 preserves the standard ROBERT scientific outputs for this test case.\n" + ] + } + ], + "source": [ + "# Final validation summary\n", + "\n", + "dat_pass = all(normalized_dat_results.values())\n", + "\n", + "csv_failures = []\n", + "\n", + "for relative_path, result in csv_comparison_results.items():\n", + " if relative_path == \"CURATE/CURATE_options.csv\":\n", + " continue\n", + "\n", + " if not (\n", + " result[\"same_shape\"]\n", + " and result[\"same_columns\"]\n", + " and result[\"numeric_differences\"] == 0\n", + " and result[\"text_differences\"] == 0\n", + " ):\n", + " csv_failures.append(relative_path)\n", + "\n", + "print(\"ROBERT OUTPUT VALIDATION SUMMARY\")\n", + "print(\"================================\")\n", + "\n", + "print(f\"Normalized DAT comparison: {'PASS' if dat_pass else 'FAIL'}\")\n", + "print(f\"Scientific CSV comparison: {'PASS' if not csv_failures else 'FAIL'}\")\n", + "\n", + "print(\n", + " \"Expected metadata difference: \"\n", + " \"CURATE/CURATE_options.csv records a different input CSV path.\"\n", + ")\n", + "\n", + "if csv_failures:\n", + " print(\"\\nFiles requiring investigation:\")\n", + " for path in csv_failures:\n", + " print(f\" {path}\")\n", + "else:\n", + " print(\n", + " \"\\nConclusion: The ChatBob-modified ROBERT 2.1.2 preserves \"\n", + " \"the standard ROBERT scientific outputs for this test case.\"\n", + " )" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "cheminf", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.15" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/json-output-for-agent/AGENTS.md b/json-output-for-agent/AGENTS.md new file mode 100644 index 0000000..f390c16 --- /dev/null +++ b/json-output-for-agent/AGENTS.md @@ -0,0 +1,590 @@ +# AGENTS.md — ROBERT JSON Output Agent Guide + +## Mission + +We are modifying ROBERT in the smallest safe way possible so that ROBERT can export structured JSON files during a run. + +The goal is to capture information ROBERT already generates, including information written to `.dat` files, shown in the PDF report, saved as images, or produced during CURATE, GENERATE, VERIFY, PREDICT, and REPORT. + +The JSON files will later support: + +- a user interface, +- chemist-friendly result interpretation, +- easier retrieval of ROBERT run evidence, +- optional LLM explanations. + +This project must not change ROBERT's scientific behavior. + +ROBERT remains the authority. + +The agent helps expose ROBERT evidence. It does not replace ROBERT, rescore ROBERT, or invent new diagnostics. + +--- + +## Core Principle + +Minimize changes to existing ROBERT code. + +When changes are necessary, make them: + +- local, +- small, +- reversible, +- easy to explain, +- additive, +- well commented, +- tested when practical. + +The preferred strategy is to add helper functions in one project-specific helper file and then call those helpers from existing ROBERT modules with as few added lines as possible. + +--- + +## Runtime Evidence Capture Rule + +The JSON-output-for-agent layer must capture structured evidence at runtime, not merely summarize files after the run when the relevant values are available during the run. + +Whenever ROBERT writes meaningful scientific or workflow evidence to a module `.dat` file, the same underlying values should also be captured in a structured in-memory audit object and written to a module audit JSON file. + +Existing `.dat` behavior must be preserved exactly. + +Do not remove, rewrite, reroute, or replace existing `self.args.log.write(...)` calls. + +Add JSON capture beside existing logging, not instead of existing logging. + +The preferred pattern is: + +1. ROBERT computes or identifies an important value. +2. ROBERT builds the same text it already writes to `.dat`. +3. ROBERT writes that text to `.dat` exactly as before. +4. The JSON-output-for-agent layer stores the same underlying value as structured evidence. + +Evidence must be labeled conceptually as: + +- `direct`: captured directly from ROBERT runtime values, +- `derived`: computed from direct values only for convenience, +- `unavailable`: not recoverable from current runtime values without deeper instrumentation. + +Do not invent scientific interpretations, new diagnostics, new thresholds, new scores, or new model-quality judgments. + +Do not parse `.dat` files as the primary strategy when the same values are available in memory at the point of logging. `.dat` parsing may be used only as a temporary fallback and must be labeled as such. + +JSON write failures must remain fail-soft and must never change ROBERT scientific behavior, CLI behavior, or standard `.dat`, `.csv`, image, model, or report outputs. + +Example pattern: + +```python +# Existing ROBERT behavior: preserve this exactly. +self.args.log.write(txt) + +# Additive JSON-output-for-agent behavior: capture the same event as structured evidence. +self.args.curate_audit = audit_event( + self.args.curate_audit, + event_type="correlation_filter_removed_descriptor", + payload={ + "removed": removed_descriptor, + "kept": kept_descriptor, + "r2": r2_value, + "reason": "high correlation with kept descriptor", + }, + evidence_level="direct", + dat_text=txt, +) +``` + +The purpose of the JSON artifacts is not to replace ROBERT outputs. The purpose is to expose ROBERT's existing evidence in a structured form so ChatBob can explain completed runs to chemists. + +The first proof of concept is: + +```text +JSON/curate_audit.json +``` + +This file must capture the important evidence currently written to `CURATE_data.dat`, including: + +- input counts, +- target column, +- names column, +- ignored columns, +- categorical handling, +- duplicate filtering, +- constant descriptor removals, +- correlated descriptor removals with removed descriptor, kept descriptor, and R², +- RFECV applied/skipped status and reason, +- final descriptor counts and final descriptor list, +- curated files written, +- Pearson heatmap generated/skipped status, +- runtime status. + +This pattern has now been scaled through PREDICT. +Current source-verified status: + +- implemented: CURATE, GENERATE, VERIFY, PREDICT, +- not implemented: AQME, EVALUATE, +- REPORT currently captures figure provenance only. + +--- + +## Likely JSON Artifacts + +Module-level JSON files should be named as audit files, not generic summaries, because their role is to expose ROBERT decision evidence in structured form. + +Preferred module-level files: + +```text +JSON/curate_audit.json +JSON/generate_audit.json +JSON/verify_audit.json +JSON/predict_audit.json +JSON/aqme_audit.json +JSON/evaluate_audit.json +JSON/report_audit.json +``` + +Use `JSON/report_audit.json` only if report-specific provenance is needed. +The currently implemented REPORT artifact is: + +```text +JSON/report_figure_provenance.json +``` + +Supporting JSON artifacts may also exist, but they have different purposes: + +```text +JSON/dataset_profile.json +``` + +Profiles the raw incoming dataset before ROBERT changes it. This is not a CURATE decision audit. + +```text +*_manifest.json +``` + +Inventories generated files, paths, sizes, timestamps, and artifact types. This is not scientific decision evidence. + +```text +JSON/*_json_output_audit.json +``` + +Records whether JSON artifacts were attempted and whether writing succeeded or failed. This is not ROBERT scientific evidence. + +Possible run-level files: + +```text +run_context.json +run_summary.json +``` + +The exact names should be confirmed after inspecting existing ROBERT conventions. + +Architecture decision: + +- all ChatBob runtime JSON artifacts are written to the top-level `JSON/` folder, +- module folders (`CURATE/`, `GENERATE/`, `VERIFY/`, `PREDICT/`, `AQME/`, `EVALUATE/`, `REPORT/`) remain standard ROBERT outputs, +- wrapper archive manifests are generated in timestamped run archives and are not module-native runtime audits. + +--- + +## Standard Output Preservation Rule + +Normal ROBERT outputs must remain in their default root locations. + +This includes `.dat`, `.csv`, image files, and `ROBERT_report.pdf` behavior. + +If a timestamped run archive is used, it must be copy-only: + +- run ROBERT normally, +- do not reroute ROBERT output destinations, +- do not modify default output paths, +- after completion, duplicate generated files into `json-output-for-agent/runs/_/`. + +The wrapper archive is a project-side mirror, not a replacement output path. + +--- + +## Project Folder Convention + +Use one dedicated project folder for this work: + +```text +json-output-for-agent/ +``` + +This folder may contain: + +```text +json-output-for-agent/ + README.md + PROJECT_RULES.md + TASKS.md + task_tracker_plain_english.md + json_schema_notes.md + example_outputs/ +``` + +Python helper code should not use hyphens in the filename. + +Use: + +```text +robert/json_output_for_agent.py +``` + +Do not use: + +```text +robert/json-output-for-agent.py +``` + +Reason: Python imports work cleanly with underscores, not hyphens. + +--- + +## Source of Truth + +The existing ROBERT codebase is the source of truth for: + +- how ROBERT runs, +- how folders and output files are created, +- how reports are generated, +- how the ROBERT score is computed, +- existing command-line behavior, +- current tests and documentation. + +Do not infer behavior if it can be determined from the code. + +Inspect the relevant files first. + +--- + +## Mandatory Planning Rule + +Before modifying code, always propose a short plan that includes: + +1. files to inspect, +2. files likely to change, +3. expected behavior, +4. risks, +5. how to test the change. + +Do not implement until the user explicitly approves the plan. + +If a change seems necessary, propose it first. + +Do not implement without approval. + +--- + +## Mandatory Execution Checkpoint + +Before any implementation work begins, the agent must: + +1. present the plan to the user, +2. receive explicit user approval, +3. only then execute edits or commands. + +This applies to: + +- code, +- notebooks, +- scripts, +- documentation, +- tests, +- folder reorganization. + +--- + +## Communication Rule + +Explain progress in plain language for a chemist. + +Avoid unnecessary software jargon. + +When technical terms are necessary, define them briefly. + +For every meaningful change, explain: + +- what changed, +- why it changed, +- what file changed, +- how we know it worked, +- what remains uncertain. + +--- + +## Code Comment Rule + +Every new code block added to existing ROBERT files must include one or preferably two plain-language comment lines. + +The comments should explain what the new code does and why it is being added. + +Example: + +```python +# Save a small JSON summary of this ROBERT step for later UI or agent use. +# This does not change the ROBERT calculation; it only records information already produced. +``` + +Avoid clever or vague comments. + +Use plain English. + +--- + +## What We Are Trying to Capture + +We want to capture structured information from the full ROBERT pipeline, including: + +- input files, +- run settings, +- generated output files, +- `.dat` files, +- report values, +- images produced by ROBERT, +- model metrics, +- verification results, +- warning messages, +- score-related evidence, +- feature importance outputs, +- outlier information, +- file timestamps, +- run folder structure. + +The goal is to create a structured JSON record associated with each ROBERT run. + +--- + +## Desired JSON Behavior + +JSON outputs should be: + +- structured, +- stable, +- readable, +- useful for a UI, +- useful for later LLM explanation, +- traceable back to ROBERT outputs. + +Each JSON artifact should include a schema version. + +Example: + +```json +{ + "schema_version": "0.1", + "module": "VERIFY", + "artifact_type": "module_summary", + "status": "completed", + "files_created": [], + "values": {}, + "warnings": [], + "notes": [] +} +``` +--- + +## File Timestamp Rule + +For each ROBERT run, capture a structured inventory of generated files. + +For each file, record when practical: + +- file path, +- file name, +- module or folder, +- file extension, +- file size, +- last modified timestamp, +- whether it is JSON, DAT, CSV, PDF, PNG, SVG, TXT, or another type. + +This manifest should help a future UI or agent retrieve the correct files later. + +--- + +## Direct Values vs Derived Values + +JSON values should be labeled conceptually as either: + +1. Direct ROBERT values + Values already produced by ROBERT. + +2. Derived convenience values + Simple helper values calculated only to organize ROBERT output. + +Acceptable derived convenience values include: + +- number of files created, +- list of available output files, +- timestamp inventory, +- whether a known output file exists, +- number of warnings captured from an existing ROBERT output. + +Do not create new scientific diagnostics unless explicitly approved. + +Do not create a new score. + +Do not override the ROBERT score. + +--- + +## Helper Function Strategy + +Prefer a single helper file: + +```text +robert/json_output_for_agent.py +``` + +This file should contain reusable helper functions for: + +- safe JSON writing, +- timestamp capture, +- file manifest creation, +- conversion of NumPy/pandas values into JSON-safe values, +- module summary creation, +- run context assembly. + +Existing ROBERT modules should call these helpers with minimal added code. + +Avoid scattering JSON-writing logic throughout the existing codebase. + +--- + +## Rules for Changing Existing ROBERT Code + +Do not make broad edits. + +Do not refactor unrelated code. + +Do not change existing ROBERT behavior unless explicitly requested. + +Do not remove existing functionality. + +Do not rename public functions, CLI arguments, folders, or output files unless explicitly approved. + +Do not change scoring logic. + +Do not change model-selection logic. + +Do not change default thresholds. + +Do not introduce network calls into the standard ROBERT workflow. + +Do not add required LLM or API dependencies. + +Any LLM/API functionality must remain optional and outside the default ROBERT workflow. + +--- + +## Testing Expectations + +For each new JSON output, test that: + +- the expected JSON file is created, +- the JSON file can be opened, +- the JSON file is valid, +- expected top-level fields exist, +- the normal ROBERT output still appears, +- existing behavior is unchanged. + +If full tests are too slow, run the smallest meaningful test and explain what was not tested. + +--- + +## Controlled Run Comparison Rule + +When validating that JSON-output changes do not alter ROBERT behavior, compare the modified branch against an unmodified ROBERT baseline. + +The comparison is valid only when both runs use: + +- the same ROBERT version or commit, +- identical input CSV contents, +- the same command-line options, +- the same model settings and random seed, +- and, where practical, the same Python dependency environment. + +The preferred comparison checks are: + +1. Compare standard file inventories. +2. Compare primary `.dat` files exactly. +3. Normalize only expected run-specific metadata: + - timestamps, + - absolute paths, + - execution times, + - trailing whitespace. +4. Compare CSV files by: + - row and column structure, + - exact text values, + - numeric values within a strict tolerance. +5. Treat changes to descriptors, models, data splits, metrics, predictions, uncertainty values, verification outcomes, or outliers as scientific differences requiring investigation. + +Do not claim that ChatBob changes preserve ROBERT behavior when the baseline and modified runs use different ROBERT versions. + +--- + +## Upstream Integration Rule + +Before a long-lived feature branch diverges substantially from upstream: + +- fetch the latest upstream branch, +- inspect new upstream changes, +- merge or rebase only after protecting local work, +- resolve conflicts by preserving current upstream ROBERT behavior plus additive JSON capture, +- rerun controlled output comparisons, +- and open a pull request for review before additional dependent work accumulates. + +--- + +## Documentation Tracking Requirement + +After meaningful progress, update: + +```text +json-output-for-agent/task_tracker_plain_english.md +``` + +This file is the running plain-English record of accomplishments. + +Also update: + +```text +json-output-for-agent/TASKS.md +``` + +when tasks are completed, added, postponed, or changed. + +The task tracker should be understandable to a chemist who does not want to inspect the code. + +--- + +## First Development Task + +Do not write code immediately. + +First inspect the repository and identify: + +1. where CURATE writes outputs, +2. where GENERATE writes outputs, +3. where VERIFY writes outputs, +4. where PREDICT writes outputs, +5. where REPORT writes the PDF, +6. where `.dat` files are created, +7. where images are created, +8. where the ROBERT score is calculated, +9. which outputs already exist and can be captured, +10. where the smallest JSON helper calls could be inserted. + +Then propose the smallest safe implementation plan. + +--- + +## Current Strategic Direction + +The current goal is to make ROBERT easier to use through a future UI. + +The immediate technical step is to make ROBERT produce structured JSON evidence as it runs. + +The durable contribution of this branch should be: + +> ROBERT can produce stable, structured, machine-readable evidence files without changing its scientific behavior. + +The UI architecture can be decided later. diff --git a/json-output-for-agent/PROJECT_RULES.md b/json-output-for-agent/PROJECT_RULES.md new file mode 100644 index 0000000..595ec90 --- /dev/null +++ b/json-output-for-agent/PROJECT_RULES.md @@ -0,0 +1,231 @@ +# PROJECT_RULES.md — ROBERT JSON Output Project Rules + +## Process Rules + +1. Work in small steps, not giant rewrites. +2. Prefer readable code over clever code. +3. Every major change must be explained in plain language. +4. State assumptions explicitly. +5. Do not invent data or results. +6. If uncertain, mark uncertainty and suggest how to test it. +7. Update `task_tracker_plain_english.md` after meaningful progress. +8. Update `TASKS.md` when tasks change. +9. Keep scientific reasoning separate from speculation. +10. Preserve existing ROBERT behavior unless explicitly approved. + +--- + +## Standard Workflow + +For each step: + +1. State the goal of the step. +2. State the scientific and coding assumptions. +3. Inspect relevant files before changing anything. +4. Propose a short implementation plan. +5. Wait for explicit approval. +6. Make one small, focused change. +7. Add plain-English comments to new code. +8. Test the change. +9. Explain what changed in plain language. +10. Record confirmed results separately from interpretation. +11. Update `task_tracker_plain_english.md`. +12. Update `TASKS.md` if needed. + +--- + +## Planning Requirement + +Before modifying code, provide: + +1. files inspected, +2. files likely to change, +3. proposed change, +4. expected behavior, +5. risks, +6. testing plan. + +Do not implement until the user approves. + +--- + +## Code Change Rules + +Allowed: + +- small additive changes, +- helper functions, +- JSON export calls, +- file inventory creation, +- documentation updates, +- focused tests. + +Not allowed without explicit approval: + +- broad refactoring, +- changing ROBERT score logic, +- changing model-selection logic, +- changing thresholds, +- changing existing CLI behavior, +- deleting existing outputs, +- renaming public files or functions, +- introducing network/API calls, +- making LLM calls part of standard ROBERT execution. + +For timestamped run archives: + +- preserve standard ROBERT root output behavior, +- do not change default output destinations, +- use copy-only duplication into `json-output-for-agent/runs/_/`. + +--- + +## Comment Rule + +Every new code section added to existing ROBERT files must include one or two plain-English comments. + +The comments should explain: + +- what the code does, +- why it is safe, +- whether it changes behavior or only records information. + +Preferred style: + +```python +# Record a JSON summary for later UI or agent use. +# This only saves information ROBERT already created; it does not change the calculation. +``` + +--- + +## JSON Output Rules + +All JSON outputs should: + +- include `schema_version`, +- include the ROBERT module name when relevant, +- include clear field names, +- avoid ambiguous abbreviations, +- distinguish direct values from derived helper values when practical, +- be valid JSON, +- be readable by humans, +- be useful to a future UI. + +--- + +## File Manifest Rules + +For each ROBERT run, aim to capture a file manifest that records generated files. + +When practical, include: + +- path, +- filename, +- extension, +- file size, +- modified timestamp, +- likely ROBERT module, +- short description if known. + +The manifest should help future tools locate outputs without guessing. + +--- + +## Testing Rules + +For each new JSON artifact: + +1. Confirm the file is created. +2. Confirm it opens as valid JSON. +3. Confirm expected top-level keys exist. +4. Confirm normal ROBERT outputs are still created. +5. Confirm no scientific result changed. + +If a test is not run, say so clearly and explain why. + +--- + +## Controlled Scientific Comparison Rule + +A claim that JSON-output changes preserve ROBERT behavior must be based on a controlled comparison. + +Before comparing outputs, confirm: + +- both runs use the same ROBERT version or commit, +- both input CSV files have identical contents, +- both runs use the same command-line options, +- model seeds and relevant settings match, +- and dependency environments are sufficiently comparable. + +Compare: + +- the four primary `.dat` files, +- curated CSV files, +- model parameter and model database CSV files, +- selected best-model files, +- prediction CSV files. + +Raw `.dat` files may differ in timestamps, paths, execution times, and trailing whitespace. These values may be normalized for comparison, but scientific values must not be removed or normalized. + +Any difference in descriptors, data splits, selected models, metrics, verification outcomes, predictions, uncertainty values, or outlier identification must be investigated before declaring the implementation behavior-preserving. + +--- + +## Pull Request Review Checkpoint + +When a feature branch has produced a validated, behavior-preserving milestone: + +- update the project documentation, +- synchronize with the latest upstream branch, +- rerun controlled comparisons, +- open a pull request before substantial additional work accumulates, +- and use the pull request to agree on integration, naming, output location, and schema direction. + +A pull request may remain open for design review before final merge approval. + +--- + +## Plain-English Update Rule + +After meaningful progress, update: + +```text +task_tracker_plain_english.md +``` + +Each update should include: + +- date, +- what was done, +- files changed, +- what was tested, +- result, +- what remains uncertain, +- next suggested step. + +Use plain language for a chemist. + +--- + +## Scientific Integrity Rules + +Do not claim that a model is good or bad unless the evidence supports it. + +Do not invent explanations. + +Do not invent thresholds. + +Do not invent descriptor meanings. + +Do not invent missing values. + +If something is unknown, say it is unknown. + +If something needs testing, propose a test. + +--- + +## Current Project Priority + +The priority is to identify where ROBERT already creates output information and then record that information in structured JSON with minimal changes to existing code. diff --git a/json-output-for-agent/README.md b/json-output-for-agent/README.md new file mode 100644 index 0000000..8eaa7ec --- /dev/null +++ b/json-output-for-agent/README.md @@ -0,0 +1,114 @@ +# ROBERT JSON Output Project + +This folder tracks the project to add structured JSON outputs to ROBERT. + +The goal is to help future tools understand ROBERT runs without fragile parsing of PDFs or loosely formatted text files. + +The project does not change ROBERT's scientific behavior. + +## Important Files + +- `AGENTS.md` + Instructions for AI coding agents working on this project. + +- `PROJECT_RULES.md` + Process rules for how changes should be proposed, approved, implemented, tested, and documented. + +- `TASKS.md` + Current project to-do list. + +- `task_tracker_plain_english.md` + Plain-English record of completed work. + +- `json_schema_notes.md` + Working notes about possible JSON structures. + +### Current Priority + +The immediate priority is to prepare the JSON-output work for upstream review. + +Current source-verified module-native runtime audit status: + +- Implemented: CURATE, GENERATE, VERIFY, PREDICT +- Not implemented: AQME, EVALUATE +- REPORT: figure provenance only (`JSON/report_figure_provenance.json`), no full report audit yet + +Next steps: + +1. Open a pull request from `json-output-for-agent`. +2. Agree with Juanvi on the integration branch and JSON output location. +3. Reconcile the documented implementation status of all module audit files. +4. Review the current JSON structures against a small set of likely ChatBob questions. +5. Build deterministic question-to-evidence retrieval before adding an LLM answer layer. + +## Validation Source-Data Policy (Temporary) + +For JSON-output validation runs, `databases/` is currently treated as a protected source-data folder. + +Rules: +- Do not edit CSV files in `databases/`. +- Do not move or rename files in `databases/`. +- Do not overwrite source CSV files. +- Do not write generated ROBERT outputs into `databases/`. + +Generated outputs must remain in normal root output folders and optional copy-only archives. + + +## 2026-07-13 Controlled Run Comparison + +The ChatBob JSON-output branch was synchronized with upstream ROBERT 2.1.2 and compared against an unmodified ROBERT 2.1.2 run. + +Validation result: + +- all four primary ROBERT `.dat` files were identical after normalizing only timestamps, file paths, execution times, and trailing whitespace; +- all scientific values in 28 matching CSV files were identical; +- the only expected CSV difference was the recorded input file path in `CURATE/CURATE_options.csv`. + +For the tested regression case, the JSON-output implementation did not alter standard ROBERT scientific outputs. + +## 2026-07-14 In-Repo Verification Update + +Additional manual verification was completed in this repository after the 2026-07-13 comparison milestone. + +Confirmed: + +- structured JSON files were generated and reviewed for CURATE, GENERATE, VERIFY, PREDICT, and REPORT stages; +- when the updated ROBERT version in this repo was run and compared against DAT files from original ROBERT, no DAT differences were observed. + +This confirmation supports the additive-behavior claim for the verified workflow scope. + + +## Comparison Notebook + +`comparison/compare_robert_outputs.ipynb` compares two completed ROBERT runs. + +The copied ROBERT output folders are local validation data and are not committed. + +## 2026-06-08 Validation Snapshot + +Validated with timestamped wrapper: +- Regression input: `databases/Regression/AQME-ROBERT_A_predict_solubility.csv` +- Classification input: `databases/Clasification/F_predict_outcome.csv` + +Pass/fail summary: +- PASS: Standard root outputs are still produced for CURATE/GENERATE/VERIFY/PREDICT. +- PASS: JSON artifacts are produced (`JSON/dataset_profile.json`, `JSON/curate_audit.json`, `JSON/curate_json_output_audit.json`, archive manifests, `run_summary.json`). +- PASS: JSON files open and required top-level fields are present. +- PASS: `JSON/dataset_profile.json` includes `schema_version` and works for both regression and classification targets. +- PASS: No JSON-layer status text detected in standard `.dat` files. +- PASS: Copy-only archive behavior confirmed via `wrapper_run.json` copy-mode statement. +- PASS: Missing module folders are represented in manifests with `module_dir_exists=false` and `file_count=0`. +- PARTIAL: REPORT PDF was not generated because required WeasyPrint system libraries are missing in this environment. + +## Git Ignore Guidance + +Guidance for validation workflow: +- Do not ignore `databases/`. +- Ignore generated ROBERT run outputs in root folders and timestamped archives. + +## Open TODO + +Decide whether `databases/` should: +- remain in the repository, +- move to `tests/fixtures/`, or +- be removed after formal JSON unit tests are created. diff --git a/json-output-for-agent/TASKS.md b/json-output-for-agent/TASKS.md new file mode 100644 index 0000000..3e21c04 --- /dev/null +++ b/json-output-for-agent/TASKS.md @@ -0,0 +1,535 @@ +# TASKS.md — ROBERT JSON Output Tasks + +## Project Status + +Status: Active module-native audit implementation. + +Goal: Add structured JSON outputs to ROBERT with minimal changes to existing code. + +Core principle: + +* ROBERT remains the scientific source of truth. +* JSON files record runtime evidence that ROBERT already creates. +* JSON writes are additive and fail-soft. +* Standard ROBERT `.dat`, `.csv`, image, and report behavior should remain unchanged. +* No ROBERT model logic, descriptor logic, thresholds, scoring logic, prediction logic, plotting logic, or CLI behavior should be changed unless explicitly approved. + +--- + +## Current Priority + +Prepare the additive JSON-output implementation for upstream review while keeping the schema experimental. + +Immediate priorities: + +1. Open a pull request before additional upstream conflicts accumulate. +2. Document the controlled ROBERT 2.1.2 run comparison. +3. Complete documentation reconciliation with source-verified module status. +4. Review the current JSON structures against a small set of ChatBob user questions. +5. Agree with Juanvi on output location, branch integration, and the scope of the first schema. + +--- + +## Validation Standard for Each Module + +For each module-native audit artifact in `JSON/`, confirm: + +* [ ] Standard `.dat` output is unchanged except timestamp/runtime. +* [ ] Audit JSON opens as valid JSON. +* [ ] Module `*_json_output_audit.json` records successful write. +* [ ] JSON-layer text does not appear in standard `.dat` output. +* [ ] Git diff shows additive audit capture only. +* [ ] No ROBERT thresholds changed. +* [ ] No ROBERT scoring logic changed. +* [ ] No ROBERT descriptor logic changed. +* [ ] No ROBERT model-selection logic changed. +* [ ] No ROBERT prediction logic changed. +* [ ] No ROBERT plotting behavior changed. +* [ ] No ROBERT CLI behavior changed. + +--- + +## Module-Native Audit Status + +This section tracks JSON files written during the ROBERT module run in `JSON/`, not archive-side manifests created after copying outputs. + +| Module | Audit artifact | Status | Validation status | Notes | +| -------- | ---------------------------------------------------- | --------------- | ----------------- | --------------------------------------------------------------------------------------------------------------------------------------- | +| CURATE | `JSON/curate_audit.json` | Implemented | Validated | Captures CURATE runtime evidence and curated-output metadata. | +| GENERATE | `JSON/generate_audit.json` | Implemented | Validated | Captures model generation, BO, PFI, best-model selection, outputs, and heatmap artifacts. | +| VERIFY | `JSON/verify_audit.json` | Implemented | Validated | Captures VERIFY branch context, model context, y-mean, y-shuffle, one-hot tests, thresholds, pass/fail status, plots, and summary text. | +| PREDICT | `JSON/predict_audit.json` | Implemented | Validated (manual) | Source implementation confirmed; manual artifact review and DAT parity verification completed in-repo. | +| AQME | `JSON/aqme_audit.json` | Not implemented | Not validated | Later module. | +| EVALUATE | `JSON/evaluate_audit.json` | Not implemented | Not validated | Later module. | +| REPORT | `JSON/report_figure_provenance.json` | Partial | Not validated | Figure provenance is implemented; full module-native report audit is not implemented. | + +Notes: + +* Archive manifests and `run_summary.json` are copy-side artifacts. +* Module-native audits are runtime evidence artifacts. +* Module-native audits and JSON write-status files are written to the top-level `JSON/` folder. +* Both are useful, but they answer different questions. +* Archive manifests tell us what files were produced and copied. +* Module-native audits tell us what ROBERT knew while it was running. + +--- + +## Phase 1 — Inspect Existing ROBERT Outputs + +### To Do + +* [x] Identify where CURATE writes output files. +* [x] Identify where GENERATE writes output files. +* [x] Identify where VERIFY writes output files. +* [x] Identify where PREDICT writes output files. +* [x] Identify where REPORT writes the PDF report. +* [x] Identify where `.dat` files are created. +* [x] Identify where images are created. +* [x] Identify where the ROBERT score is calculated. +* [x] Identify which values are already available in memory before being written to files. +* [x] Identify the smallest safe places to call JSON helper functions. + +### Done + +* [x] Output map inspection completed on 2026-05-27. +* [x] Created `json-output-for-agent/output_map.md` with: + + * module-by-module output write map, + * `.dat` file creation map, + * plot/image output map, + * ROBERT score/report flow map, + * safest future JSON insertion points, + * risks and first implementation proposal. + +--- + +## Phase 1.5 — Incoming Dataset Profile JSON + +This phase added one JSON artifact that profiles observable facts from the incoming dataset before ROBERT modifies the data. + +### To Do + +* [x] Confirm insertion point in `curate.__init__`. +* [x] Confirm helper file location: `robert/json_output_for_agent.py`. +* [x] Confirm JSON output location: `JSON/dataset_profile.json`. +* [x] Enforce standard ROBERT output protection rule for JSON layer. +* [x] Implement `profile_input_dataset` helper function. +* [x] Implement dataset profile measurement helper functions. +* [x] Implement safe JSON writing helper function. +* [x] Implement JSON output audit helper function. +* [x] Add single JSON helper call in `curate.__init__`. +* [x] Confirm `JSON/dataset_profile.json` is created. +* [x] Confirm CURATE still completes when JSON write fails. +* [x] Confirm JSON status is written only to `JSON/curate_json_output_audit.json`. +* [x] Confirm `CURATE/CURATE_data.dat` contains no JSON-layer status messages. + +### Done + +* [x] Implemented `robert/json_output_for_agent.py`. +* [x] Added fail-soft hook in `robert/curate.py`. +* [x] Validated baseline run creates `JSON/dataset_profile.json`. +* [x] Injected simulated JSON failure and confirmed standard CURATE outputs still complete. +* [x] Removed JSON-layer writes to standard CURATE logger output. +* [x] Added `JSON/curate_json_output_audit.json` as the only JSON-layer status channel. + +--- + +## Phase 2 — JSON Structure and Schema Notes + +### To Do + +* [ ] Decide final project folder name. +* [ ] Decide whether JSON export is always on or controlled by an option. +* [ ] Draft stable module-audit JSON schema after more modules are implemented. +* [ ] Draft stable run-level JSON schema after module-native audits are better understood. +* [ ] Decide how much full-value data should be included versus summarized. +* [ ] Decide long-term naming conventions for direct ROBERT values and JSON-only explanatory aliases. +* [ ] Decide whether timestamps should always use UTC. +* [ ] Decide whether module-native audit artifacts should eventually be indexed by a run-level context file. + +### Done + +* [x] Experimental schema notes started in `json_schema_notes.md`. +* [x] Current schema version marked as experimental: `schema_version: 0.1`. +* [x] Pattern established: + + * module name, + * artifact type, + * status, + * input/settings section, + * event stream, + * runtime section, + * JSON-layer write status in `json_output_audit.json`. + +--- + +## Phase 3 — Build JSON Helper + +### To Do + +* [x] Create `robert/json_output_for_agent.py`. +* [x] Add safe JSON writing helper. +* [x] Add JSON-safe conversion helper for common Python, NumPy, pandas, and pathlib objects. +* [x] Add file manifest helper. +* [x] Add module summary helper. +* [x] Add run-level summary helper. +* [x] Add module audit helper pattern: + + * `init_module_audit` + * `audit_set` + * `audit_event` + * `finalize_module_audit` + * `write_json_output_audit` +* [ ] Add plain-English comments to helper functions where needed. +* [ ] Add simple tests for helper functions. + +### Done + +* [x] Core dataset-profile helper implemented. +* [x] Archive manifest and run summary helpers implemented. +* [x] Module-native audit helper pattern implemented and reused across CURATE, GENERATE, and VERIFY. + +--- + +## Phase 4 — CURATE Module-Native Audit + +### To Do + +* [x] Choose CURATE as first module-native audit target. +* [x] Propose exact files and functions to change. +* [x] Wait for user approval. +* [x] Implement `JSON/curate_audit.json`. +* [x] Confirm existing ROBERT CURATE output is unchanged. +* [x] Confirm audit JSON output is created. +* [x] Confirm JSON output audit is created. +* [x] Confirm no JSON text appears in `CURATE/CURATE_data.dat`. + +### Done + +* [x] `JSON/curate_audit.json` implemented. +* [x] CURATE runtime evidence captured. +* [x] CURATE JSON write status isolated to `JSON/curate_json_output_audit.json`. +* [x] CURATE validation completed. +* [x] CURATE fault-injection validation completed for audit write failure. + +--- + +## Phase 5 — GENERATE Module-Native Audit + +### To Do + +* [x] Propose implementation plan for GENERATE audit. +* [x] Wait for user approval. +* [x] Implement `JSON/generate_audit.json`. +* [x] Capture inputs and model-scan context. +* [x] Capture BO workflow evidence. +* [x] Capture PFI workflow evidence. +* [x] Capture best-model selection evidence. +* [x] Capture heatmap/image artifact metadata. +* [x] Capture outputs and runtime. +* [x] Confirm existing `GENERATE_data.dat` is unchanged except timestamp/runtime and non-scientific formatting differences. +* [x] Confirm key CSV outputs are unchanged. +* [x] Confirm audit JSON validates. +* [x] Confirm JSON output audit validates. +* [x] Confirm no JSON text appears in `GENERATE/GENERATE_data.dat`. + +### Done + +* [x] `JSON/generate_audit.json` implemented. +* [x] `JSON/generate_json_output_audit.json` implemented for audit-write status. +* [x] Standard GENERATE behavior validated against accepted baseline. +* [x] GENERATE audit captures: + + * input settings, + * model list, + * model-specific CURATE file use, + * BO workflow, + * optimized parameters, + * PFI descriptor filtering, + * best-model selection, + * heatmap artifact metadata, + * output file paths, + * runtime. + +--- + +## Phase 6 — VERIFY Module-Native Audit + +### To Do + +* [x] Propose implementation plan for VERIFY audit. +* [x] Wait for user approval. +* [x] Implement `JSON/verify_audit.json`. +* [x] Capture inputs and threshold constants. +* [x] Capture No_PFI and PFI branch context. +* [x] Capture model context. +* [x] Capture y-mean test evidence. +* [x] Capture y-shuffle test evidence. +* [x] Capture one-hot test evidence. +* [x] Capture threshold/status analysis. +* [x] Capture VERIFY plot artifact metadata. +* [x] Capture final `.dat` summary preview. +* [x] Add JSON-only explanatory descriptor fields where useful. +* [x] Confirm existing `VERIFY_data.dat` is unchanged except timestamp/runtime. +* [x] Confirm audit JSON validates. +* [x] Confirm JSON output audit validates. +* [x] Confirm no JSON text appears in `VERIFY/VERIFY_data.dat`. + +### Done + +* [x] `JSON/verify_audit.json` implemented. +* [x] `JSON/verify_json_output_audit.json` implemented for audit-write status. +* [x] Standard VERIFY behavior validated against accepted baseline. +* [x] VERIFY audit captures: + + * branch context, + * model context, + * active descriptor evidence, + * cross-validation and sorted-CV metrics, + * y-mean test, + * y-shuffle test, + * one-hot test, + * thresholds, + * PASS/UNCLEAR/FAILED status, + * plot artifact metadata, + * final summary text preview, + * runtime. +* [x] Confirmed descriptor-related JSON fields are explanatory JSON metadata only and do not rename or change original ROBERT variables. + +--- + +## Phase 7 — PREDICT Module-Native Audit + +### To Do + +* [x] Inspect current `predict.py` and `predict_utils.py` insertion points. +* [x] Propose implementation plan before coding. +* [x] Wait for user approval. +* [x] Implement `JSON/predict_audit.json`. +* [x] Capture input settings and model/source context. +* [x] Capture prediction CSV output metadata. +* [x] Capture prediction summary metrics. +* [x] Capture train/validation/test/external prediction evidence when available. +* [x] Capture uncertainty or variability evidence when available. +* [x] Capture SHAP artifact metadata when available. +* [x] Capture PFI artifact metadata when available. +* [x] Capture outlier evidence when available. +* [x] Capture y-distribution artifact metadata when available. +* [x] Capture Pearson heatmap metadata when available. +* [x] Capture generated plot/image paths. +* [x] Capture final `.dat` summary preview if useful. +* [ ] Confirm existing `PREDICT_data.dat` is unchanged except timestamp/runtime. +* [ ] Confirm prediction CSV outputs are unchanged. +* [ ] Confirm image outputs are unchanged or only differ by normal timestamp/rendering metadata. +* [ ] Confirm audit JSON validates. +* [ ] Confirm JSON output audit validates. +* [ ] Confirm no JSON text appears in `PREDICT/PREDICT_data.dat`. + +### Done + +* [x] `JSON/predict_audit.json` implemented. +* [x] `JSON/predict_json_output_audit.json` implemented for audit-write status. +* [x] Manual review confirmed structured JSON generation for PREDICT in the current repo. +* [x] Manual in-repo comparison against original ROBERT DAT files found no DAT differences. +* [ ] Add formal automated validation tests for PREDICT audit content. + +--- + +## Phase 8 — AQME Module-Native Audit + +### To Do + +* [ ] Inspect current `aqme.py` insertion points. +* [ ] Propose implementation plan before coding. +* [ ] Wait for user approval. +* [ ] Implement `JSON/aqme_audit.json`. +* [ ] Capture input settings. +* [ ] Capture descriptor-generation provenance where available. +* [ ] Capture output file metadata. +* [ ] Capture any generated `.dat` summary preview if useful. +* [ ] Confirm existing AQME outputs are unchanged. +* [ ] Confirm audit JSON validates. +* [ ] Confirm JSON output audit validates. +* [ ] Confirm no JSON text appears in standard AQME outputs. + +### Done + +* [ ] Not started. + +--- + +## Phase 9 — EVALUATE Module-Native Audit + +### To Do + +* [ ] Inspect current `evaluate.py` insertion points. +* [ ] Propose implementation plan before coding. +* [ ] Wait for user approval. +* [ ] Implement `JSON/evaluate_audit.json`. +* [ ] Capture input settings. +* [ ] Capture model evaluation context. +* [ ] Capture model parameter/output metadata. +* [ ] Capture evaluation metrics where available. +* [ ] Confirm existing EVALUATE outputs are unchanged. +* [ ] Confirm audit JSON validates. +* [ ] Confirm JSON output audit validates. +* [ ] Confirm no JSON text appears in standard EVALUATE outputs. + +### Done + +* [ ] Not started. + +--- + +## Phase 10 — REPORT and Run-Level Context + +### To Do + +* [ ] Revisit REPORT only after upstream module-native audits are stable. +* [ ] Decide whether REPORT needs its own `JSON/report_audit.json`. +* [ ] Document the currently implemented report artifact `JSON/report_figure_provenance.json`. +* [ ] Decide whether a run-level context JSON can replace some fragile report parsing. +* [ ] Investigate current REPORT PDF/report parser failure separately. +* [ ] Confirm required WeasyPrint/system-library environment for PDF generation if PDF validation is needed. +* [ ] Decide whether REPORT should consume module-native JSON audits in a future design. + +### Done + +* [ ] REPORT module-native audit not started. +* [x] `JSON/report_figure_provenance.json` is implemented as report-figure provenance capture. +* [ ] Current REPORT issue identified as separate from CURATE, GENERATE, VERIFY, and PREDICT audit validation. + +--- + +## Phase 11 — Timestamped Archive Wrapper + +Goal: + +* Keep normal ROBERT root outputs unchanged. +* Duplicate generated outputs into a timestamped project archive folder. + +### To Do + +* [x] Confirm feasibility outside ROBERT core. +* [x] Implement wrapper script in project folder. +* [x] Ensure wrapper runs ROBERT in normal root mode. +* [x] Ensure wrapper copies outputs into timestamped run folder. +* [x] Generate per-module JSON manifests from copied run artifacts. +* [x] Generate run-level `run_summary.json` in the archive. +* [x] Document explicit copy-only rule in AGENTS and project rules docs. +* [x] Validate on full runs and inspect copied archive contents. +* [ ] Decide long-term role of archive manifests now that module-native audits are being implemented. + +### Done + +* [x] Added wrapper script: `json-output-for-agent/scripts/run_robert_timestamped.py`. +* [x] Added policy language that standard output paths remain unchanged and archive behavior is copy-only. +* [x] Added archive-side manifest generation and run summary generation after copy. +* [x] Confirmed copy-only behavior. + +--- + +## Phase 12 — Documentation and Tracking + +### To Do + +* [x] Document what JSON files are created. +* [x] Document where JSON files are saved. +* [x] Document what each top-level field means. +* [x] Document whether the JSON schema is experimental. +* [x] Add a short plain-English explanation for chemists. +* [x] Update `task_tracker_plain_english.md`. +* [x] Update `TASKS.md`. +* [ ] Update `json_schema_notes.md` to reflect CURATE, GENERATE, and VERIFY implementation status. +* [ ] Update `output_map.md` to mark implemented module-native hooks. +* [ ] Update `README.md` to reflect current module-native audit status. +* [ ] Update `PROJECT_RULES.md` only if process rules change. + +### Done + +* [x] Initial validation and schema documentation updated on 2026-06-08. +* [x] `TASKS.md` rewritten to reflect current module-native audit workflow. + +--- + +## Phase 13 — Controlled ROBERT Run Equivalence Validation + +### To Do + +* [x] Confirm both comparison runs use the same ROBERT version. +* [x] Synchronize the ChatBob branch with upstream ROBERT 2.1.2. +* [x] Run the same regression dataset through original and ChatBob ROBERT. +* [x] Compare the four primary `.dat` files exactly. +* [x] Normalize only timestamps, paths, runtime, and trailing whitespace. +* [x] Confirm normalized `.dat` files are identical. +* [x] Compare matching CSV files by shape, columns, text, and numeric values. +* [x] Confirm best-model CSV files are identical. +* [x] Confirm prediction CSV values are identical. +* [x] Record the expected `csv_name` path difference in `CURATE_options.csv`. +* [ ] Repeat the controlled comparison for one classification example. +* [ ] Convert the comparison notebook checks into formal automated tests if appropriate. + +### Done + +* [x] Regression comparison completed using ROBERT 2.1.2 for both runs. +* [x] Four normalized `.dat` files were identical. +* [x] Scientific values in 28 matching CSV files were identical. +* [x] ChatBob JSON-output additions did not change standard ROBERT scientific outputs for this test case. +* [x] Additional manual in-repo verification (2026-07-14) reported no DAT differences versus original ROBERT. + +--- + +## Phase 14 — Upstream Integration and Schema Review + +### To Do + +* [ ] Open a pull request from `json-output-for-agent`. +* [ ] Share the run-comparison evidence with Juanvi. +* [ ] Agree on whether the feature should merge into `master`, a development branch, or remain behind a feature branch temporarily. +* [ ] Agree on the official JSON output location. +* [ ] Inventory current JSON filenames, top-level keys, event types, and payload structures. +* [ ] Define five initial user questions ChatBob should answer. +* [ ] Map each question to its JSON evidence and original ROBERT source. +* [ ] Identify the minimum schema changes required. +* [ ] Avoid freezing a large schema before the question-to-evidence map is reviewed. +* [ ] Build deterministic evidence-selection functions before adding LLM answer generation. + +--- + + +## Validation Source-Data Policy + +For JSON-output validation runs, `databases/` is currently treated as a protected source-data folder. + +Rules: + +* Do not edit CSV files in `databases/`. +* Do not move or rename files in `databases/`. +* Do not overwrite source CSV files. +* Do not write generated ROBERT outputs into `databases/`. + +Generated outputs must remain in normal root output folders and optional copy-only archives. + +Open decision: + +* Decide whether `databases/` should: + + * remain in the repository, + * move to `tests/fixtures/`, or + * be removed after formal JSON unit tests are created. + +--- + +## Parking Lot + +Ideas to revisit later: + +* UI architecture: Dash, EasyROB, Vercel, Streamlit, or another approach. +* Whether JSON export should become an official ROBERT feature. +* Whether JSON output should be optional or default. +* Whether future ChatBob should read only module-native audit JSON files, only run-level context JSON, or both. +* Whether images should be copied, referenced, or summarized in JSON. +* Whether output directories should use a timestamped run ID. +* Whether multiple ROBERT runs should be indexed by a central run registry. +* Whether REPORT should eventually read structured JSON rather than parsing `.dat` files. +* Whether formal unit tests should replace some manual validation checks. diff --git a/json-output-for-agent/json_schema_notes.md b/json-output-for-agent/json_schema_notes.md new file mode 100644 index 0000000..6f286a6 --- /dev/null +++ b/json-output-for-agent/json_schema_notes.md @@ -0,0 +1,271 @@ +# json_schema_notes.md + +This file records working ideas for ROBERT JSON outputs. + +The schema is experimental until confirmed through implementation and testing. + +--- + +## Design Goals + +The JSON files should be: + +- easy for humans to read, +- easy for a UI to load, +- easy for an LLM to explain, +- traceable back to ROBERT outputs, +- stable enough to use across runs, +- explicit about what is known and unknown. + +--- + +## Basic Module Summary Schema + +```json +{ + "schema_version": "0.1", + "module": "CURATE", + "artifact_type": "module_summary", + "status": "completed", + "inputs": {}, + "outputs": { + "files_created": [] + }, + "parameters": {}, + "direct_robert_values": {}, + "derived_helper_values": {}, + "warnings": [], + "notes": [] +} +``` + +--- + +## Basic Run Context Schema + +```json +{ + "schema_version": "0.1", + "artifact_type": "run_context", + "robert_version": null, + "run_id": null, + "created_at": null, + "command": null, + "modules": { + "curate": {}, + "generate": {}, + "verify": {}, + "predict": {}, + "report": {} + }, + "score": {}, + "warnings": [], + "file_manifest": [] +} +``` + +--- + +## Basic File Manifest Schema + +```json +{ + "schema_version": "0.1", + "artifact_type": "file_manifest", + "run_id": null, + "created_at": null, + "files": [ + { + "path": "PREDICT/PREDICT_data.dat", + "name": "PREDICT_data.dat", + "extension": ".dat", + "module": "PREDICT", + "size_bytes": null, + "modified_at": null, + "description": "ROBERT prediction summary data" + } + ] +} +``` + +--- + +## Open Schema Questions + +- Should each module write its own JSON file? +- Should REPORT assemble the full `run_context.json`? +- Should JSON export be controlled by a command-line option? +- Should JSON files include full values or only summaries? +- Should images be listed only, or should image metadata be captured? +- Should `.dat` file contents be copied into JSON or only referenced? +- Should file timestamps use local time or UTC? +- Should output folders be timestamped by run? + +--- + +## Confirmed Baseline (2026-06-08) + +- `JSON/dataset_profile.json` is the raw-intake artifact and is captured before ROBERT modifies the incoming dataset. +- Archive manifests and run summary JSONs are copy-only mirrors of generated outputs. +- JSON schema is still experimental (`schema_version: 0.1`). + +## Confirmed Artifact Location Policy (2026-07-13) + +- all ChatBob runtime JSON artifacts are written to the top-level `JSON/` folder; +- module-native runtime audit files in `JSON/` are distinct from wrapper-generated archive manifests; +- standard ROBERT scientific outputs remain in module folders (`CURATE/`, `GENERATE/`, `VERIFY/`, `PREDICT/`, `AQME/`, `EVALUATE/`, `REPORT/`). + +--- + +## Long-Term Module Audit Pattern (Design Direction) + +Planned pattern: +- Keep `JSON/dataset_profile.json` as raw incoming data evidence. +- Add one module-specific audit JSON per ROBERT module over time. +- Keep writes additive and fail-soft. +- Keep JSON-layer status only in JSON audit files. +- Never write JSON-layer status to standard `.dat`, `.csv`, image, or PDF outputs. + +Proposed future module audit files: +- `JSON/curate_audit.json` +- `JSON/generate_audit.json` +- `JSON/verify_audit.json` +- `JSON/predict_audit.json` +- `JSON/aqme_audit.json` +- `JSON/evaluate_audit.json` +- `JSON/report_audit.json` + +--- + +## CURATE Audit Schema: Original Design and Current Implementation + +Purpose: +- Capture observable CURATE evidence without changing ROBERT behavior. +- Record unavailable values as unavailable rather than guessed. + +```json +{ + "schema_version": "0.1", + "artifact_type": "curate_audit", + "module": "CURATE", + "status": "completed_or_partial", + "captured_utc": "ISO-8601", + "inputs": { + "source_csv": "path string", + "target_column": "string", + "names_column": "string or null", + "ignored_columns": [], + "discarded_columns_requested": [] + }, + "observed_counts": { + "rows_before_curate": "int or unavailable", + "rows_after_curate": "int or unavailable", + "columns_before_curate": "int or unavailable", + "columns_after_curate": "int or unavailable", + "descriptors_removed_duplicate_filter": "int or unavailable", + "descriptors_removed_missingness": "int or unavailable", + "descriptors_removed_categorical_transform": "int or unavailable", + "descriptors_removed_correlation_filter": "int or unavailable", + "descriptors_removed_other": "int or unavailable" + }, + "files_written": [ + { + "path": "CURATE/...", + "artifact_type": "csv|dat|png|json|other", + "exists": true + } + ], + "standard_output_paths": { + "curate_dat": "CURATE/CURATE_data.dat", + "curate_options_csv": "CURATE/CURATE_options.csv", + "curated_csvs": [] + }, + "notes": [], + "unavailable_fields": [], + "json_layer_audit": { + "attempted": true, + "succeeded": true, + "error_type": null, + "error_message": null + } +} +``` + +Design constraints: +- No guessed explanations for why descriptors were removed. +- If a reason/count is not observable from existing ROBERT state, mark as unavailable. +- Audit status belongs only in JSON audit artifacts. + +Implementation status (2026-07-13): +- `JSON/curate_audit.json`, `JSON/generate_audit.json`, `JSON/verify_audit.json`, and `JSON/predict_audit.json` are implemented. +- `JSON/aqme_audit.json` and `JSON/evaluate_audit.json` are not implemented. +- REPORT has figure provenance capture in `JSON/report_figure_provenance.json`; a full `JSON/report_audit.json` is not implemented. +- Current implementation records direct observable fields and marks step-specific descriptor-removal counters as `unavailable`. +- JSON-layer write status for this artifact is recorded in `JSON/curate_json_output_audit.json` with `artifact="curate_audit.json"`. + +--- + +## Standard Output Preservation Validation + +A controlled regression comparison was completed on 2026-07-13 using: + +- unmodified ROBERT 2.1.2, and +- ChatBob-modified ROBERT 2.1.2. + +Results: + +- all four primary `.dat` files were identical after normalizing only timestamps, paths, execution times, and trailing whitespace; +- all scientific values in 28 matching CSV files were identical; +- the only CSV text difference was the expected input path stored in `CURATE_options.csv`. + +This confirms that the JSON-output implementation is additive for the tested regression case. + +This result does not by itself prove that every JSON value is correct. JSON evidence must still be validated against the relevant ROBERT DAT, CSV, and in-memory source values. + +--- + +## Question-Driven Schema Review + +The next schema step should begin with a small set of user questions rather than a large universal schema. + +Initial candidate questions: + +1. What model was selected? +2. Which descriptors were used? +3. How well did the model perform? +4. Did the model pass the verification tests? +5. Which points were identified as outliers? + +For each question, document: + +- the required JSON artifact, +- the event type or section, +- the required fields, +- whether each field is direct, derived, or unavailable, +- the original ROBERT DAT or CSV source, +- the expected answer structure. + +Only make schema changes that are needed to answer these initial questions reliably. + +## Minimal Common Audit Envelope + +A possible common structure for module-native audit files is: + +```json +{ + "schema_version": "0.1", + "module": "GENERATE", + "artifact_type": "generate_audit", + "status": "completed", + "source_files": [], + "events": [ + { + "event_type": "prepare_sets", + "evidence_level": "direct_or_derived", + "source_reference": null, + "payload": {} + } + ], + "runtime": {}, + "unavailable_fields": [] +} diff --git a/json-output-for-agent/output_map.md b/json-output-for-agent/output_map.md new file mode 100644 index 0000000..6864b12 --- /dev/null +++ b/json-output-for-agent/output_map.md @@ -0,0 +1,230 @@ +# ROBERT Output Map + +## Plain-English Summary + +This inspection mapped where ROBERT currently writes outputs without changing ROBERT code. + +Main findings: +- CURATE writes curated CSV files and a CURATE options CSV in the CURATE folder, and also creates a Pearson heatmap image. +- GENERATE writes model databases and parameter CSV files in GENERATE/Raw_data, copies best files to GENERATE/Best_model, and creates heatmap images. +- VERIFY mainly writes a verification plot image and logs all test values to VERIFY/VERIFY_data.dat. +- PREDICT writes prediction CSVs, many analysis images (results, SHAP, PFI, outliers, distribution, Pearson), and logs metrics/warnings to PREDICT/PREDICT_data.dat. +- REPORT writes ROBERT_report.pdf in the working directory (plus temporary report.css and optional report_debug.txt). +- ROBERT score is calculated in report_utils.calc_score by combining parsed values from PREDICT_data.dat and VERIFY_data.dat. +- The values needed for JSON export already exist in memory in dictionaries/dataframes before file writes (for example: csv_df, options_df, bo_data, PFI_dict, verify_results, verify_metrics, Xy_data, data_score). + +## Files Inspected + +- robert/curate.py +- robert/generate.py +- robert/generate_utils.py +- robert/verify.py +- robert/predict.py +- robert/predict_utils.py +- robert/report.py +- robert/report_utils.py +- robert/utils.py + +## Output Map + +| ROBERT module | file/function that creates outputs | output folder | output files created | values that could be captured in JSON | possible JSON insertion point | +|---|---|---|---|---|---| +| CURATE | robert/curate.py -> save_curate | CURATE | _CURATE.csv, optional _CURATE_.csv, CURATE_options.csv | filtered descriptor list, model-specific descriptor lists, y, ignore, names, class label mapping, output file names | At end of save_curate right after each to_csv call and after options_df to_csv | +| CURATE | robert/utils.py -> pearson_map (called from curate.__init__) | CURATE | Pearson_heatmap.png (if descriptors <= 30) | descriptor count, correlation matrix summary, max abs correlation, generated file path | In pearson_map after plt.savefig and before log write | +| CURATE | robert/utils.py -> Logger via load_variables + finish_print | CURATE | CURATE_data.dat | all plain-text curation messages, counts, warnings, elapsed time | In finish_print just before log.finalize | +| GENERATE | robert/generate_utils.py -> BO_workflow | GENERATE/Raw_data/No_PFI | _db.csv, .csv | bo_data (model/type/kfold/repeat_kfolds/seed/error_type/y/names/X_descriptors/params/combined metric), set split labels | In BO_workflow immediately after db csv and params csv writes | +| GENERATE | robert/generate_utils.py -> save_pfi_csv | GENERATE/Raw_data/PFI | _PFI.csv, optional _PFI_db.csv | selected descriptors after PFI, combined metric with PFI, class label mapping, split | In save_pfi_csv after csv_PFI_df.to_csv and after _PFI_db write | +| GENERATE | robert/generate_utils.py -> detect_best | GENERATE/Best_model/(No_PFI or PFI) | copied best params csv and best _db.csv | winning model file name, best combined metric, selection criterion (min for rmse/mae, max otherwise) | In detect_best after best file copy operations | +| GENERATE | robert/utils.py -> create_heatmap (via heatmap_workflow) | GENERATE/Raw_data | Heatmap_ML_models_No_PFI.png, Heatmap_ML_models_PFI.png | per-model combined metric table used for heatmap, image path | In create_heatmap after plt.savefig | +| GENERATE | robert/utils.py -> Logger via load_variables + finish_print | GENERATE | GENERATE_data.dat | BO summary lines, per-model logs, elapsed time | In finish_print before finalize | +| VERIFY | robert/verify.py -> verify.__init__/analyze_tests/print_verify | VERIFY (log) | VERIFY_data.dat | verify_results dict (CV_score, sorted_CV_score, y_mean, y_shuffle, onehot, thresholds, pass/fail), verify_metrics dict | In verify.__init__ after analyze_tests and before print_verify | +| VERIFY | robert/utils.py -> plot_metrics | VERIFY | VERIFY_tests__.png | verify_metrics bars/colors/thresholds, plotted metric values | In plot_metrics after plt.savefig | +| PREDICT | robert/predict_utils.py -> save_predictions | PREDICT and PREDICT/csv_test | _.csv, csv_test/__.csv | df_results table, external predictions table, conformal half-width, class label reconversion, set labels | In save_predictions immediately after each to_csv | +| PREDICT | robert/utils.py -> graph_reg/graph_clas | PREDICT and PREDICT/csv_test | Results_*.png, CV_variability_*.png, CV_train_valid_predict_*.png | y_true/y_pred arrays, prediction SD arrays, plotted set type, pred range tracked in Xy_data | In graph_reg/graph_clas after plt.savefig | +| PREDICT | robert/utils.py -> shap_analysis | PREDICT | SHAP_*.png | SHAP min/max per descriptor, descriptor ranking | In shap_analysis right after shap values computed and after plt.savefig | +| PREDICT | robert/utils.py -> PFI_plot | PREDICT | PFI_*.png | permutation importances means/stds, sorted descriptor importance | In PFI_plot after plt.savefig | +| PREDICT | robert/utils.py -> outlier_plot | PREDICT | Outliers_*.png | outlier SD values, outlier names, outlier counts/percentages | In outlier_plot after outlier_filter and after plt.savefig | +| PREDICT | robert/utils.py -> distribution_plot | PREDICT | y_distribution_*.png | quartile/class distribution counts, imbalance warnings | In distribution_plot after y_dist_dict built and after final file move | +| PREDICT | robert/utils.py -> pearson_map (via pearson_map_predict) | PREDICT | Pearson_heatmap_No_PFI.png, Pearson_heatmap_PFI.png (if <= 30 descriptors) | correlation matrix, max abs correlation pair, warning level | In pearson_map after plt.savefig | +| PREDICT | robert/utils.py -> Logger via load_variables + finish_print | PREDICT | PREDICT_data.dat | all prediction summaries, metrics, warnings, elapsed time | In finish_print before finalize | +| REPORT | robert/report_utils.py -> make_report | working directory | ROBERT_report.pdf | final assembled HTML string, css filename list, output path | In make_report after make_pdf and write_bytes | +| REPORT | robert/report.py -> report.__init__ | working directory | temporary report.css, optional report_debug.txt | report_html string, module section fragments, eval_only flag | In report.__init__ after report_html completion and before make_report | +| REPORT | robert/report.py + robert/report_utils.py -> print_score/calc_score | in-memory during report creation | no direct score file (score appears in report and section data) | data_score dict with all intermediate score components and final robert_score_ | In report.print_score after calc_score call for each suffix | + +## .dat File Map + +Primary creation mechanism: +- robert/utils.py defines Logger(filein, append, suffix="dat") which opens file path filein_append.dat in write mode. +- robert/utils.py load_variables creates Logger at module startup for all modules except REPORT. +- robert/utils.py finish_print writes elapsed time and closes the dat log with finalize(). + +Expected dat outputs from core modules: +- CURATE/CURATE_data.dat +- GENERATE/GENERATE_data.dat +- VERIFY/VERIFY_data.dat +- PREDICT/PREDICT_data.dat + +Other dat usage: +- report_utils.repro_info reads /_data.dat files to build report sections. +- report.py reads PREDICT/PREDICT_data.dat in multiple sections (warnings, metrics, predictions). + +## Plot/Image Output Map + +Main producers and files: +- CURATE: Pearson_heatmap.png from robert/utils.py pearson_map. +- GENERATE: Heatmap_ML_models_No_PFI.png and Heatmap_ML_models_PFI.png from robert/utils.py create_heatmap. +- VERIFY: VERIFY_tests_*.png from robert/utils.py plot_metrics. +- PREDICT: + - Results_*.png and/or CV_train_valid_predict_*.png from graph_reg/graph_clas. + - CV_variability_*.png from graph_reg(sd_graph=True). + - SHAP_*.png from shap_analysis. + - PFI_*.png from PFI_plot. + - Outliers_*.png from outlier_plot. + - y_distribution_*.png from distribution_plot. + - Pearson_heatmap_No_PFI.png and Pearson_heatmap_PFI.png from pearson_map. + - External-set plots in PREDICT/csv_test from graph_reg/graph_clas with csv_test=True. + +## ROBERT Score and Report Map + +Where score is calculated: +- robert/report.py print_score calls report_utils.calc_score(dat_files, suffix, pred_type, data_score). +- robert/report_utils.py calc_score combines: + - get_predict_scores output parsed from PREDICT_data.dat. + - get_verify_scores output parsed from VERIFY_data.dat. + - Additional arithmetic for regression/classification to compute robert_score_No PFI and robert_score_PFI. + +Where PDF is written: +- robert/report_utils.py make_report writes ROBERT_report.pdf to current working directory. +- robert/report.py prepares report_html and temporary report.css before make_report. + +## Safest Future JSON Export Points + +Smallest safe insertion points with minimal behavioral risk: +- CURATE: end of save_curate after CSV writes (already has all dataframes/options in hand). +- GENERATE: + - end of BO_workflow after writing model db and params CSV. + - end of save_pfi_csv after writing PFI CSV(s). + - end of detect_best after best-file copy. +- VERIFY: in verify.__init__ after analyze_tests returns verify_results and verify_metrics, before print_verify. +- PREDICT: + - end of save_predictions after base and external CSV writes. + - after each analytics function computes in-memory outputs (shap_analysis, PFI_plot, outlier_plot, distribution_plot), before/after image save. +- REPORT: + - inside print_score after each calc_score call to capture fully assembled data_score. + - inside make_report immediately after PDF write for final report artifact metadata. + +Why these are safest: +- They are post-computation points where all needed values are already assembled. +- They are immediately adjacent to existing output writes. +- They avoid changing model training, prediction, and scoring algorithms. + +## Risks + +- REPORT score currently parses text from dat logs; changing dat text format later could silently break score extraction. +- Some outputs are conditional (for example Pearson heatmap skipped when descriptor count > 30, external csv_test artifacts only when csv_test provided). +- Ordering assumptions exist in report image pairing (No_PFI vs PFI order handling). +- If JSON writes are added directly into critical loops, accidental performance impact is possible. +- If JSON serialization is attempted on raw numpy/pandas objects without conversion, write failures can occur. + +## Current Runtime Audit Status (2026-07-13) + +- Canonical ChatBob runtime JSON location is the top-level `JSON/` folder. +- Implemented module-native runtime audits: `JSON/curate_audit.json`, `JSON/generate_audit.json`, `JSON/verify_audit.json`, `JSON/predict_audit.json`. +- Not implemented: `JSON/aqme_audit.json`, `JSON/evaluate_audit.json`. +- REPORT currently provides `JSON/report_figure_provenance.json` only (no full report audit). +- Wrapper-generated `*_manifest.json` and `run_summary.json` in timestamped archives are copy-side metadata, not module-native runtime audits. + +## Proposed First Implementation Step + +Implement one read-only helper module first (no hooks yet), for example robert/json_output_for_agent.py, with: +- safe_json_dump(data, path) +- to_json_safe(obj) for pandas/numpy/pathlib/scalars/lists/dicts +- write_module_artifact(module_name, artifact_type, payload, destination) + +Then add only one hook in CURATE save_curate after CURATE_options.csv write, because that point is simple, deterministic, and already has clean metadata. + +## Dataset Profile JSON Artifact (Implemented) + +What was added: +- New helper module: robert/json_output_for_agent.py +- New CURATE hook: robert/curate.py in `curate.__init__`, immediately after `load_variables(...)` and before `load_database(...)` +- New output file in normal runs: JSON/dataset_profile.json + +Why this insertion point was selected: +- It is the earliest practical place with access to `csv_name`, `y`, and `ignore`. +- The helper reads the input file independently and writes one additive artifact. +- The hook is fail-soft (try/except + JSON audit record), so CURATE behavior is unchanged if JSON writing fails. + +Standard ROBERT output protection rule (implemented): +- The JSON layer does not write status messages to standard ROBERT outputs. +- JSON-layer success/failure is written only to `JSON/curate_json_output_audit.json`. +- Standard outputs such as `CURATE/CURATE_data.dat`, curated CSVs, options CSV, and images remain untouched by JSON-layer status. + +Failure-tolerance validation completed: +- Baseline run created normal CURATE outputs plus `JSON/dataset_profile.json`. +- A simulated JSON write failure was injected by monkeypatching `write_json` to raise. +- CURATE still completed and produced standard outputs: + - CURATE_data.dat + - CURATE_options.csv + - Pearson_heatmap.png + - model-specific curated CSVs and general curated CSV +- JSON-layer status was recorded in `JSON/curate_json_output_audit.json`: + - success events for normal runs + - failure events with error type/message for simulated failures + +## Open Questions + +- Should JSON export be always on, or behind an option flag? +- JSON file location has been decided: use the top-level `JSON/` folder. +- Should each module write one summary JSON or multiple artifact JSON files? +- Do we need a run-level manifest file that indexes every generated artifact path? +- For REPORT, should we store only final data_score or also parsed intermediate values from get_predict_scores/get_verify_scores? +- Should JSON writes fail-soft (warn and continue) to guarantee no behavior change in scientific outputs? + + +## Controlled Output Comparison Update (2026-07-13) + +A completed unmodified ROBERT 2.1.2 run was compared with a completed ChatBob-modified ROBERT 2.1.2 run. + +Primary DAT comparison: + +- `CURATE/CURATE_data.dat`: identical after expected metadata normalization +- `GENERATE/GENERATE_data.dat`: identical after expected metadata normalization +- `VERIFY/VERIFY_data.dat`: identical after expected metadata normalization +- `PREDICT/PREDICT_data.dat`: identical after expected metadata normalization + +Expected normalized differences: + +- run timestamps, +- absolute input and output paths, +- module execution times, +- trailing whitespace. + +CSV comparison: + +- 28 matching CSV files were compared. +- All scientific numeric and text values matched. +- The only expected text difference was `csv_name` in `CURATE/CURATE_options.csv`. +- Best-model and prediction CSV files were identical. + +Conclusion: + +For this regression test case, the additive JSON-output hooks did not change ROBERT's standard scientific outputs. + + +## Implementation Update (2026-05-28) + +Implemented now (wrapper-side, archive-only): +- `robert/json_output_for_agent.py` now includes helpers to build module file manifests from copied archive outputs. +- `json-output-for-agent/scripts/run_robert_timestamped.py` now generates: + - `_manifest.json` for CURATE, GENERATE, VERIFY, PREDICT, REPORT (and AQME/EVALUATE when present), + - `run_summary.json` with command, return code, module manifest index, and top-level artifact manifest, + - `json_output_audit.json` with manifest write status. + +What this means for capture timing: +- Current implementation captures after ROBERT run completion and after copy into timestamped archive. +- This keeps normal ROBERT outputs untouched and ensures JSON reflects final saved artifacts. + +What is not implemented yet as of 2026-05-28: +- Module-native simultaneous JSON writes during `.dat`/CSV/image write points inside CURATE/GENERATE/VERIFY/PREDICT/REPORT. +- That path is still optional for later if real-time event streaming is required. diff --git a/json-output-for-agent/scripts/run_robert_timestamped.py b/json-output-for-agent/scripts/run_robert_timestamped.py new file mode 100644 index 0000000..f0ab04c --- /dev/null +++ b/json-output-for-agent/scripts/run_robert_timestamped.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +"""Timestamped ROBERT run wrapper that does not change ROBERT output behavior. + +This wrapper runs ROBERT exactly as usual from the repository root so standard +outputs are still produced in their normal locations (CURATE/, GENERATE/, etc.). +After ROBERT finishes, it copies those generated outputs into a timestamped run +archive under json-output-for-agent/runs/ for project-side tracking. +""" + +from __future__ import annotations + +import argparse +import json +import re +import shutil +import subprocess +import sys +from datetime import datetime +from pathlib import Path +from typing import List, Optional + + +PATH_OPTIONS = {"csv_name", "csv_test", "varfile", "params_dir"} +COPY_DIRS = ["CURATE", "GENERATE", "VERIFY", "PREDICT", "REPORT", "AQME", "EVALUATE", "JSON"] +COPY_FILES = ["ROBERT_report.pdf", "report.css", "report_debug.txt"] + + +def _sanitize_name(raw: str) -> str: + cleaned = re.sub(r"[^A-Za-z0-9_-]+", "_", raw.strip()) + cleaned = re.sub(r"_+", "_", cleaned).strip("_") + return cleaned or "run" + + +def _extract_option_value(args: List[str], option: str) -> Optional[str]: + long_opt = f"--{option}" + prefix = f"{long_opt}=" + for i, arg in enumerate(args): + if arg.startswith(prefix): + return arg.split("=", 1)[1] + if arg == long_opt and i + 1 < len(args): + return args[i + 1] + return None + + +def _absolutize_path_options(args: List[str], base_dir: Path) -> List[str]: + out = list(args) + i = 0 + while i < len(out): + arg = out[i] + if not arg.startswith("--"): + i += 1 + continue + + if "=" in arg: + key, value = arg[2:].split("=", 1) + if key in PATH_OPTIONS and value not in {"", "None"}: + out[i] = f"--{key}={(base_dir / value).resolve() if not Path(value).is_absolute() else Path(value).resolve()}" + i += 1 + continue + + key = arg[2:] + if key in PATH_OPTIONS and i + 1 < len(out): + val = out[i + 1] + if val not in {"", "None"} and not val.startswith("--"): + p = Path(val) + out[i + 1] = str((base_dir / p).resolve() if not p.is_absolute() else p.resolve()) + i += 1 + + return out + + +def _make_run_dir(runs_root: Path, label: str) -> Path: + stamp = datetime.now().strftime("%Y-%m-%d_%H%M%S") + base = runs_root / f"{stamp}_{label}" + candidate = base + n = 2 + while candidate.exists(): + candidate = runs_root / f"{stamp}_{label}_{n}" + n += 1 + candidate.mkdir(parents=True, exist_ok=False) + return candidate + + +def _copy_outputs(repo_root: Path, run_dir: Path) -> None: + for name in COPY_DIRS: + src = repo_root / name + if src.exists() and src.is_dir(): + shutil.copytree(src, run_dir / name, dirs_exist_ok=True) + + for name in COPY_FILES: + src = repo_root / name + if src.exists() and src.is_file(): + shutil.copy2(src, run_dir / name) + + +def _move_json_artifacts(source_dir: Path, json_dir: Path, filenames: List[str]) -> None: + for filename in filenames: + src = source_dir / filename + if not src.exists() or not src.is_file(): + continue + dst = json_dir / filename + try: + if dst.exists(): + if dst.is_dir(): + shutil.rmtree(dst) + else: + dst.unlink() + shutil.move(str(src), str(dst)) + except Exception: + pass + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Run ROBERT from repository root, then copy standard outputs into a " + "timestamped archive under json-output-for-agent/runs/." + ) + ) + parser.add_argument( + "--wrapper-runs-root", + default="json-output-for-agent/runs", + help="Archive root relative to repo root or absolute path.", + ) + parser.add_argument( + "--wrapper-python", + default=sys.executable, + help="Python executable used to run 'python -m robert'.", + ) + parser.add_argument( + "--wrapper-label", + default=None, + help="Optional run label override. Default comes from --csv_name stem.", + ) + parser.add_argument( + "--wrapper-dry-run", + action="store_true", + help="Print planned actions without running ROBERT.", + ) + + known, robert_args = parser.parse_known_args() + if robert_args and robert_args[0] == "--": + robert_args = robert_args[1:] + + if not robert_args: + print("x No ROBERT arguments were provided. Pass standard ROBERT args after wrapper args.") + return 2 + + invocation_cwd = Path.cwd().resolve() + repo_root = Path(__file__).resolve().parents[2] + + csv_name_value = _extract_option_value(robert_args, "csv_name") + label = known.wrapper_label + if label is None: + if not csv_name_value: + print("x Could not infer run label because --csv_name was not provided.") + print(" Provide --csv_name or set --wrapper-label explicitly.") + return 2 + label = _sanitize_name(Path(csv_name_value).stem) + + runs_root = Path(known.wrapper_runs_root) + if not runs_root.is_absolute(): + runs_root = (repo_root / runs_root).resolve() + + normalized_robert_args = _absolutize_path_options(robert_args, invocation_cwd) + cmd = [known.wrapper_python, "-m", "robert", *normalized_robert_args] + + run_dir = _make_run_dir(runs_root, _sanitize_name(label)) + json_dir = run_dir / "JSON" + json_dir.mkdir(parents=True, exist_ok=True) + if str(repo_root) not in sys.path: + sys.path.insert(0, str(repo_root)) + + try: + from robert.json_output_for_agent import write_archive_manifests, write_run_summary, write_json + except Exception: + write_archive_manifests = None + write_run_summary = None + write_json = None + + metadata = { + "timestamp": datetime.now().isoformat(), + "repo_root": str(repo_root), + "invocation_cwd": str(invocation_cwd), + "run_dir": str(run_dir), + "command": cmd, + "copy_mode": ( + "ROBERT normal root outputs are preserved; this wrapper only duplicates " + "generated artifacts into run_dir." + ), + } + + if known.wrapper_dry_run: + metadata["dry_run"] = True + (json_dir / "wrapper_run.json").write_text(json.dumps(metadata, indent=2), encoding="utf-8") + print(f"o Dry run created archive folder: {run_dir}") + print(f"o Planned command: {' '.join(cmd)}") + return 0 + + print(f"o Running ROBERT from repo root: {repo_root}") + print(f"o Archive folder: {run_dir}") + completed = subprocess.run(cmd, cwd=repo_root) + + _copy_outputs(repo_root, run_dir) + metadata["return_code"] = completed.returncode + metadata_path = json_dir / "wrapper_run.json" + metadata_path.write_text(json.dumps(metadata, indent=2), encoding="utf-8") + + if write_archive_manifests is not None and write_json is not None: + manifest_status = write_archive_manifests(run_dir) + _ = write_json(manifest_status, json_dir / "json_output_audit.json") + _move_json_artifacts( + run_dir, + json_dir, + [ + "CURATE_manifest.json", + "GENERATE_manifest.json", + "VERIFY_manifest.json", + "PREDICT_manifest.json", + "REPORT_manifest.json", + "AQME_manifest.json", + "EVALUATE_manifest.json", + ], + ) + + if write_run_summary is not None: + _ = write_run_summary( + run_dir, + command=cmd, + return_code=completed.returncode, + wrapper_metadata_path=metadata_path, + ) + _move_json_artifacts(run_dir, json_dir, ["run_summary.json"]) + + print(f"o Copied generated outputs to: {run_dir}") + return completed.returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/json-output-for-agent/task_tracker_plain_english.md b/json-output-for-agent/task_tracker_plain_english.md new file mode 100644 index 0000000..1c44196 --- /dev/null +++ b/json-output-for-agent/task_tracker_plain_english.md @@ -0,0 +1,686 @@ +# task_tracker_plain_english.md + +This file records what has been done in plain language. + +It should be updated after every meaningful project step. + +The goal is that a chemist can read this file and understand the project history without opening the code. + +--- + +## Current Project Goal + +We are adding structured JSON outputs to ROBERT. + +These JSON files should record information ROBERT already creates during a run. + +The purpose is to make ROBERT results easier to inspect later through a user interface or optional explanation tool. + +We are not changing ROBERT's scientific behavior. + +--- + +## Running Log + +### Entry 001 + +Date: YYYY-MM-DD + +Status: Project setup started. + +What we decided: +- We will work in small steps. +- The existing ROBERT code is the source of truth. +- We will inspect before changing. +- The agent must propose a plan before implementation. +- The user must approve before code changes. +- We will minimize changes to existing ROBERT files. +- We will use helper functions where possible. +- We will track progress in this file. + +Files created or planned: +- `AGENTS.md` +- `PROJECT_RULES.md` +- `TASKS.md` +- `task_tracker_plain_english.md` + +Confirmed results: +- No code has been changed yet. + +Uncertainties: +- We have not yet inspected where all ROBERT outputs are created. +- We have not yet decided the final JSON schema. +- We have not yet decided whether JSON export should always run or be controlled by an option. + +Next step: +- Inspect the ROBERT code to find where CURATE, GENERATE, VERIFY, PREDICT, and REPORT create outputs. + +### Entry 002 + +Date: 2026-05-27 + +Goal of this step: +- Inspect ROBERT outputs and create a durable output map before any code modification. + +What changed: +- We inspected CURATE, GENERATE, VERIFY, PREDICT, REPORT, and shared utility/report helper files. +- We documented where CSV files, image files, dat files, and the PDF report are created. +- We documented where ROBERT score values are calculated. +- We documented where values are already in memory before being written. +- We identified minimal and safe future points where JSON helper calls can be added later. + +Files changed: +- `json-output-for-agent/output_map.md` +- `json-output-for-agent/task_tracker_plain_english.md` +- `json-output-for-agent/TASKS.md` + +Why this change was made: +- To prepare a low-risk JSON export plan that does not change ROBERT scientific behavior. + +How this was tested: +- Manual code inspection only. +- No ROBERT source code was edited. +- No runtime behavior was changed. + +Confirmed results: +- Output creation points were mapped for all requested modules. +- Dat file creation flow was mapped through the shared Logger and finish_print functions. +- Image/plot creation points were mapped across CURATE, GENERATE, VERIFY, and PREDICT. +- PDF report creation and ROBERT score calculation flow were mapped in REPORT/report_utils. + +What remains uncertain: +- Final JSON schema details are still undecided. +- Whether JSON export should always run or be optional is still undecided. +- Exact naming/location convention for JSON artifacts is still undecided. + +Next suggested step: +- Propose one minimal first implementation: add a JSON helper module and one JSON write call in CURATE after `CURATE_options.csv` is written. + +### Entry 003 + +Date: 2026-05-28 + +Goal of this step: +- Implement the first JSON artifact as an incoming dataset profile. +- Confirm that CURATE still finishes normally if JSON writing fails. + +What changed: +- Added a new helper module: robert/json_output_for_agent.py. +- Added a fail-soft JSON hook in robert/curate.py immediately after load_variables(). +- The helper now writes CURATE/dataset_profile.json during normal CURATE runs. +- The CURATE hook is wrapped in try/except and does not stop execution. + +Files changed: +- `robert/json_output_for_agent.py` +- `robert/curate.py` +- `json-output-for-agent/TASKS.md` +- `json-output-for-agent/task_tracker_plain_english.md` + +Why this change was made: +- To add the first machine-readable artifact requested for UI/agent workflows while preserving existing ROBERT behavior. + +How this was tested: +- Baseline CURATE run: + - `python -m robert --curate --csv_name tests/Robert_example.csv --y Target_values --ignore Name --names Name` + - Confirmed normal CURATE outputs plus `CURATE/dataset_profile.json`. +- Fault-injection CURATE run: + - Monkeypatched `write_json` to raise `RuntimeError('simulated json write failure')`. + - Confirmed CURATE completed and generated standard outputs. + - Confirmed JSON failure was recorded in `CURATE/json_output_audit.json`. + +### Entry 004 + +Date: 2026-05-28 + +Goal of this step: +- Enforce the standard ROBERT output protection rule. +- Ensure JSON-layer status is never written into standard ROBERT files. + +What changed: +- Removed JSON-layer status logging to the standard CURATE logger in robert/curate.py. +- Added write_json_output_audit(...) helper in robert/json_output_for_agent.py. +- Added dedicated audit artifact CURATE/json_output_audit.json for JSON-layer events. +- CURATE now records dataset profile success/failure only in the audit JSON file. + +Files changed: +- `robert/curate.py` +- `robert/json_output_for_agent.py` +- `json-output-for-agent/TASKS.md` +- `json-output-for-agent/task_tracker_plain_english.md` +- `json-output-for-agent/output_map.md` + +Why this change was made: +- To keep the JSON layer invisible to all standard ROBERT outputs. +- To guarantee that .dat/.csv/image and other standard files are not modified by JSON-layer status messages. + +How this was tested: +- Baseline CURATE run confirmed: + - Standard CURATE outputs are produced. + - CURATE/dataset_profile.json is produced. + - CURATE/json_output_audit.json records success. + - CURATE/CURATE_data.dat contains no JSON-layer status text. +- Fault-injection run (write_json monkeypatched to raise) confirmed: + - CURATE still completes. + - Standard CURATE outputs are still produced. + - CURATE/CURATE_data.dat still contains no JSON-layer status text. + - CURATE/json_output_audit.json records failure with error type and message. + +Confirmed results: +- JSON-layer status is now isolated to JSON audit artifacts. +- Standard ROBERT outputs are unchanged by JSON success/failure state. + +What remains uncertain: +- An automated unit test dedicated to JSON audit edge cases is not yet added. + +Next suggested step: +- Stop here per scope; do not extend beyond this protection fix. + +### Entry 005 + +Date: 2026-05-28 + +Goal of this step: +- Add a timestamped archive wrapper without changing normal ROBERT output behavior. + +What changed: +- Added a wrapper script: `json-output-for-agent/scripts/run_robert_timestamped.py`. +- Wrapper runs ROBERT from the repository root exactly as normal. +- After ROBERT finishes, wrapper copies generated outputs into: + - `json-output-for-agent/runs/_/` +- Added explicit copy-only policy text in `AGENTS.md` and `PROJECT_RULES.md`. + +Files changed: +- `json-output-for-agent/scripts/run_robert_timestamped.py` +- `json-output-for-agent/AGENTS.md` +- `json-output-for-agent/PROJECT_RULES.md` +- `json-output-for-agent/TASKS.md` +- `json-output-for-agent/task_tracker_plain_english.md` +- `json-output-for-agent/README.md` + +Why this change was made: +- To keep the established ROBERT audience and workflows fully intact while creating a project-side run archive for JSON/export tracking. + +How this was tested: +- Wrapper dry-run check to confirm command build, folder creation, and metadata write without executing ROBERT. + +Confirmed results: +- Wrapper policy is explicit: normal root outputs are preserved; archive is duplicate-copy only. + +What remains uncertain: +- A full end-to-end wrapper run validation is still pending. + +Next suggested step: +- Execute one full wrapper run and verify archive completeness for CURATE/GENERATE/VERIFY/PREDICT/REPORT artifacts. + +### Entry 006 + +Date: 2026-05-28 + +Goal of this step: +- Start implementation of full-run JSON mirroring with minimal complexity and no change to standard ROBERT behavior. + +What changed: +- Extended `robert/json_output_for_agent.py` with archive-manifest helpers. +- Added file metadata capture helpers (path, extension, size, modified time, sha256). +- Added safe text preview extraction for textual files and best-effort PDF text preview extraction. +- Added `collect_module_manifest(...)` and `write_module_manifest(...)`. +- Added `write_archive_manifests(...)` to generate one manifest per module folder in a run archive. +- Added `collect_run_summary(...)` and `write_run_summary(...)`. +- Updated wrapper script `json-output-for-agent/scripts/run_robert_timestamped.py` to generate manifests and run summary after copy. +- Added archive-level JSON write status artifact: `json_output_audit.json` in each run folder. + +Files changed: +- `robert/json_output_for_agent.py` +- `json-output-for-agent/scripts/run_robert_timestamped.py` +- `json-output-for-agent/TASKS.md` +- `json-output-for-agent/task_tracker_plain_english.md` + +Why this change was made: +- To deliver immediate full-run JSON mirrors from exact copied output artifacts while avoiding high-risk changes inside CURATE/GENERATE/VERIFY/PREDICT/REPORT internals. +- To keep standard outputs as source of truth and maintain copy-only archive behavior. + +How this was tested: +- Wrapper dry run: + - `python json-output-for-agent/scripts/run_robert_timestamped.py --wrapper-dry-run --csv_name tests/Robert_example.csv` +- Full wrapper run (CURATE example): + - `python json-output-for-agent/scripts/run_robert_timestamped.py --curate --discard "['xtest']" --y Target_values --csv_name tests/Robert_example.csv --names Name` +- Verified archive folder contains: + - `CURATE_manifest.json` + - `GENERATE_manifest.json` + - `VERIFY_manifest.json` + - `PREDICT_manifest.json` + - `REPORT_manifest.json` + - `run_summary.json` + - `json_output_audit.json` + - `wrapper_run.json` + +Confirmed results: +- Standard ROBERT execution and outputs remain unchanged. +- JSON mirrors are generated in timestamped archive only. +- Manifest records include hashes and file-level metadata suitable for exact comparisons. +- Missing module folders produce valid manifests with `module_dir_exists=false` and `file_count=0`. + +What remains uncertain: +- Module-native write-time hooks are still not added. +- Automated tests for helper-manifest functions are still not added. + +Next suggested step: +- Add small helper tests for manifest and run-summary functions, then (if still needed) add module-native hooks incrementally behind a low-risk option. + +### Entry 007 + +Date: 2026-06-08 + +Goal of this step: +- Run a full validation checklist using protected source-data in `databases/`. +- Record pass/fail outcomes. +- Add a design-only proposal for a future `CURATE/curate_audit.json` hook. + +What changed: +- Completed one regression wrapper validation run: + - `databases/Regression/AQME-ROBERT_A_predict_solubility.csv` + - `y=solubility`, `names=code_name`, `ignore=code_name` +- Completed one classification wrapper validation run: + - `databases/Clasification/F_predict_outcome.csv` + - `y=Outcome`, `names=Name`, `ignore=Name`, `type=clas` +- Verified no JSON-layer text appears in standard `.dat` files. +- Verified `CURATE/dataset_profile.json` and `CURATE/json_output_audit.json` exist and are valid JSON. +- Verified required top-level fields in archive `run_summary.json` and CURATE JSON artifacts. +- Verified copy-only archive behavior from `wrapper_run.json`. +- Verified missing module folders are represented safely in manifests with: + - `module_dir_exists=false` + - `file_count=0` +- Added design-only proposal for future `CURATE/curate_audit.json` (no code hook implemented yet). + +Files changed: +- `json-output-for-agent/README.md` +- `json-output-for-agent/json_schema_notes.md` +- `json-output-for-agent/TASKS.md` +- `json-output-for-agent/task_tracker_plain_english.md` +- `.gitignore` + +Why this change was made: +- To establish a clear, reproducible validation baseline before adding more module-native JSON hooks. +- To preserve ROBERT scientific behavior while preparing a low-risk future audit pattern. + +How this was tested: +- Wrapper regression run and classification run from repository root. +- JSON validity and required-field checks by Python scripts. +- DAT pollution check by searching for JSON-layer markers in standard `.dat` files. +- Archive checks for copy-only behavior and missing-module handling. + +Confirmed results: +- PASS: Standard root outputs for CURATE/GENERATE/VERIFY/PREDICT were produced. +- PASS: JSON artifacts were created in standard and archive paths. +- PASS: JSON files are valid and required fields are present. +- PASS: `dataset_profile.json` includes `schema_version` and works for both regression and classification input targets. +- PASS: No JSON-layer status messages were found in standard `.dat` files. +- PASS: Copy-only archive behavior confirmed. +- PASS: Missing-file/module behavior handled in manifests. +- PARTIAL: REPORT PDF generation failed in this environment due missing WeasyPrint system libraries. + +What remains uncertain: +- We still need to install/report-support system libraries if PDF generation must be included in this validation matrix. +- Module-native audit hooks beyond dataset profile remain unimplemented by design. + +Next suggested step: +- Propose the smallest safe implementation plan for `CURATE/curate_audit.json` and wait for explicit approval before coding. + +### Entry 008 + +Date: 2026-06-08 + +Goal of this step: +- Implement the approved minimal hook for `CURATE/curate_audit.json`. +- Keep the implementation fail-soft and additive only. + +What changed: +- Added reusable helper `build_curate_audit_payload(...)` in `robert/json_output_for_agent.py`. +- Generalized `write_json_output_audit(...)` to accept an `artifact` field so events can identify either: + - `dataset_profile.json`, or + - `curate_audit.json`. +- Added CURATE hook in `robert/curate.py` to write `CURATE/curate_audit.json`. +- Hook records only observable evidence and marks unavailable values explicitly. +- Hook is wrapped in try/except and writes status only to `CURATE/json_output_audit.json`. + +Files changed: +- `robert/json_output_for_agent.py` +- `robert/curate.py` +- `json-output-for-agent/TASKS.md` +- `json-output-for-agent/task_tracker_plain_english.md` + +Why this change was made: +- To start module-native audit pattern in the lowest-risk module (CURATE). +- To capture module-level evidence without changing ROBERT scientific behavior. + +How this was tested: +- CURATE-only regression run: + - `python -m robert --curate --csv_name databases/Regression/AQME-ROBERT_A_predict_solubility.csv --y solubility --names code_name --ignore code_name --model "['RF']"` +- CURATE-only classification run: + - `python -m robert --curate --csv_name databases/Clasification/F_predict_outcome.csv --y Outcome --names Name --ignore Name --type clas --model "['RF']"` +- Checked artifacts created in `CURATE/`: + - `dataset_profile.json` + - `curate_audit.json` + - `json_output_audit.json` +- Verified JSON audit events include both artifacts: + - `artifact=dataset_profile.json` + - `artifact=curate_audit.json` +- Verified no JSON-layer markers in `CURATE/CURATE_data.dat`. + +Confirmed results: +- `curate_audit.json` is created for regression and classification CURATE runs. +- `curate_audit.json` includes `schema_version` and the approved top-level sections. +- Unknown step-level descriptor-removal counters are recorded as `unavailable` (not guessed). +- Standard CURATE outputs remain unchanged. +- JSON-layer status remains isolated to JSON audit artifacts. + +What remains uncertain: +- Step-specific descriptor-removal counters per filtering stage are not yet directly exposed as in-memory counters. + +Next suggested step: +- If desired, add low-risk direct counters for specific CURATE filter stages so future audit files can replace selected `unavailable` fields with observed values. + +### Entry 009 + +Date: 2026-06-08 + +Goal of this step: +- Run fault-injection validation specifically for `curate_audit.json` write failure. + +What changed: +- Ran CURATE with a temporary monkeypatch on the exact reference used by CURATE: + - patched `robert.curate.write_json` + - raised `RuntimeError('simulated curate_audit write failure')` only when output path ended with `curate_audit.json` + - allowed all other JSON writes (`dataset_profile.json`, `json_output_audit.json`) to proceed + +Files changed: +- `json-output-for-agent/task_tracker_plain_english.md` + +Why this change was made: +- To confirm fail-soft behavior for `curate_audit.json` writing without affecting standard CURATE outputs. + +How this was tested: +- Command run: + - `python` inline script patching `robert.curate.write_json` and executing CURATE on `databases/Regression/AQME-ROBERT_A_predict_solubility.csv`. +- Verified: + - CURATE completed (`FAULT_INJECTION_RUN_DONE`). + - Standard outputs existed: + - `CURATE/CURATE_data.dat` + - `CURATE/CURATE_options.csv` + - curated CSV files. + - `CURATE/dataset_profile.json` existed with schema version `0.1`. + - `CURATE/json_output_audit.json` recorded failed `curate_audit.json` event: + - `error_type=RuntimeError` + - `error_message=simulated curate_audit write failure` + - `CURATE/CURATE_data.dat` contained no JSON-layer status text. + +Confirmed results: +- Fail-soft behavior for `curate_audit.json` write failure is validated. +- Standard CURATE behavior/output remained unchanged. +- JSON-layer status remained isolated to JSON audit files. + +What remains uncertain: +- None specific to this fault-injection case. + +Next suggested step: +- Proceed to next approved module-native audit workstream when ready. + +### Entry 010 + +Date: 2026-06-09 + +Goal of this step: + +* Record progress after implementing module-native runtime audits for CURATE, GENERATE, and VERIFY. +* Reset the task list so the next workstream is clear before starting PREDICT. + +What changed: + +* `TASKS.md` was rewritten to reflect the current module-native audit workflow. +* The task list now separates archive-side manifests from module-native runtime audit JSON files. +* The task list now identifies PREDICT as the next module to implement. +* CURATE, GENERATE, and VERIFY are marked as implemented and validated module-native audits. + +Files changed: + +* `json-output-for-agent/TASKS.md` +* `json-output-for-agent/task_tracker_plain_english.md` + +Why this change was made: + +* The previous task list still reflected the early CURATE-first phase of the project. +* The project has now moved into a module-by-module runtime audit pattern. +* Updating the tracker now prevents confusion before starting PREDICT. + +How this was tested: + +* Documentation update only. +* No ROBERT source code was changed in this step. +* No ROBERT run was required for this documentation checkpoint. + +Confirmed results: + +* CURATE runtime audit has been implemented and validated. +* GENERATE runtime audit has been implemented and validated. +* VERIFY runtime audit has been implemented and validated. +* Standard `.dat` behavior was preserved during validation, with differences limited to timestamp/runtime or non-scientific formatting. +* JSON-layer status remains isolated to `json_output_audit.json`. +* No original ROBERT scientific variables, thresholds, descriptor logic, model logic, prediction logic, plotting logic, or CLI behavior were intentionally changed. + +What remains uncertain: + +* PREDICT module-native audit implementation has not started. +* AQME and EVALUATE module-native audits have not started. +* REPORT remains deferred and has a separate report-generation/parser issue to revisit later. +* The long-term role of archive manifests versus module-native audits still needs to be decided. + +Next suggested step: + +* Commit the documentation checkpoint. +* Begin PREDICT planning only, with no implementation until the plan is reviewed and approved. + + +### Entry 011 + +Date: 2026-07-13 + +Goal of this step: +- Synchronize the ChatBob JSON-output branch with the current upstream ROBERT version. +- Compare an unmodified ROBERT run with a ChatBob-modified ROBERT run. +- Confirm whether the JSON-output additions change standard ROBERT scientific outputs. + +What changed: +- Fetched the current upstream ROBERT repository. +- Confirmed that upstream `master` contains ROBERT version 2.1.2. +- Merged `upstream/master` into the `json-output-for-agent` branch. +- Resolved one merge conflict in `robert/generate.py`. +- Preserved both: + - the upstream classification handling added in ROBERT 2.1.2, and + - the additive ChatBob GENERATE audit capture. +- Added a Jupyter notebook for comparing completed ROBERT output folders. +- Compared an unmodified ROBERT 2.1.2 run with the ChatBob-modified ROBERT 2.1.2 run. + +Files changed: +- ROBERT files brought in through the merge from `upstream/master`. +- `robert/generate.py`, where the merge conflict was resolved. +- `comparison/compare_robert_outputs.ipynb`. +- Project documentation files associated with this update. + +Why this change was made: +- An initial comparison mistakenly compared ROBERT 2.1.2 against the ChatBob branch while it was still based on ROBERT 2.1.0. +- That version mismatch produced differences in prediction values and outlier reporting. +- Synchronizing both runs to ROBERT 2.1.2 was necessary to isolate the effect of the ChatBob JSON additions. + +How this was tested: +- Ran the same regression dataset through: + - unmodified ROBERT 2.1.2, and + - ChatBob-modified ROBERT 2.1.2. +- Compared the four primary `.dat` files: + - `CURATE/CURATE_data.dat` + - `GENERATE/GENERATE_data.dat` + - `VERIFY/VERIFY_data.dat` + - `PREDICT/PREDICT_data.dat` +- Normalized only expected run-specific values: + - timestamps, + - input paths, + - output paths, + - execution times, + - trailing whitespace. +- Compared 28 matching CSV files. +- Numeric CSV values were compared within a strict floating-point tolerance. +- Text values were compared exactly. + +Confirmed results: +- PASS: All four normalized `.dat` files were identical. +- PASS: All scientific CSV files were identical within numerical tolerance. +- PASS: CURATE outputs were unchanged. +- PASS: GENERATE model files and selected best-model files were unchanged. +- PASS: VERIFY outputs were unchanged. +- PASS: PREDICT values and uncertainty values were unchanged. +- EXPECTED DIFFERENCE: `CURATE/CURATE_options.csv` recorded a different input file path. +- EXPECTED DIFFERENCE: Raw `.dat` files contained different timestamps, paths, and execution times. +- The earlier PREDICT differences were caused by comparing ROBERT 2.1.0 with ROBERT 2.1.2, not by the ChatBob JSON additions. + +What this establishes: +- For this regression test case, the ChatBob JSON-output implementation is additive. +- It does not change the standard ROBERT scientific `.dat` or CSV outputs. + +What remains uncertain: +- The same controlled comparison has not yet been documented for classification. +- The current JSON structure still needs to be reviewed against the questions ChatBob should answer. +- The long-term location and integration of the JSON outputs should be agreed with Juanvi. +- A pull request should be reviewed before the branch diverges further from upstream. + +Next suggested step: +- Open a pull request from `json-output-for-agent` for review. +- Share the validation result with Juanvi. +- Agree on the JSON output location and integration strategy. +- Inspect the current JSON structures and map a small set of likely user questions to the evidence required to answer them. + + +### Entry 012 + +Date: 2026-07-13 + +Goal of this step: +- Reconcile documentation with current source implementation status after post-session coding. +- Apply the confirmed architecture decision for artifact location. + +What changed: +- Confirmed from source that module-native runtime audits are implemented for: + - CURATE, + - GENERATE, + - VERIFY, + - PREDICT. +- Confirmed from source that module-native runtime audits are not implemented for: + - AQME, + - EVALUATE. +- Confirmed REPORT currently writes figure provenance only (no full report audit). +- Confirmed ChatBob runtime JSON artifacts are written to the top-level `JSON/` folder. +- Updated project docs to remove stale status/path ambiguity and preserve the distinction between: + - module-native runtime audits, + - JSON write-status audits, + - wrapper-generated archive manifests, + - standard ROBERT outputs. + +Files changed: +- `json-output-for-agent/TASKS.md` +- `json-output-for-agent/AGENTS.md` +- `json-output-for-agent/json_schema_notes.md` +- `json-output-for-agent/output_map.md` +- `json-output-for-agent/README.md` +- `json-output-for-agent/task_tracker_plain_english.md` + +Why this change was made: +- Earlier task/status documents were stale after implementation continued outside the previous agent session. +- Documentation now reflects source-verified implementation status and canonical JSON artifact location. + +How this was tested: +- Source inspection only (no ROBERT run, no tests). + +Confirmed results: +- Runtime audit paths are documented using `JSON/` examples. +- PREDICT is documented as implemented with validation still incomplete. +- AQME and EVALUATE remain documented as not implemented. +- REPORT is documented as figure-provenance-only at this stage. + +What remains uncertain: +- Full audit-content validation for PREDICT remains pending. +- AQME and EVALUATE audit validation remains pending because implementation is not present. + +Next suggested step: +- Prepare a draft pull request with explicit validation boundaries and open design decisions. + + +### Entry 013 + +Date: 2026-07-14 + +Goal of this step: +- Record newly completed manual verification performed outside the agent session. + +What changed: +- Confirmed manual review that structured JSON files were generated for: + - CURATE, + - GENERATE, + - VERIFY, + - PREDICT, + - REPORT. +- Confirmed manual in-repo run comparison against original ROBERT DAT files showed no DAT differences. +- Updated status docs so this verification is explicitly documented. + +Files changed: +- `json-output-for-agent/README.md` +- `json-output-for-agent/TASKS.md` +- `json-output-for-agent/task_tracker_plain_english.md` + +Why this change was made: +- The verification was completed outside the agent and needed to be captured as project evidence. + +How this was tested: +- Documentation update only in this step. +- Verification evidence source: user-confirmed manual run and DAT comparison in this repository. + +Confirmed results: +- Structured JSON generation was reviewed for major workflow stages. +- DAT comparison against original ROBERT reported no differences. + +What remains uncertain: +- Automated tests for module-audit content are still pending. +- AQME and EVALUATE module-native audits remain not implemented. + +Next suggested step: +- Prepare draft pull request text that includes both the 2026-07-13 controlled comparison and 2026-07-14 in-repo manual verification. + + +--- + +## Template for Future Entries + +### Entry XXX + +Date: YYYY-MM-DD + +Goal of this step: +- + +What changed: +- + +Files changed: +- + +Why this change was made: +- + +How this was tested: +- + +Confirmed results: +- + +What remains uncertain: +- + +Next suggested step: +- diff --git a/robert/curate.py b/robert/curate.py index 91562c3..f7255e5 100644 --- a/robert/curate.py +++ b/robert/curate.py @@ -63,6 +63,16 @@ import os import json import pandas as pd +from robert.json_output_for_agent import ( + audit_event, + audit_set, + agent_json_path, + finalize_module_audit, + init_module_audit, + profile_input_dataset, + write_json, + write_json_output_audit, +) from robert.utils import (load_variables, finish_print, load_database, pearson_map, check_clas_problem, categorical_transform, correlation_filter) @@ -84,8 +94,65 @@ def __init__(self, **kwargs): # load default and user-specified variables self.args = load_variables(kwargs, "curate") + # Initialize runtime CURATE audit capture with direct input evidence. + # This records structured evidence beside existing .dat logging. + self.args.curate_audit = init_module_audit( + module="CURATE", + artifact_type="curate_audit", + source_files=[str(self.args.csv_name)], + command_line=None, + ) + self.args.curate_audit = audit_set(self.args.curate_audit, "inputs", "source_csv", self.args.csv_name) + self.args.curate_audit = audit_set(self.args.curate_audit, "inputs", "target_column", self.args.y) + self.args.curate_audit = audit_set(self.args.curate_audit, "inputs", "names_column", self.args.names) + self.args.curate_audit = audit_set(self.args.curate_audit, "inputs", "ignored_columns", self.args.ignore) + self.args.curate_audit = audit_set(self.args.curate_audit, "inputs", "discarded_columns_requested", self.args.discard) + + # Save a raw-input dataset profile JSON for downstream UI/agent workflows. + # This is additive and fail-soft: any JSON issue must not affect CURATE outputs. + dataset_profile_path = agent_json_path("dataset_profile.json") + curate_audit_path = agent_json_path("curate_audit.json") + json_audit_path = agent_json_path("curate_json_output_audit.json") + try: + dataset_profile = profile_input_dataset(self.args.csv_name, self.args.y, self.args.ignore) + json_write_ok = write_json(dataset_profile, dataset_profile_path) + if not json_write_ok: + _ = write_json_output_audit( + json_audit_path, + module="CURATE", + attempted_output_path=dataset_profile_path, + attempted=True, + succeeded=False, + artifact="dataset_profile.json", + error=RuntimeError("dataset_profile_json_write_returned_false"), + ) + else: + _ = write_json_output_audit( + json_audit_path, + module="CURATE", + attempted_output_path=dataset_profile_path, + attempted=True, + succeeded=True, + artifact="dataset_profile.json", + error=None, + ) + except Exception as json_error: + _ = write_json_output_audit( + json_audit_path, + module="CURATE", + attempted_output_path=dataset_profile_path, + attempted=True, + succeeded=False, + artifact="dataset_profile.json", + error=json_error, + ) + # load database, discard user-defined descriptors and perform data checks csv_df,_,_ = load_database(self,self.args.csv_name,"curate") + rows_before_curate = int(len(csv_df)) + columns_before_curate = int(len(csv_df.columns)) + self.args.curate_audit = audit_set(self.args.curate_audit, "counts", "rows_before_curate", rows_before_curate) + self.args.curate_audit = audit_set(self.args.curate_audit, "counts", "columns_before_curate", columns_before_curate) # adjust options of classification problems and detects whether the right type of problem was used self = check_clas_problem(self,csv_df) @@ -118,6 +185,49 @@ def __init__(self, **kwargs): # create Pearson heatmap (use the general filtered dataframe) _ = pearson_map(self,csv_df,'curate') + self.args.curate_audit = audit_set(self.args.curate_audit, "counts", "rows_after_curate", int(len(csv_df))) + self.args.curate_audit = audit_set(self.args.curate_audit, "counts", "columns_after_curate", int(len(csv_df.columns))) + self.args.curate_audit = audit_set( + self.args.curate_audit, + "runtime", + "module_runtime_seconds", + round(time.time() - start_time, 2), + ) + + # Save the CURATE runtime audit JSON without affecting standard ROBERT outputs. + try: + audit_write_ok = finalize_module_audit(self.args.curate_audit, curate_audit_path, status="completed") + if not audit_write_ok: + _ = write_json_output_audit( + json_audit_path, + module="CURATE", + attempted_output_path=curate_audit_path, + attempted=True, + succeeded=False, + artifact="curate_audit.json", + error=RuntimeError("curate_audit_json_write_returned_false"), + ) + else: + _ = write_json_output_audit( + json_audit_path, + module="CURATE", + attempted_output_path=curate_audit_path, + attempted=True, + succeeded=True, + artifact="curate_audit.json", + error=None, + ) + except Exception as json_error: + _ = write_json_output_audit( + json_audit_path, + module="CURATE", + attempted_output_path=curate_audit_path, + attempted=True, + succeeded=False, + artifact="curate_audit.json", + error=json_error, + ) + # finish the printing of the CURATE info file _ = finish_print(self,start_time,'CURATE') @@ -146,6 +256,17 @@ def dup_filter(self,csv_df_dup): csv_df_dup.reset_index(drop=True) self.args.log.write(txt_dup) + self.args.curate_audit = audit_set(self.args.curate_audit, "duplicate_filter", "activated", True) + self.args.curate_audit = audit_set(self.args.curate_audit, "duplicate_filter", "removed_datapoint_indices", datapoint_drop) + self.args.curate_audit = audit_set(self.args.curate_audit, "duplicate_filter", "removed_count", len(datapoint_drop)) + self.args.curate_audit = audit_event( + self.args.curate_audit, + event_type="duplicate_filter", + payload={"removed_count": len(datapoint_drop), "removed_datapoint_indices": datapoint_drop}, + evidence_level="direct", + dat_text=txt_dup, + ) + return csv_df_dup @@ -159,6 +280,9 @@ def save_curate(self,csv_df): # Check if csv_df is a tuple (csv_df_filtered, csv_df_per_model) from correlation_filter if isinstance(csv_df, tuple): csv_df_filtered, csv_df_per_model = csv_df + + model_descriptor_counts = {} + model_descriptors = {} # Save model-specific curated databases (sorted for reproducibility) for model in csv_df_per_model: @@ -177,6 +301,22 @@ def save_curate(self,csv_df): txt_csv = f'\n o Model {model}: {len(csv_df_per_model[model].columns)-len(self.args.ignore)-count_y} descriptors remaining:\n' txt_csv += ' ' + ', '.join(f'{var}' for var in csv_df_per_model[model].columns if var not in self.args.ignore and var != self.args.y) self.args.log.write(txt_csv) + + final_descs = [var for var in csv_df_per_model[model].columns if var not in self.args.ignore and var != self.args.y] + model_descriptor_counts[str(model)] = len(final_descs) + model_descriptors[str(model)] = final_descs + self.args.curate_audit = audit_event( + self.args.curate_audit, + event_type="save_curate_model_output", + payload={ + "model": str(model), + "curated_csv_path": str(csv_curate_name), + "final_descriptor_count": len(final_descs), + "final_descriptors": final_descs, + }, + evidence_level="direct", + dat_text=txt_csv, + ) self.args.log.write(f'\no Model-specific curated databases were stored in {self.args.destination}') @@ -186,6 +326,9 @@ def save_curate(self,csv_df): csv_df_to_save_general = csv_df_filtered.reset_index(drop=True).sort_values(by=self.args.y).reset_index(drop=True) _ = csv_df_to_save_general.to_csv(f'{csv_curate_name_general}', index=None, header=True) + self.args.curate_audit = audit_set(self.args.curate_audit, "outputs", "model_descriptor_count", model_descriptor_counts) + self.args.curate_audit = audit_set(self.args.curate_audit, "outputs", "model_descriptors", model_descriptors) + else: # Original behavior: save single curated database csv_curate_name_general = f'{csv_basename}_CURATE.csv' @@ -194,6 +337,10 @@ def save_curate(self,csv_df): path_reduced = '/'.join(f'{csv_curate_name_general}'.replace('\\','/').split('/')[-2:]) self.args.log.write(f'\no The curated database was stored in {path_reduced}.') + final_descs = [var for var in csv_df.columns if var not in self.args.ignore and var != self.args.y] + self.args.curate_audit = audit_set(self.args.curate_audit, "outputs", "model_descriptor_count", {"default": len(final_descs)}) + self.args.curate_audit = audit_set(self.args.curate_audit, "outputs", "model_descriptors", {"default": final_descs}) + # Save important options used in CURATE options_name = f'CURATE_options.csv' options_name = self.args.destination.joinpath(options_name) @@ -211,3 +358,12 @@ def save_curate(self,csv_df): options_df['class_1_label'] = [self.args.class_1_label] _ = options_df.to_csv(f'{options_name}', index=None, header=True) + + self.args.curate_audit = audit_set(self.args.curate_audit, "outputs", "general_curated_csv_path", str(csv_curate_name_general)) + self.args.curate_audit = audit_set(self.args.curate_audit, "outputs", "curate_options_csv_path", str(options_name)) + self.args.curate_audit = audit_set( + self.args.curate_audit, + "outputs", + "curated_csv_paths", + [str(fp) for fp in sorted(self.args.destination.glob(f"{csv_basename}_CURATE*.csv"))], + ) diff --git a/robert/generate.py b/robert/generate.py index 91f4550..806244e 100644 --- a/robert/generate.py +++ b/robert/generate.py @@ -89,6 +89,15 @@ import os import time +import glob +from robert.json_output_for_agent import ( + init_module_audit, + audit_set, + audit_event, + finalize_module_audit, + agent_json_path, + write_json_output_audit, +) from robert.utils import ( load_variables, finish_print, @@ -121,8 +130,47 @@ def __init__(self, **kwargs): # load default and user-specified variables self.args = load_variables(kwargs, "generate") + # Initialize GENERATE runtime audit capture (additive and fail-soft). + self.args.generate_audit = init_module_audit( + module="GENERATE", + artifact_type="generate_audit", + source_files=[str(self.args.csv_name)], + command_line=None, + ) + self.args.generate_audit = audit_set(self.args.generate_audit, "inputs", "source_csv", self.args.csv_name) + self.args.generate_audit = audit_set(self.args.generate_audit, "inputs", "target_column", self.args.y) + self.args.generate_audit = audit_set(self.args.generate_audit, "inputs", "names_column", self.args.names) + self.args.generate_audit = audit_set(self.args.generate_audit, "inputs", "ignored_columns", getattr(self.args, "ignore", [])) + self.args.generate_audit = audit_set(self.args.generate_audit, "inputs", "discarded_columns_requested", getattr(self.args, "discard", [])) + self.args.generate_audit = audit_set(self.args.generate_audit, "inputs", "model_list", list(self.args.model)) + self.args.generate_audit = audit_set(self.args.generate_audit, "inputs", "prediction_type", self.args.type) + self.args.generate_audit = audit_set(self.args.generate_audit, "inputs", "seed", self.args.seed) + self.args.generate_audit = audit_set(self.args.generate_audit, "inputs", "error_type", self.args.error_type) + self.args.generate_audit = audit_set(self.args.generate_audit, "inputs", "init_points", self.args.init_points) + self.args.generate_audit = audit_set(self.args.generate_audit, "inputs", "n_iter", self.args.n_iter) + self.args.generate_audit = audit_set(self.args.generate_audit, "inputs", "pfi_filter", self.args.pfi_filter) + self.args.generate_audit = audit_set(self.args.generate_audit, "inputs", "pfi_epochs", self.args.pfi_epochs) + self.args.generate_audit = audit_set(self.args.generate_audit, "inputs", "pfi_threshold", self.args.pfi_threshold) + self.args.generate_audit = audit_set(self.args.generate_audit, "inputs", "pfi_max", self.args.pfi_max) + self.args.generate_audit = audit_set(self.args.generate_audit, "inputs", "test_set", self.args.test_set) + self.args.generate_audit = audit_set(self.args.generate_audit, "inputs", "kfold", self.args.kfold) + self.args.generate_audit = audit_set(self.args.generate_audit, "inputs", "repeat_kfolds", self.args.repeat_kfolds) + self.args.generate_audit = audit_set(self.args.generate_audit, "inputs", "split", self.args.split) + self.args.generate_audit = audit_set(self.args.generate_audit, "inputs", "destination", str(self.args.destination)) + + generate_audit_path = agent_json_path("generate_audit.json") + json_audit_path = agent_json_path("generate_json_output_audit.json") + # load database, discard user-defined descriptors and perform data checks csv_df, _, _ = load_database(self,self.args.csv_name,"generate") + initial_desc_count = len([col for col in csv_df.columns if col not in self.args.ignore and col != self.args.y]) + ignored_desc_count = len([col for col in csv_df.columns if col in self.args.ignore]) + self.args.generate_audit = audit_set(self.args.generate_audit, "load_database", "datapoints_loaded", int(len(csv_df))) + self.args.generate_audit = audit_set(self.args.generate_audit, "load_database", "descriptors_loaded", int(initial_desc_count)) + self.args.generate_audit = audit_set(self.args.generate_audit, "load_database", "ignored_descriptors_loaded", int(ignored_desc_count)) + self.args.generate_audit = audit_set(self.args.generate_audit, "load_database", "discarded_descriptors_loaded", int(len(getattr(self.args, "discard", [])))) + self.args.generate_audit = audit_set(self.args.generate_audit, "load_database", "source_csv_used", self.args.csv_name) + self.args.generate_audit = audit_set(self.args.generate_audit, "load_database", "source_csv_role", "initial_source_csv") # adjust classification labels and auto-detect binary classification if self.args.type.lower() == 'clas' or (self.args.type.lower() == 'reg' and self.args.auto_type): @@ -137,10 +185,17 @@ def __init__(self, **kwargs): self.args.log.write(txt_heatmap) # scan different ML models - self.args.log.write(f''' o Starting BO-based hyperoptimization using the combined target: + bo_target_txt = f''' o Starting BO-based hyperoptimization using the combined target: \n 1. 50% = {self.args.error_type.upper()} from a {self.args.repeat_kfolds}x repeated {self.args.kfold}-fold CV (interpoplation) \n 2. 50% = {self.args.error_type.upper()} from the bottom or top (worst performing) fold in a sorted {self.args.kfold}-fold CV (extrapolation) - \n''') + \n''' + self.args.log.write(bo_target_txt) + self.args.generate_audit = audit_set(self.args.generate_audit, "model_scan", "number_of_ml_models", len(self.args.model)) + self.args.generate_audit = audit_set(self.args.generate_audit, "model_scan", "model_list", list(self.args.model)) + self.args.generate_audit = audit_set(self.args.generate_audit, "model_scan", "error_type", self.args.error_type) + self.args.generate_audit = audit_set(self.args.generate_audit, "model_scan", "repeated_kfold_setting", self.args.repeat_kfolds) + self.args.generate_audit = audit_set(self.args.generate_audit, "model_scan", "kfold_setting", self.args.kfold) + self.args.generate_audit = audit_set(self.args.generate_audit, "model_scan", "bo_target_explanation", bo_target_txt.strip()) for ML_model in self.args.model: @@ -160,7 +215,8 @@ def __init__(self, **kwargs): # Store the original csv_name temporarily original_csv_name = self.args.csv_name - if os.path.exists(csv_model_specific): + model_specific_found = os.path.exists(csv_model_specific) + if model_specific_found: csv_to_load = csv_model_specific # Temporarily update csv_name to the model-specific CSV self.args.csv_name = str(csv_model_specific) @@ -168,6 +224,20 @@ def __init__(self, **kwargs): else: csv_to_load = self.args.csv_name self.args.log.write(f' x Using general database (model-specific not found): {os.path.basename(self.args.csv_name)}') + + self.args.generate_audit = audit_event( + self.args.generate_audit, + event_type="model_run_start", + payload={ + "cycle_number": int(cycle), + "model_name": str(ML_model), + "model_specific_curate_csv_found": bool(model_specific_found), + "model_specific_csv_path_checked": str(csv_model_specific), + "csv_path_used": str(csv_to_load), + "fell_back_to_general_csv": bool(not model_specific_found), + }, + evidence_level="direct", + ) # load database, discard user-defined descriptors and perform data checks csv_df, csv_X, csv_y = load_database(self,csv_to_load,"generate",print_info=False) @@ -175,8 +245,39 @@ def __init__(self, **kwargs): self = check_clas_problem(self,csv_df) csv_y = csv_df[self.args.y] + descriptor_count_after_load = len([col for col in csv_df.columns if col not in self.args.ignore and col != self.args.y]) + self.args.generate_audit = audit_event( + self.args.generate_audit, + event_type="model_csv_loaded", + payload={ + "model_name": str(ML_model), + "descriptor_count_after_loading_model_csv": int(descriptor_count_after_load), + "source_csv_role": "model_specific_curate_csv" if model_specific_found else "general_csv_fallback", + "source_csv_used": str(csv_to_load), + }, + evidence_level="direct", + ) + + # standardizes and separates an external test set Xy_data = prepare_sets(self,csv_df,csv_X,csv_y,None,self.args.names,None,None,None,BO_opt=True) + train_count = len(Xy_data['y_train']) if 'y_train' in Xy_data else 0 + test_count = len(Xy_data['test_points']) if 'test_points' in Xy_data else 0 + total_count = train_count + test_count + test_fraction = float(test_count / total_count) if total_count > 0 else 0.0 + self.args.generate_audit = audit_event( + self.args.generate_audit, + event_type="prepare_sets", + payload={ + "model_name": str(ML_model), + "train_count": int(train_count), + "test_count": int(test_count), + "test_fraction": test_fraction, + "descriptor_count": int(len(Xy_data.get('X_descriptors', []))), + "split_method": str(self.args.split), + }, + evidence_level="derived", + ) # hyperopt process for ML models _ = BO_workflow(self, Xy_data, csv_df, ML_model) @@ -201,17 +302,91 @@ def __init__(self, **kwargs): # detects best combinations dir_csv = self.args.destination.joinpath(f"Raw_data") - _ = detect_best(f'{dir_csv}/No_PFI') + best_no_pfi = detect_best(f'{dir_csv}/No_PFI') + if best_no_pfi is not None: + self.args.generate_audit = audit_event( + self.args.generate_audit, + event_type="best_model_selection", + payload=best_no_pfi, + evidence_level="direct", + ) # create heatmap plot(s) - _ = heatmap_workflow(self,"No_PFI") + hm_no_pfi = heatmap_workflow(self,"No_PFI") + if hm_no_pfi is not None: + self.args.generate_audit = audit_event( + self.args.generate_audit, + event_type="image_artifact", + payload=hm_no_pfi, + evidence_level="direct", + ) # detect best and create heatmap for PFI models if self.args.pfi_filter: try: # if no models were found - _ = detect_best(f'{dir_csv}/PFI') - _ = heatmap_workflow(self,"PFI") + best_pfi = detect_best(f'{dir_csv}/PFI') + if best_pfi is not None: + self.args.generate_audit = audit_event( + self.args.generate_audit, + event_type="best_model_selection", + payload=best_pfi, + evidence_level="direct", + ) + hm_pfi = heatmap_workflow(self,"PFI") + if hm_pfi is not None: + self.args.generate_audit = audit_event( + self.args.generate_audit, + event_type="image_artifact", + payload=hm_pfi, + evidence_level="direct", + ) except UnboundLocalError: pass + raw_no_pfi_paths = sorted(glob.glob(self.args.destination.joinpath("Raw_data/No_PFI/*.csv").as_posix())) + raw_pfi_paths = sorted(glob.glob(self.args.destination.joinpath("Raw_data/PFI/*.csv").as_posix())) + best_no_pfi_paths = sorted(glob.glob(self.args.destination.joinpath("Best_model/No_PFI/*.csv").as_posix())) + best_pfi_paths = sorted(glob.glob(self.args.destination.joinpath("Best_model/PFI/*.csv").as_posix())) + heatmap_paths = sorted(glob.glob(self.args.destination.joinpath("Raw_data/Heatmap_ML_models*.png").as_posix())) + self.args.generate_audit = audit_set(self.args.generate_audit, "outputs", "raw_data_no_pfi_csv_paths", raw_no_pfi_paths) + self.args.generate_audit = audit_set(self.args.generate_audit, "outputs", "raw_data_pfi_csv_paths", raw_pfi_paths) + self.args.generate_audit = audit_set(self.args.generate_audit, "outputs", "best_model_no_pfi_paths", best_no_pfi_paths) + self.args.generate_audit = audit_set(self.args.generate_audit, "outputs", "best_model_pfi_paths", best_pfi_paths) + self.args.generate_audit = audit_set(self.args.generate_audit, "outputs", "heatmap_image_paths", heatmap_paths) + self.args.generate_audit = audit_set(self.args.generate_audit, "outputs", "generated_audit_path", str(generate_audit_path)) + self.args.generate_audit = audit_set(self.args.generate_audit, "runtime", "module_runtime_seconds", round(time.time() - start_time, 2)) + + try: + audit_write_ok = finalize_module_audit(self.args.generate_audit, generate_audit_path, status="completed") + if not audit_write_ok: + _ = write_json_output_audit( + json_audit_path, + module="GENERATE", + attempted_output_path=generate_audit_path, + attempted=True, + succeeded=False, + artifact="generate_audit.json", + error=RuntimeError("generate_audit_json_write_returned_false"), + ) + else: + _ = write_json_output_audit( + json_audit_path, + module="GENERATE", + attempted_output_path=generate_audit_path, + attempted=True, + succeeded=True, + artifact="generate_audit.json", + error=None, + ) + except Exception as json_error: + _ = write_json_output_audit( + json_audit_path, + module="GENERATE", + attempted_output_path=generate_audit_path, + attempted=True, + succeeded=False, + artifact="generate_audit.json", + error=json_error, + ) + _ = finish_print(self,start_time,'GENERATE') diff --git a/robert/generate_utils.py b/robert/generate_utils.py index c5fa580..d830710 100644 --- a/robert/generate_utils.py +++ b/robert/generate_utils.py @@ -8,6 +8,7 @@ import numpy as np import glob import json +from robert.json_output_for_agent import audit_event, audit_set from robert.utils import ( load_params, PFI_filter, @@ -34,6 +35,8 @@ def BO_workflow(self, Xy_data, csv_df, ML_model): 'names': self.args.names, 'X_descriptors': Xy_data['X_descriptors']} + bo_ran = ML_model.upper() != 'MVL' + if ML_model.upper() != 'MVL': bo_data['params'], bo_data[f"combined_{bo_data['error_type']}"] = BO_optimizer(self,bo_data,Xy_data) bo_data['params'] = model_adjust_params(self, bo_data['model'], bo_data['params']) @@ -72,6 +75,36 @@ def BO_workflow(self, Xy_data, csv_df, ML_model): bo_data_df = pd.DataFrame([bo_data_to_save]) _ = bo_data_df.to_csv(f'{params_name}.csv', index = None, header=True) + if hasattr(self.args, 'generate_audit'): + self.args.generate_audit = audit_event( + self.args.generate_audit, + event_type="bo_workflow", + payload={ + "model": bo_data['model'], + "type": bo_data['type'], + "error_type": bo_data['error_type'], + "kfold": bo_data['kfold'], + "repeat_kfolds": bo_data['repeat_kfolds'], + "seed": bo_data['seed'], + "y": bo_data['y'], + "names": bo_data['names'], + "descriptor_count": len(bo_data['X_descriptors']), + "descriptor_list": bo_data['X_descriptors'], + "bo_was_run": bool(bo_ran), + "bo_skipped_reason": None if bo_ran else "model_is_mvl", + "optimized_parameters": bo_data.get('params', {}), + "combined_metric_value": bo_data.get(f"combined_{bo_data['error_type']}", None), + "no_pfi_parameter_csv_path": f'{params_name}.csv', + "no_pfi_database_csv_path": f'{db_name}.csv', + "class_label_mapping": { + "class_0_label": getattr(self.args, 'class_0_label', None), + "class_1_label": getattr(self.args, 'class_1_label', None), + }, + "split_type": self.args.split, + }, + evidence_level="direct", + ) + return bo_data @@ -90,8 +123,10 @@ def PFI_workflow(self, csv_df, ML_model, Xy_data): desc_keep = calc_desc_keep(self,Xy_data,PFI_discard_cols) discard_idx, descriptors_PFI = [],[] + fallback_reason = None # if no descriptors pass the filter, just choose them based on importance until having the number of descps from desc_keep if len(PFI_discard_cols) == len(descp_cols_pfi): + fallback_reason = "all_descriptors_below_pfi_threshold_using_desc_keep_fallback" PFI_discard_cols = [] for _,column in enumerate(descp_cols_pfi): @@ -115,6 +150,34 @@ def PFI_workflow(self, csv_df, ML_model, Xy_data): # save CSV file _ = save_pfi_csv(self,csv_df,name_csv_hyperopt,PFI_dict,Xy_data_PFI,ML_model) + if hasattr(self.args, 'generate_audit'): + pfi_params_csv_path = self.args.destination.joinpath(f"Raw_data/PFI/{ML_model}_PFI.csv") + pfi_db_csv_path = self.args.destination.joinpath(f"Raw_data/PFI/{ML_model}_PFI_db.csv") + self.args.generate_audit = audit_event( + self.args.generate_audit, + event_type="pfi_workflow", + payload={ + "pfi_activated": True, + "model": str(ML_model), + "pfi_threshold": self.args.pfi_threshold, + "pfi_max": self.args.pfi_max, + "pfi_epochs": self.args.pfi_epochs, + "descriptor_count_before_pfi": len(descp_cols_pfi), + "descriptors_discarded_by_pfi": PFI_discard_cols, + "descriptor_count_discarded": len(PFI_discard_cols), + "descriptors_retained_after_pfi": descriptors_PFI, + "descriptor_count_after_pfi": len(descriptors_PFI), + "desc_keep": int(desc_keep), + "desc_keep_label": "internal_pfi_keep_limit", + "desc_keep_note": "Internal ROBERT variable used during PFI descriptor selection. It is not necessarily the final number of descriptors retained. Use descriptor_count_after_pfi and descriptors_retained_after_pfi for the final PFI result.", + "fallback_reason": fallback_reason, + "combined_metric_after_pfi": PFI_dict.get(f"combined_{PFI_dict['error_type']}", None), + "pfi_parameter_csv_path": str(pfi_params_csv_path), + "pfi_database_csv_path": str(pfi_db_csv_path), + }, + evidence_level="direct", + ) + def calc_desc_keep(self,Xy_data,PFI_discard_cols): ''' @@ -197,12 +260,16 @@ def detect_best(folder): # detect files file_list = glob.glob(f'{folder}/*.csv') + candidate_param_csv_files = [] + candidate_metric_values = {} errors = [] for file in file_list: if '_db' not in file: results_model = pd.read_csv(f'{file}', encoding='utf-8') training_error = results_model[f"combined_{results_model['error_type'][0]}"][0] errors.append(training_error) + candidate_param_csv_files.append(file) + candidate_metric_values[file] = float(training_error) else: errors.append(np.nan) # detect best result and copy files to the Best_model folder @@ -216,6 +283,20 @@ def detect_best(folder): shutil.copyfile(f'{best_name}', f'{best_name}'.replace('Raw_data','Best_model')) shutil.copyfile(f'{best_db}', f'{best_db}'.replace('Raw_data','Best_model')) + return { + "folder_scanned": str(folder), + "candidate_parameter_csv_files": candidate_param_csv_files, + "candidate_metric_values": candidate_metric_values, + "error_type": str(results_model['error_type'][0]), + "selection_rule": "minimum" if results_model['error_type'][0].lower() in ['mae', 'rmse'] else "maximum", + "best_parameter_csv_path": str(best_name), + "best_database_csv_path": str(best_db), + "copied_destination_parameter_path": str(best_name).replace('Raw_data', 'Best_model'), + "copied_destination_database_path": str(best_db).replace('Raw_data', 'Best_model'), + "best_metric_value": float(np.nanmin(errors)) if results_model['error_type'][0].lower() in ['mae', 'rmse'] else float(np.nanmax(errors)), + "pfi_status": "PFI" if '/PFI' in str(folder).replace('\\', '/') else "No_PFI", + } + def heatmap_workflow(self,folder_hm): """ @@ -248,3 +329,19 @@ def heatmap_workflow(self,folder_hm): suffix = 'PFI' _ = create_heatmap(self,csv_df,suffix,path_raw) + title_fig = f'Heatmap ML models {suffix}' + name_fig = '_'.join(title_fig.split()) + image_path = path_raw.joinpath(f'{name_fig}.png') + + return { + "plot_type": "heatmap", + "pfi_status": "No_PFI" if folder_hm == "No_PFI" else "PFI", + "source_folder": str(path_raw.joinpath(folder_hm)), + "model_scores_included": {str(k): float(v) for k, v in csv_data.items()}, + "heatmap_suffix": suffix, + "image_path": str(image_path), + "image_generated": bool(os.path.exists(image_path)), + "fail_soft_reason": None if os.path.exists(image_path) else "heatmap_output_not_detected_after_creation", + "short_interpretation": "model-screening heatmap comparing optimized models", + } + diff --git a/robert/json_output_for_agent.py b/robert/json_output_for_agent.py new file mode 100644 index 0000000..e8a7fa6 --- /dev/null +++ b/robert/json_output_for_agent.py @@ -0,0 +1,1151 @@ +"""Shared helpers for writing ROBERT evidence into JSON files. + +These functions support ChatBob by turning ROBERT outputs, dataset facts, +and run artifacts into structured files that a user interface or LLM can read. + +The helpers are intentionally additive and fail-soft: +they should never change ROBERT calculations, scoring, model selection, +standard output files, or normal run behavior. +""" + +from __future__ import annotations + +import os +import json +import hashlib +from pathlib import Path +from datetime import datetime, timezone +from typing import Any, Dict, List + +import numpy as np +import pandas as pd + + +_MANIFEST_SCHEMA_VERSION = "0.1" +_SUMMARY_SCHEMA_VERSION = "0.1" +_DEFAULT_MODULES = ["CURATE", "GENERATE", "VERIFY", "PREDICT", "REPORT", "AQME", "EVALUATE"] + + +def agent_json_path(filename: str) -> Path: + """Return a top-level JSON/ path for an agent-facing artifact.""" + + output_dir = Path(os.getcwd()) / "JSON" + try: + output_dir.mkdir(parents=True, exist_ok=True) + except Exception: + pass + return output_dir / filename + + +def _to_unavailable() -> str: + """Return the standard marker used when ROBERT evidence is not currently captured.""" + return "unavailable" + + +def _iso_timestamp_from_epoch(epoch_seconds: float) -> str: + return datetime.fromtimestamp(epoch_seconds, tz=timezone.utc).isoformat() + + +def _sha256_file(file_path: Path, chunk_size: int = 1024 * 1024) -> str | None: + """Create a file fingerprint so ChatBob can tell whether an output file changed.""" + hasher = hashlib.sha256() + try: + with file_path.open("rb") as handle: + while True: + chunk = handle.read(chunk_size) + if not chunk: + break + hasher.update(chunk) + return hasher.hexdigest() + except Exception: + return None + + +def _extract_pdf_text_preview(file_path: Path, max_chars: int) -> str | None: + """Try to extract a short text preview from a PDF without interrupting ROBERT if it fails.""" + try: + import fitz + + doc = fitz.open(str(file_path)) + chunks: List[str] = [] + chars = 0 + for page in doc: + if chars >= max_chars: + break + text = page.get_text("text") + if not text: + continue + remaining = max_chars - chars + fragment = text[:remaining] + chunks.append(fragment) + chars += len(fragment) + doc.close() + out = "\n".join(chunks).strip() + return out if out else None + except Exception: + return None + + +def _extract_text_preview(file_path: Path, max_chars: int = 4000) -> str | None: + """Read a short preview from text-like ROBERT output files for later search or display.""" + suffix = file_path.suffix.lower() + try: + if suffix in {".dat", ".txt", ".log", ".csv", ".json", ".yaml", ".yml"}: + text = file_path.read_text(encoding="utf-8", errors="replace") + return text[:max_chars] if text else None + if suffix == ".pdf": + return _extract_pdf_text_preview(file_path, max_chars=max_chars) + except Exception: + return None + return None + + +def _infer_artifact_type(file_path: Path) -> str: + """Label a ROBERT output file as CSV, DAT, image, PDF, text, or other.""" + suffix = file_path.suffix.lower() + if suffix == ".csv": + return "csv" + if suffix == ".dat": + return "dat" + if suffix in {".png", ".jpg", ".jpeg", ".svg", ".tif", ".tiff", ".gif", ".webp"}: + return "image" + if suffix == ".pdf": + return "pdf" + if suffix in {".json", ".yaml", ".yml", ".txt", ".log"}: + return "textual" + return "binary_or_other" + + +def _build_file_record(file_path: Path, run_dir: Path) -> Dict[str, Any]: + """Describe one output file using its path, type, size, timestamp, fingerprint, and preview.""" + stats = file_path.stat() + rel = file_path.relative_to(run_dir) + artifact_type = _infer_artifact_type(file_path) + + return { + "path": str(file_path), + "path_relative_to_run": str(rel).replace("\\", "/"), + "artifact_type": artifact_type, + "extension": file_path.suffix.lower(), + "size_bytes": int(stats.st_size), + "modified_utc": _iso_timestamp_from_epoch(stats.st_mtime), + "sha256": _sha256_file(file_path), + "text_preview": _extract_text_preview(file_path), + } + + +def collect_module_manifest(module_name: str, run_dir: str | Path, max_files: int = 1000) -> Dict[str, Any]: + """Build a structured inventory of files produced by one ROBERT module. + + This helps ChatBob answer questions such as which files were created, + where they are stored, and which outputs are available for inspection. + """ + + run_root = Path(run_dir) + module_dir = run_root / module_name + payload: Dict[str, Any] = { + "schema_version": _MANIFEST_SCHEMA_VERSION, + "artifact_type": "module_manifest", + "module": str(module_name), + "run_dir": str(run_root), + "module_dir": str(module_dir), + "module_dir_exists": bool(module_dir.exists() and module_dir.is_dir()), + "captured_utc": datetime.now(timezone.utc).isoformat(), + "file_count": 0, + "files": [], + } + + if not (module_dir.exists() and module_dir.is_dir()): + return payload + + files: List[Path] = [] + for candidate in module_dir.rglob("*"): + if candidate.is_file(): + files.append(candidate) + + files = sorted(files, key=lambda p: str(p).lower()) + if max_files > 0: + files = files[:max_files] + + records = [_build_file_record(fp, run_root) for fp in files] + payload["file_count"] = int(len(records)) + payload["files"] = records + return _to_json_safe(payload) + + +def write_module_manifest(module_name: str, run_dir: str | Path) -> bool: + """Write the file inventory for one ROBERT module as a JSON manifest.""" + + run_root = Path(run_dir) + output_path = run_root / f"{module_name}_manifest.json" + manifest = collect_module_manifest(module_name, run_root) + return write_json(manifest, output_path) + + +def write_archive_manifests(run_dir: str | Path, modules: List[str] | None = None) -> Dict[str, Any]: + """Create file inventories for the ROBERT modules in one completed run. + + A ROBERT run can produce many outputs across folders such as CURATE, + GENERATE, VERIFY, PREDICT, REPORT, AQME, and EVALUATE. This helper + asks each module folder to write a manifest that lists its available + files. + + The returned status tells ChatBob which module inventories were written + successfully, so the interface can know what evidence is available before + trying to explain the run. + """ + + run_root = Path(run_dir) + module_names = modules or list(_DEFAULT_MODULES) + + status: Dict[str, Any] = { + "schema_version": _MANIFEST_SCHEMA_VERSION, + "artifact_type": "archive_manifest_write_status", + "run_dir": str(run_root), + "captured_utc": datetime.now(timezone.utc).isoformat(), + "module_results": [], + } + + for module_name in module_names: + output_path = run_root / f"{module_name}_manifest.json" + succeeded = write_module_manifest(module_name, run_root) + status["module_results"].append( + { + "module": str(module_name), + "manifest_path": str(output_path), + "succeeded": bool(succeeded), + } + ) + + return _to_json_safe(status) + + +def collect_run_summary( + run_dir: str | Path, + command: List[str] | None = None, + return_code: int | None = None, + wrapper_metadata_path: str | Path | None = None, +) -> Dict[str, Any]: + """Create a high-level map of one completed ROBERT run. + A completed ROBERT run can contain many module folders, manifest files, + plots, CSV files, reports, and other outputs. + This helper gathers the top-level information needed to understand + what is available in the run. ChatBob can use this summary as a starting + point before it reads the more detailed CURATE, GENERATE, + VERIFY, PREDICT, or REPORT evidence. + """ + + run_root = Path(run_dir) + manifest_paths = sorted(run_root.glob("*_manifest.json")) + + summary: Dict[str, Any] = { + "schema_version": _SUMMARY_SCHEMA_VERSION, + "artifact_type": "run_summary", + "run_dir": str(run_root), + "captured_utc": datetime.now(timezone.utc).isoformat(), + "return_code": return_code, + "command": command or [], + "wrapper_metadata_path": str(wrapper_metadata_path) if wrapper_metadata_path is not None else None, + "module_manifests": [], + "top_level_files": [], + } + + for manifest_path in manifest_paths: + try: + manifest_data = json.loads(manifest_path.read_text(encoding="utf-8")) + except Exception: + manifest_data = {} + + summary["module_manifests"].append( + { + "module": manifest_data.get("module"), + "manifest_path": str(manifest_path), + "file_count": manifest_data.get("file_count"), + "module_dir_exists": manifest_data.get("module_dir_exists"), + } + ) + + top_level_files = [] + for file_path in sorted(run_root.glob("*")): + if file_path.is_file(): + top_level_files.append(_build_file_record(file_path, run_root)) + summary["top_level_files"] = top_level_files + + return _to_json_safe(summary) + + +def write_run_summary( + run_dir: str | Path, + command: List[str] | None = None, + return_code: int | None = None, + wrapper_metadata_path: str | Path | None = None, +) -> bool: + """Write the high-level run summary as run_summary.json at the run root. + This gives ChatBob a single file that points to the main outputs of a completed ROBERT run. + The summary does not replace the original ROBERT files; + it simply helps locate and organize them. + """ + + run_root = Path(run_dir) + output_path = run_root / "run_summary.json" + summary = collect_run_summary( + run_root, + command=command, + return_code=return_code, + wrapper_metadata_path=wrapper_metadata_path, + ) + return write_json(summary, output_path) + + +def _read_raw_csv(csv_path: str | Path) -> pd.DataFrame: + """Read the incoming CSV with separator autodetection to preserve raw intake facts. + This is done before ROBERT changes the dataset. + CURATE may later remove, reorder, transform, or filter parts of the data. + This helper preserves the starting point so ChatBob can explain what the user originally gave to ROBERT. + """ + + return pd.read_csv(csv_path, sep=None, engine="python", encoding="utf-8") + + +def _to_json_safe(obj: Any) -> Any: + """Convert scientific Python objects (pandas/numpy/path values) into values that can be written as JSON. + + ROBERT uses pandas, NumPy, file paths, timestamps, and missing values. + Many of these objects cannot be written directly into JSON. + This helper translates them into plain strings, numbers, lists, + dictionaries, or nulls that ChatBob can read reliably. + """ + + if isinstance(obj, Path): + return str(obj) + + if obj is None: + return None + + if isinstance(obj, (pd.Series, pd.Index)): + return [_to_json_safe(x) for x in obj.tolist()] + + if isinstance(obj, np.ndarray): + return [_to_json_safe(x) for x in obj.tolist()] + + if pd.api.types.is_scalar(obj) and pd.isna(obj): + return None + + if isinstance(obj, (np.bool_, bool)): + return bool(obj) + + if isinstance(obj, (np.integer,)): + return int(obj) + + if isinstance(obj, (np.floating, float)): + if np.isfinite(obj): + return float(obj) + return None + + if isinstance(obj, (pd.Timestamp,)): + return obj.isoformat() + + if isinstance(obj, (list, tuple, set)): + return [_to_json_safe(x) for x in obj] + + if isinstance(obj, dict): + return {str(k): _to_json_safe(v) for k, v in obj.items()} + + return obj + + +def _is_boolean_like(series: pd.Series) -> bool: + """Detect columns that behave like booleans even when encoded as strings or 0/1. + + Detect whether a column behaves like yes/no or true/false data. + Some datasets store boolean information as 0/1, yes/no, true/false, + or similar text labels. + This helper identifies those simple two-state columns so the + dataset profile can describe them more clearly.""" + + non_null = series.dropna() + if non_null.empty: + return False + + lowered = non_null.astype(str).str.strip().str.lower() + allowed = {"0", "1", "true", "false", "yes", "no", "y", "n", "t", "f"} + uniques = set(lowered.unique()) + return len(uniques) <= 2 and uniques.issubset(allowed) + + +def _type_label(series: pd.Series) -> str: + """Assign a simple observable type label to one dataset column. + This helper labels a column as numeric, categorical, text-like, + boolean-like, or unknown based only on what is visible in the data. + It does not make a scientific judgment; it creates a plain-language + description that ChatBob can use when explaining the dataset.""" + + if pd.api.types.is_bool_dtype(series) or _is_boolean_like(series): + return "boolean_like" + + if pd.api.types.is_numeric_dtype(series): + return "numeric" + + non_null = series.dropna() + if non_null.empty: + return "unknown" + + if pd.api.types.is_string_dtype(series) or series.dtype == "object": + unique_count = non_null.nunique(dropna=True) + ratio = unique_count / max(len(non_null), 1) + avg_len = non_null.astype(str).str.len().mean() + if ratio > 0.5 or avg_len > 30: + return "text" + return "categorical" + + return "categorical" + + +def measure_dataset_shape(raw_df: pd.DataFrame) -> Dict[str, Any]: + + """Record the basic size and layout of the original dataset. + This captures the number of rows, number of columns, column names, column order, + duplicate rows, and duplicate columns before CURATE changes anything. + It helps ChatBob answer the basic question: what did the user start with? + """ + + return { + "number_of_rows": int(len(raw_df)), + "number_of_columns": int(len(raw_df.columns)), + "column_names": [str(c) for c in raw_df.columns], + "column_order": [str(c) for c in raw_df.columns], + "duplicate_row_count": int(raw_df.duplicated().sum()), + "duplicate_column_count": int(raw_df.T.duplicated().sum()), + } + + +def measure_missingness(raw_df: pd.DataFrame) -> Dict[str, Any]: + """Measure missing values in the original dataset. + Missing values can affect how ROBERT curates and models a dataset. + This helper records how many values are missing, which columns are affected, + and whether missingness is small or substantial. + """ + total_cells = int(raw_df.shape[0] * raw_df.shape[1]) + missing_by_col = raw_df.isna().sum() + total_missing = int(missing_by_col.sum()) + rows_with_any_missing = int(raw_df.isna().any(axis=1).sum()) + + def _cols_over(threshold: float) -> List[str]: + pct = (missing_by_col / max(len(raw_df), 1)) * 100.0 + return [str(col) for col in pct.index if pct.loc[col] > threshold] + + return { + "total_missing_values": total_missing, + "percent_missing_overall": float((100.0 * total_missing / total_cells) if total_cells else 0.0), + "columns_with_missing_values": [str(c) for c in missing_by_col.index if int(missing_by_col[c]) > 0], + "missing_values_by_column": {str(c): int(v) for c, v in missing_by_col.items()}, + "rows_with_any_missing_value": rows_with_any_missing, + "percent_rows_with_missing": float((100.0 * rows_with_any_missing / len(raw_df)) if len(raw_df) else 0.0), + "columns_over_5_percent_missing": _cols_over(5.0), + "columns_over_20_percent_missing": _cols_over(20.0), + "columns_over_50_percent_missing": _cols_over(50.0), + } + + +def measure_column_types(raw_df: pd.DataFrame) -> Dict[str, Any]: + """Group the original dataset columns into simple observable categories. + The helper identifies columns that appear numeric, categorical, text-like, + boolean-like, constant, mostly constant, or mixed in type. + This gives ChatBob a practical way to explain the character of the starting dataset. + """ + numeric_columns: List[str] = [] + categorical_columns: List[str] = [] + text_columns: List[str] = [] + boolean_like_columns: List[str] = [] + constant_columns: List[str] = [] + mostly_constant_columns: List[str] = [] + mixed_type_columns: List[str] = [] + + for col in raw_df.columns: + series = raw_df[col] + label = _type_label(series) + + if label == "numeric": + numeric_columns.append(str(col)) + elif label == "text": + text_columns.append(str(col)) + elif label == "boolean_like": + boolean_like_columns.append(str(col)) + else: + categorical_columns.append(str(col)) + + non_null = series.dropna() + if not non_null.empty: + unique_values = non_null.nunique(dropna=True) + if unique_values == 1: + constant_columns.append(str(col)) + else: + top_freq = non_null.value_counts(dropna=True).iloc[0] + if (top_freq / len(non_null)) >= 0.95: + mostly_constant_columns.append(str(col)) + + python_types = {type(v).__name__ for v in non_null.head(1000).tolist()} + if len(python_types) > 1: + mixed_type_columns.append(str(col)) + + return { + "numeric_columns": numeric_columns, + "categorical_columns": categorical_columns, + "text_columns": text_columns, + "boolean_like_columns": boolean_like_columns, + "constant_columns": constant_columns, + "mostly_constant_columns": mostly_constant_columns, + "mixed_type_columns": mixed_type_columns, + } + + +def measure_per_column_stats(raw_df: pd.DataFrame) -> Dict[str, Dict[str, Any]]: + """Create a compact summary for each column in the original dataset. + Each column summary includes its inferred type, number of non-missing values, + number of unique values, and a few example values. + This helps ChatBob explain individual columns without needing to display the full dataset. + """ + per_column: Dict[str, Dict[str, Any]] = {} + + for col in raw_df.columns: + series = raw_df[col] + non_null = series.dropna() + example_values = [_to_json_safe(v) for v in non_null.drop_duplicates().head(5).tolist()] + + per_column[str(col)] = { + "inferred_type": _type_label(series), + "non_null_count": int(non_null.shape[0]), + "unique_value_count": int(non_null.nunique(dropna=True)), + "example_values": example_values, + } + + return per_column + + +def measure_target_profile(raw_df: pd.DataFrame, y: str) -> Dict[str, Any]: + """Describe the target column before ROBERT changes the dataset. + The target column is the property ROBERT is trying to model. + This helper records missing values, unique values, numeric range, average, spread, + likely problem type, and class counts when the target appears categorical. + """ + if y not in raw_df.columns: + return { + "target_name": y, + "target_missing_count": None, + "target_unique_count": None, + "target_min": None, + "target_max": None, + "target_mean": None, + "target_median": None, + "target_standard_deviation": None, + "target_skew_estimate": None, + "target_value_counts_if_categorical": {}, + "target_type_guess": "missing", + "possible_problem_type": "unknown", + "regression_likely": False, + "classification_likely": False, + "ambiguous": True, + } + + series = raw_df[y] + non_null = series.dropna() + unique_count = int(non_null.nunique(dropna=True)) + target_label = _type_label(series) + + numeric_non_null = pd.to_numeric(non_null, errors="coerce").dropna() + has_numeric = not numeric_non_null.empty and len(numeric_non_null) == len(non_null) + + classification_likely = bool((target_label in {"categorical", "boolean_like"}) or unique_count <= 10) + regression_likely = bool(has_numeric and unique_count > 10) + ambiguous = bool(classification_likely == regression_likely) + + if classification_likely: + value_counts = {str(k): int(v) for k, v in non_null.value_counts(dropna=True).to_dict().items()} + else: + value_counts = {} + + if has_numeric and len(numeric_non_null) > 0: + target_min = float(numeric_non_null.min()) + target_max = float(numeric_non_null.max()) + target_mean = float(numeric_non_null.mean()) + target_median = float(numeric_non_null.median()) + target_std = float(numeric_non_null.std()) if len(numeric_non_null) > 1 else 0.0 + target_skew = float(numeric_non_null.skew()) if len(numeric_non_null) > 2 else 0.0 + else: + target_min = None + target_max = None + target_mean = None + target_median = None + target_std = None + target_skew = None + + if classification_likely and not regression_likely: + problem_type = "classification" + elif regression_likely and not classification_likely: + problem_type = "regression" + else: + problem_type = "ambiguous" + + return { + "target_name": str(y), + "target_missing_count": int(series.isna().sum()), + "target_unique_count": unique_count, + "target_min": target_min, + "target_max": target_max, + "target_mean": target_mean, + "target_median": target_median, + "target_standard_deviation": target_std, + "target_skew_estimate": target_skew, + "target_value_counts_if_categorical": value_counts, + "target_type_guess": target_label, + "possible_problem_type": problem_type, + "regression_likely": regression_likely, + "classification_likely": classification_likely, + "ambiguous": ambiguous, + } + + +def measure_descriptor_counts(raw_df: pd.DataFrame, y: str, ignore: List[str]) -> Dict[str, Any]: + """Count the starting descriptor columns before CURATE begins. + Descriptors are the input features available to build a model, + excluding the target column and ignored columns. + This helper records how many descriptors exist and how many appear numeric, + categorical, text-like, constant, near-constant, or high in missing values. + """ + ignore_set = set(ignore or []) + descriptors = [c for c in raw_df.columns if c != y and c not in ignore_set] + descriptor_df = raw_df[descriptors] if descriptors else pd.DataFrame(index=raw_df.index) + + numeric = [] + categorical = [] + text = [] + constant = [] + near_constant = [] + high_missing = [] + + for col in descriptors: + series = raw_df[col] + label = _type_label(series) + if label == "numeric": + numeric.append(str(col)) + elif label == "text": + text.append(str(col)) + else: + categorical.append(str(col)) + + non_null = series.dropna() + if not non_null.empty: + unique_vals = non_null.nunique(dropna=True) + if unique_vals == 1: + constant.append(str(col)) + else: + top_freq = non_null.value_counts(dropna=True).iloc[0] + if (top_freq / len(non_null)) >= 0.95: + near_constant.append(str(col)) + + missing_pct = (float(series.isna().sum()) / len(raw_df) * 100.0) if len(raw_df) else 0.0 + if missing_pct > 20.0: + high_missing.append(str(col)) + + return { + "initial_descriptor_count": int(len(descriptors)), + "numeric_descriptor_count": int(len(numeric)), + "categorical_descriptor_count": int(len(categorical)), + "text_descriptor_count": int(len(text)), + "constant_descriptor_count": int(len(constant)), + "near_constant_descriptor_count": int(len(near_constant)), + "high_missing_descriptor_count": int(len(high_missing)), + } + + +def profile_input_dataset(csv_path: str | Path, y: str, ignore: List[str] | None = None) -> Dict[str, Any]: + """Build the raw dataset profile used to explain the starting data. + This helper re-reads the original CSV before CURATE changes it + and gathers the dataset shape, missingness, column types, per-column summaries, + target profile, and descriptor counts into one JSON-ready record. + ChatBob can use this file to explain what the user gave ROBERT before + any curation or model-building decisions were made. + """ + + raw_df = _read_raw_csv(csv_path) + ignore = ignore or [] + + payload = { + "schema_version": "0.1", + "artifact_type": "incoming_dataset_profile", + "source_csv": str(csv_path), + "dataset_shape": measure_dataset_shape(raw_df), + "missingness": measure_missingness(raw_df), + "column_types": measure_column_types(raw_df), + "per_column": measure_per_column_stats(raw_df), + "target_profile": measure_target_profile(raw_df, y), + "descriptor_counts": measure_descriptor_counts(raw_df, y, ignore), + } + return _to_json_safe(payload) + + +def write_json(data: Dict[str, Any], output_path: str | Path) -> bool: + """Write a JSON file without risking the ROBERT run. + This helper writes structured evidence to disk and returns True if the write succeeds + or False if it fails. A JSON failure should not stop ROBERT or alter any standard ROBERT output. + """ + + out = Path(output_path) + try: + out.parent.mkdir(parents=True, exist_ok=True) + with out.open("w", encoding="utf-8") as handle: + json.dump(_to_json_safe(data), handle, indent=2, sort_keys=False, ensure_ascii=False) + return True + except Exception: + return False + + +def init_module_audit( + module: str, + artifact_type: str, + source_files: List[str] | None = None, + command_line: str | None = None, +) -> Dict[str, Any]: + """Start a structured audit record for one ROBERT module. + CURATE, GENERATE, VERIFY, PREDICT, and REPORT each produce evidence + that ChatBob may need to explain later. + This helper creates a common starting structure so each module can record what happened in a consistent way. + """ + + try: + payload = { + "schema_version": "0.1", + "module": str(module), + "artifact_type": str(artifact_type), + "status": "in_progress", + "started_utc": datetime.now(timezone.utc).isoformat(), + "source_files": [str(p) for p in (source_files or [])], + "command_line": str(command_line) if command_line is not None else None, + "sections": {}, + "events": [], + "notes": [], + } + return _to_json_safe(payload) + except Exception: + return { + "schema_version": "0.1", + "module": str(module), + "artifact_type": str(artifact_type), + "status": "in_progress", + "sections": {}, + "events": [], + } + + +def audit_event( + audit: Dict[str, Any], + event_type: str, + payload: Dict[str, Any] | None = None, + evidence_level: str = "direct", + dat_text: str | None = None, +) -> Dict[str, Any]: + """Add one event to a module audit without interrupting ROBERT. + An event is something that happened during a module, such as a filtering step, + warning, file creation, model check, or result. + ChatBob can later use these events to explain the sequence of what ROBERT did. + """ + + try: + if not isinstance(audit, dict): + return audit + + event = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "event_type": str(event_type), + "evidence_level": str(evidence_level), + "payload": _to_json_safe(payload or {}), + } + if dat_text is not None: + event["dat_text_preview"] = str(dat_text)[:1000] + + events = audit.get("events", []) + if not isinstance(events, list): + events = [] + events.append(event) + audit["events"] = events + return audit + except Exception: + return audit + + +def audit_set( + audit: Dict[str, Any], + section: str, + key: str, + value: Any, + evidence_level: str = "direct", +) -> Dict[str, Any]: + """Store one named value inside a module audit section. + This helper lets the JSON audit record important values in organized sections + such as inputs, counts, warnings, model results, or files created. + The evidence level marks whether the value came directly from ROBERT + or was added as a simple helper summary for ChatBob. + """ + + try: + if not isinstance(audit, dict): + return audit + + sections = audit.get("sections", {}) + if not isinstance(sections, dict): + sections = {} + + sec = sections.get(str(section), {}) + if not isinstance(sec, dict): + sec = {} + + sec[str(key)] = { + "value": _to_json_safe(value), + "evidence_level": str(evidence_level), + } + sections[str(section)] = sec + audit["sections"] = sections + return audit + except Exception: + return audit + + +def finalize_module_audit( + audit: Dict[str, Any], + output_path: str | Path, + status: str = "completed", +) -> bool: + """Finish and write a module audit JSON file. + This helper marks the audit as completed, adds an ending timestamp, + and writes the JSON file using the fail-soft JSON writer. + It gives each module a consistent way to close its evidence record. + """ + + try: + if not isinstance(audit, dict): + return False + audit["status"] = str(status) + audit["finished_utc"] = datetime.now(timezone.utc).isoformat() + return write_json(audit, output_path) + except Exception: + return False + + +def write_json_output_audit( + audit_path: str | Path, + module: str, + attempted_output_path: str | Path, + attempted: bool, + succeeded: bool, + artifact: str = "dataset_profile.json", + error: Exception | None = None, +) -> bool: + """Record whether the extra JSON evidence layer succeeded or failed. + This audit is about the ChatBob-facing JSON files only. + It records which JSON artifact was attempted, whether it succeeded, + and what error occurred if it failed. + It must not write messages into standard ROBERT outputs such as DAT, CSV, image, model, or report files. + """ + + event = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "module": str(module), + "layer": "json-output-for-agent", + "layer_note": "This audit belongs to the JSON-output-for-agent layer and does not modify standard ROBERT outputs.", + "artifact": str(artifact), + "attempted": bool(attempted), + "succeeded": bool(succeeded), + "attempted_output_path": str(attempted_output_path), + "error_type": type(error).__name__ if error is not None else None, + "error_message": str(error) if error is not None else None, + } + + out = Path(audit_path) + try: + out.parent.mkdir(parents=True, exist_ok=True) + payload: Dict[str, Any] = { + "schema_version": "0.1", + "artifact_type": "json_output_audit", + "module": str(module), + "events": [], + } + if out.exists(): + try: + with out.open("r", encoding="utf-8") as handle: + existing = json.load(handle) + if isinstance(existing, dict): + payload.update(existing) + except Exception: + pass + + events = payload.get("events", []) + if not isinstance(events, list): + events = [] + events.append(event) + payload["events"] = events + + with out.open("w", encoding="utf-8") as handle: + json.dump(_to_json_safe(payload), handle, indent=2, sort_keys=False, ensure_ascii=False) + return True + except Exception: + return False + + +def init_report_figure_provenance() -> Dict[str, Any]: + """Create the initial payload for REPORT figure provenance capture.""" + + return { + "schema_version": "1.0", + "artifact_type": "figure_provenance", + "report_pdf_path": None, + "report_pdf_exists": False, + "figures": [], + } + + +def _infer_report_source_module(path_text: str) -> str: + upper_path = path_text.upper().replace("\\", "/") + for module in ["CURATE", "GENERATE", "VERIFY", "PREDICT", "AQME"]: + if f"/{module}/" in f"/{upper_path}/": + return module + return "unknown" + + +def _infer_report_branch_key(path_text: str) -> str: + upper_path = path_text.upper() + if "NO_PFI" in upper_path: + return "No_PFI" + if "PFI" in upper_path: + return "PFI" + if "CUSTOM" in upper_path: + return "custom" + return "unknown" + + +def _infer_report_figure_role(path_text: str) -> str: + lower_path = path_text.lower() + file_name = Path(path_text).name.lower() + + if "cv_variability" in file_name or "cv_variability" in lower_path: + return "cv_variability_plot" + if "results_" in file_name or "results_" in lower_path or "cv_variability" in lower_path: + return "prediction_plot" + if "shap_" in file_name or "/shap_" in lower_path: + return "shap_plot" + if "pfi_" in file_name or "/pfi_" in lower_path: + return "pfi_plot" + if "pearson" in file_name or "pearson" in lower_path: + return "pearson_map" + if "outlier" in file_name or "outlier" in lower_path: + return "outlier_plot" + if "distribution" in file_name or "distribution" in lower_path: + return "distribution_plot" + if "heatmap" in file_name or "heatmap" in lower_path: + return "model_screening_plot" + if "verify_tests" in file_name or "verify_tests" in lower_path: + return "verify_plot" + return "unknown" + + +def _infer_supporting_json_path(source_module: str) -> str | None: + module_to_artifact = { + "CURATE": "curate_audit.json", + "GENERATE": "generate_audit.json", + "VERIFY": "verify_audit.json", + "PREDICT": "predict_audit.json", + "AQME": "aqme_audit.json", + } + artifact_name = module_to_artifact.get(source_module) + if artifact_name is None: + return None + return str(Path(os.getcwd()) / source_module / artifact_name) + + +def _infer_supporting_csv_path(path_text: str, source_module: str, branch_key: str) -> str | None: + lower_path = path_text.lower().replace("\\", "/") + if source_module == "PREDICT" and "/predict/csv_test/" in f"/{lower_path}": + csv_test_dir = Path(os.getcwd()) / "PREDICT" / "csv_test" + if branch_key == "No_PFI": + matches = sorted(csv_test_dir.glob("*_No_PFI.csv")) + if len(matches) == 1: + return str(matches[0]) + return None + if branch_key == "PFI": + matches = sorted(csv_test_dir.glob("*_PFI.csv")) + if len(matches) == 1: + return str(matches[0]) + return None + + if source_module == "PREDICT" and "results_" in lower_path: + stem = Path(path_text).stem + if stem.startswith("Results_"): + remaining_stem = stem[len("Results_") :] + if remaining_stem: + candidate_csv = Path(os.getcwd()) / "PREDICT" / f"{remaining_stem}.csv" + if candidate_csv.exists(): + return str(candidate_csv) + + return None + + +def infer_report_figure_provenance(figure_path: str | Path, report_section: str | None = None) -> Dict[str, Any]: + """Infer conservative provenance for one figure path from path and filename patterns only.""" + + path_obj = Path(figure_path) + path_text = str(path_obj) + source_module = _infer_report_source_module(path_text) + branch_key = _infer_report_branch_key(path_text) + + return { + "figure_path": path_text, + "figure_filename": path_obj.name, + "exists": bool(path_obj.exists()), + "report_section": report_section, + "source_module": source_module, + "branch_key": branch_key, + "figure_role": _infer_report_figure_role(path_text), + "supporting_json_path": _infer_supporting_json_path(source_module), + "supporting_csv_path": _infer_supporting_csv_path(path_text, source_module, branch_key), + } + + +def add_report_figure_record( + provenance_payload: Dict[str, Any], + figure_path: str | Path, + report_section: str | None = None, +) -> Dict[str, Any]: + """Append one figure provenance record in fail-soft mode.""" + + try: + if not isinstance(provenance_payload, dict): + return provenance_payload + + figures = provenance_payload.get("figures", []) + if not isinstance(figures, list): + figures = [] + + figures.append(infer_report_figure_provenance(figure_path, report_section=report_section)) + provenance_payload["figures"] = figures + return provenance_payload + except Exception: + return provenance_payload + + +def finalize_report_figure_provenance( + provenance_payload: Dict[str, Any], + output_path: str | Path, + report_pdf_path: str | Path, + report_pdf_exists: bool, +) -> bool: + """Finalize and write REPORT figure provenance in fail-soft mode.""" + + try: + if not isinstance(provenance_payload, dict): + return False + + provenance_payload["report_pdf_path"] = str(report_pdf_path) + provenance_payload["report_pdf_exists"] = bool(report_pdf_exists) + return write_json(provenance_payload, output_path) + except Exception: + return False + + +# Legacy fallback payload builder; runtime audit capture is preferred when available. +def build_curate_audit_payload( + source_csv: str | Path, + destination_dir: str | Path, + target_column: str, + names_column: str | None, + ignored_columns: List[str] | None, + discarded_columns_requested: List[str] | None, + rows_before_curate: int | None, + rows_after_curate: int | None, + columns_before_curate: int | None, + columns_after_curate: int | None, +) -> Dict[str, Any]: + """Build the CURATE audit JSON from values that are directly observable. + This helper records the input file, target column, ignored columns, + requested discarded columns, before-and-after dataset counts, expected CURATE output files, + and fields that are not yet available in structured form. + It is intentionally conservative. + If ROBERT has not exposed a value in a reliable structured way, + the audit marks that value as unavailable rather than guessing. + """ + + src = Path(source_csv) + dst = Path(destination_dir) + ignored_columns = ignored_columns or [] + discarded_columns_requested = discarded_columns_requested or [] + + csv_stem = src.stem + files_written: List[Dict[str, Any]] = [] + + candidate_paths: List[Path] = [ + dst / f"{csv_stem}_CURATE.csv", + dst / "CURATE_options.csv", + dst / "CURATE_data.dat", + dst / "Pearson_heatmap.png", + ] + candidate_paths.extend(sorted(dst.glob(f"{csv_stem}_CURATE_*.csv"))) + + seen: set[str] = set() + for file_path in candidate_paths: + key = str(file_path.resolve()) + if key in seen: + continue + seen.add(key) + files_written.append( + { + "path": str(file_path), + "artifact_type": _infer_artifact_type(file_path), + "exists": bool(file_path.exists()), + } + ) + + payload = { + "schema_version": "0.1", + "artifact_type": "curate_audit", + "module": "CURATE", + "status": "completed", + "captured_utc": datetime.now(timezone.utc).isoformat(), + "inputs": { + "source_csv": str(source_csv), + "target_column": str(target_column), + "names_column": str(names_column) if names_column else None, + "ignored_columns": [str(col) for col in ignored_columns], + "discarded_columns_requested": [str(col) for col in discarded_columns_requested], + }, + "observed_counts": { + "rows_before_curate": rows_before_curate, + "rows_after_curate": rows_after_curate, + "columns_before_curate": columns_before_curate, + "columns_after_curate": columns_after_curate, + "descriptors_removed_duplicate_filter": _to_unavailable(), + "descriptors_removed_missingness": _to_unavailable(), + "descriptors_removed_categorical_transform": _to_unavailable(), + "descriptors_removed_correlation_filter": _to_unavailable(), + "descriptors_removed_other": _to_unavailable(), + }, + "files_written": files_written, + "standard_output_paths": { + "curate_dat": str(dst / "CURATE_data.dat"), + "curate_options_csv": str(dst / "CURATE_options.csv"), + "curated_csvs": [ + str(fp) + for fp in sorted(dst.glob(f"{csv_stem}_CURATE*.csv")) + ], + }, + "unavailable_fields": [ + "observed_counts.descriptors_removed_duplicate_filter", + "observed_counts.descriptors_removed_missingness", + "observed_counts.descriptors_removed_categorical_transform", + "observed_counts.descriptors_removed_correlation_filter", + "observed_counts.descriptors_removed_other", + ], + "notes": [ + "Step-specific descriptor removal counts are marked unavailable until direct in-memory counters are exposed.", + "This artifact records observable CURATE evidence only and does not modify standard ROBERT outputs.", + ], + } + + return _to_json_safe(payload) diff --git a/robert/predict.py b/robert/predict.py index c5e1f30..3ea9f1f 100644 --- a/robert/predict.py +++ b/robert/predict.py @@ -37,11 +37,20 @@ import os import time +import math from robert.predict_utils import (plot_predictions, save_predictions, print_predict, pearson_map_predict ) +from robert.json_output_for_agent import ( + init_module_audit, + audit_set, + audit_event, + finalize_module_audit, + agent_json_path, + write_json_output_audit, +) from robert.utils import (load_variables, load_db_n_params, load_n_predict, @@ -72,6 +81,29 @@ def __init__(self, **kwargs): # load default and user-specified variables self.args = load_variables(kwargs, "predict") + source_files = [str(self.args.params_dir)] + if self.args.csv_test != '': + source_files.append(str(self.args.csv_test)) + + self.args.predict_audit = init_module_audit( + module="PREDICT", + artifact_type="predict_audit", + source_files=source_files, + command_line=getattr(self.args, "command_line", None), + ) + predict_audit_path = agent_json_path("predict_audit.json") + json_audit_path = agent_json_path("predict_json_output_audit.json") + + self.args.predict_audit = audit_set(self.args.predict_audit, "inputs", "params_dir", self.args.params_dir) + self.args.predict_audit = audit_set(self.args.predict_audit, "inputs", "csv_test", self.args.csv_test) + self.args.predict_audit = audit_set(self.args.predict_audit, "inputs", "t_value", self.args.t_value) + self.args.predict_audit = audit_set(self.args.predict_audit, "inputs", "alpha", self.args.alpha) + self.args.predict_audit = audit_set(self.args.predict_audit, "inputs", "shap_show", self.args.shap_show) + self.args.predict_audit = audit_set(self.args.predict_audit, "inputs", "pfi_show", self.args.pfi_show) + self.args.predict_audit = audit_set(self.args.predict_audit, "inputs", "pfi_epochs", self.args.pfi_epochs) + self.args.predict_audit = audit_set(self.args.predict_audit, "inputs", "names", self.args.names) + self.args.predict_audit = audit_set(self.args.predict_audit, "inputs", "destination", str(self.args.destination)) + # if params_dir = '', the program performs the tests for the No_PFI and PFI folders if 'GENERATE/Best_model' in self.args.params_dir: params_dirs = [f'{self.args.params_dir}/No_PFI',f'{self.args.params_dir}/PFI'] @@ -79,15 +111,86 @@ def __init__(self, **kwargs): suffix_titles = ['No_PFI','PFI'] else: params_dirs = [self.args.params_dir] - suffix = ['custom'] + suffixes = ['custom'] + suffix_titles = ['custom'] for (params_dir,suffix,suffix_title) in zip(params_dirs,suffixes,suffix_titles): - if os.path.exists(params_dir): + branch_key = f"{suffix_title}::{params_dir}" + params_dir_exists = bool(os.path.exists(params_dir)) + is_pfi_filtered = bool(str(suffix_title) == 'PFI') + + self.args.predict_audit = audit_event( + self.args.predict_audit, + event_type="predict_branch", + payload={ + "branch_key": str(branch_key), + "params_dir_used": str(params_dir), + "suffix": str(suffix), + "suffix_title": str(suffix_title), + "params_dir_exists": params_dir_exists, + "branch_is_pfi_filtered": is_pfi_filtered, + "workflow_includes_pfi_branch": bool('GENERATE/Best_model' in self.args.params_dir), + }, + evidence_level="direct", + ) + if params_dir_exists: _ = print_pfi(self,params_dir) # load the Xy databse and model parameters Xy_data, model_data, suffix_title = load_db_n_params(self,params_dir,suffix,suffix_title,"verify",True) # module 'verify' since PREDICT follows similar protocols + + active_descriptor_list = [str(desc) for desc in model_data.get('X_descriptors', [])] if isinstance(model_data.get('X_descriptors', []), list) else [str(desc) for desc in model_data.get('X_descriptors', [])] + active_descriptor_count = int(len(active_descriptor_list)) + + train_datapoints = None + test_datapoints = None + total_datapoints_loaded = None + if 'y_train' in Xy_data: + train_datapoints = int(len(Xy_data['y_train'])) + elif 'X_train' in Xy_data and hasattr(Xy_data['X_train'], 'shape'): + train_datapoints = int(Xy_data['X_train'].shape[0]) + + if 'y_test' in Xy_data: + test_datapoints = int(len(Xy_data['y_test'])) + elif 'X_test' in Xy_data and hasattr(Xy_data['X_test'], 'shape'): + test_datapoints = int(Xy_data['X_test'].shape[0]) + + if 'X' in Xy_data and hasattr(Xy_data['X'], 'shape'): + total_datapoints_loaded = int(Xy_data['X'].shape[0]) + elif 'y' in Xy_data: + total_datapoints_loaded = int(len(Xy_data['y'])) + + external_available = bool( + 'y_external' in Xy_data + and hasattr(Xy_data['y_external'], 'isnull') + and not Xy_data['y_external'].isnull().values.any() + and len(Xy_data['y_external']) > 0 + ) + + self.args.predict_audit = audit_event( + self.args.predict_audit, + event_type="model_context", + payload={ + "branch_key": str(branch_key), + "params_dir_used": str(params_dir), + "suffix": str(suffix), + "suffix_title": str(suffix_title), + "model_name": str(model_data['model']), + "model_type": str(model_data['type']), + "error_type": str(model_data['error_type']), + "y_column": str(model_data['y']), + "names_column": str(model_data['names']), + "active_descriptor_count": active_descriptor_count, + "active_descriptor_list": active_descriptor_list, + "active_descriptor_source": "model_data['X_descriptors']", + "train_datapoints": train_datapoints, + "test_datapoints": test_datapoints, + "total_datapoints_loaded": total_datapoints_loaded, + "external_test_available": external_available, + }, + evidence_level="direct", + ) # get results from training, test and external test (if any) Xy_data = load_n_predict(self, model_data, Xy_data, BO_opt=False) @@ -102,29 +205,345 @@ def __init__(self, **kwargs): overwrite_predictions=True, ) + def _mean_safe(values): + if values is None: + return None + try: + if len(values) == 0: + return None + mean_val = sum(values) / len(values) + if isinstance(mean_val, float) and (math.isnan(mean_val) or math.isinf(mean_val)): + return None + return float(mean_val) + except Exception: + return None + + measured_y_range = None + if all(key in Xy_data for key in ['pred_min', 'pred_max', 'pred_range']): + measured_y_range = { + "min": float(Xy_data['pred_min']), + "max": float(Xy_data['pred_max']), + "range": float(Xy_data['pred_range']), + } + + self.args.predict_audit = audit_event( + self.args.predict_audit, + event_type="prediction_result_context", + payload={ + "branch_key": str(branch_key), + "model_type": str(model_data['type']), + "regression_metrics": { + "cv": { + "r2": Xy_data.get('r2_train', None), + "mae": Xy_data.get('mae_train', None), + "rmse": Xy_data.get('rmse_train', None), + }, + "test": { + "r2": Xy_data.get('r2_test', None), + "mae": Xy_data.get('mae_test', None), + "rmse": Xy_data.get('rmse_test', None), + }, + "external": { + "r2": Xy_data.get('r2_external', None), + "mae": Xy_data.get('mae_external', None), + "rmse": Xy_data.get('rmse_external', None), + } if external_available else None, + }, + "classification_metrics": { + "cv": { + "acc": Xy_data.get('acc_train', None), + "f1": Xy_data.get('f1_train', None), + "mcc": Xy_data.get('mcc_train', None), + }, + "test": { + "acc": Xy_data.get('acc_test', None), + "f1": Xy_data.get('f1_test', None), + "mcc": Xy_data.get('mcc_test', None), + }, + "external": { + "acc": Xy_data.get('acc_external', None), + "f1": Xy_data.get('f1_external', None), + "mcc": Xy_data.get('mcc_external', None), + } if external_available else None, + }, + "prediction_sd_summary": { + "train_mean_sd": _mean_safe(Xy_data.get('y_pred_train_sd', None)), + "test_mean_sd": _mean_safe(Xy_data.get('y_pred_test_sd', None)), + "external_mean_sd": _mean_safe(Xy_data.get('y_pred_external_sd', None)) if external_available else None, + }, + "conformal_half_width": Xy_data.get('conformal_half_width', None), + "measured_y_range": measured_y_range, + }, + evidence_level="direct", + ) + # save predictions for all sets - path_n_suffix, name_points, Xy_data = save_predictions(self,Xy_data,model_data,suffix_title) + path_n_suffix, name_points, Xy_data, save_metadata = save_predictions(self,Xy_data,model_data,suffix_title) + + self.args.predict_audit = audit_event( + self.args.predict_audit, + event_type="save_predictions", + payload={ + "branch_key": str(branch_key), + "metadata": save_metadata, + }, + evidence_level="direct", + ) # represent y vs predicted y colors = plot_predictions(self,model_data,Xy_data,path_n_suffix) + if all(key in Xy_data for key in ['pred_min', 'pred_max', 'pred_range']): + self.args.predict_audit = audit_event( + self.args.predict_audit, + event_type="measured_y_range_post_plot", + payload={ + "branch_key": str(branch_key), + "measured_y_range": { + "min": float(Xy_data['pred_min']), + "max": float(Xy_data['pred_max']), + "range": float(Xy_data['pred_range']), + }, + }, + evidence_level="direct", + ) + else: + self.args.predict_audit = audit_event( + self.args.predict_audit, + event_type="measured_y_range_post_plot", + payload={ + "branch_key": str(branch_key), + "state": "skipped", + "reason": "pred_range_keys_not_available_after_plot_predictions", + }, + evidence_level="derived", + ) + + base_name = os.path.basename(path_n_suffix) + base_dir = os.path.dirname(path_n_suffix) + graph_type = 'regression' if model_data['type'].lower() == 'reg' else 'classification' + external_plots_applicable = bool( + 'y_external' in Xy_data + and hasattr(Xy_data['y_external'], 'isnull') + and not Xy_data['y_external'].isnull().values.any() + and len(Xy_data['y_external']) > 0 + ) + + if graph_type == 'regression': + expected_plots = { + "results": f"{base_dir}/Results_{base_name}.png", + "cv_variability": f"{base_dir}/CV_variability_{base_name}.png", + "external": f"{base_dir}/csv_test/CV_variability_{base_name}_external.png" if external_plots_applicable else None, + } + else: + expected_plots = { + "cv_train_valid": f"{base_dir}/CV_train_valid_predict_{base_name}.png", + "test": f"{base_dir}/Results_{base_name}_test.png", + "external": f"{base_dir}/csv_test/Results_{base_name}_external.png" if external_plots_applicable else None, + } + + plot_status = {} + for plot_key, plot_path in expected_plots.items(): + if plot_path is None: + plot_status[plot_key] = {"path": None, "state": "not_applicable", "exists": False} + else: + exists = bool(os.path.exists(plot_path)) + plot_status[plot_key] = { + "path": str(plot_path), + "state": "created" if exists else "skipped", + "exists": exists, + } + + self.args.predict_audit = audit_event( + self.args.predict_audit, + event_type="predict_plots", + payload={ + "branch_key": str(branch_key), + "graph_type": graph_type, + "plots": plot_status, + }, + evidence_level="derived", + ) + # print results - _ = print_predict(self,Xy_data,model_data,suffix_title) + print_results_text = print_predict(self,Xy_data,model_data,suffix_title) + + self.args.predict_audit = audit_event( + self.args.predict_audit, + event_type="print_predict_summary", + payload={ + "branch_key": str(branch_key), + "model_type": str(model_data['type']), + "cv_metrics": { + "r2": Xy_data.get('r2_train', None), + "mae": Xy_data.get('mae_train', None), + "rmse": Xy_data.get('rmse_train', None), + "acc": Xy_data.get('acc_train', None), + "f1": Xy_data.get('f1_train', None), + "mcc": Xy_data.get('mcc_train', None), + }, + "test_metrics": { + "r2": Xy_data.get('r2_test', None), + "mae": Xy_data.get('mae_test', None), + "rmse": Xy_data.get('rmse_test', None), + "acc": Xy_data.get('acc_test', None), + "f1": Xy_data.get('f1_test', None), + "mcc": Xy_data.get('mcc_test', None), + }, + "external_metrics": { + "r2": Xy_data.get('r2_external', None), + "mae": Xy_data.get('mae_external', None), + "rmse": Xy_data.get('rmse_external', None), + "acc": Xy_data.get('acc_external', None), + "f1": Xy_data.get('f1_external', None), + "mcc": Xy_data.get('mcc_external', None), + } if external_plots_applicable else None, + "dat_text_preview_deferred": bool(not isinstance(print_results_text, str) or print_results_text == ''), + }, + evidence_level="direct", + dat_text=print_results_text if isinstance(print_results_text, str) and print_results_text != '' else None, + ) # SHAP analysis _ = shap_analysis(self,Xy_data,model_data,path_n_suffix) + shap_path = f"{base_dir}/SHAP_{base_name}.png" + self.args.predict_audit = audit_event( + self.args.predict_audit, + event_type="shap_artifact", + payload={ + "branch_key": str(branch_key), + "path": str(shap_path), + "exists": bool(os.path.exists(shap_path)), + "state": "created" if os.path.exists(shap_path) else "skipped", + }, + evidence_level="derived", + ) + # PFI analysis _ = PFI_plot(self,Xy_data,model_data,path_n_suffix) + pfi_path = f"{base_dir}/PFI_{base_name}.png" + self.args.predict_audit = audit_event( + self.args.predict_audit, + event_type="pfi_artifact", + payload={ + "branch_key": str(branch_key), + "path": str(pfi_path), + "exists": bool(os.path.exists(pfi_path)), + "state": "created" if os.path.exists(pfi_path) else "skipped", + }, + evidence_level="derived", + ) + # create Pearson heatmap _ = pearson_map_predict(self,Xy_data,params_dir) + pearson_path = None + pearson_state = "not_applicable" + if str(suffix_title) in ['No_PFI', 'PFI']: + pearson_path = str(self.args.destination.joinpath(f"Pearson_heatmap_{suffix_title}.png")) + pearson_exists = bool(os.path.exists(pearson_path)) + pearson_state = "created" if pearson_exists else "skipped" + else: + pearson_exists = False + + self.args.predict_audit = audit_event( + self.args.predict_audit, + event_type="pearson_artifact", + payload={ + "branch_key": str(branch_key), + "path": pearson_path, + "exists": pearson_exists, + "state": pearson_state, + }, + evidence_level="derived", + ) + # Outlier analysis if model_data['type'].lower() == 'reg': _ = outlier_plot(self,Xy_data,path_n_suffix,name_points,colors) + outlier_path = f"{base_dir}/Outliers_{base_name}.png" + outlier_exists = bool(os.path.exists(outlier_path)) + self.args.predict_audit = audit_event( + self.args.predict_audit, + event_type="outlier_artifact", + payload={ + "branch_key": str(branch_key), + "path": str(outlier_path), + "exists": outlier_exists, + "state": "created" if outlier_exists else "skipped", + }, + evidence_level="derived", + ) + else: + self.args.predict_audit = audit_event( + self.args.predict_audit, + event_type="outlier_artifact", + payload={ + "branch_key": str(branch_key), + "path": None, + "exists": False, + "state": "not_applicable", + }, + evidence_level="derived", + ) # y distribution _ = distribution_plot(self,Xy_data,path_n_suffix,model_data) + distribution_path = f"{base_dir}/y_distribution_{base_name}.png" + distribution_exists = bool(os.path.exists(distribution_path)) + self.args.predict_audit = audit_event( + self.args.predict_audit, + event_type="distribution_artifact", + payload={ + "branch_key": str(branch_key), + "path": str(distribution_path), + "exists": distribution_exists, + "state": "created" if distribution_exists else "skipped", + }, + evidence_level="derived", + ) + + self.args.predict_audit = audit_set( + self.args.predict_audit, + "runtime", + "module_runtime_seconds", + round(time.time() - start_time, 2), + ) + + try: + audit_write_ok = finalize_module_audit(self.args.predict_audit, predict_audit_path, status="completed") + if not audit_write_ok: + _ = write_json_output_audit( + json_audit_path, + module="PREDICT", + attempted_output_path=predict_audit_path, + attempted=True, + succeeded=False, + artifact="predict_audit.json", + error=RuntimeError("predict_audit_json_write_returned_false"), + ) + else: + _ = write_json_output_audit( + json_audit_path, + module="PREDICT", + attempted_output_path=predict_audit_path, + attempted=True, + succeeded=True, + artifact="predict_audit.json", + error=None, + ) + except Exception as json_error: + _ = write_json_output_audit( + json_audit_path, + module="PREDICT", + attempted_output_path=predict_audit_path, + attempted=True, + succeeded=False, + artifact="predict_audit.json", + error=json_error, + ) + _ = finish_print(self,start_time,'PREDICT') diff --git a/robert/predict_utils.py b/robert/predict_utils.py index 76dcb84..c93dc1e 100644 --- a/robert/predict_utils.py +++ b/robert/predict_utils.py @@ -156,6 +156,19 @@ def save_predictions(self,Xy_data,model_data,suffix_title): base_csv_path = f"{Path(os.getcwd()).joinpath(base_csv_name)}" path_n_suffix = f'{base_csv_path}' _ = df_results.to_csv(f'{base_csv_path}.csv', index = None, header=True) + + save_metadata = { + 'base_prediction_csv_path': f'{base_csv_path}.csv', + 'base_prediction_csv_created': bool(os.path.exists(f'{base_csv_path}.csv')), + 'base_row_count': int(len(df_results)), + 'base_column_names': [str(col) for col in df_results.columns], + 'external_csv_path': None, + 'external_csv_created': False, + 'external_row_count': None, + 'external_column_names': None, + 'external_state': 'not_applicable' if self.args.csv_test == '' else 'skipped', + 'class_labels_reconverted': bool(reconvert_labels), + } # also save results for performance of individual folds (useful for t-tests and Wilcoxon tests between the folds) error1, error2, error3 = get_error_labels(model_data['type']) @@ -209,6 +222,12 @@ def save_predictions(self,Xy_data,model_data,suffix_title): _ = Xy_external.to_csv(name_external, index = None, header=True) print_preds += f'\n - External set with predicted results: PREDICT/csv_test/{csv_name_external}' + save_metadata['external_csv_path'] = str(name_external) + save_metadata['external_csv_created'] = bool(os.path.exists(name_external)) + save_metadata['external_row_count'] = int(len(Xy_external)) + save_metadata['external_column_names'] = [str(col) for col in Xy_external.columns] + save_metadata['external_state'] = 'created' if save_metadata['external_csv_created'] else 'skipped' + self.args.log.write(print_preds) # store the names of the datapoints @@ -222,7 +241,7 @@ def save_predictions(self,Xy_data,model_data,suffix_title): name_points['train'] = df_results[model_data['names']][df_results.Set == 'CV'] name_points['test'] = df_results[model_data['names']][df_results.Set == 'Test'] - return path_n_suffix, name_points, Xy_data + return path_n_suffix, name_points, Xy_data, save_metadata def print_predict(self,Xy_data,model_data,suffix_title): @@ -268,6 +287,8 @@ def print_predict(self,Xy_data,model_data,suffix_title): self.args.log.write(print_results) + return print_results + def pearson_map_predict(self,Xy_data,params_dir): ''' diff --git a/robert/report.py b/robert/report.py index 8a167c0..3905d2f 100644 --- a/robert/report.py +++ b/robert/report.py @@ -28,6 +28,12 @@ from robert.utils import (load_variables, pd_to_dict, ) +from robert.json_output_for_agent import ( + agent_json_path, + init_report_figure_provenance, + add_report_figure_record, + finalize_report_figure_provenance, +) from robert.report_utils import ( get_csv_names, get_col_score, @@ -92,6 +98,7 @@ def __init__(self, **kwargs): # load default and user-specified variables self.args = load_variables(kwargs, "report") + self.figure_provenance = init_report_figure_provenance() eval_only = False # if EVALUATE is activated, no PFI models are generated @@ -178,6 +185,17 @@ def __init__(self, **kwargs): else: _ = make_report(report_html,HTML) + report_pdf_path = f'{os.getcwd()}/ROBERT_report.pdf' + try: + _ = finalize_report_figure_provenance( + self.figure_provenance, + agent_json_path('report_figure_provenance.json'), + report_pdf_path, + os.path.exists(report_pdf_path), + ) + except Exception: + pass + # Remove report.css file os.remove("report.css") @@ -237,7 +255,7 @@ def print_score(self,dat_files,pred_type,eval_only,spacing_PFI): height = 221 if pred_type == 'clas': height += diff_height - score_dat += self.print_img('Results',-5,height,'PREDICT',pred_type,eval_only,diff_names=True) + score_dat += self.print_img('Results',-5,height,'PREDICT',pred_type,eval_only,diff_names=True,report_section='Section A. ROBERT Score') for suffix in ['No PFI','PFI']: spacing = get_spacing_col(suffix,spacing_PFI) @@ -578,7 +596,7 @@ def print_adv_anal(self,pred_type,eval_only,spacing_PFI,data_score): height = 223 if pred_type == 'clas': height -= 15 - adv_score_dat += self.print_img('VERIFY_tests',13,height,'VERIFY',pred_type,eval_only) + adv_score_dat += self.print_img('VERIFY_tests',13,height,'VERIFY',pred_type,eval_only,report_section='Section B. Advanced Score Analysis') # page break to second page adv_score_dat += '
' @@ -586,7 +604,7 @@ def print_adv_anal(self,pred_type,eval_only,spacing_PFI,data_score): adv_score_dat += section_separator elif section == 'adv_cv_sd' and pred_type == 'reg': - adv_score_dat += self.print_img('CV_variability',10,221,'PREDICT',pred_type,eval_only) + adv_score_dat += self.print_img('CV_variability',10,221,'PREDICT',pred_type,eval_only,report_section='Section B. Advanced Score Analysis') elif section == 'adv_cv_diff' and pred_type == 'clas': adv_score_dat += section_separator @@ -653,7 +671,7 @@ def print_outliers(self,pred_type,eval_only,spacing_PFI): # add corresponding images height = 217 - outlier_dat += self.print_img('Outliers',-5,height,'PREDICT',pred_type,eval_only) + outlier_dat += self.print_img('Outliers',-5,height,'PREDICT',pred_type,eval_only,report_section='Section E. Outlier Analysis') # add separator line and page break outlier_dat += '
' @@ -673,7 +691,7 @@ def print_y_distrib(self,pred_type,eval_only,spacing_PFI,warnings_dict): # add corresponding images height = 220 - distrib_dat += self.print_img('y_distribution',-5,height,'PREDICT',pred_type,eval_only) + distrib_dat += self.print_img('y_distribution',-5,height,'PREDICT',pred_type,eval_only,report_section='Section C. Distribution of y Values') columns_y_distrib = [] # get two columns to combine and print @@ -752,16 +770,20 @@ def print_features(self,warnings_dict,eval_only,spacing_PFI): pair_list = f'

Pearson maps not created if >30 descriptors.' pair_list += f'{(" ")*15}' if len(image_pair) == 1: + self.figure_provenance = add_report_figure_record(self.figure_provenance, image_pair[0], report_section='Section D. Feature Importances') pair_list += f'

' elif len(image_pair) == 0: pair_list += f'{(" ")*15}' pair_list += f'Pearson maps not created if >30 descriptors.

' elif eval_only: if len(image_pair) == 1: + self.figure_provenance = add_report_figure_record(self.figure_provenance, image_pair[0], report_section='Section D. Feature Importances') pair_list = f'

' elif len(image_pair) == 0: pair_list = f'

Pearson maps not created if >30 descriptors.

' else: + self.figure_provenance = add_report_figure_record(self.figure_provenance, image_pair[0], report_section='Section D. Feature Importances') + self.figure_provenance = add_report_figure_record(self.figure_provenance, image_pair[1], report_section='Section D. Feature Importances') pair_list = f'

' pair_list += f'{(" ")*22}' pair_list += f'

' @@ -806,7 +828,7 @@ def print_generate(self,pred_type,eval_only): # add corresponding images if not eval_only: height = 236 - generate_dat += self.print_img('Heatmap',-5,height,'GENERATE',pred_type,eval_only) + generate_dat += self.print_img('Heatmap',-5,height,'GENERATE',pred_type,eval_only,report_section='Section F. Model Screening') generate_dat += '

' @@ -1088,7 +1110,7 @@ def print_predictions(self,pred_type,eval_only,spacing_PFI): prefix_img = 'Results' height += 17 if len(glob.glob(f'{os.getcwd()}/PREDICT/csv_test/{prefix_img}*.png')) > 0: - pred_dat += self.print_img(prefix_img,-5,height,'PREDICT/csv_test',pred_type,eval_only) + pred_dat += self.print_img(prefix_img,-5,height,'PREDICT/csv_test',pred_type,eval_only,report_section='Section J. New Predictions') # add separator line and page break pred_dat += '
' @@ -1164,7 +1186,7 @@ def module_lines(self,module,module_data,pred_type='reg',eval_only=False): return title_line - def print_img(self,file_name,margin_top,height,module,pred_type,eval_only,test_set=False,diff_names=False): + def print_img(self,file_name,margin_top,height,module,pred_type,eval_only,test_set=False,diff_names=False,report_section=None): """ Generates the string that includes couples of images to print """ @@ -1188,6 +1210,11 @@ def print_img(self,file_name,margin_top,height,module,pred_type,eval_only,test_s # keep the ordering (No_PFI in the left, PFI in the right of the PDF) results_images = revert_list(results_images) + + if len(results_images) > 0: + self.figure_provenance = add_report_figure_record(self.figure_provenance, results_images[0], report_section=report_section) + if not eval_only and len(results_images) > 1: + self.figure_provenance = add_report_figure_record(self.figure_provenance, results_images[1], report_section=report_section) # add the graphs width = 100 diff --git a/robert/utils.py b/robert/utils.py index 59f68d9..ada935c 100644 --- a/robert/utils.py +++ b/robert/utils.py @@ -58,6 +58,7 @@ from sklearn.inspection import permutation_importance from sklearn.exceptions import ConvergenceWarning from robert.argument_parser import set_options, var_dict +from robert.json_output_for_agent import audit_event, audit_set from bayes_opt import BayesianOptimization from bayes_opt import acquisition import warnings # this avoids warnings from sklearn @@ -664,6 +665,12 @@ def correlation_filter(self, csv_df): """ txt_corr = '' + constant_descs_removed = [] + low_y_corr_descs_removed = [] + high_intercorr_events = [] + rfecv_selection_method = {} + rfecv_descriptors_selected = {} + rfecv_applied = False # Sort columns alphabetically and rows by y value for reproducibility descriptor_cols = [col for col in csv_df.columns if col not in self.args.ignore and col != self.args.y] @@ -687,6 +694,7 @@ def correlation_filter(self, csv_df): # Remove descriptors where all values are the same if len(set(csv_df[column])) == 1: descriptors_drop.append(column) + constant_descs_removed.append(column) txt_corr += f'\n - {column}: all the values are the same' # Remove descriptors with low correlation to the response values @@ -697,6 +705,7 @@ def correlation_filter(self, csv_df): rsquared_y = res_y.rvalue**2 if rsquared_y < self.args.thres_y: descriptors_drop.append(column) + low_y_corr_descs_removed.append(column) txt_corr += f'\n - {column}: R**2 = {rsquared_y:.2} with the {self.args.y} values' self.args.log.write(txt_corr) @@ -750,6 +759,16 @@ def correlation_filter(self, csv_df): keep_col = col_name_1 descriptors_drop.append(drop_col) + high_intercorr_events.append( + { + "removed": drop_col, + "kept": keep_col, + "r2_between_descriptors": float(max_r2), + "removed_r2_to_target": float(r2_with_y[drop_col]), + "kept_r2_to_target": float(r2_with_y[keep_col]), + "reason": "lower_r2_to_target_or_alphabetical_tie_break", + } + ) txt_corr += f'\n - {drop_col} removed (R2 = {max_r2:.2f} with {keep_col}), kept more predictive descriptor' upper = upper.drop(index=drop_col, columns=drop_col) @@ -776,6 +795,7 @@ def correlation_filter(self, csv_df): num_descriptors = round(len(csv_df[self.args.y]) / 3) if n_descps > num_descriptors: + rfecv_applied = True cv_type = f'{self.args.repeat_kfolds}x {self.args.kfold}_fold_cv' txt_corr += f'\no There are more descriptors than one-third of the data points. A Recursive Feature Elimination with Cross-Validation (RFECV) or permutation feature importance (PFI) using {cv_type} will be performed to select the most relevant descriptors for each model' self.args.log.write(txt_corr) @@ -833,6 +853,8 @@ def correlation_filter(self, csv_df): # Sort final list alphabetically for consistent ordering in output descriptors_used[model] = sorted(descriptors_used[model]) + rfecv_selection_method[str(model)] = "PFI" + rfecv_descriptors_selected[str(model)] = list(descriptors_used[model]) txt_corr += f'\n - {model}: {len(descriptors_used[model])} descriptors selected (using PFI)' @@ -869,6 +891,8 @@ def correlation_filter(self, csv_df): # Sort final list alphabetically for consistent ordering in output descriptors_used[model] = sorted(descriptors_used[model]) + rfecv_selection_method[str(model)] = "RFECV" + rfecv_descriptors_selected[str(model)] = list(descriptors_used[model]) txt_corr += f'\n - {model}: {len(descriptors_used[model])} descriptors selected (using RFECV)' @@ -892,6 +916,41 @@ def correlation_filter(self, csv_df): self.args.log.write(txt_corr) + if hasattr(self.args, 'curate_audit'): + if rfecv_applied: + rfecv_skip_reason = None + else: + current_desc = int(len(csv_df_filtered.columns) - len(self.args.ignore) - 1) + rfecv_skip_reason = f"descriptor_count_not_above_threshold ({current_desc} <= {num_descriptors})" + + self.args.curate_audit = audit_set(self.args.curate_audit, "correlation_filter", "activated", True) + self.args.curate_audit = audit_set(self.args.curate_audit, "correlation_filter", "thres_x", self.args.thres_x) + self.args.curate_audit = audit_set(self.args.curate_audit, "correlation_filter", "thres_y", self.args.thres_y if self.args.corr_filter_y else None) + self.args.curate_audit = audit_set(self.args.curate_audit, "correlation_filter", "descriptor_count_before_filter", n_descps) + self.args.curate_audit = audit_set(self.args.curate_audit, "correlation_filter", "constant_descriptors_removed", constant_descs_removed) + self.args.curate_audit = audit_set(self.args.curate_audit, "correlation_filter", "constant_descriptor_count_removed", len(constant_descs_removed)) + self.args.curate_audit = audit_set(self.args.curate_audit, "correlation_filter", "low_y_correlation_descriptors_removed", low_y_corr_descs_removed) + self.args.curate_audit = audit_set(self.args.curate_audit, "correlation_filter", "low_y_correlation_descriptor_count_removed", len(low_y_corr_descs_removed)) + self.args.curate_audit = audit_set(self.args.curate_audit, "correlation_filter", "high_intercorrelation_removals", high_intercorr_events) + self.args.curate_audit = audit_set(self.args.curate_audit, "correlation_filter", "high_intercorrelation_descriptor_count_removed", len(high_intercorr_events)) + self.args.curate_audit = audit_set(self.args.curate_audit, "correlation_filter", "descriptors_removed_correlation_filter", len(high_intercorr_events)) + self.args.curate_audit = audit_set(self.args.curate_audit, "correlation_filter", "rfecv_applied", rfecv_applied) + self.args.curate_audit = audit_set(self.args.curate_audit, "correlation_filter", "rfecv_selection_method_by_model", rfecv_selection_method) + self.args.curate_audit = audit_set(self.args.curate_audit, "correlation_filter", "rfecv_descriptors_selected_by_model", rfecv_descriptors_selected) + self.args.curate_audit = audit_set(self.args.curate_audit, "correlation_filter", "rfecv_skip_reason", rfecv_skip_reason) + self.args.curate_audit = audit_event( + self.args.curate_audit, + event_type="correlation_filter", + payload={ + "constant_removed": len(constant_descs_removed), + "low_y_corr_removed": len(low_y_corr_descs_removed), + "high_intercorr_removed": len(high_intercorr_events), + "rfecv_applied": rfecv_applied, + }, + evidence_level="direct", + dat_text=txt_corr, + ) + # Return both the general filtered dataframe and model-specific dataframes return csv_df_filtered, csv_df_per_model @@ -1262,6 +1321,10 @@ def load_database(self,csv_load,module,print_info=True,external_test=False): csv_df = pd.read_csv(csv_load, encoding='utf-8') # Missing data handling: robust strategy for columns and rows (optional KNN imputer) + cols_to_drop = [] + n_removed_rows = 0 + cols_with_missing = [] + knn_applied = False target_col = self.args.y descriptor_cols = [col for col in csv_df.columns if col not in self.args.ignore+self.args.discard and col != self.args.y] min_count = int(0.9 * len(csv_df)) @@ -1289,6 +1352,7 @@ def load_database(self,csv_load,module,print_info=True,external_test=False): if csv_df[numeric_columns].isna().any().any(): imputer = KNNImputer(n_neighbors=5) csv_df[numeric_columns] = pd.DataFrame(imputer.fit_transform(csv_df[numeric_columns]), columns=numeric_columns, index=csv_df.index) + knn_applied = True if module.lower() == 'curate': txt_load += f"\n - Applied KNN imputer to columns with missing values\n" else: @@ -1334,6 +1398,29 @@ def load_database(self,csv_load,module,print_info=True,external_test=False): self.args.log.write(f"\nx The aren't any valid descriptors! Check the messages above to see whether the filters have discarded descriptors") sys.exit() + if module.lower() == 'curate' and hasattr(self.args, 'curate_audit'): + self.args.curate_audit = audit_set(self.args.curate_audit, "load_database", "datapoints_loaded", int(len(csv_df))) + self.args.curate_audit = audit_set(self.args.curate_audit, "load_database", "accepted_descriptors_loaded", int(len([c for c in csv_df.columns if c not in self.args.ignore and c != self.args.y]))) + self.args.curate_audit = audit_set(self.args.curate_audit, "load_database", "ignored_descriptors_loaded", int(len([c for c in csv_df.columns if c in self.args.ignore]))) + self.args.curate_audit = audit_set(self.args.curate_audit, "load_database", "discarded_descriptors_loaded", int(len(self.args.discard))) + self.args.curate_audit = audit_set(self.args.curate_audit, "load_database", "columns_removed_lt90pct_data", list(cols_to_drop)) + self.args.curate_audit = audit_set(self.args.curate_audit, "load_database", "rows_removed_gt50pct_missing", int(n_removed_rows)) + self.args.curate_audit = audit_set(self.args.curate_audit, "load_database", "columns_removed_any_missing", list(cols_with_missing)) + self.args.curate_audit = audit_set(self.args.curate_audit, "load_database", "knn_imputer_applied", bool(knn_applied)) + self.args.curate_audit = audit_event( + self.args.curate_audit, + event_type="load_database", + payload={ + "datapoints_loaded": int(len(csv_df)), + "accepted_descriptors_loaded": int(len([c for c in csv_df.columns if c not in self.args.ignore and c != self.args.y])), + "columns_removed_lt90pct_data": len(cols_to_drop), + "rows_removed_gt50pct_missing": int(n_removed_rows), + "knn_imputer_applied": bool(knn_applied), + }, + evidence_level="direct", + dat_text=txt_load, + ) + # Sort columns alphabetically for reproducibility across ALL modules if module.lower() not in ['aqme', 'aqme_test']: # Get descriptor columns (excluding y and ignore) @@ -1415,6 +1502,25 @@ def categorical_transform(self,csv_df,module): self.args.log.write(f'{txt_categor}') + if module.lower() == 'curate' and hasattr(self.args, 'curate_audit'): + self.args.curate_audit = audit_set(self.args.curate_audit, "categorical_transform", "activated", True) + self.args.curate_audit = audit_set(self.args.curate_audit, "categorical_transform", "categorical_variables", list(categorical_vars)) + self.args.curate_audit = audit_set(self.args.curate_audit, "categorical_transform", "categorical_variables_found", len(categorical_vars) > 0) + self.args.curate_audit = audit_set(self.args.curate_audit, "categorical_transform", "generated_descriptors", list(new_categor_desc)) + self.args.curate_audit = audit_set(self.args.curate_audit, "categorical_transform", "mode", self.args.categorical) + self.args.curate_audit = audit_set(self.args.curate_audit, "categorical_transform", "descriptors_removed_categorical_transform", len(descriptors_to_drop)) + self.args.curate_audit = audit_event( + self.args.curate_audit, + event_type="categorical_transform", + payload={ + "categorical_variables_count": len(categorical_vars), + "generated_descriptors_count": len(new_categor_desc), + "mode": self.args.categorical, + }, + evidence_level="direct", + dat_text=txt_categor if module.lower() == 'curate' else None, + ) + return csv_df @@ -1427,6 +1533,8 @@ def create_folders(folder_names): def finish_print(self,start_time,module): elapsed_time = round(time.time() - start_time, 2) + if module.upper() == 'CURATE' and hasattr(self.args, 'curate_audit'): + self.args.curate_audit = audit_set(self.args.curate_audit, "runtime", "module_runtime_seconds", float(elapsed_time)) self.args.log.write(f"\nTime {module.upper()}: {elapsed_time} seconds\n") self.args.log.finalize() @@ -3278,6 +3386,20 @@ def pearson_map(self,csv_df_pearson,module,params_dir=None): self.args.log.write(f'\nx The Pearson heatmap was not generated because the number of features and the y value ({len(csv_df_pearson.columns)}) is higher than 30.') if module.lower() == 'predict': self.args.log.write(f'\n x The Pearson heatmap was not generated because the number of features and the y value ({len(csv_df_pearson.columns)}) is higher than 30.') + if module.lower() == 'curate' and hasattr(self.args, 'curate_audit'): + self.args.curate_audit = audit_set(self.args.curate_audit, "pearson_map", "generated", False) + self.args.curate_audit = audit_set(self.args.curate_audit, "pearson_map", "descriptor_count_for_map", int(len(csv_df_pearson.columns))) + self.args.curate_audit = audit_set(self.args.curate_audit, "pearson_map", "output_path", None) + self.args.curate_audit = audit_set(self.args.curate_audit, "pearson_map", "skip_reason", f"descriptor_count_exceeds_limit ({len(csv_df_pearson.columns)} > 30)") + self.args.curate_audit = audit_event( + self.args.curate_audit, + event_type="pearson_map", + payload={ + "generated": False, + "descriptor_count_for_map": int(len(csv_df_pearson.columns)), + }, + evidence_level="direct", + ) else: sb.set(font_scale=1.2, style='ticks') @@ -3330,6 +3452,22 @@ def pearson_map(self,csv_df_pearson,module,params_dir=None): elif module.lower() == 'predict': self.args.log.write(f'\n o The Pearson heatmap was stored in {path_reduced}.') + if module.lower() == 'curate' and hasattr(self.args, 'curate_audit'): + self.args.curate_audit = audit_set(self.args.curate_audit, "pearson_map", "generated", True) + self.args.curate_audit = audit_set(self.args.curate_audit, "pearson_map", "descriptor_count_for_map", int(len(csv_df_pearson.columns))) + self.args.curate_audit = audit_set(self.args.curate_audit, "pearson_map", "output_path", str(heatmap_path)) + self.args.curate_audit = audit_set(self.args.curate_audit, "pearson_map", "skip_reason", None) + self.args.curate_audit = audit_event( + self.args.curate_audit, + event_type="pearson_map", + payload={ + "generated": True, + "descriptor_count_for_map": int(len(csv_df_pearson.columns)), + "output_path": str(heatmap_path), + }, + evidence_level="direct", + ) + return corr_matrix diff --git a/robert/verify.py b/robert/verify.py index ac07ada..51f00d3 100644 --- a/robert/verify.py +++ b/robert/verify.py @@ -25,6 +25,14 @@ import time import numpy as np from statistics import mode +from robert.json_output_for_agent import ( + init_module_audit, + audit_set, + audit_event, + finalize_module_audit, + agent_json_path, + write_json_output_audit, +) from robert.utils import (load_variables, load_db_n_params, load_n_predict, @@ -56,6 +64,27 @@ def __init__(self, **kwargs): # load default and user-specified variables self.args = load_variables(kwargs, "verify") + command_line_value = getattr(self.args, "command_line", None) + if not isinstance(command_line_value, str) or command_line_value.strip() == "": + command_line_value = None + + self.args.verify_audit = init_module_audit( + module="VERIFY", + artifact_type="verify_audit", + source_files=[str(self.args.params_dir)], + command_line=command_line_value, + ) + verify_audit_path = agent_json_path("verify_audit.json") + json_audit_path = agent_json_path("verify_json_output_audit.json") + + self.args.verify_audit = audit_set(self.args.verify_audit, "inputs", "params_dir", self.args.params_dir) + self.args.verify_audit = audit_set(self.args.verify_audit, "inputs", "seed", self.args.seed) + self.args.verify_audit = audit_set(self.args.verify_audit, "inputs", "kfold", self.args.kfold) + self.args.verify_audit = audit_set(self.args.verify_audit, "inputs", "repeat_kfolds", self.args.repeat_kfolds) + self.args.verify_audit = audit_set(self.args.verify_audit, "inputs", "destination", str(self.args.destination)) + self.args.verify_audit = audit_set(self.args.verify_audit, "inputs", "thres_test_pass", thres_test_pass) + self.args.verify_audit = audit_set(self.args.verify_audit, "inputs", "thres_test_unclear", thres_test_unclear) + # if params_dir = '', the program performs the tests for the No_PFI and PFI folders if 'GENERATE/Best_model' in self.args.params_dir: params_dirs = [f'{self.args.params_dir}/No_PFI',f'{self.args.params_dir}/PFI'] @@ -63,15 +92,90 @@ def __init__(self, **kwargs): suffix_titles = ['No_PFI','PFI'] else: params_dirs = [self.args.params_dir] - suffix = ['custom'] + suffixes = ['custom'] + suffix_titles = ['custom'] for (params_dir,suffix,suffix_title) in zip(params_dirs,suffixes,suffix_titles): + is_pfi_filtered = bool(str(suffix_title) == 'PFI') + self.args.verify_audit = audit_event( + self.args.verify_audit, + event_type="verify_branch", + payload={ + "params_dir_used": str(params_dir), + "suffix": str(suffix), + "suffix_title": str(suffix_title), + "params_dir_exists": bool(os.path.exists(params_dir)), + "pfi_section_active": is_pfi_filtered, + "branch_is_pfi_filtered": is_pfi_filtered, + "workflow_includes_pfi_branch": bool('GENERATE/Best_model' in self.args.params_dir), + }, + evidence_level="direct", + ) if os.path.exists(params_dir): _ = print_pfi(self,params_dir) # load the Xy databse and model parameters Xy_data, model_data, suffix_title = load_db_n_params(self,params_dir,suffix,suffix_title,"verify",True) + + train_datapoints = None + total_datapoints_loaded = None + descriptor_list = None + accepted_descriptor_count = None + ignored_descriptor_count = None + discarded_descriptor_count = None + + if isinstance(Xy_data, dict): + if 'X_train' in Xy_data and hasattr(Xy_data['X_train'], 'shape'): + train_datapoints = int(Xy_data['X_train'].shape[0]) + if hasattr(Xy_data['X_train'], 'columns'): + descriptor_list = [str(col) for col in Xy_data['X_train'].columns] + elif 'y_train' in Xy_data and hasattr(Xy_data['y_train'], '__len__'): + train_datapoints = int(len(Xy_data['y_train'])) + + if 'X' in Xy_data and hasattr(Xy_data['X'], 'shape'): + total_datapoints_loaded = int(Xy_data['X'].shape[0]) + elif 'y' in Xy_data and hasattr(Xy_data['y'], '__len__'): + total_datapoints_loaded = int(len(Xy_data['y'])) + + if isinstance(model_data, dict): + if 'X_descriptors' in model_data and model_data['X_descriptors'] is not None: + accepted_descriptor_count = int(len(model_data['X_descriptors'])) + if descriptor_list is None: + descriptor_list = [str(col) for col in model_data['X_descriptors']] + if 'X_descriptors_ignored' in model_data and model_data['X_descriptors_ignored'] is not None: + ignored_descriptor_count = int(len(model_data['X_descriptors_ignored'])) + if 'X_descriptors_discarded' in model_data and model_data['X_descriptors_discarded'] is not None: + discarded_descriptor_count = int(len(model_data['X_descriptors_discarded'])) + + self.args.verify_audit = audit_event( + self.args.verify_audit, + event_type="model_context", + payload={ + "params_dir_used": str(params_dir), + "suffix": str(suffix), + "suffix_title": str(suffix_title), + "model_name": str(model_data['model']), + "model_type": str(model_data['type']), + "error_type": str(model_data['error_type']), + "descriptor_count": int(len(model_data['X_descriptors'])), + "y_column": str(model_data['y']), + "names_column": str(model_data['names']), + "kfold": int(model_data['kfold']), + "repeat_kfolds": int(model_data['repeat_kfolds']), + "seed": int(model_data['seed']), + "train_datapoints": train_datapoints, + "total_datapoints_loaded": total_datapoints_loaded, + "accepted_descriptor_count": accepted_descriptor_count, + "ignored_descriptor_count": ignored_descriptor_count, + "discarded_descriptor_count": discarded_descriptor_count, + "descriptor_list": descriptor_list, + "active_descriptor_count": accepted_descriptor_count, + "active_descriptor_list": descriptor_list, + "active_descriptor_source": "model_data['X_descriptors']", + }, + evidence_level="direct", + ) # this dictionary will keep the results of the tests verify_results = {'error_type': model_data['error_type']} @@ -89,6 +193,28 @@ def __init__(self, **kwargs): verify_results[f'f1_train_sorted_CV'] = [float(f"{val:.2f}") for val in Xy_data[f'f1_train_sorted_CV']] verify_results[f'mcc_train_sorted_CV'] = [float(f"{val:.2f}") for val in Xy_data[f'mcc_train_sorted_CV']] + self.args.verify_audit = audit_event( + self.args.verify_audit, + event_type="repeated_sorted_cv", + payload={ + "params_dir_used": str(params_dir), + "suffix_title": str(suffix_title), + "original_cv_score": verify_results['CV_score'], + "sorted_cv_score": verify_results['sorted_CV_score'], + "sorted_metrics_regression": { + "r2": verify_results.get('r2_train_sorted_CV', None), + "mae": verify_results.get('mae_train_sorted_CV', None), + "rmse": verify_results.get('rmse_train_sorted_CV', None), + }, + "sorted_metrics_classification": { + "acc": verify_results.get('acc_train_sorted_CV', None), + "f1": verify_results.get('f1_train_sorted_CV', None), + "mcc": verify_results.get('mcc_train_sorted_CV', None), + }, + }, + evidence_level="direct", + ) + # load the Xy databse and model parameters Xy_data, model_data, suffix_title = load_db_n_params(self,params_dir,suffix,suffix_title,"verify",False) @@ -112,10 +238,64 @@ def __init__(self, **kwargs): # plot a bar graph with the results print_ver = plot_metrics(model_data,suffix_title,verify_metrics,verify_results) + csv_name = os.path.basename(model_data['model']).split('_db.csv')[0] + verify_plot_file = f"{os.getcwd()}/VERIFY/VERIFY_tests_{csv_name}_{suffix_title}.png" + self.args.verify_audit = audit_event( + self.args.verify_audit, + event_type="verify_plot", + payload={ + "plot_type": "VERIFY flawed-model test plot", + "suffix_title": str(suffix_title), + "image_path": str(verify_plot_file), + "image_generation_completed": bool(os.path.exists(verify_plot_file)), + "short_interpretation": "comparison of original model vs flawed-model VERIFY tests", + }, + evidence_level="derived", + ) # print and save results _ = self.print_verify(results_print,verify_results,print_ver,model_data) + self.args.verify_audit = audit_set( + self.args.verify_audit, + "runtime", + "module_runtime_seconds", + round(time.time() - start_time, 2), + ) + + try: + audit_write_ok = finalize_module_audit(self.args.verify_audit, verify_audit_path, status="completed") + if not audit_write_ok: + _ = write_json_output_audit( + json_audit_path, + module="VERIFY", + attempted_output_path=verify_audit_path, + attempted=True, + succeeded=False, + artifact="verify_audit.json", + error=RuntimeError("verify_audit_json_write_returned_false"), + ) + else: + _ = write_json_output_audit( + json_audit_path, + module="VERIFY", + attempted_output_path=verify_audit_path, + attempted=True, + succeeded=True, + artifact="verify_audit.json", + error=None, + ) + except Exception as json_error: + _ = write_json_output_audit( + json_audit_path, + module="VERIFY", + attempted_output_path=verify_audit_path, + attempted=True, + succeeded=False, + artifact="verify_audit.json", + error=json_error, + ) + _ = finish_print(self,start_time,'VERIFY') @@ -137,6 +317,19 @@ def ymean_test(self,verify_results,Xy_data,model_data): verify_results['y_mean'] = Xy_ymean[f'{verify_results["error_type"]}_train'] + baseline_prediction = float(Xy_ymean['y_train'].mean()) if model_data['type'].lower() == 'reg' else int(mode(Xy_ymean['y_train'])) + self.args.verify_audit = audit_event( + self.args.verify_audit, + event_type="verify_test", + payload={ + "test_name": "y_mean", + "baseline_prediction_used": baseline_prediction, + "resulting_metric": verify_results['y_mean'], + "error_type": verify_results['error_type'], + }, + evidence_level="direct", + ) + return verify_results @@ -152,6 +345,18 @@ def yshuffle_test(self,verify_results,Xy_data,model_data): verify_results['y_shuffle'] = Xy_yshuffle[f'{verify_results["error_type"]}_train'] + self.args.verify_audit = audit_event( + self.args.verify_audit, + event_type="verify_test", + payload={ + "test_name": "y_shuffle", + "random_seed_used": int(model_data['seed']), + "resulting_metric": verify_results['y_shuffle'], + "error_type": verify_results['error_type'], + }, + evidence_level="direct", + ) + return verify_results @@ -182,6 +387,18 @@ def onehot_test(self,verify_results,Xy_data,model_data): Xy_onehot = load_n_predict(self, model_data, Xy_onehot, BO_opt=False) verify_results['onehot'] = Xy_onehot[f'{verify_results["error_type"]}_train'] + + self.args.verify_audit = audit_event( + self.args.verify_audit, + event_type="verify_test", + payload={ + "test_name": "onehot", + "descriptor_count_tested": int(len(Xy_onehot['X_train'].columns)), + "resulting_metric": verify_results['onehot'], + "error_type": verify_results['error_type'], + }, + evidence_level="direct", + ) return verify_results @@ -244,6 +461,35 @@ def analyze_tests(self,verify_results): 'unclear_higher_thres': verify_results['unclear_higher_thres'], 'unclear_lower_thres': verify_results['unclear_lower_thres'], } + + per_test_status = {} + for test_ver, result_text in zip(['y_mean', 'y_shuffle', 'onehot'], results_print): + if 'FAILED' in result_text: + per_test_status[test_ver] = 'FAILED' + elif 'UNCLEAR' in result_text: + per_test_status[test_ver] = 'UNCLEAR' + else: + per_test_status[test_ver] = 'PASSED' + + self.args.verify_audit = audit_event( + self.args.verify_audit, + event_type="analyze_tests", + payload={ + "higher_threshold": verify_results['higher_thres'], + "unclear_higher_threshold": verify_results['unclear_higher_thres'], + "lower_threshold": verify_results['lower_thres'], + "unclear_lower_threshold": verify_results['unclear_lower_thres'], + "per_test_status": per_test_status, + "per_test_metric": { + "y_mean": verify_results['y_mean'], + "y_shuffle": verify_results['y_shuffle'], + "onehot": verify_results['onehot'], + }, + "scoring_direction": "lower_is_better" if verify_results['error_type'].lower() in ['mae', 'rmse'] else "higher_is_better", + "colors": verify_metrics['colors'], + }, + evidence_level="derived", + ) return results_print,verify_results,verify_metrics @@ -268,4 +514,29 @@ def print_verify(self,results_print,verify_results,print_ver,model_data): elif model_data['type'].lower() == 'clas': print_ver += f"\n - Sorted CV : Accuracy = {verify_results['acc_train_sorted_CV']}, F1 score = {verify_results['f1_train_sorted_CV']}, MCC = {verify_results['mcc_train_sorted_CV']}" + self.args.verify_audit = audit_event( + self.args.verify_audit, + event_type="print_verify_summary", + payload={ + "original_cv_metric": verify_results['CV_score'], + "y_mean_result": verify_results['y_mean'], + "y_shuffle_result": verify_results['y_shuffle'], + "onehot_result": verify_results['onehot'], + "sorted_cv_metrics": { + "regression": { + "r2": verify_results.get('r2_train_sorted_CV', None), + "mae": verify_results.get('mae_train_sorted_CV', None), + "rmse": verify_results.get('rmse_train_sorted_CV', None), + }, + "classification": { + "acc": verify_results.get('acc_train_sorted_CV', None), + "f1": verify_results.get('f1_train_sorted_CV', None), + "mcc": verify_results.get('mcc_train_sorted_CV', None), + }, + }, + }, + evidence_level="direct", + dat_text=print_ver, + ) + self.args.log.write(print_ver)