Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
4d4c156
feat(config): add trusted registry-key resolver (refs #301)
stanlrt Jul 8, 2026
c860256
feat(config): register use: nodes and target-fqn map (refs #301)
stanlrt Jul 8, 2026
4b7f89b
refactor(misc): rename adapter import kwarg library to import_name (r…
stanlrt Jul 8, 2026
dbcdd43
feat(config): generate JSON Schema from adapter registry (refs #301)
stanlrt Jul 8, 2026
d614542
feat(config): resolve transparency and robustness adapters by use key…
stanlrt Jul 8, 2026
89421fe
feat(reporting): resolve reporter by use key and reject _target_ (ref…
stanlrt Jul 8, 2026
86d30d9
feat(deps): infer extras from use keys via static registry scan (refs…
stanlrt Jul 8, 2026
3fe1945
feat(metrics): resolve metric computer by use key (refs #301)
stanlrt Jul 8, 2026
0c72195
feat(transparency): resolve evaluator by use key (refs #301)
stanlrt Jul 8, 2026
acd213e
feat(tracking): resolve tracker by use key and reject _target_ (refs …
stanlrt Jul 8, 2026
8d89391
docs(config): generate adapter reference tables from registry (refs #…
stanlrt Jul 8, 2026
2b8ae5e
feat(data): resolve label and input parsers by use key (refs #301)
stanlrt Jul 8, 2026
330d71e
fix(config): reject nested _target_ at any depth (refs #301)
stanlrt Jul 8, 2026
d03f48a
refactor(config): drop unused resolve_target prefix helper (refs #301)
stanlrt Jul 8, 2026
5c938d3
feat(config): add config-schema CLI and ship committed schema (refs #…
stanlrt Jul 8, 2026
65eccaa
docs(config): note tracking group omitted from schema pending fix (re…
stanlrt Jul 8, 2026
06157df
feat(config)!: migrate shipped configs to use keys (refs #301)
stanlrt Jul 8, 2026
1061acc
test(config): migrate adapter tests to use keys (refs #301)
stanlrt Jul 8, 2026
acc98cf
docs(config): document use selector and config-schema (refs #301)
stanlrt Jul 8, 2026
1c0acb1
docs(transparency): fix README to use: selector (refs #301)
stanlrt Jul 8, 2026
1bf7fea
refactor(config): extract shared scanner iterator and use-key partial…
stanlrt Jul 8, 2026
fe29c43
test(transparency): fix pyright library kwarg and dict arg-type (refs…
stanlrt Jul 8, 2026
30a2e56
docs(config): make generated-comment string concat explicit (refs #301)
stanlrt Jul 8, 2026
1eff8e4
test(deps): skip test-registered stubs in scanner drift guard (refs #…
stanlrt Jul 8, 2026
0c2e540
fix(config): restore flat visualiser kwargs on python and yaml surfac…
stanlrt Jul 8, 2026
55cc71c
refactor(config): dedup resolver helpers and fix deferred review item…
stanlrt Jul 8, 2026
560f5d8
test(misc): add plugin end-to-end tests and runnable example-plugin (…
stanlrt Jul 8, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .github/workflows/code-quality.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,11 @@ jobs:
- name: Pyright
run: uv run pyright

- name: Config schema freshness
run: |
uv run raitap config-schema -o /tmp/raitap.schema.json
git diff --exit-code --no-index -- src/raitap/schema/raitap.schema.json /tmp/raitap.schema.json

- name: Pytest on 3.13 (with coverage)
run: uv run pytest -m "not e2e and not cuda" -v --cov=src/raitap --cov-report=term-missing --cov-report=html

Expand Down
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -100,3 +100,6 @@ contributor-configs/*/images/
/metrics/
/reports/
/transparency/

# Generated at docs build by docs/_ext/raitap_adapters.py (issue #301)
docs/modules/_generated_adapters.md
8 changes: 4 additions & 4 deletions contributor-configs/fasterrcnn-udacity/assessment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -40,15 +40,15 @@ data:
source: ${hydra:runtime.cwd}/contributor-configs/fasterrcnn-udacity/labels/udacity-boxes.json

metrics:
_target_: DetectionMetrics
use: detection

transparency:
# One per-box Integrated Gradients run. ``call.target`` must be 0 — the
# ScalarDetectionWrapper exposes a single scalar channel, so ``auto_pred``
# is rejected (see docs/modules/transparency/configuration.md "Detection
# knobs"). ``max_boxes: 3`` keeps the K-loop small enough for CPU CI.
detection_ig:
_target_: CaptumExplainer
use: captum
algorithm: IntegratedGradients
call:
target: 0
Expand All @@ -65,10 +65,10 @@ transparency:
layout: NCHW
show_sample_names: true
visualisers:
- _target_: DetectionImageVisualiser
- use: detection_image

reporting:
_target_: HTMLReporter
use: html
filename: fasterrcnn_udacity_report
# Force all 4 samples into the local-detail section (default cap is 3).
sample_selection: [0, 1, 2, 3]
Expand Down
10 changes: 5 additions & 5 deletions contributor-configs/imagecorruptions-imagenet/assessment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -36,20 +36,20 @@ metrics:
robustness:
# Average-case: ImageNet-C common corruption at a fixed severity.
gaussian_noise:
_target_: ImageCorruptionsAssessor
use: imagecorruptions
algorithm: gaussian_noise # any of the 15 ImageNet-C corruptions
constructor:
severity: 3 # 1..5
visualisers:
- _target_: CorruptionAccuracyVisualiser
- use: corruption_accuracy
# Worst-case: bounded PGD adversarial attack (torchattacks).
pgd:
_target_: TorchattacksAssessor
use: torchattacks
algorithm: PGD
constructor:
eps: 0.03
alpha: 0.008
steps: 10
visualisers:
- _target_: ImagePairVisualiser
- _target_: PerturbationHeatmapVisualiser
- use: image_pair
- use: perturbation_heatmap
38 changes: 19 additions & 19 deletions contributor-configs/lwise-ham10000/assessment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,12 @@ data:
column: label

metrics:
_target_: MulticlassClassificationMetrics
use: multiclass_classification
num_classes: 7

transparency:
gradcam_localisation:
_target_: CaptumExplainer
use: captum
algorithm: LayerGradCam
constructor:
layer_path: 1.layer4.2.conv3
Expand All @@ -49,7 +49,7 @@ transparency:
layout: NCHW
show_sample_names: true
visualisers:
- _target_: CaptumImageVisualiser
- use: captum_image
constructor:
method: heat_map
sign: positive
Expand All @@ -60,7 +60,7 @@ transparency:
max_samples: 4

saliency_evidence:
_target_: CaptumExplainer
use: captum
algorithm: Saliency
call:
target: auto_pred
Expand All @@ -72,7 +72,7 @@ transparency:
layout: NCHW
show_sample_names: true
visualisers:
- _target_: CaptumImageVisualiser
- use: captum_image
constructor:
method: blended_heat_map
sign: positive
Expand All @@ -81,7 +81,7 @@ transparency:
title: "Positive lesion evidence"
call:
max_samples: 4
- _target_: CaptumImageVisualiser
- use: captum_image
constructor:
method: masked_image
sign: positive
Expand All @@ -92,7 +92,7 @@ transparency:
max_samples: 4

occlusion_check:
_target_: CaptumExplainer
use: captum
algorithm: Occlusion
call:
target: auto_pred
Expand All @@ -106,7 +106,7 @@ transparency:
layout: NCHW
show_sample_names: true
visualisers:
- _target_: CaptumImageVisualiser
- use: captum_image
constructor:
method: heat_map
sign: absolute_value
Expand All @@ -117,7 +117,7 @@ transparency:
max_samples: 4

integrated_evidence_detail:
_target_: CaptumExplainer
use: captum
algorithm: IntegratedGradients
call:
target: auto_pred
Expand All @@ -130,7 +130,7 @@ transparency:
layout: NCHW
show_sample_names: true
visualisers:
- _target_: CaptumImageVisualiser
- use: captum_image
constructor:
method: blended_heat_map
sign: absolute_value
Expand All @@ -142,7 +142,7 @@ transparency:

robustness:
fgsm_linf_fast:
_target_: TorchattacksAssessor
use: torchattacks
algorithm: FGSM
constructor:
eps: 0.03
Expand All @@ -153,15 +153,15 @@ robustness:
layout: NCHW
show_sample_names: true
visualisers:
- _target_: ImagePairVisualiser
- use: image_pair
constructor:
max_samples: 4
- _target_: PerturbationHeatmapVisualiser
- use: perturbation_heatmap
constructor:
max_samples: 4

pgd_linf_small:
_target_: TorchattacksAssessor
use: torchattacks
algorithm: PGD
constructor:
eps: 0.03
Expand All @@ -174,25 +174,25 @@ robustness:
layout: NCHW
show_sample_names: true
visualisers:
- _target_: ImagePairVisualiser
- use: image_pair
constructor:
max_samples: 4
- _target_: PerturbationHeatmapVisualiser
- use: perturbation_heatmap
constructor:
max_samples: 4
marabou_linf:
_target_: MarabouAssessor
use: marabou
algorithm: linf-box
constructor:
epsilon: 0.05
norm: Linf
timeout_s: 300
visualisers:
- _target_: VerdictSummaryVisualiser
- use: verdict_summary


reporting:
_target_: HTMLReporter
use: html
filename: lwise_ham10000_report.pdf
include_config: true
include_metadata: true
Expand Down
2 changes: 1 addition & 1 deletion contributor-configs/lwise-ham10000/assessment_mlflow.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ defaults:
- _self_

tracking:
_target_: MLFlowTracker
use: mlflow
output_forwarding_url: http://127.0.0.1:5001
backend_store_uri: sqlite:///mlflow/mlflow.db
default_artifact_root: ./mlflow/artifacts
Expand Down
4 changes: 2 additions & 2 deletions contributor-configs/marabou-mnist/assessment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,11 @@ model:

robustness:
marabou_linf:
_target_: MarabouAssessor
use: marabou
algorithm: linf-box
constructor:
epsilon: 0.05
norm: Linf
timeout_s: 300
visualisers:
- _target_: VerdictSummaryVisualiser
- use: verdict_summary
10 changes: 5 additions & 5 deletions contributor-configs/noisetunnel-smoothgrad/assessment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ metrics:

transparency:
smoothgrad_over_saliency:
_target_: CaptumExplainer
use: captum
algorithm: NoiseTunnel
constructor:
base_algorithm: Saliency
Expand All @@ -49,7 +49,7 @@ transparency:
layout: NCHW
show_sample_names: true
visualisers:
- _target_: CaptumImageVisualiser
- use: captum_image
constructor:
method: blended_heat_map
sign: absolute_value
Expand All @@ -60,7 +60,7 @@ transparency:
max_samples: 4

saliency_baseline:
_target_: CaptumExplainer
use: captum
algorithm: Saliency
call:
target: auto_pred
Expand All @@ -72,7 +72,7 @@ transparency:
layout: NCHW
show_sample_names: true
visualisers:
- _target_: CaptumImageVisualiser
- use: captum_image
constructor:
method: blended_heat_map
sign: absolute_value
Expand All @@ -83,7 +83,7 @@ transparency:
max_samples: 4

reporting:
_target_: HTMLReporter
use: html
filename: noisetunnel_smoothgrad_report
include_config: true
include_metadata: true
8 changes: 4 additions & 4 deletions contributor-configs/quantus-eval/assessment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ metrics:

transparency:
integrated_gradients:
_target_: CaptumExplainer
use: captum
algorithm: IntegratedGradients
call:
target: auto_pred
Expand All @@ -60,7 +60,7 @@ transparency:
layout: NCHW
show_sample_names: true
visualisers:
- _target_: CaptumImageVisualiser
- use: captum_image
constructor:
method: blended_heat_map
sign: absolute_value
Expand All @@ -69,13 +69,13 @@ transparency:
call:
max_samples: 4
evaluation:
_target_: raitap.transparency.QuantusEvaluator
use: quantus
metrics: [faithfulness_correlation, sparseness]
constructor:
faithfulness_correlation: {nr_runs: 10, subset_size: 32}

reporting:
_target_: HTMLReporter
use: html
filename: quantus_eval_report
include_config: true
include_metadata: true
6 changes: 3 additions & 3 deletions contributor-configs/text-classification-sst2/assessment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ metrics:

transparency:
token_ig:
_target_: CaptumExplainer
use: captum
algorithm: LayerIntegratedGradients
constructor:
# LayerIntegratedGradients attributes wrt. the embedding layer output.
Expand All @@ -61,10 +61,10 @@ transparency:
kind: text
layout: TOKENS
visualisers:
- _target_: CaptumTextVisualiser
- use: captum_text

reporting:
_target_: HTMLReporter
use: html
filename: text_classification_sst2_report
include_config: true
include_metadata: true
6 changes: 3 additions & 3 deletions contributor-configs/tree-xgboost-shap/assessment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ data:

transparency:
treeshap:
_target_: ShapExplainer
use: shap
algorithm: TreeExplainer
call:
# Binary XGBoost SHAP returns stacked (B, F, 2); pick the positive class.
Expand All @@ -39,12 +39,12 @@ transparency:
layout: "(B,F)"
show_sample_names: true
visualisers:
- _target_: ShapBarVisualiser
- use: shap_bar
constructor:
feature_names: [f0, f1, f2, f3, f4, f5]

reporting:
_target_: HTMLReporter
use: html
filename: tree_xgboost_shap_report
include_config: true
include_metadata: true
Loading
Loading