From 5e356eb12396dee8ba34c28364ca899597bf6aee Mon Sep 17 00:00:00 2001 From: Max Wipfli Date: Fri, 6 Mar 2026 20:35:03 +0100 Subject: [PATCH 1/8] [evaluation] Add run_evaluation.py script Builds Dynamatic via ninja, then runs the embedded .dyn script for each kernel in parallel. --- tools/evaluation/run_evaluation.py | 174 +++++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 tools/evaluation/run_evaluation.py diff --git a/tools/evaluation/run_evaluation.py b/tools/evaluation/run_evaluation.py new file mode 100644 index 000000000..b5fdcb782 --- /dev/null +++ b/tools/evaluation/run_evaluation.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +""" +Dynamatic performance/resource-usage evaluation script. + +Builds Dynamatic, then runs the embedded .dyn script for each kernel, +collecting exit codes and logging failures. Supports parallel execution through +the -j flag. +""" + +import argparse +import logging +import shutil +import subprocess +import sys +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path + +# ────────────────────────────────────────────────────────────────────────────── +# Kernel list +# ────────────────────────────────────────────────────────────────────────────── +KERNELS = [ + "atax", + "atax_float", + "bicg", + "bicg_float", + "covariance", + "gaussian", + "gemver", + "gemver_float", + "get_tanh", + "histogram", + "insertion_sort", + "jacobi_1d_imper", + "kernel_2mm", + "kernel_2mm_float", + "kernel_3mm", + "kernel_3mm_float", + "kmp", + "loop_array", + "lu", + "matching", + "matching_2", + "matrix_power", + "pivot", + "polyn_mult", + "symm_float", + "syr2k_float", + "threshold", + "triangular", + "while_loop_1", +] + +# ────────────────────────────────────────────────────────────────────────────── +# .dyn script template: {src} is substituted with the kernel source path +# ────────────────────────────────────────────────────────────────────────────── +DYN_SCRIPT = """\ +set-src {src} +compile --buffer-algorithm fpga20 +write-hdl --hdl vhdl +simulate +synthesize +exit +""" + +# ────────────────────────────────────────────────────────────────────────────── +# Paths +# ────────────────────────────────────────────────────────────────────────────── +REPO_ROOT = Path(__file__).resolve().parent.parent + + +def setup_logging() -> None: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)-8s %(message)s", + datefmt="%H:%M:%S", + stream=sys.stdout, + ) + + +def build() -> None: + """Run ninja to build Dynamatic.""" + logging.info("Building Dynamatic (ninja -C build) …") + result = subprocess.run( + ["ninja", "-C", "build"], + cwd=REPO_ROOT, + ) + if result.returncode != 0: + logging.error("Build failed with exit code %d — aborting.", result.returncode) + sys.exit(result.returncode) + logging.info("Build succeeded.") + + +def run_kernel(kernel: str) -> tuple[str, int]: + """ + Run the .dyn script for *kernel*, writing stdout/stderr directly to + integration-test/{kernel}/out/dynamatic_{out,err}.txt, and return + (kernel, returncode). + """ + src = f"integration-test/{kernel}/{kernel}.c" + script = DYN_SCRIPT.format(src=src) + + out_dir = REPO_ROOT / "integration-test" / kernel / "out" + # ensure a clean output directory for the kernel + if out_dir.exists(): + shutil.rmtree(out_dir) + out_dir.mkdir(parents=True) + + with ( + open(out_dir / "dynamatic_out.txt", "w") as out_f, + open(out_dir / "dynamatic_err.txt", "w") as err_f, + ): + result = subprocess.run( + ["bin/dynamatic"], + input=script, + text=True, + stdout=out_f, + stderr=err_f, + cwd=REPO_ROOT, + ) + + return kernel, result.returncode + + +def main() -> None: + setup_logging() + + parser = argparse.ArgumentParser( + description="Run Dynamatic evaluation for a list of kernels." + ) + parser.add_argument( + "-j", + type=int, + default=1, + metavar="JOBS", + help="Number of kernels to run in parallel (default: 1).", + ) + args = parser.parse_args() + + build() + + logging.info("Running %d kernel(s) with -j %d …", len(KERNELS), args.j) + + passed: list[str] = [] + failed: list[tuple[str, int]] = [] + + with ThreadPoolExecutor(max_workers=args.j) as executor: + futures = {executor.submit(run_kernel, k): k for k in KERNELS} + for future in as_completed(futures): + kernel, returncode = future.result() + if returncode == 0: + logging.info("[PASS] %s", kernel) + passed.append(kernel) + else: + logging.error("[FAIL] %s (exit code %d)", kernel, returncode) + failed.append((kernel, returncode)) + + # Summary + total = len(KERNELS) + logging.info( + "Results: %d/%d passed, %d failed.", + len(passed), + total, + len(failed), + ) + if failed: + logging.error( + "Failed kernels: %s", + ", ".join(f"{k} (rc={rc})" for k, rc in sorted(failed)), + ) + sys.exit(1) + + +if __name__ == "__main__": + main() From a461e67c8bd4d07f2ed360e75367826c4b611704 Mon Sep 17 00:00:00 2001 From: Max Wipfli Date: Fri, 6 Mar 2026 20:41:15 +0100 Subject: [PATCH 2/8] [evaluation] Add --output-dir option to run_evaluation.py Archives each kernel's output directory after evaluation finishes. --- tools/evaluation/run_evaluation.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tools/evaluation/run_evaluation.py b/tools/evaluation/run_evaluation.py index b5fdcb782..181fd917b 100644 --- a/tools/evaluation/run_evaluation.py +++ b/tools/evaluation/run_evaluation.py @@ -85,7 +85,7 @@ def build() -> None: cwd=REPO_ROOT, ) if result.returncode != 0: - logging.error("Build failed with exit code %d — aborting.", result.returncode) + logging.error("Build failed with exit code %d. Aborting...", result.returncode) sys.exit(result.returncode) logging.info("Build succeeded.") @@ -134,8 +134,22 @@ def main() -> None: metavar="JOBS", help="Number of kernels to run in parallel (default: 1).", ) + parser.add_argument( + "-o", + "--output-dir", + type=Path, + default=None, + metavar="DIR", + help="Copy kernel out/ directories to DIR/{kernel}/out after each run.", + ) args = parser.parse_args() + if args.output_dir is not None: + if args.output_dir.exists(): + logging.error("Output directory %s already exists. Aborting...", args.output_dir) + sys.exit(1) + args.output_dir.mkdir(parents=True) + build() logging.info("Running %d kernel(s) with -j %d …", len(KERNELS), args.j) @@ -154,6 +168,11 @@ def main() -> None: logging.error("[FAIL] %s (exit code %d)", kernel, returncode) failed.append((kernel, returncode)) + if args.output_dir is not None: + src = REPO_ROOT / "integration-test" / kernel / "out" + dst = args.output_dir / kernel / "out" + shutil.move(src, dst) + # Summary total = len(KERNELS) logging.info( From a8cc31cc5acf5b726f28a807cb20e06461934f9d Mon Sep 17 00:00:00 2001 From: Max Wipfli Date: Fri, 6 Mar 2026 20:49:19 +0100 Subject: [PATCH 3/8] [evaluation] Minor fixes --- tools/evaluation/run_evaluation.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) mode change 100644 => 100755 tools/evaluation/run_evaluation.py diff --git a/tools/evaluation/run_evaluation.py b/tools/evaluation/run_evaluation.py old mode 100644 new mode 100755 index 181fd917b..692a9f764 --- a/tools/evaluation/run_evaluation.py +++ b/tools/evaluation/run_evaluation.py @@ -65,7 +65,7 @@ # ────────────────────────────────────────────────────────────────────────────── # Paths # ────────────────────────────────────────────────────────────────────────────── -REPO_ROOT = Path(__file__).resolve().parent.parent +REPO_ROOT = Path(__file__).resolve().parents[2] def setup_logging() -> None: @@ -129,6 +129,7 @@ def main() -> None: ) parser.add_argument( "-j", + "--jobs", type=int, default=1, metavar="JOBS", @@ -152,12 +153,12 @@ def main() -> None: build() - logging.info("Running %d kernel(s) with -j %d …", len(KERNELS), args.j) + logging.info("Running %d kernel(s) with %d parallel job(s) …", len(KERNELS), args.jobs) passed: list[str] = [] failed: list[tuple[str, int]] = [] - with ThreadPoolExecutor(max_workers=args.j) as executor: + with ThreadPoolExecutor(max_workers=args.jobs) as executor: futures = {executor.submit(run_kernel, k): k for k in KERNELS} for future in as_completed(futures): kernel, returncode = future.result() From 349a4cffaf0793acb04fe2d8170a94a3311b93f9 Mon Sep 17 00:00:00 2001 From: Max Wipfli Date: Fri, 6 Mar 2026 20:57:15 +0100 Subject: [PATCH 4/8] [evaluation] Add more immediate logging feedback while running kernels --- tools/evaluation/run_evaluation.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/evaluation/run_evaluation.py b/tools/evaluation/run_evaluation.py index 692a9f764..7c5230ce5 100755 --- a/tools/evaluation/run_evaluation.py +++ b/tools/evaluation/run_evaluation.py @@ -96,6 +96,7 @@ def run_kernel(kernel: str) -> tuple[str, int]: integration-test/{kernel}/out/dynamatic_{out,err}.txt, and return (kernel, returncode). """ + logging.info("Running kernel %s...", kernel) src = f"integration-test/{kernel}/{kernel}.c" script = DYN_SCRIPT.format(src=src) @@ -118,6 +119,10 @@ def run_kernel(kernel: str) -> tuple[str, int]: cwd=REPO_ROOT, ) + if result.returncode == 0: + logging.info("[PASS] %s", kernel) + else: + logging.error("[FAIL] %s (exit code %d)", kernel, result.returncode) return kernel, result.returncode @@ -163,10 +168,8 @@ def main() -> None: for future in as_completed(futures): kernel, returncode = future.result() if returncode == 0: - logging.info("[PASS] %s", kernel) passed.append(kernel) else: - logging.error("[FAIL] %s (exit code %d)", kernel, returncode) failed.append((kernel, returncode)) if args.output_dir is not None: From d97b143eccd52ecdef9a71c673e58d1ab0541dc8 Mon Sep 17 00:00:00 2001 From: Max Wipfli Date: Fri, 6 Mar 2026 21:01:24 +0100 Subject: [PATCH 5/8] [evaluation] Add total time measurement --- tools/evaluation/run_evaluation.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tools/evaluation/run_evaluation.py b/tools/evaluation/run_evaluation.py index 7c5230ce5..1ec41b726 100755 --- a/tools/evaluation/run_evaluation.py +++ b/tools/evaluation/run_evaluation.py @@ -12,6 +12,7 @@ import shutil import subprocess import sys +import time from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path @@ -79,7 +80,7 @@ def setup_logging() -> None: def build() -> None: """Run ninja to build Dynamatic.""" - logging.info("Building Dynamatic (ninja -C build) …") + logging.info("Building Dynamatic...") result = subprocess.run( ["ninja", "-C", "build"], cwd=REPO_ROOT, @@ -158,7 +159,8 @@ def main() -> None: build() - logging.info("Running %d kernel(s) with %d parallel job(s) …", len(KERNELS), args.jobs) + logging.info("Running %d kernel(s) with %d parallel job(s)...", len(KERNELS), args.jobs) + start_time = time.time() passed: list[str] = [] failed: list[tuple[str, int]] = [] @@ -178,6 +180,11 @@ def main() -> None: shutil.move(src, dst) # Summary + elapsed = int(time.time() - start_time) // 60 + hours = elapsed // 60 + minutes = elapsed % 60 + logging.info("Total time: %dh %02dm.", hours, minutes) + total = len(KERNELS) logging.info( "Results: %d/%d passed, %d failed.", From 0d3544b8631958451aa1d36ad696a7843e13594d Mon Sep 17 00:00:00 2001 From: Max Wipfli Date: Mon, 9 Mar 2026 20:46:40 +0100 Subject: [PATCH 6/8] [evaluation] Improve kernel pass/fail checks (catch silent failures) Check for FATAL in stdout, and "C and VHDL outputs match" in sim/report.txt. --- tools/evaluation/run_evaluation.py | 53 +++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/tools/evaluation/run_evaluation.py b/tools/evaluation/run_evaluation.py index 1ec41b726..574d7b369 100755 --- a/tools/evaluation/run_evaluation.py +++ b/tools/evaluation/run_evaluation.py @@ -91,11 +91,11 @@ def build() -> None: logging.info("Build succeeded.") -def run_kernel(kernel: str) -> tuple[str, int]: +def run_kernel(kernel: str) -> tuple[str, str | None]: """ Run the .dyn script for *kernel*, writing stdout/stderr directly to integration-test/{kernel}/out/dynamatic_{out,err}.txt, and return - (kernel, returncode). + (kernel, failure_reason). failure_reason is None on success. """ logging.info("Running kernel %s...", kernel) src = f"integration-test/{kernel}/{kernel}.c" @@ -107,10 +107,10 @@ def run_kernel(kernel: str) -> tuple[str, int]: shutil.rmtree(out_dir) out_dir.mkdir(parents=True) - with ( - open(out_dir / "dynamatic_out.txt", "w") as out_f, - open(out_dir / "dynamatic_err.txt", "w") as err_f, - ): + out_path = out_dir / "dynamatic_out.txt" + err_path = out_dir / "dynamatic_err.txt" + + with open(out_path, "w") as out_f, open(err_path, "w") as err_f: result = subprocess.run( ["bin/dynamatic"], input=script, @@ -120,11 +120,32 @@ def run_kernel(kernel: str) -> tuple[str, int]: cwd=REPO_ROOT, ) - if result.returncode == 0: - logging.info("[PASS] %s", kernel) - else: - logging.error("[FAIL] %s (exit code %d)", kernel, result.returncode) - return kernel, result.returncode + if result.returncode != 0: + reason = f"exit code {result.returncode}" + logging.error("[FAIL] %s (%s)", kernel, reason) + return kernel, reason + + # Check stdout for FATAL messages + if "FATAL" in out_path.read_text(errors="replace"): + reason = "FATAL in stdout" + logging.error("[FAIL] %s (%s)", kernel, reason) + return kernel, reason + + # Check simulation report + report_path = out_dir / "sim" / "report.txt" + if not report_path.exists(): + reason = "sim/report.txt not found" + logging.error("[FAIL] %s (%s)", kernel, reason) + return kernel, reason + + report_text = report_path.read_text(errors="replace") + if "C and VHDL outputs match" not in report_text: + reason = 'sim/report.txt missing "C and VHDL outputs match"' + logging.error("[FAIL] %s (%s)", kernel, reason) + return kernel, reason + + logging.info("[PASS] %s", kernel) + return kernel, None def main() -> None: @@ -163,16 +184,16 @@ def main() -> None: start_time = time.time() passed: list[str] = [] - failed: list[tuple[str, int]] = [] + failed: list[tuple[str, str | None]] = [] with ThreadPoolExecutor(max_workers=args.jobs) as executor: futures = {executor.submit(run_kernel, k): k for k in KERNELS} for future in as_completed(futures): - kernel, returncode = future.result() - if returncode == 0: + kernel, failure_reason = future.result() + if failure_reason is None: passed.append(kernel) else: - failed.append((kernel, returncode)) + failed.append((kernel, failure_reason)) if args.output_dir is not None: src = REPO_ROOT / "integration-test" / kernel / "out" @@ -195,7 +216,7 @@ def main() -> None: if failed: logging.error( "Failed kernels: %s", - ", ".join(f"{k} (rc={rc})" for k, rc in sorted(failed)), + ", ".join(f"{k} ({reason})" for k, reason in sorted(failed)), ) sys.exit(1) From ad2538737bd617a102aa738fef80dd3b70f5a9c2 Mon Sep 17 00:00:00 2001 From: Max Wipfli Date: Mon, 9 Mar 2026 20:47:53 +0100 Subject: [PATCH 7/8] [evaluation] Increase clock period to 5 ns There are some issues with the default (4 ns) clock period. For example, histogram takes almost twice as many cycles for some reason when reducing the period from 5 ns to 4 ns. --- tools/evaluation/run_evaluation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/evaluation/run_evaluation.py b/tools/evaluation/run_evaluation.py index 574d7b369..5216ff4ac 100755 --- a/tools/evaluation/run_evaluation.py +++ b/tools/evaluation/run_evaluation.py @@ -56,6 +56,7 @@ # ────────────────────────────────────────────────────────────────────────────── DYN_SCRIPT = """\ set-src {src} +set-clock-period 5 compile --buffer-algorithm fpga20 write-hdl --hdl vhdl simulate From 461149773a0afb77941e999bb6b5ec28dc88a412 Mon Sep 17 00:00:00 2001 From: Max Wipfli Date: Tue, 10 Mar 2026 10:36:56 +0100 Subject: [PATCH 8/8] [evaluation] Add script to extract key metrics from evaluation data into CSV --- tools/evaluation/extract_evaluation_data.py | 124 ++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 tools/evaluation/extract_evaluation_data.py diff --git a/tools/evaluation/extract_evaluation_data.py b/tools/evaluation/extract_evaluation_data.py new file mode 100644 index 000000000..104df0cfd --- /dev/null +++ b/tools/evaluation/extract_evaluation_data.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +""" +Dynamatic evaluation data extraction script + +Extracts key metrics from evaluation data and writes it to a CSV file. +""" + +import argparse +import csv +import re +import sys +from pathlib import Path + + +def parse_sim_report(path: Path): + """Return (cycle_count, passed) from a simulation report.txt.""" + text = path.read_text() + passed = "C and VHDL outputs match" in text + m = re.search(r"Simulation done!\s+Latency\s*=\s*(\d+)\s+cycles", text) + cycle_count = int(m.group(1)) if m else None + return cycle_count, passed + + +def parse_utilization(path: Path): + """Return (slices, luts, ffs) from a utilization report.""" + slices = luts = ffs = None + for line in path.read_text().splitlines(): + # Match table rows like "| Slice LUTs | 4029 | ..." + # Use the most-specific patterns first. + if m := re.match(r"\|\s*Slice LUTs\s*\|\s*(\d+)", line): + luts = int(m.group(1)) + elif m := re.match(r"\|\s*Slice Registers\s*\|\s*(\d+)", line): + ffs = int(m.group(1)) + elif m := re.match(r"\|\s*Slice\s*\|\s*(\d+)", line): + slices = int(m.group(1)) + return slices, luts, ffs + + +def parse_timing(path: Path): + """Return (cp_ns, slack_ns, cp_src, cp_dst) from a timing report.""" + cp_ns = slack_ns = cp_src = cp_dst = None + for line in path.read_text().splitlines(): + if m := re.match(r"\s*Slack\s*\([^)]*\)\s*:\s*(-?[\d.]+)ns", line): + slack_ns = float(m.group(1)) + elif m := re.match(r"\s*Data Path Delay:\s*([\d.]+)ns", line): + cp_ns = float(m.group(1)) + elif m := re.match(r"\s*Source:\s*(\S+)", line): + cp_src = m.group(1) + elif m := re.match(r"\s*Destination:\s*(\S+)", line): + cp_dst = m.group(1) + return cp_ns, slack_ns, cp_src, cp_dst + + +def main(): + parser = argparse.ArgumentParser( + description="Extract evaluation metrics from run_evaluation.py output directory." + ) + parser.add_argument("eval_dir", help="Path to the evaluation output directory") + args = parser.parse_args() + + eval_dir = Path(args.eval_dir) + if not eval_dir.is_dir(): + print(f"Error: {eval_dir} is not a directory", file=sys.stderr) + sys.exit(1) + + columns = [ + "kernel", + "cycle_count", + "utilization_slice", + "utilization_lut", + "utilization_ff", + "timing_cp_ns", + "timing_slack_ns", + "timing_cp_src", + "timing_cp_dst", + ] + + writer = csv.DictWriter(sys.stdout, fieldnames=columns) + writer.writeheader() + + for kernel_dir in sorted(eval_dir.iterdir()): + if not kernel_dir.is_dir(): + continue + kernel = kernel_dir.name + + row: dict = {"kernel": kernel} + + # Simulation report + sim_report = kernel_dir / "out" / "sim" / "report.txt" + if not sim_report.exists(): + print(f"Error: {sim_report} not found", file=sys.stderr) + sys.exit(1) + cycle_count, passed = parse_sim_report(sim_report) + if not passed: + print(f"Error: {kernel}: C and VHDL outputs do not match", file=sys.stderr) + sys.exit(1) + row["cycle_count"] = cycle_count + + # Utilization report + util_rpt = kernel_dir / "out" / "synth" / "utilization_post_pr.rpt" + if not util_rpt.exists(): + print(f"Error: {util_rpt} not found", file=sys.stderr) + sys.exit(1) + slices, luts, ffs = parse_utilization(util_rpt) + row["utilization_slice"] = slices + row["utilization_lut"] = luts + row["utilization_ff"] = ffs + + # Timing report + timing_rpt = kernel_dir / "out" / "synth" / "timing_post_pr.rpt" + if not timing_rpt.exists(): + print(f"Error: {timing_rpt} not found", file=sys.stderr) + sys.exit(1) + cp_ns, slack_ns, cp_src, cp_dst = parse_timing(timing_rpt) + row["timing_cp_ns"] = cp_ns + row["timing_slack_ns"] = slack_ns + row["timing_cp_src"] = cp_src + row["timing_cp_dst"] = cp_dst + + writer.writerow(row) + + +if __name__ == "__main__": + main()