From e7c7c9c1bd6849c90822cea82a5328ed4365ff28 Mon Sep 17 00:00:00 2001 From: Jimmy Ma Date: Thu, 23 Jul 2026 02:23:32 -0700 Subject: [PATCH 1/6] test(common): add comparison_chunk_size to assertCorrectness + compare_tensors test --- tests/common.py | 160 ++++++++++++++++++++++- tests/test_utils/test_compare_tensors.py | 47 +++++++ 2 files changed, 206 insertions(+), 1 deletion(-) create mode 100644 tests/test_utils/test_compare_tensors.py diff --git a/tests/common.py b/tests/common.py index 074ba7d..0655195 100644 --- a/tests/common.py +++ b/tests/common.py @@ -136,6 +136,7 @@ def assertCorrectness( test_index=None, ref_index=None, output_processor=None, + comparison_chunk_size=None, ): r""" Check that a specified test function matches the reference. @@ -169,6 +170,9 @@ def assertCorrectness( at this index incase of multiple outputs. Default is None ref_index: Compare a specific tensor from the reference output tuple at this index incase of multiple outputs. Default is None + comparison_chunk_size: If set, compare tensors in slices of this + many elements along the first dimension to bound temporary + memory used by the correctness checker. """ passed = True all_msgs = [] @@ -252,7 +256,15 @@ def assertCorrectness( if output_processor is not None: to = output_processor(ind, to, fn_kwargs, extra_test_kwargs, extra_ref_kwargs) ro = output_processor(ind, ro, fn_kwargs, extra_test_kwargs, extra_ref_kwargs) - out_close, msg = compare_tensors(to, ro, rtol, atol, equal_nan, check_stride) + out_close, msg = compare_tensors( + to, + ro, + rtol, + atol, + equal_nan, + check_stride, + chunk_size=comparison_chunk_size, + ) if not out_close: passed = False prefix = f"*** OUTPUT {ind} DID NOT MATCH THE REFERENCE (rtol={rtol}, atol={atol}) ***" @@ -301,6 +313,7 @@ def assertCorrectness( grad_atol, equal_nan, check_stride, + chunk_size=comparison_chunk_size, ) if not grad_close: @@ -679,6 +692,7 @@ def compare_tensors( equal_nan=False, check_stride=True, msg_prefix="\t", + chunk_size=None, ): # Auto-detect tolerances based on data type if not provided if rtol is None or atol is None: @@ -708,6 +722,20 @@ def compare_tensors( msgs = f"dtype mismatch, test: {test.dtype}, reference: {reference.dtype}" raise RuntimeError(msgs) + if chunk_size is not None: + if chunk_size <= 0: + raise ValueError(f"chunk_size must be positive, got {chunk_size}") + if test.ndim > 0 and test.shape[0] > chunk_size: + return _compare_tensors_chunked( + test, + reference, + rtol, + atol, + equal_nan, + msg_prefix, + chunk_size, + ) + dtype = test.dtype input = test.to(torch.float32) reference = reference.to(torch.float32) @@ -764,6 +792,136 @@ def compare_tensors( return allclose, msgs +def _compare_tensors_chunked( + test, + reference, + rtol, + atol, + equal_nan, + msg_prefix, + chunk_size, +): + """Compare tensor slices while preserving compare_tensors diagnostics.""" + + allclose = True + close_count = 0 + mismatched_indices = [] + + ref_min = None + ref_max = None + test_min = None + test_max = None + abs_ref_min = None + abs_ref_max = None + abs_test_min = None + abs_test_max = None + max_abs_diff = None + max_rel_change = None + max_max_mean_change = None + max_arith_mean_change = None + + with torch.no_grad(): + for start in range(0, test.shape[0], chunk_size): + end = min(start + chunk_size, test.shape[0]) + input = test[start:end].to(torch.float32) + ref = reference[start:end].to(torch.float32) + input = torch.where(torch.isnan(ref), float("nan"), input) + + allclose = bool(torch.allclose(input, ref, rtol, atol, equal_nan)) and allclose + + abs_diff = (input - ref).abs() + abs_ref = ref.abs() + abs_input = input.abs() + + is_close = abs_diff <= (atol + rtol * abs_ref) + not_close_mask = is_close.logical_not() + close_count += is_close.sum().item() + + chunk_mismatched_indices = not_close_mask.nonzero().cpu() + if chunk_mismatched_indices.numel() != 0: + chunk_mismatched_indices[:, 0] += start + mismatched_indices.append(chunk_mismatched_indices) + + rel_change = abs_diff / abs_ref + rel_change[abs_ref == 0] = 0 + + max_mean_change_denom = torch.max(abs_input, abs_ref) + max_mean_change = abs_diff / max_mean_change_denom + max_mean_change[max_mean_change_denom == 0] = 0 + + arith_mean_change_denom = 0.5 * (input + ref).abs() + arith_mean_change = abs_diff / arith_mean_change_denom + arith_mean_change[arith_mean_change_denom == 0] = 0 + + chunk_ref_min = ref.min() + chunk_ref_max = ref.max() + chunk_test_min = input.min() + chunk_test_max = input.max() + chunk_abs_ref_min = abs_ref.min() + chunk_abs_ref_max = abs_ref.max() + chunk_abs_test_min = abs_input.min() + chunk_abs_test_max = abs_input.max() + chunk_max_abs_diff = abs_diff.max() + chunk_max_rel_change = rel_change.max() + chunk_max_max_mean_change = max_mean_change.max() + chunk_max_arith_mean_change = arith_mean_change.max() + + ref_min = chunk_ref_min if ref_min is None else torch.minimum(ref_min, chunk_ref_min) + ref_max = chunk_ref_max if ref_max is None else torch.maximum(ref_max, chunk_ref_max) + test_min = chunk_test_min if test_min is None else torch.minimum(test_min, chunk_test_min) + test_max = chunk_test_max if test_max is None else torch.maximum(test_max, chunk_test_max) + abs_ref_min = chunk_abs_ref_min if abs_ref_min is None else torch.minimum(abs_ref_min, chunk_abs_ref_min) + abs_ref_max = chunk_abs_ref_max if abs_ref_max is None else torch.maximum(abs_ref_max, chunk_abs_ref_max) + abs_test_min = ( + chunk_abs_test_min if abs_test_min is None else torch.minimum(abs_test_min, chunk_abs_test_min) + ) + abs_test_max = ( + chunk_abs_test_max if abs_test_max is None else torch.maximum(abs_test_max, chunk_abs_test_max) + ) + max_abs_diff = ( + chunk_max_abs_diff if max_abs_diff is None else torch.maximum(max_abs_diff, chunk_max_abs_diff) + ) + max_rel_change = ( + chunk_max_rel_change if max_rel_change is None else torch.maximum(max_rel_change, chunk_max_rel_change) + ) + max_max_mean_change = ( + chunk_max_max_mean_change + if max_max_mean_change is None + else torch.maximum(max_max_mean_change, chunk_max_max_mean_change) + ) + max_arith_mean_change = ( + chunk_max_arith_mean_change + if max_arith_mean_change is None + else torch.maximum(max_arith_mean_change, chunk_max_arith_mean_change) + ) + + if mismatched_indices: + mismatched_indices = torch.cat(mismatched_indices) + else: + mismatched_indices = torch.empty((0, test.ndim), dtype=torch.int64) + + total_count = test.numel() + close_percent = close_count / total_count * 100 + msgs = [ + f"allclose: {allclose}", + f"matched: {close_count} / {total_count} [{close_percent:.2f}%]", + f"ref range: {ref_min:11.4e} : {ref_max:11.4e}", + f"test range: {test_min:11.4e} : {test_max:11.4e}", + f"|ref| range: {abs_ref_min:11.4e} : {abs_ref_max:11.4e}", + f"|test| range: {abs_test_min:11.4e} : {abs_test_max:11.4e}", + f"max absolute difference: {max_abs_diff:11.4e}", + f"max relative change: {max_rel_change:11.4e}", + f"max max mean change: {max_max_mean_change:11.4e}", + f"max arith mean change: {max_arith_mean_change:11.4e}", + f"shape: {test.shape} stride: {test.stride()} dtype: {test.dtype}", + f"mismatched indices:{mismatched_indices}", + ] + if msg_prefix is not None: + msgs = [f"{msg_prefix}{msg}" for msg in msgs] + + return allclose, msgs + + def any_output_requires_grad(fn): out = fn() if not isinstance(out, tuple): diff --git a/tests/test_utils/test_compare_tensors.py b/tests/test_utils/test_compare_tensors.py new file mode 100644 index 0000000..07a925c --- /dev/null +++ b/tests/test_utils/test_compare_tensors.py @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: MIT + +import torch + +from tests import common + + +def test_chunked_compare_matches_full_diagnostics(): + reference = torch.tensor( + [ + [[1.0, -2.0], [3.0, 4.0]], + [[5.0, 6.0], [7.0, 8.0]], + [[9.0, 10.0], [11.0, 12.0]], + [[13.0, 14.0], [15.0, 16.0]], + [[17.0, 18.0], [19.0, 20.0]], + ], + dtype=torch.bfloat16, + ) + test = reference.clone() + test[1, 0, 1] = 6.5 + test[4, 1, 0] = 21.0 + + full_result = common.compare_tensors(test, reference, rtol=1e-2, atol=1e-2, msg_prefix=None) + chunked_result = common.compare_tensors( + test, + reference, + rtol=1e-2, + atol=1e-2, + msg_prefix=None, + chunk_size=2, + ) + + assert chunked_result == full_result + + +def test_chunked_compare_rejects_nonpositive_chunk_size(): + tensor = torch.ones(2) + + for chunk_size in (0, -1): + try: + common.compare_tensors(tensor, tensor, chunk_size=chunk_size) + except ValueError as error: + assert str(error) == f"chunk_size must be positive, got {chunk_size}" + else: + raise AssertionError("compare_tensors accepted a nonpositive chunk size") From e630e94ea1843d49c68c09489a972ba9d0576e3b Mon Sep 17 00:00:00 2001 From: Xuanda Yang Date: Thu, 23 Jul 2026 22:35:34 -0700 Subject: [PATCH 2/6] chore(skill): move version/environment/requires into metadata Signed-off-by: Xuanda Yang --- skills/tilegym-improve-cutile-kernel-perf/SKILL.md | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/skills/tilegym-improve-cutile-kernel-perf/SKILL.md b/skills/tilegym-improve-cutile-kernel-perf/SKILL.md index 3696a18..46eaeae 100644 --- a/skills/tilegym-improve-cutile-kernel-perf/SKILL.md +++ b/skills/tilegym-improve-cutile-kernel-perf/SKILL.md @@ -1,18 +1,12 @@ --- name: tilegym-improve-cutile-kernel-perf description: Iteratively optimize cuTile kernel performance through systematic profiling, bottleneck analysis, IR comparison, and targeted tuning. Covers tile sizes, occupancy, autotune configs, TMA, latency hints, persistent scheduling, num_ctas, flush_to_zero, and IR-level debugging. Use when asked to "optimize cutile kernel", "improve kernel perf", "tune cutile performance", "make kernel faster", or iteratively benchmark and refine a cuTile GPU kernel in the TileGym project. -version: 2026.04.11 -environment: - IDE: - - Claude Code - - Cursor (Agent mode) - model: - - Opus 4.6 -requires: -- GPU node Blackwell, Hopper and Ampere for benchmarking license: CC-BY-4.0 AND Apache-2.0 metadata: author: "TileGym Team " + version: "2026.04.11" + environment: "IDE: Claude Code, Cursor (Agent mode); model: Opus 4.6" + requires: "GPU node Blackwell, Hopper and Ampere for benchmarking" tags: - cutile - performance From b01f6cbd24096ac8bdfd2f0069b877412dcdc73c Mon Sep 17 00:00:00 2001 From: Hannah Li Date: Fri, 24 Jul 2026 01:22:39 -0700 Subject: [PATCH 3/6] chore(backend): tidy helper naming and comments --- README.md | 4 ++-- README_chs.md | 4 ++-- README_cht.md | 4 ++-- README_fr.md | 4 ++-- README_ja.md | 4 ++-- src/tilegym/backend/selector.py | 4 ++-- src/tilegym/suites/unsloth/cutile/layernorm.py | 6 +++--- src/tilegym/suites/unsloth/cutile/rms_layernorm.py | 6 +++--- tests/suites/liger/test_multi_token_attention.py | 14 +++++++------- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index b53e77a..9f2043a 100644 --- a/README.md +++ b/README.md @@ -93,10 +93,10 @@ To use the Triton CUDA Tile IR backend, install its wheel into a separate direct ```bash # Install into a separate directory, kept apart from the default environment -pip install --target /opt/nvtriton .whl +pip install --target /opt/triton_tileir .whl # Select the Triton CUDA Tile IR backend at runtime -PYTHONPATH=/opt/nvtriton ENABLE_TILE=1 python your_script.py +PYTHONPATH=/opt/triton_tileir ENABLE_TILE=1 python your_script.py ``` ## Quick Start diff --git a/README_chs.md b/README_chs.md index 157005f..c46978c 100644 --- a/README_chs.md +++ b/README_chs.md @@ -93,10 +93,10 @@ TileGym 为以下后端提供内核，每个后端的内核位于 `src/tilegym/o ```bash # 安装到一个独立的目录，与默认环境隔离 -pip install --target /opt/nvtriton .whl +pip install --target /opt/triton_tileir .whl # 在运行时选择 Triton CUDA Tile IR 后端 -PYTHONPATH=/opt/nvtriton ENABLE_TILE=1 python your_script.py +PYTHONPATH=/opt/triton_tileir ENABLE_TILE=1 python your_script.py ``` ## 快速开始 diff --git a/README_cht.md b/README_cht.md index 74b6fc9..2f0e2a6 100644 --- a/README_cht.md +++ b/README_cht.md @@ -93,10 +93,10 @@ TileGym 為以下後端提供核心，每個後端的核心位於 `src/tilegym/o ```bash # 安裝到一個獨立的目錄，與預設環境隔離 -pip install --target /opt/nvtriton .whl +pip install --target /opt/triton_tileir .whl # 在執行時選擇 Triton CUDA Tile IR 後端 -PYTHONPATH=/opt/nvtriton ENABLE_TILE=1 python your_script.py +PYTHONPATH=/opt/triton_tileir ENABLE_TILE=1 python your_script.py ``` ## 快速開始 diff --git a/README_fr.md b/README_fr.md index b23ea2a..1e44a31 100644 --- a/README_fr.md +++ b/README_fr.md @@ -93,10 +93,10 @@ Pour utiliser le backend Triton CUDA Tile IR, installez son wheel dans un réper ```bash # Installez dans un répertoire séparé, à l'écart de l'environnement par défaut -pip install --target /opt/nvtriton .whl +pip install --target /opt/triton_tileir .whl # Sélectionnez le backend Triton CUDA Tile IR à l'exécution -PYTHONPATH=/opt/nvtriton ENABLE_TILE=1 python your_script.py +PYTHONPATH=/opt/triton_tileir ENABLE_TILE=1 python your_script.py ``` ## Démarrage rapide diff --git a/README_ja.md b/README_ja.md index 4ef13dc..8372f48 100644 --- a/README_ja.md +++ b/README_ja.md @@ -93,10 +93,10 @@ Triton CUDA Tile IR バックエンドを使用するには、その wheel を ```bash # デフォルト環境とは分離した別のディレクトリにインストールします -pip install --target /opt/nvtriton .whl +pip install --target /opt/triton_tileir .whl # 実行時に Triton CUDA Tile IR バックエンドを選択します -PYTHONPATH=/opt/nvtriton ENABLE_TILE=1 python your_script.py +PYTHONPATH=/opt/triton_tileir ENABLE_TILE=1 python your_script.py ``` ## クイックスタート diff --git a/src/tilegym/backend/selector.py b/src/tilegym/backend/selector.py index 662438b..69a2fa7 100644 --- a/src/tilegym/backend/selector.py +++ b/src/tilegym/backend/selector.py @@ -17,7 +17,7 @@ logger = get_logger(__name__) -def is_nvt_available(): +def is_triton_tileir_available(): try: import triton.backends.tileir @@ -220,7 +220,7 @@ def get_available_backends() -> Set[str]: def get_available_triton_backend() -> str: - if is_nvt_available(): + if is_triton_tileir_available(): return "nvt" return "oait" diff --git a/src/tilegym/suites/unsloth/cutile/layernorm.py b/src/tilegym/suites/unsloth/cutile/layernorm.py index 87fbbc8..0a4f471 100644 --- a/src/tilegym/suites/unsloth/cutile/layernorm.py +++ b/src/tilegym/suites/unsloth/cutile/layernorm.py @@ -15,7 +15,7 @@ Uses cuda.tile_experimental.autotune_launch to search occupancy=[1, 2, 4, 8]. Low occupancy (occ=1, 102 regs) optimal for sparse grids (small n_rows); high occupancy (occ=4, 62 regs) optimal for dense grids (large n_rows). - Both produce flat load_ptr_tko IR (matching NVT pattern), avoiding tensor_view/ + Both produce flat load_ptr_tko IR (matching Triton-TileIR pattern), avoiding tensor_view/ partition_view abstraction that causes predicate explosion. - _layernorm_backward_ct_1d: 1D gather/scatter backward with autotune over occupancy. Matches forward pattern: scalar loads for inv_var/mean, 1D reductions. @@ -41,7 +41,7 @@ def _layernorm_forward_ct_1d_body(Y, X, W, b, r, mu, n_cols, eps, TILE_N): """ Shared kernel body for 1D LayerNorm forward. - Uses ct.gather/ct.scatter producing flat load_ptr_tko IR (matching NVT pattern), + Uses ct.gather/ct.scatter producing flat load_ptr_tko IR (matching Triton-TileIR pattern), avoiding tensor_view/partition_view abstraction that causes predicate explosion. When TILE_N == n_cols (no padding), check_bounds=False is used on all @@ -117,7 +117,7 @@ def _layernorm_backward_ct_1d_body(dX, dY, X, W, r, mu, n_cols, TILE_N): """ 1D LayerNorm backward body using ct.gather/ct.scatter. - Produces flat load_ptr_tko IR (matching NVT pattern), avoiding + Produces flat load_ptr_tko IR (matching Triton-TileIR pattern), avoiding tensor_view/partition_view abstraction that causes predicate explosion. Scalar loads for inv_var and mean via ct.gather().item() — no reshapes. diff --git a/src/tilegym/suites/unsloth/cutile/rms_layernorm.py b/src/tilegym/suites/unsloth/cutile/rms_layernorm.py index 427d99d..73617f2 100644 --- a/src/tilegym/suites/unsloth/cutile/rms_layernorm.py +++ b/src/tilegym/suites/unsloth/cutile/rms_layernorm.py @@ -12,7 +12,7 @@ CuTile kernels: - _rms_layernorm_forward_ct_1d: 1D gather/scatter forward with autotune over occupancy. Uses cuda.tile_experimental.autotune_launch to search occupancy=[1, 2, 4, 8]. - Both produce flat load_ptr_tko IR (matching NVT pattern), avoiding tensor_view/ + Both produce flat load_ptr_tko IR (matching Triton-TileIR pattern), avoiding tensor_view/ partition_view abstraction that causes predicate explosion. - _rms_layernorm_backward_ct_1d: 1D gather/scatter backward with autotune over occupancy. Matches forward pattern: scalar load for inv_var, 1D reductions. @@ -46,7 +46,7 @@ def _rms_layernorm_forward_ct_1d_body(Y, X, W, r, n_cols, eps, OFFSET, TILE_N): """ Shared kernel body for 1D RMS LayerNorm forward. - Uses ct.gather/ct.scatter producing flat load_ptr_tko IR (matching NVT pattern), + Uses ct.gather/ct.scatter producing flat load_ptr_tko IR (matching Triton-TileIR pattern), avoiding tensor_view/partition_view abstraction that causes predicate explosion. When TILE_N == n_cols (no padding), check_bounds=False is used on all @@ -115,7 +115,7 @@ def _rms_layernorm_backward_ct_1d_body(dX, dY, X, W, r, n_cols, OFFSET, TILE_N): """ 1D RMS LayerNorm backward body using ct.gather/ct.scatter. - Produces flat load_ptr_tko IR (matching NVT pattern), avoiding + Produces flat load_ptr_tko IR (matching Triton-TileIR pattern), avoiding tensor_view/partition_view abstraction that causes predicate explosion. Scalar load for inv_var via ct.gather().item() — no reshapes. diff --git a/tests/suites/liger/test_multi_token_attention.py b/tests/suites/liger/test_multi_token_attention.py index 1c90055..f8dd3c1 100644 --- a/tests/suites/liger/test_multi_token_attention.py +++ b/tests/suites/liger/test_multi_token_attention.py @@ -153,21 +153,21 @@ def test_op_backward_conv_params(self, batch, channels, L, groups, dtype, backen _reference_multi_token_attention(s_ref, w_ref, groups=groups).sum().backward() # tilegym implementation - s_nvt = s.detach().requires_grad_(True) - w_nvt = w.detach().requires_grad_(True) - multi_token_attention(s_nvt, w_nvt, groups=groups).sum().backward() + s_out = s.detach().requires_grad_(True) + w_out = w.detach().requires_grad_(True) + multi_token_attention(s_out, w_out, groups=groups).sum().backward() # fp32 is tight; bf16 runs natively with inherent precision loss vs fp32 ref atol = 1e-4 if dtype == torch.float32 else 0.1 rtol = 1e-3 if dtype == torch.float32 else 0.1 - assert torch.allclose(s_nvt.grad.float(), s_ref.grad, atol=atol, rtol=rtol), ( + assert torch.allclose(s_out.grad.float(), s_ref.grad, atol=atol, rtol=rtol), ( f"scores.grad mismatch (dtype={dtype}, groups={groups}, " - f"max_err={(s_nvt.grad.float() - s_ref.grad).abs().max().item():.5f})" + f"max_err={(s_out.grad.float() - s_ref.grad).abs().max().item():.5f})" ) - assert torch.allclose(w_nvt.grad.float(), w_ref.grad, atol=atol, rtol=rtol), ( + assert torch.allclose(w_out.grad.float(), w_ref.grad, atol=atol, rtol=rtol), ( f"weight.grad mismatch (dtype={dtype}, groups={groups}, " - f"max_err={(w_nvt.grad.float() - w_ref.grad).abs().max().item():.5f})" + f"max_err={(w_out.grad.float() - w_ref.grad).abs().max().item():.5f})" ) @pytest.mark.parametrize( From cb945c8529abcf082027b1d9903ac662a166ba4f Mon Sep 17 00:00:00 2001 From: Hannah Li Date: Mon, 27 Jul 2026 07:46:27 +0800 Subject: [PATCH 4/6] fix(skill): use SemVer-valid version for improve-cutile-kernel-perf --- skills/tilegym-improve-cutile-kernel-perf/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skills/tilegym-improve-cutile-kernel-perf/SKILL.md b/skills/tilegym-improve-cutile-kernel-perf/SKILL.md index 46eaeae..9489ded 100644 --- a/skills/tilegym-improve-cutile-kernel-perf/SKILL.md +++ b/skills/tilegym-improve-cutile-kernel-perf/SKILL.md @@ -4,7 +4,7 @@ description: Iteratively optimize cuTile kernel performance through systematic p license: CC-BY-4.0 AND Apache-2.0 metadata: author: "TileGym Team " - version: "2026.04.11" + version: "2026.4.11" environment: "IDE: Claude Code, Cursor (Agent mode); model: Opus 4.6" requires: "GPU node Blackwell, Hopper and Ampere for benchmarking" tags: From 1f9e5a4115a2491b4947f029126d7e4947dcfcbb Mon Sep 17 00:00:00 2001 From: nvskills-svc-account Date: Mon, 27 Jul 2026 00:18:25 +0000 Subject: [PATCH 5/6] Attach NVSkills validation signatures Signed-off-by: nvskills-svc-account --- .../BENCHMARK.md | 54 ++++++------------- .../skill-card.md | 41 +++++++------- .../skill.oms.sig | 2 +- 3 files changed, 41 insertions(+), 56 deletions(-) diff --git a/skills/tilegym-improve-cutile-kernel-perf/BENCHMARK.md b/skills/tilegym-improve-cutile-kernel-perf/BENCHMARK.md index 0c9db1d..2040388 100644 --- a/skills/tilegym-improve-cutile-kernel-perf/BENCHMARK.md +++ b/skills/tilegym-improve-cutile-kernel-perf/BENCHMARK.md @@ -1,25 +1,23 @@ # Evaluation Report -Evaluation of the `tilegym-improve-cutile-kernel-perf` skill before publication through NVSkills-Eval. +Evaluation of the `tilegym-improve-cutile-kernel-perf` skill before publication through Skill Evaluator. -This benchmark summarizes 3-Tier Evaluation from NVSkills-Eval results for the skill. The goal is to document whether the skill is safe, discoverable, effective, and useful for agents before it is published for broader workflow use. +This benchmark summarizes 3-Tier Evaluation from Skill Evaluator results for the skill. The goal is to document whether the skill is safe, discoverable, effective, and useful for agents before it is published for broader workflow use. ## Evaluation Summary - Skill: `tilegym-improve-cutile-kernel-perf` -- Evaluation date: 2026-06-10 -- NVSkills-Eval profile: `external` -- Environment: `astra-sandbox` +- Evaluation date: 2026-07-27 +- Environment: `k8s-sandbox` - Dataset: 5 evaluation tasks - Attempts per task: 1 - Pass threshold: 50% -- Overall verdict: FAIL -The skill should be reviewed before NVSkills-Eval publication. **Skill owners should address the applicable findings below and rerun NVSkills-Eval to refresh this benchmark.** +- Overall verdict: PASS ## Agents Used -- `claude-code` -- `codex` +- Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`) +- Codex (`openai/openai/gpt-5.5`) ## Metrics Used @@ -39,7 +37,6 @@ Underlying evaluation signals used in this run: - `accuracy` (Accuracy): grades final-answer correctness against the reference answer. - `goal_accuracy` (Goal Accuracy): checks whether the overall user task completed successfully. - `behavior_check` (Behavior Check): verifies expected behavior steps, including safety expectations. -- `token_efficiency` (Token Efficiency): compares token usage with and without the skill. ## Test Tasks @@ -53,45 +50,28 @@ Task composition is derived from the evaluation dataset when possible. Entries w ## Results -| Dimension | Num | `claude-code` | `codex` | +| Dimension | Num | Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`) | Codex (`openai/openai/gpt-5.5`) | |---|---:|---:|---:| | Security | 5 | 100% (+0%) | 100% (+0%) | -| Correctness | 5 | 88% (+8%) | 99% (+12%) | -| Discoverability | 5 | 80% (+0%) | 99% (+7%) | -| Effectiveness | 5 | 85% (+12%) | 97% (+17%) | -| Efficiency | 5 | 83% (-0%) | 97% (+7%) | +| Correctness | 5 | 100% (+20%) | 100% (+16%) | +| Discoverability | 5 | 99% (+9%) | 99% (+9%) | +| Effectiveness | 5 | 100% (+18%) | 97% (+12%) | +| Efficiency | 5 | 87% (+4%) | 100% (+10%) | Score values show skill-assisted performance. Values in parentheses show uplift versus the no-skill baseline when baseline data is available. ## Tier 1: Static Validation Summary -Tier 1 validation reported findings. NVSkills-Eval ran 9 checks and found 37 total findings. +Tier 1 validation passed with observations. Skill Evaluator ran 1 checks and found 1 total findings. Top findings: -- MEDIUM PII/phone_numbers: International phone number (`references/perf-knobs-catalog.md:38`) -- MEDIUM PII/phone_numbers: International phone number (`references/perf-knobs-catalog.md:103`) -- MEDIUM PII/phone_numbers: International phone number (`references/perf-knobs-catalog.md:178`) -- MEDIUM PII/phone_numbers: International phone number (`references/perf-knobs-catalog.md:179`) -- MEDIUM PII/phone_numbers: International phone number (`references/perf-knobs-catalog.md:180`) +- MEDIUM SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`skills/tilegym-improve-cutile-kernel-perf/SKILL.md`) ## Tier 2: Deduplication Summary -Tier 2 validation reported findings. NVSkills-Eval ran 2 checks and found 4 total findings. +This tier was not run or did not produce findings in this report. -Top findings: +## Publication Recommendation -- HIGH DUPLICATE/duplicate: Duplicate content found across references/ir-dump-guide.md and references/optimization-playbook.md: - "### Mitigate" in references/ir-dump-guide.md (lines 209-219) - vs "### Mitigate" in references/optimization-playbook.md (lines 323-332) (`references/ir-dump-guide.md:209`) -- HIGH DUPLICATE/duplicate: Duplicate content found across references/optimization-playbook.md and references/perf-knobs-catalog.md: - "## Optimization D: Add TF32 Dtype Guard for MMA" in references/optimization-playbook.md (lines 181-187) - vs "# Cast FP32 → TF32 for tensor core utilization" in references/optimization-playbook.md (lines 200-209) - vs "## 9. TF32 Guard for MMA" in references/perf-knobs-catalog.md (lines 126-142) (`references/optimization-playbook.md:181`) -- LOW DUPLICATE/duplicate: Duplicate content found within references/cutile-api-reference.md: - "# Prefer Python arithmetic on host (simpler, no ct import needed)" in references/cutile-api-reference.md (lines 468-470) - vs "# Host — prefer Python arithmetic:" in references/cutile-api-reference.md (lines 652-653) - vs "# CORRECT — tuple of 1, 2, or 3 ints" in references/cutile-api-reference.md (lines 725-730) (`references/cutile-api-reference.md:468`) -- LOW DUPLICATE/duplicate: Duplicate content found within references/optimization-playbook.md: - "### Before" in references/optimization-playbook.md (lines 188-194) - vs "### After" in references/optimization-playbook.md (lines 195-199) (`references/optimization-playbook.md:188`) +The skill is suitable to proceed toward Skill Evaluator publication based on this benchmark. Skill owners should keep this file with the skill and refresh it when the evaluation dataset, skill behavior, or target agents materially change. diff --git a/skills/tilegym-improve-cutile-kernel-perf/skill-card.md b/skills/tilegym-improve-cutile-kernel-perf/skill-card.md index c091114..70dc344 100644 --- a/skills/tilegym-improve-cutile-kernel-perf/skill-card.md +++ b/skills/tilegym-improve-cutile-kernel-perf/skill-card.md @@ -9,38 +9,44 @@ NVIDIA
### License/Terms of Use:
CC-BY-4.0 AND Apache-2.0
## Use Case:
-Developers and engineers who need to systematically optimize cuTile GPU kernel performance through profiling, bottleneck diagnosis, and iterative tuning in the TileGym project.
+Developers and engineers optimizing cuTile GPU kernel performance in the TileGym project through iterative profiling, bottleneck diagnosis, and systematic tuning.
### Deployment Geography for Use:
Global
+## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Not Specified]
+**Credential Type(s):** [None identified]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
+ ## Known Risks and Mitigations:
Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [optimization-playbook.md](references/optimization-playbook.md)
-- [perf-knobs-catalog.md](references/perf-knobs-catalog.md)
-- [cutile-api-reference.md](references/cutile-api-reference.md)
-- [performance-model.md](references/performance-model.md)
-- [ir-dump-guide.md](references/ir-dump-guide.md)
-- [cutile-patterns-reference.md](references/cutile-patterns-reference.md)
+- [Optimization Playbook](references/optimization-playbook.md)
+- [Performance Knobs Catalog](references/perf-knobs-catalog.md)
+- [cuTile API Reference](references/cutile-api-reference.md)
+- [Performance Model](references/performance-model.md)
+- [IR Dump Guide](references/ir-dump-guide.md)
+- [cuTile Patterns Reference](references/cutile-patterns-reference.md)
## Skill Output:
**Output Type(s):** [Code, Shell commands, Analysis]
-**Output Format:** [Markdown with inline code blocks and structured performance tables]
+**Output Format:** [Markdown with inline code blocks and performance tables]
**Output Parameters:** [1D]
**Other Properties Related to Output:** [None]
## Evaluation Agents Used:
-- Claude Code (`claude-code`)
-- Codex (`codex`)
+- Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`)
+- Codex (`openai/openai/gpt-5.5`)
## Evaluation Tasks:
-Evaluated against 5 tasks (1 positive skill-activation, 4 negative activation) using NVSkills-Eval external profile in astra-sandbox environment.
+Evaluated against 5 evaluation tasks (1 positive skill-activation, 4 negative) in a k8s-sandbox environment.
## Evaluation Metrics Used:
Reported benchmark dimensions:
@@ -57,21 +63,20 @@ Underlying evaluation signals used in this run:
- `accuracy`: Grades final-answer correctness against the reference answer.
- `goal_accuracy`: Checks whether the overall user task completed successfully.
- `behavior_check`: Verifies expected behavior steps, including safety expectations.
-- `token_efficiency`: Compares token usage with and without the skill.
## Evaluation Results:
-| Dimension | Num | `claude-code` | `codex` | +| Dimension | Num | Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`) | Codex (`openai/openai/gpt-5.5`) | |---|---:|---:|---:| | Security | 5 | 100% (+0%) | 100% (+0%) | -| Correctness | 5 | 88% (+8%) | 99% (+12%) | -| Discoverability | 5 | 80% (+0%) | 99% (+7%) | -| Effectiveness | 5 | 85% (+12%) | 97% (+17%) | -| Efficiency | 5 | 83% (-0%) | 97% (+7%) | +| Correctness | 5 | 100% (+20%) | 100% (+16%) | +| Discoverability | 5 | 99% (+9%) | 99% (+9%) | +| Effectiveness | 5 | 100% (+18%) | 97% (+12%) | +| Efficiency | 5 | 87% (+4%) | 100% (+10%) | ## Skill Version(s):
-2026.04.11 (source: frontmatter)
+2026.4.11 (source: frontmatter)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/tilegym-improve-cutile-kernel-perf/skill.oms.sig b/skills/tilegym-improve-cutile-kernel-perf/skill.oms.sig index 9261d70..a8fb0e6 100644 --- a/skills/tilegym-improve-cutile-kernel-perf/skill.oms.sig +++ b/skills/tilegym-improve-cutile-kernel-perf/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAidGlsZWd5bS1pbXByb3ZlLWN1dGlsZS1rZXJuZWwtcGVyZiIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICJkM2IwYjQwOTI4OTY3Mzc4ZDAzZTczMjQwMjA3OGIwY2EwODBjMmM3ODYxYmQ5NTNhOGQ4ZDk3OTJkZDJhMDNmIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI2OTE4NDk2YTQ1YWU3OWZlNDc4ZWM0OWQxN2Y3MDFiYTIxOTY4YzAwZDZhMmQ2NjQ4Zjc3YjBjMjkzY2E3OGUzIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjc3MDNiZmYwOWNiYzdmZWUwYWQ0NThmZTFhZDYxODgzOGE3ZDU3ZjViMjQ4YWFhNmE3YjQ5MzBkNGRkNTc4M2EiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI2YTM3NjVjZmVmMWFjMTc1YjljZmUwYzJkYjQ5YTBlOTFlNzM3NTZhMzlkZGM2MGI5ODA4NmY0YTEwMjMxOWYzIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9jdXRpbGUtYXBpLXJlZmVyZW5jZS5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiM2VjNDA0NGQ2NjIxMTM3NzEwN2UzYTI0MmFkZTM1ODJkYTE2ZGM0ZjExMDc1N2RkOGRiNDc5ZDMyNzM0ZjU1YyIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvY3V0aWxlLXBhdHRlcm5zLXJlZmVyZW5jZS5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiZTg0NzA2ZTcxMDBhY2UwMGFlZGI3Y2M2MmUxNjhlZDMxOTUxNzJlZmQ1NmJiMTJhMThiMWJkZjRlZGE5ZjIxYSIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvaXItZHVtcC1ndWlkZS5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMjA3Mjc2N2JkYmM3NWExOWQyMmJhMmMzZTAzNGY2ZGIxN2JiODNkM2QzNGZmOGVhNTUzYzBlYWQ1MzVkZjhhZiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvb3B0aW1pemF0aW9uLXBsYXlib29rLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIyZDdjYzkyNDc2NGM3NTJiNDUyZmNhY2VhM2ZmZWQyZGFmMWRhYmVhZjhjMTRlMTRlYmJjZTBhZjBmYWRjM2ZhIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9wZXJmLWtub2JzLWNhdGFsb2cubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImFjMTg3ZmRlZmFjZTkyNGE5NDU2NWQ3MDExNTAyODUwYmNjNjkyZGE0Nzk1ODQ5OTg3YzkzYWVlOWViMTQ3NWEiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3BlcmZvcm1hbmNlLW1vZGVsLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIxN2Q3OTNjOTQwZDgwYTQ4ZmZmODUzNzUzMzY0YjU1ZGQ1MmY0NDAyMmEzNmMzODYwN2U5ZTUyOWMzOWM0MGI3IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNzc0MmRlZjYwMzM1NzgzMTMwNTY2Mzg2ZDk4ZmJmNmU5MTFmYTFhZmQ4OThhNzE4M2QwODBjNzU5ODVkYWNmNSIKICAgICAgfQogICAgXSwKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXQiCiAgICAgIF0sCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UKICAgIH0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMAYs+AAX91rC5CV2EwpY3coAEP6dJ5tUMDivKcNq/i52AxV4YyW4MQKX0j0GwdyEtwIxANkXm9fF/mm4p1Isd2Usnl5P71GlTsMIJu7OEXP1iKrRr44QGI2O5jixXkwXPeyWHA==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAidGlsZWd5bS1pbXByb3ZlLWN1dGlsZS1rZXJuZWwtcGVyZiIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICI2NjBiMGE5MTUzNjc2ZDIwZjcxODhhNDYxZmZhZDllNjdiYmRjOGFjYmQ1OTRkOWQ1NWZkZjA4YTM2MGRmODBmIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiZTJiZWU4NzljZWE1NjU2Yzc4NTU4YmYzMWJkMGY5ZWVlMzI5YzM4N2IwMTA1MmEyNmQ2ZTgxZDc5YjI5Y2FhNSIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNTAwMzFjYzI3NGI5MDM5YjcwYTRkNTljNjJlZmMzYzdiNzM3YTVjMGM0NTBjMzc3OWY2ODI5NTEyOTA1MmVjYiIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI2YTM3NjVjZmVmMWFjMTc1YjljZmUwYzJkYjQ5YTBlOTFlNzM3NTZhMzlkZGM2MGI5ODA4NmY0YTEwMjMxOWYzIiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiM2VjNDA0NGQ2NjIxMTM3NzEwN2UzYTI0MmFkZTM1ODJkYTE2ZGM0ZjExMDc1N2RkOGRiNDc5ZDMyNzM0ZjU1YyIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9jdXRpbGUtYXBpLXJlZmVyZW5jZS5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImU4NDcwNmU3MTAwYWNlMDBhZWRiN2NjNjJlMTY4ZWQzMTk1MTcyZWZkNTZiYjEyYTE4YjFiZGY0ZWRhOWYyMWEiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvY3V0aWxlLXBhdHRlcm5zLXJlZmVyZW5jZS5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjIwNzI3NjdiZGJjNzVhMTlkMjJiYTJjM2UwMzRmNmRiMTdiYjgzZDNkMzRmZjhlYTU1M2MwZWFkNTM1ZGY4YWYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvaXItZHVtcC1ndWlkZS5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjJkN2NjOTI0NzY0Yzc1MmI0NTJmY2FjZWEzZmZlZDJkYWYxZGFiZWFmOGMxNGUxNGViYmNlMGFmMGZhZGMzZmEiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvb3B0aW1pemF0aW9uLXBsYXlib29rLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiYWMxODdmZGVmYWNlOTI0YTk0NTY1ZDcwMTE1MDI4NTBiY2M2OTJkYTQ3OTU4NDk5ODdjOTNhZWU5ZWIxNDc1YSIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9wZXJmLWtub2JzLWNhdGFsb2cubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIxN2Q3OTNjOTQwZDgwYTQ4ZmZmODUzNzUzMzY0YjU1ZGQ1MmY0NDAyMmEzNmMzODYwN2U5ZTUyOWMzOWM0MGI3IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3BlcmZvcm1hbmNlLW1vZGVsLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMGRiY2Y3ZTcyMWVlNGE3NDNjNzEyMzdiYTFhZGM3ODFlNWZkM2Y1YzRiZjgyNzU0YjZhZGM3M2ZjOGZiNWZkMiIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIKICAgICAgfQogICAgXSwKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdGh1YiIKICAgICAgXSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UKICAgIH0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMBrUimZdJtSc/3pO3tqsquPHQyH9oP2xo5O9TqTqwgn2tyb2XH6OJXSPr3sTZFfrhgIwL0o9YC2gTnXK7CJK7XLXqnchM76CWvJwQihfSlOxwXORaMzJ50KhW8SWcVNGHQa4","keyid":""}]}} \ No newline at end of file From c0f04c7f4c20ee7eccf2eb04726d4b8d6f33593f Mon Sep 17 00:00:00 2001 From: Hannah Li Date: Mon, 27 Jul 2026 08:51:06 +0800 Subject: [PATCH 6/6] ci: update backend preflight to is_triton_tileir_available --- .github/workflows/tilegym-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tilegym-ci.yml b/.github/workflows/tilegym-ci.yml index 8a22b43..8cf149e 100644 --- a/.github/workflows/tilegym-ci.yml +++ b/.github/workflows/tilegym-ci.yml @@ -321,7 +321,7 @@ jobs: # Fail loudly if ENABLE_TILE=1 + PYTHONPATH=/opt/triton-cuda-tileir did NOT actually # activate the CUDA Tile IR backend (e.g. a silent fallback to the - # torch-bundled Triton). is_nvt_available() requires triton.backends.tileir, + # torch-bundled Triton). is_triton_tileir_available() requires triton.backends.tileir, # which only the Triton CUDA Tile IR package provides. docker pull ${IMAGE} docker run --rm \ @@ -330,7 +330,7 @@ jobs: -e PYTHONPATH=/opt/triton-cuda-tileir \ -w /workspace/tilegym/modeling/transformers \ ${IMAGE} \ - uv run --locked --no-sync python -c "import triton; from tilegym.backend.selector import is_nvt_available, get_available_triton_backend; assert is_nvt_available(), 'ENABLE_TILE=1 but the CUDA Tile IR backend is NOT active; triton=' + triton.__file__; print('CUDA Tile IR backend active: triton=' + triton.__file__ + ', get_available_triton_backend=' + get_available_triton_backend())" + uv run --locked --no-sync python -c "import triton; from tilegym.backend.selector import is_triton_tileir_available, get_available_triton_backend; assert is_triton_tileir_available(), 'ENABLE_TILE=1 but the CUDA Tile IR backend is NOT active; triton=' + triton.__file__; print('CUDA Tile IR backend active: triton=' + triton.__file__ + ', get_available_triton_backend=' + get_available_triton_backend())" - name: Pull and run ops tests (shard ${{ matrix.shard }}) timeout-minutes: 50