From 4bfb66470fc8678b88ba06f9a174faca1bd63794 Mon Sep 17 00:00:00 2001 From: Pablo Gonzalez Date: Fri, 22 May 2026 10:32:05 -0500 Subject: [PATCH] Add private ids to checker and final report --- tools/submission/README.md | 1 + tools/submission/generate_final_report.py | 1 + .../configuration/configuration.py | 2 ++ .../submission_checker/constants.py | 7 +++++ tools/submission/submission_checker/loader.py | 31 ++++++++++++++++++- tools/submission/submission_checker/main.py | 23 +++++++------- .../submission/submission_checker/results.py | 2 ++ tools/submission/submission_checker/utils.py | 31 +++++++++++++++++++ 8 files changed, 85 insertions(+), 13 deletions(-) diff --git a/tools/submission/README.md b/tools/submission/README.md index 6d620233b2..e25ea6c36d 100644 --- a/tools/submission/README.md +++ b/tools/submission/README.md @@ -69,6 +69,7 @@ python3 -m inference.tools.submission.submission_checker.main [--scenarios-to-skip] [--skip-all-systems-have-results-check] [--skip-calibration-check] + [--private-ids] ``` ### implemented checks diff --git a/tools/submission/generate_final_report.py b/tools/submission/generate_final_report.py index 2d4454e137..5f77bfaf47 100644 --- a/tools/submission/generate_final_report.py +++ b/tools/submission/generate_final_report.py @@ -117,6 +117,7 @@ def main(): "ID", "Unique ID (e.g. for Audit)", "ColorKey", + "PrivateID", "Submitter", "Availability", "System", diff --git a/tools/submission/submission_checker/configuration/configuration.py b/tools/submission/submission_checker/configuration/configuration.py index 97a20bc4a2..37abb6b802 100644 --- a/tools/submission/submission_checker/configuration/configuration.py +++ b/tools/submission/submission_checker/configuration/configuration.py @@ -20,6 +20,7 @@ def __init__( skip_calibration_check=False, skip_dataset_size_check=False, submitter=None, + private_ids=False, ): self.base = MODEL_CONFIG.get(version) self.extra_model_benchmark_map = extra_model_benchmark_map @@ -38,6 +39,7 @@ def __init__( self.skip_all_systems_have_results_check = skip_all_systems_have_results_check self.skip_calibration_check = skip_calibration_check self.skip_dataset_size_check = skip_dataset_size_check + self.private_ids = private_ids self.load_config(version) def load_config(self, version): diff --git a/tools/submission/submission_checker/constants.py b/tools/submission/submission_checker/constants.py index 2f4abd87f8..24b677e7f9 100644 --- a/tools/submission/submission_checker/constants.py +++ b/tools/submission/submission_checker/constants.py @@ -1748,3 +1748,10 @@ "v6.0": "{division}/{submitter}/src", "default": "{division}/{submitter}/src", } + +PRIVATE_ID_PATH = { + "v5.0": "{division}/{submitter}/results/{system}/privateid.json", + "v5.1": "{division}/{submitter}/results/{system}/privateid.json", + "v6.0": "{division}/{submitter}/results/{system}/privateid.json", + "default": "{division}/{submitter}/results/{system}/privateid.json", +} diff --git a/tools/submission/submission_checker/loader.py b/tools/submission/submission_checker/loader.py index 79cfdce73a..61c176efa7 100644 --- a/tools/submission/submission_checker/loader.py +++ b/tools/submission/submission_checker/loader.py @@ -1,6 +1,6 @@ import os from .constants import * -from .utils import list_dir +from .utils import list_dir, generate_private_id from .parsers.loadgen_parser import LoadgenParser from typing import Generator, Literal from .utils import * @@ -127,6 +127,9 @@ def __init__(self, root, version, config: Config) -> None: self.src_path = os.path.join( self.root, SRC_PATH.get( version, SRC_PATH["default"])) + self.private_id_path = os.path.join( + self.root, PRIVATE_ID_PATH.get( + version, PRIVATE_ID_PATH["default"])) self.filter_submitter = self.config.get_submitter() def get_measurement_path(self, path, division, @@ -271,6 +274,30 @@ def load(self) -> Generator[SubmissionLogs, None, None]: system_json = self.load_single_log( system_json_path, "System") system_type = system_json.get("system_type") + + private_id = "" + if self.config.private_ids: + private_id_json_path = self.private_id_path.format( + division=division, submitter=submitter, system=system) + try: + private_id_json = self.load_single_log( + private_id_json_path, "System") + private_id = private_id_json[system] + except: + self.logger.warning( + "%s Private id not cached for system %s", + system_path, + system + ) + private_id = generate_private_id(system) + with open(private_id_json_path, "w") as f: + json.dump({system: private_id}, f, indent=4) + self.logger.warning( + "%s Private id generated and saved at %s", + system_path, + private_id_json_path + ) + for benchmark in list_dir(system_path): benchmark_path = os.path.join(system_path, benchmark) if division.lower() in ["closed", "network"]: @@ -464,5 +491,7 @@ def load(self) -> Generator[SubmissionLogs, None, None]: "TEST08_acc_result": test08_acc_result, "TEST07_acc_result": test07_acc_result, "TEST09_acc_result": test09_acc_result, + # Private ID + "private_id": private_id } yield SubmissionLogs(perf_log, acc_log, acc_result, acc_json, system_json, measurements_json, loader_data) diff --git a/tools/submission/submission_checker/main.py b/tools/submission/submission_checker/main.py index babb1193be..5805853bc0 100644 --- a/tools/submission/submission_checker/main.py +++ b/tools/submission/submission_checker/main.py @@ -1,4 +1,5 @@ import argparse +from collections import Counter import logging import os import sys @@ -113,6 +114,11 @@ def get_args(): action="store_true", help="skips dataset size check, only for backwards compatibility", ) + parser.add_argument( + "--private-ids", + action="store_true", + help="generate and persist private IDs for each system in privateid.json", + ) args = parser.parse_args() return args @@ -134,7 +140,7 @@ def main(): args.version, args.extra_model_benchmark_map, ignore_uncommited=args.submission_exceptions, - skip_compliance=args.skip_power_check, + skip_compliance=args.skip_compliance, skip_power_check=args.skip_power_check, skip_meaningful_fields_emptiness_check=args.skip_meaningful_fields_emptiness_check, skip_check_power_measure_files=args.skip_check_power_measure_files, @@ -145,6 +151,7 @@ def main(): skip_calibration_check=args.skip_calibration_check, skip_dataset_size_check=args.skip_dataset_size_check, submitter=args.submitter, + private_ids=args.private_ids ) if args.scenarios_to_skip: @@ -253,13 +260,8 @@ def main(): ) def merge_two_dict(x, y): - z = x.copy() - for key in y: - if key not in z: - z[key] = y[key] - else: - z[key] += y[key] - return z + z = Counter(x) + Counter(y) + return dict(z) # systems can be repeating in open, closed and network unique_closed_systems = merge_two_dict( @@ -285,10 +287,7 @@ def merge_two_dict(x, y): # Counting the number of closed,open and network results def sum_dict_values(x): - count = 0 - for key in x: - count += x[key] - return count + return sum(x.values()) count_closed_power_results = sum_dict_values(closed_power_systems) count_closed_non_power_results = sum_dict_values(closed_non_power_systems) diff --git a/tools/submission/submission_checker/results.py b/tools/submission/submission_checker/results.py index 990172b989..52b514eaf5 100644 --- a/tools/submission/submission_checker/results.py +++ b/tools/submission/submission_checker/results.py @@ -55,6 +55,7 @@ def __init__(self, csv_path, config: Config) -> None: "has_power", "Units", "weight_data_types", + "PrivateID", ] self.rows = [] self.csv_path = csv_path @@ -124,6 +125,7 @@ def add_result(self, submission_logs: SubmissionLogs): ) row["Units"] = unit row["weight_data_types"] = submission_logs.measurements_json["weight_data_types"] + row["PrivateID"] = submission_logs.loader_data.get("private_id", "") self.rows.append(row.copy()) if row["has_power"]: row["Result"] = submission_logs.loader_data["power_metric"] diff --git a/tools/submission/submission_checker/utils.py b/tools/submission/submission_checker/utils.py index 6435b9e165..12fe13a770 100644 --- a/tools/submission/submission_checker/utils.py +++ b/tools/submission/submission_checker/utils.py @@ -1,6 +1,7 @@ import os from .constants import * from .parsers.loadgen_parser import LoadgenParser +import hashlib def list_dir(*path): @@ -332,3 +333,33 @@ def get_power_metric(config, scenario_fixed, log_path, is_valid, res): avg_power_efficiency = (samples_per_query * 1000) / power_metric return is_valid, power_metric, scenario, avg_power_efficiency + +_ADJECTIVES = [ + "amber", "azure", "bold", "brave", "calm", "clear", "cool", "crisp", + "dark", "deep", "deft", "epic", "fair", "fast", "firm", "flat", + "free", "full", "gold", "gray", "hard", "high", "keen", "kind", + "lean", "light", "long", "loud", "mild", "mint", "neat", "noble", + "open", "pale", "pure", "quick", "quiet", "rare", "rich", "safe", + "sage", "slim", "slow", "soft", "star", "still", "stone", "sure", + "swift", "teal", "trim", "true", "vast", "warm", "wide", "wise", + "wood", "worn", "young", "zeal", "zinc", "zest", "blue", "core", +] +_NOUNS = [ + "apex", "arch", "atom", "bay", "beam", "blade", "bloom", "bolt", + "bridge", "brook", "bud", "cap", "cedar", "cloud", "coral", "core", + "crest", "dawn", "delta", "dome", "dune", "echo", "edge", "elm", + "ember", "fern", "field", "flame", "flare", "fleet", "flow", "foam", + "ford", "forge", "frost", "gate", "glade", "glow", "grove", "helm", + "hill", "horn", "iris", "isle", "jade", "lake", "lark", "leaf", + "ledge", "lens", "lime", "lynx", "mast", "mesa", "mill", "moon", + "moor", "moss", "nook", "opal", "peak", "pine", "reef", "vale", +] + + +def generate_private_id(system_id: str) -> str: + h = hashlib.sha256(system_id.encode()).digest() + adj = _ADJECTIVES[h[0] % len(_ADJECTIVES)] + noun = _NOUNS[h[1] % len(_NOUNS)] + suffix = h[2:4].hex() + return f"{adj}-{noun}-{suffix}" +