Add private ids to checker and final report

tools/submission/README.md

@@ -69,6 +69,7 @@ python3 -m inference.tools.submission.submission_checker.main
     [--scenarios-to-skip]
     [--skip-all-systems-have-results-check]
     [--skip-calibration-check]
+    [--private-ids]
 ```
 
 ### implemented checks

tools/submission/generate_final_report.py

@@ -117,6 +117,7 @@ def main():
         "ID",
         "Unique ID (e.g. for Audit)",
         "ColorKey",
+        "PrivateID",
         "Submitter",
         "Availability",
         "System",

tools/submission/submission_checker/configuration/configuration.py

@@ -20,6 +20,7 @@ def __init__(
         skip_calibration_check=False,
         skip_dataset_size_check=False,
         submitter=None,
+        private_ids=False,
     ):
         self.base = MODEL_CONFIG.get(version)
         self.extra_model_benchmark_map = extra_model_benchmark_map
@@ -38,6 +39,7 @@ def __init__(
         self.skip_all_systems_have_results_check = skip_all_systems_have_results_check
         self.skip_calibration_check = skip_calibration_check
         self.skip_dataset_size_check = skip_dataset_size_check
+        self.private_ids = private_ids
         self.load_config(version)
 
     def load_config(self, version):

tools/submission/submission_checker/constants.py

@@ -1748,3 +1748,10 @@
     "v6.0": "{division}/{submitter}/src",
     "default": "{division}/{submitter}/src",
 }
+
+PRIVATE_ID_PATH = {
+    "v5.0": "{division}/{submitter}/results/{system}/privateid.json",
+    "v5.1": "{division}/{submitter}/results/{system}/privateid.json",
+    "v6.0": "{division}/{submitter}/results/{system}/privateid.json",
+    "default": "{division}/{submitter}/results/{system}/privateid.json",
+}

tools/submission/submission_checker/loader.py

@@ -1,6 +1,6 @@
 import os
 from .constants import *
-from .utils import list_dir
+from .utils import list_dir, generate_private_id
 from .parsers.loadgen_parser import LoadgenParser
 from typing import Generator, Literal
 from .utils import *
@@ -127,6 +127,9 @@ def __init__(self, root, version, config: Config) -> None:
         self.src_path = os.path.join(
             self.root, SRC_PATH.get(
                 version, SRC_PATH["default"]))
+        self.private_id_path = os.path.join(
+            self.root, PRIVATE_ID_PATH.get(
+                version, PRIVATE_ID_PATH["default"]))
         self.filter_submitter = self.config.get_submitter()
 
     def get_measurement_path(self, path, division,
@@ -271,6 +274,30 @@ def load(self) -> Generator[SubmissionLogs, None, None]:
                     system_json = self.load_single_log(
                         system_json_path, "System")
                     system_type = system_json.get("system_type")
+
+                    private_id = ""
+                    if self.config.private_ids:
+                        private_id_json_path = self.private_id_path.format(
+                            division=division, submitter=submitter, system=system)
+                        try:
+                            private_id_json = self.load_single_log(
+                                private_id_json_path, "System")
+                            private_id = private_id_json[system]
+                        except:
+                            self.logger.warning(
+                                "%s Private id not cached for system %s",
+                                system_path,
+                                system
+                            )
+                            private_id = generate_private_id(system)
+                            with open(private_id_json_path, "w") as f:
+                                json.dump({system: private_id}, f, indent=4)
+                            self.logger.warning(
+                                "%s Private id generated and saved at %s",
+                                system_path,
+                                private_id_json_path
+                            )
+
                     for benchmark in list_dir(system_path):
                         benchmark_path = os.path.join(system_path, benchmark)
                         if division.lower() in ["closed", "network"]:
@@ -464,5 +491,7 @@ def load(self) -> Generator[SubmissionLogs, None, None]:
                                 "TEST08_acc_result": test08_acc_result,
                                 "TEST07_acc_result": test07_acc_result,
                                 "TEST09_acc_result": test09_acc_result,
+                                # Private ID
+                                "private_id": private_id
                             }
                             yield SubmissionLogs(perf_log, acc_log, acc_result, acc_json, system_json, measurements_json, loader_data)

tools/submission/submission_checker/main.py

@@ -1,4 +1,5 @@
 import argparse
+from collections import Counter
 import logging
 import os
 import sys
@@ -113,6 +114,11 @@ def get_args():
         action="store_true",
         help="skips dataset size check, only for backwards compatibility",
     )
+    parser.add_argument(
+        "--private-ids",
+        action="store_true",
+        help="generate and persist private IDs for each system in privateid.json",
+    )
     args = parser.parse_args()
     return args
 
@@ -134,7 +140,7 @@ def main():
         args.version,
         args.extra_model_benchmark_map,
         ignore_uncommited=args.submission_exceptions,
-        skip_compliance=args.skip_power_check,
+        skip_compliance=args.skip_compliance,
         skip_power_check=args.skip_power_check,
         skip_meaningful_fields_emptiness_check=args.skip_meaningful_fields_emptiness_check,
         skip_check_power_measure_files=args.skip_check_power_measure_files,
@@ -145,6 +151,7 @@ def main():
         skip_calibration_check=args.skip_calibration_check,
         skip_dataset_size_check=args.skip_dataset_size_check,
         submitter=args.submitter,
+        private_ids=args.private_ids
     )
 
     if args.scenarios_to_skip:
@@ -253,13 +260,8 @@ def main():
     )
 
     def merge_two_dict(x, y):
-        z = x.copy()
-        for key in y:
-            if key not in z:
-                z[key] = y[key]
-            else:
-                z[key] += y[key]
-        return z
+        z = Counter(x) + Counter(y)
+        return dict(z)
 
     # systems can be repeating in open, closed and network
     unique_closed_systems = merge_two_dict(
@@ -285,10 +287,7 @@ def merge_two_dict(x, y):
 
     # Counting the number of closed,open and network results
     def sum_dict_values(x):
-        count = 0
-        for key in x:
-            count += x[key]
-        return count
+        return sum(x.values())
 
     count_closed_power_results = sum_dict_values(closed_power_systems)
     count_closed_non_power_results = sum_dict_values(closed_non_power_systems)

tools/submission/submission_checker/results.py

@@ -55,6 +55,7 @@ def __init__(self, csv_path, config: Config) -> None:
             "has_power",
             "Units",
             "weight_data_types",
+            "PrivateID",
         ]
         self.rows = []
         self.csv_path = csv_path
@@ -124,6 +125,7 @@ def add_result(self, submission_logs: SubmissionLogs):
         )
         row["Units"] = unit
         row["weight_data_types"] = submission_logs.measurements_json["weight_data_types"]
+        row["PrivateID"] = submission_logs.loader_data.get("private_id", "")
         self.rows.append(row.copy())
         if row["has_power"]:
             row["Result"] = submission_logs.loader_data["power_metric"]

tools/submission/submission_checker/utils.py

@@ -1,6 +1,7 @@
 import os
 from .constants import *
 from .parsers.loadgen_parser import LoadgenParser
+import hashlib
 
 
 def list_dir(*path):
@@ -332,3 +333,33 @@ def get_power_metric(config, scenario_fixed, log_path, is_valid, res):
             avg_power_efficiency = (samples_per_query * 1000) / power_metric
 
     return is_valid, power_metric, scenario, avg_power_efficiency
+
+_ADJECTIVES = [
+    "amber", "azure", "bold", "brave", "calm", "clear", "cool", "crisp",
+    "dark", "deep", "deft", "epic", "fair", "fast", "firm", "flat",
+    "free", "full", "gold", "gray", "hard", "high", "keen", "kind",
+    "lean", "light", "long", "loud", "mild", "mint", "neat", "noble",
+    "open", "pale", "pure", "quick", "quiet", "rare", "rich", "safe",
+    "sage", "slim", "slow", "soft", "star", "still", "stone", "sure",
+    "swift", "teal", "trim", "true", "vast", "warm", "wide", "wise",
+    "wood", "worn", "young", "zeal", "zinc", "zest", "blue", "core",
+]
+_NOUNS = [
+    "apex", "arch", "atom", "bay", "beam", "blade", "bloom", "bolt",
+    "bridge", "brook", "bud", "cap", "cedar", "cloud", "coral", "core",
+    "crest", "dawn", "delta", "dome", "dune", "echo", "edge", "elm",
+    "ember", "fern", "field", "flame", "flare", "fleet", "flow", "foam",
+    "ford", "forge", "frost", "gate", "glade", "glow", "grove", "helm",
+    "hill", "horn", "iris", "isle", "jade", "lake", "lark", "leaf",
+    "ledge", "lens", "lime", "lynx", "mast", "mesa", "mill", "moon",
+    "moor", "moss", "nook", "opal", "peak", "pine", "reef", "vale",
+]
+
+
+def generate_private_id(system_id: str) -> str:
+    h = hashlib.sha256(system_id.encode()).digest()
+    adj = _ADJECTIVES[h[0] % len(_ADJECTIVES)]
+    noun = _NOUNS[h[1] % len(_NOUNS)]
+    suffix = h[2:4].hex()
+    return f"{adj}-{noun}-{suffix}"
+