From e5d7cb11f253d3608028f58852054a9f7676740d Mon Sep 17 00:00:00 2001
From: blidiselalin <blidiselalin@gmail.com>
Date: Tue, 19 May 2026 18:29:57 +0300
Subject: [PATCH 1/4] fix(deps): make medhelm[summarization] install on Python
 3.12

summ-eval pinned pyemd==0.5.1 (fails to build on 3.12) and blanc/torch<2.0,
which caused pip resolver failures. Drop summ-eval from the extra, vendor
DataStatsMetric in helm.benchmark.metrics.data_stats, and pin bert-score to
0.3.13 to avoid broken PyPI metadata in older releases.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 .gitignore                                    |  1 +
 README.md                                     |  6 +-
 docs/installation.md                          | 14 ----
 docs/medhelm.md                               |  1 -
 pyproject.toml                                | 12 +--
 .../benchmark/metrics/data_stats/__init__.py  |  3 +
 .../metrics/data_stats/data_stats_metric.py   | 66 +++++++++++++++
 .../benchmark/metrics/data_stats/fragments.py | 81 +++++++++++++++++++
 .../metrics/summarization_metrics.py          |  6 +-
 9 files changed, 157 insertions(+), 33 deletions(-)
 create mode 100644 src/helm/benchmark/metrics/data_stats/__init__.py
 create mode 100644 src/helm/benchmark/metrics/data_stats/data_stats_metric.py
 create mode 100644 src/helm/benchmark/metrics/data_stats/fragments.py

diff --git a/.gitignore b/.gitignore
index 7e37430952e..c76733df449 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
 # General Python stuff
+.venv
 venv
 __pycache__
 *.egg-info
diff --git a/README.md b/README.md
index 3f46e94e013..22e4e6b73ad 100644
--- a/README.md
+++ b/README.md
@@ -92,14 +92,12 @@ Adds heavy libraries (bert-score, rouge-score, nltk). **Install can take 2–3 m
 
 Scenarios: **DischargeMe** (hospital course summaries; requires PhysioNet `data_path`), **ACI-Bench** (clinical transcripts), **Patient-Edu** (simplifying medical jargon).
 
-**⚠️ Note on macOS with Python 3.12:** Use `pip` instead of `uv` for this tier due to build compatibility issues with `pyemd`.
-
-Install (use `pip` on Intel Mac):
+Install:
 ```sh
 pip install "medhelm[summarization]"
 ```
 
-Or with `uv` (Linux/other platforms):
+Or with `uv`:
 ```sh
 uv pip install "medhelm[summarization]"
 ```
diff --git a/docs/installation.md b/docs/installation.md
index 9cc0989c662..9256dce65ac 100644
--- a/docs/installation.md
+++ b/docs/installation.md
@@ -61,8 +61,6 @@ Scenarios: **DischargeMe** (hospital course summaries), **ACI-Bench** (clinical
 pip install "medhelm[summarization]"
 ```
 
-**Note on macOS with Python 3.12:** Use `pip` (not `uv`) for this tier due to build compatibility with `pyemd`.
-
 ### Gated / licensing tier (`[gated]`)
 
 Adds **gdown** so the code can download data from Google Drive. Install can also take longer.
@@ -92,18 +90,6 @@ pip install "medhelm[summarization,gated]"
 
 See [Quick Start](/quick_start) for running benchmarks with `medhelm-run` (after activating your environment).
 
-## Troubleshooting
-
-### macOS with Python 3.12 and the summarization tier
-
-If you encounter build errors with `pyemd` when installing `medhelm[summarization]` on macOS:
-
-```bash
-pip install "medhelm[summarization]"
-```
-
-This is a known compatibility issue. Use `pip` instead of `uv` for this tier.
-
 ## Install Multimodal Support
 
 Additional steps are required for multimodal evaluations:
diff --git a/docs/medhelm.md b/docs/medhelm.md
index 691bd8efb31..df67cf53b4d 100644
--- a/docs/medhelm.md
+++ b/docs/medhelm.md
@@ -181,7 +181,6 @@ source .venv/bin/activate
 pip install "medhelm[summarization,gated]"
 ```
 
-**Note on macOS/Python 3.12:** Use `pip` (not `uv`) for the `[summarization]` tier due to compatibility with `pyemd`.
 
 ### Alternative: Using conda
 
diff --git a/pyproject.toml b/pyproject.toml
index d6fda049d7b..44fe5db6c93 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -98,8 +98,9 @@ ranking = [
 summarization = [
     # For summarization_metrics and MedHELM Clinical NLP tier (DischargeMe, ACI-Bench, Patient-Edu)
     # Use: pip install "medhelm[summarization]" — install can take 2–3 minutes
-    "summ-eval~=0.892",
-    "bert-score~=0.3",
+    # Note: extractive stats (coverage/density/compression) are vendored in helm.benchmark.metrics.data_stats
+    # instead of summ-eval, which pins pyemd==0.5.1 (fails on Python 3.12+) and torch<2.0 via blanc.
+    "bert-score~=0.3.13",
     "rouge-score~=0.1.2",
     "nltk~=3.7,!=3.9.0",
     "sentencepiece~=0.2.0",
@@ -468,13 +469,6 @@ crfm-proxy-cli = "helm.proxy.cli:main"
 default-groups = ["dev", "types"]
 
 override-dependencies = [
-    # override torch dependency for summ-eval in "summarization" extra:
-    # - summ-eval depends on blanc 
-    # - blanc depends on torch<2.0
-    "torch~=2.1",
-    # override torch dependency for summ-eval in "summarization" extra:
-    # - summ-eval depends on blanc 
-    # - blanc depends on numpy<2.0
     "numpy>=1.26,<3",
 ]
 
diff --git a/src/helm/benchmark/metrics/data_stats/__init__.py b/src/helm/benchmark/metrics/data_stats/__init__.py
new file mode 100644
index 00000000000..79b77dd8025
--- /dev/null
+++ b/src/helm/benchmark/metrics/data_stats/__init__.py
@@ -0,0 +1,3 @@
+from helm.benchmark.metrics.data_stats.data_stats_metric import DataStatsMetric
+
+__all__ = ["DataStatsMetric"]
diff --git a/src/helm/benchmark/metrics/data_stats/data_stats_metric.py b/src/helm/benchmark/metrics/data_stats/data_stats_metric.py
new file mode 100644
index 00000000000..4a2d98ff5cd
--- /dev/null
+++ b/src/helm/benchmark/metrics/data_stats/data_stats_metric.py
@@ -0,0 +1,66 @@
+# Adapted from summ_eval.data_stats_metric (Apache 2.0 / Newsroom fragments).
+# Vendored so medhelm[summarization] does not depend on summ-eval (pyemd==0.5.1, torch<2.0).
+
+from collections import Counter
+from typing import Any, Dict, List, Optional
+
+import spacy
+
+from helm.benchmark.metrics.data_stats.fragments import Fragments
+
+_en: Optional[Any] = None
+
+
+def _get_spacy_en():
+    global _en
+    if _en is None:
+        _en = spacy.load("en_core_web_sm")
+    return _en
+
+
+def _find_ngrams(input_list: List[str], n: int):
+    return zip(*[input_list[i:] for i in range(n)])
+
+
+class DataStatsMetric:
+    """Extractive summarization statistics (coverage, density, compression)."""
+
+    def __init__(self, n_gram: int = 3, case: bool = False, tokenize: bool = True):
+        self.n_gram = n_gram
+        self.case = case
+        self.tokenize = tokenize
+
+    def evaluate_example(self, summary: str, input_text: str) -> Dict[str, float]:
+        if self.tokenize:
+            nlp = _get_spacy_en()
+            input_text = [tok.text for tok in nlp(input_text, disable=["tagger", "parser", "ner", "textcat"])]
+            summary = [tok.text for tok in nlp(summary, disable=["tagger", "parser", "ner", "textcat"])]
+        fragments = Fragments(summary, input_text, case=self.case)
+        score_dict: Dict[str, float] = {
+            "coverage": fragments.coverage(),
+            "density": fragments.density(),
+            "compression": fragments.compression(),
+            "summary_length": float(len(fragments.summary)),
+        }
+        tokenized_summary = fragments.summary
+        tokenized_text = fragments.text
+        for i in range(1, self.n_gram + 1):
+            input_ngrams = list(_find_ngrams(tokenized_text, i))
+            summ_ngrams = list(_find_ngrams(tokenized_summary, i))
+            input_ngrams_set = set(input_ngrams)
+            summ_ngrams_set = set(summ_ngrams)
+            intersect = summ_ngrams_set.intersection(input_ngrams_set)
+            try:
+                score_dict[f"percentage_novel_{i}-gram"] = (len(summ_ngrams_set) - len(intersect)) / float(
+                    len(summ_ngrams_set)
+                )
+                ngram_counter = Counter(summ_ngrams)
+                repeated = [key for key, val in ngram_counter.items() if val > 1]
+                score_dict[f"percentage_repeated_{i}-gram_in_summ"] = len(repeated) / float(len(summ_ngrams_set))
+            except ZeroDivisionError:
+                continue
+        return score_dict
+
+    @property
+    def supports_multi_ref(self) -> bool:
+        return False
diff --git a/src/helm/benchmark/metrics/data_stats/fragments.py b/src/helm/benchmark/metrics/data_stats/fragments.py
new file mode 100644
index 00000000000..b589e2e9eac
--- /dev/null
+++ b/src/helm/benchmark/metrics/data_stats/fragments.py
@@ -0,0 +1,81 @@
+# Adapted from https://github.com/lil-lab/newsroom/blob/master/newsroom/analyze/fragments.py
+# (used by summ_eval DataStatsMetric; vendored to avoid summ-eval's pyemd/torch pins)
+
+from collections import namedtuple
+
+
+def normalize(tokens, case: bool = False) -> list[str]:
+    """Lowercase tokens unless case preservation is requested."""
+    return [str(t).lower() if not case else str(t) for t in tokens]
+
+
+class Fragments:
+    Match = namedtuple("Match", ("summary", "text", "length"))
+
+    def __init__(self, summary, text, case: bool = False):
+        if isinstance(summary, str):
+            self.summary = summary.split()
+        else:
+            self.summary = summary
+        if isinstance(text, str):
+            self.text = text.split()
+        else:
+            self.text = text
+
+        self._norm_summary = normalize(self.summary, case)
+        self._norm_text = normalize(self.text, case)
+        self._matches: list[Fragments.Match] = []
+        self._match(self._norm_summary, self._norm_text)
+
+    def overlaps(self) -> list[Match]:
+        return self._matches
+
+    def coverage(self, summary_base: bool = True) -> float:
+        numerator = sum(o.length for o in self.overlaps())
+        denominator = len(self.summary) if summary_base else len(self.text)
+        if denominator == 0:
+            return 0.0
+        return numerator / denominator
+
+    def density(self, summary_base: bool = True) -> float:
+        numerator = sum(o.length**2 for o in self.overlaps())
+        denominator = len(self.summary) if summary_base else len(self.text)
+        if denominator == 0:
+            return 0.0
+        return numerator / denominator
+
+    def compression(self, text_to_summary: bool = True) -> float:
+        ratio = [len(self.text), len(self.summary)]
+        try:
+            if text_to_summary:
+                return ratio[0] / ratio[1]
+            return ratio[1] / ratio[0]
+        except ZeroDivisionError:
+            return 0.0
+
+    def _match(self, a: list[str], b: list[str]) -> None:
+        a_start = b_start = 0
+        while a_start < len(a):
+            best_match = None
+            best_match_length = 0
+            while b_start < len(b):
+                if a[a_start] == b[b_start]:
+                    a_end = a_start
+                    b_end = b_start
+                    while a_end < len(a) and b_end < len(b) and b[b_end] == a[a_end]:
+                        b_end += 1
+                        a_end += 1
+                    length = a_end - a_start
+                    if length > best_match_length:
+                        best_match = Fragments.Match(a_start, b_start, length)
+                        best_match_length = length
+                    b_start = b_end
+                else:
+                    b_start += 1
+            b_start = 0
+            if best_match:
+                if best_match_length > 0:
+                    self._matches.append(best_match)
+                a_start += best_match_length
+            else:
+                a_start += 1
diff --git a/src/helm/benchmark/metrics/summarization_metrics.py b/src/helm/benchmark/metrics/summarization_metrics.py
index 6c4d0d5465d..0f3731467e9 100644
--- a/src/helm/benchmark/metrics/summarization_metrics.py
+++ b/src/helm/benchmark/metrics/summarization_metrics.py
@@ -20,6 +20,7 @@
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
+from helm.benchmark.metrics.data_stats import DataStatsMetric
 from helm.benchmark.metrics.summac.model_summac import SummaCZS
 
 try:
@@ -67,11 +68,6 @@ def __init__(
         if not spacy.util.is_package("en_core_web_sm"):
             spacy.cli.download("en_core_web_sm")
 
-        try:
-            from summ_eval.data_stats_metric import DataStatsMetric  # type: ignore
-        except ModuleNotFoundError as e:
-            handle_module_not_found_error(e, ["summarization"])
-
         self.data_stats_metric = DataStatsMetric()
         self.task: str = task
         self.qa_fact_eval: Optional[Dict] = None

From 1903b8550fd1b3861193ddcf5ab94ba7418eb5b2 Mon Sep 17 00:00:00 2001
From: Iulian Igas <iulian@pacific.ai>
Date: Wed, 27 May 2026 10:39:42 +0300
Subject: [PATCH 2/4] fix(ci): resolve openai-whisper build failure on Python
 3.10

Use frozen uv sync in CI so the audiolm extra is not re-resolved during
install, and pin setuptools<82 for openai-whisper when updating the lockfile.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 .github/workflows/test.yml                 |   8 +-
 .github/workflows/test_scenarios.yml       |   8 +-
 .github/workflows/update-requirements.yml  |  14 +-
 .github/workflows/upgrade-requirements.yml |  12 +-
 pyproject.toml                             |   4 +-
 uv.lock                                    | 207 +--------------------
 6 files changed, 26 insertions(+), 227 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 1968517cc16..30dd6079061 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -56,12 +56,10 @@ jobs:
         with:
           version: "0.9.4"
           enable-cache: true
-      - name: Create build env with setuptools (for openai-whisper)
-        run: |
-          python -m venv .build-venv
-          .build-venv/bin/pip install setuptools wheel --quiet
       - name: Install dependencies
-        run: UV_PYTHON=".build-venv/bin/python" uv sync --extra ci --no-build-isolation-package openai-whisper
+        env:
+          UV_PREVIEW_FEATURES: extra-build-dependencies
+        run: uv sync --extra ci --frozen
       - name: Run pre-commit
         run: ./pre-commit.sh
       - name: Run tests
diff --git a/.github/workflows/test_scenarios.yml b/.github/workflows/test_scenarios.yml
index 299ee730dc0..54424ff4f6c 100644
--- a/.github/workflows/test_scenarios.yml
+++ b/.github/workflows/test_scenarios.yml
@@ -30,12 +30,10 @@ jobs:
         with:
           version: "0.9.4"
           enable-cache: true
-      - name: Create build env with setuptools (for openai-whisper)
-        run: |
-          python -m venv .build-venv
-          .build-venv/bin/pip install setuptools wheel --quiet
       - name: Install dependencies
-        run: UV_PYTHON=".build-venv/bin/python" uv sync --extra ci --extra scenarios --no-build-isolation-package openai-whisper
+        env:
+          UV_PREVIEW_FEATURES: extra-build-dependencies
+        run: uv sync --extra ci --extra scenarios --frozen
       - name: Run scenario tests
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
diff --git a/.github/workflows/update-requirements.yml b/.github/workflows/update-requirements.yml
index c3b1ba4401d..6c6107c33b3 100644
--- a/.github/workflows/update-requirements.yml
+++ b/.github/workflows/update-requirements.yml
@@ -40,14 +40,14 @@ jobs:
         with:
           version: "0.9.4"
           enable-cache: true
-      - name: Create build env with setuptools (for openai-whisper)
-        run: |
-          python -m venv .build-venv
-          .build-venv/bin/pip install setuptools wheel --quiet
       - name: Lock dependencies
-        run: UV_PYTHON=".build-venv/bin/python" uv lock --no-build-isolation-package openai-whisper
+        env:
+          UV_PREVIEW_FEATURES: extra-build-dependencies
+        run: uv lock
       - name: Install dependencies
-        run: UV_PYTHON=".build-venv/bin/python" uv sync --extra ci --no-build-isolation-package openai-whisper
+        env:
+          UV_PREVIEW_FEATURES: extra-build-dependencies
+        run: uv sync --extra ci --frozen
       - name: Write requirements.txt
         run: uv pip freeze --exclude-editable > requirements.txt
         # Need to manually run tests here because the pull request opened later will not
@@ -66,8 +66,6 @@ jobs:
         run: uv run pytest
       - name: Run helm-run
         run: uv run helm-run --suite test --run-entries simple1:model=simple/model1 --max-eval-instances 10 --exit-on-error
-      - name: Remove build venv before PR (do not commit)
-        run: rm -rf .build-venv
       - name: Create pull request
         uses: peter-evans/create-pull-request@v6
         with:
diff --git a/.github/workflows/upgrade-requirements.yml b/.github/workflows/upgrade-requirements.yml
index 6c6f44c9a96..d54870b8484 100644
--- a/.github/workflows/upgrade-requirements.yml
+++ b/.github/workflows/upgrade-requirements.yml
@@ -28,14 +28,14 @@ jobs:
         with:
           version: "0.9.4"
           enable-cache: true
-      - name: Create build env with setuptools (for openai-whisper)
-        run: |
-          python -m venv .build-venv
-          .build-venv/bin/pip install setuptools wheel --quiet
       - name: Upgrade dependencies
-        run: UV_PYTHON=".build-venv/bin/python" uv lock --upgrade --no-build-isolation-package openai-whisper
+        env:
+          UV_PREVIEW_FEATURES: extra-build-dependencies
+        run: uv lock --upgrade
       - name: Install dependencies
-        run: UV_PYTHON=".build-venv/bin/python" uv sync --extra ci --no-build-isolation-package openai-whisper
+        env:
+          UV_PREVIEW_FEATURES: extra-build-dependencies
+        run: uv sync --extra ci --frozen
       - name: Write requirements.txt
         run: uv pip freeze --exclude-editable > requirements.txt
         # Need to manually run tests here because the pull request opened later will not
diff --git a/pyproject.toml b/pyproject.toml
index 44fe5db6c93..cfd14ac0fb8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -479,9 +479,9 @@ conflicts = [
     ],
 ]
 
-# openai-whisper build needs setuptools + wheel; it doesn't declare them (CI uses --no-build-isolation-package + venv with both)
+# openai-whisper setup.py imports pkg_resources; needs setuptools<82 (pkg_resources removed in 82+)
 [tool.uv.extra-build-dependencies]
-openai-whisper = ["setuptools", "wheel"]
+openai-whisper = ["setuptools<82", "wheel"]
 
 [tool.setuptools]
 package-dir = { "" = "src" }
diff --git a/uv.lock b/uv.lock
index 620636cfa4b..f39cc00e862 100644
--- a/uv.lock
+++ b/uv.lock
@@ -19,10 +19,7 @@ conflicts = [[
 ]]
 
 [manifest]
-overrides = [
-    { name = "numpy", specifier = ">=1.26,<3" },
-    { name = "torch", specifier = "~=2.1" },
-]
+overrides = [{ name = "numpy", specifier = ">=1.26,<3" }]
 
 [[package]]
 name = "absl-py"
@@ -604,25 +601,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/4d/ea/31770a7e49f3eedfd8cd7b35e78b3a3aaad860400f8673994bc988318135/black-24.3.0-py3-none-any.whl", hash = "sha256:41622020d7120e01d377f74249e677039d20e6344ff5851de8a10f11f513bf93", size = 201493, upload-time = "2024-03-15T19:35:41.572Z" },
 ]
 
-[[package]]
-name = "blanc"
-version = "0.3.4"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "nltk" },
-    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-7-medhelm-human-evaluation' and extra == 'extra-7-medhelm-ibm')" },
-    { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-7-medhelm-human-evaluation' and extra == 'extra-7-medhelm-ibm')" },
-    { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-7-medhelm-human-evaluation' and extra == 'extra-7-medhelm-ibm')" },
-    { name = "scipy", version = "1.17.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-7-medhelm-human-evaluation' and extra == 'extra-7-medhelm-ibm')" },
-    { name = "torch" },
-    { name = "tqdm" },
-    { name = "transformers" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/12/31/c0a857c9baf4d7dd3dc879f0732e915f555281c38beee854bb6fcd36fcaf/blanc-0.3.4.tar.gz", hash = "sha256:008c10f45721bbb9ef771ac409e29b96bb90457f5c0ed7bb2960ef83a65a73be", size = 1763215, upload-time = "2024-01-04T20:01:26.974Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/d4/fd/caad0252df02b0aed73ea6f53953d2494136906e0137c2ead5226611a19a/blanc-0.3.4-py3-none-any.whl", hash = "sha256:777c703a554797051972a8eae213a94035d9884954041fa84d76fedd7df4a60d", size = 29989, upload-time = "2024-01-04T20:01:10.828Z" },
-]
-
 [[package]]
 name = "blis"
 version = "1.3.3"
@@ -1407,44 +1385,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/66/66/150e406a2db5535533aa3c946de58f0371f2e412e23f050c704588023e6e/cymem-2.0.13-cp314-cp314t-win_arm64.whl", hash = "sha256:e9027764dc5f1999fb4b4cabee1d0322c59e330c0a6485b436a68275f614277f", size = 39715, upload-time = "2025-11-14T14:58:24.773Z" },
 ]
 
-[[package]]
-name = "cython"
-version = "3.2.4"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/91/85/7574c9cd44b69a27210444b6650f6477f56c75fee1b70d7672d3e4166167/cython-3.2.4.tar.gz", hash = "sha256:84226ecd313b233da27dc2eb3601b4f222b8209c3a7216d8733b031da1dc64e6", size = 3280291, upload-time = "2026-01-04T14:14:14.473Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/a1/10/720e0fb84eab4c927c4dd6b61eb7993f7732dd83d29ba6d73083874eade9/cython-3.2.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb0cc0f23b9874ad262d7d2b9560aed9c7e2df07b49b920bda6f2cc9cb505e", size = 2960836, upload-time = "2026-01-04T14:14:51.103Z" },
-    { url = "https://files.pythonhosted.org/packages/7d/3d/b26f29092c71c36e0462752885bdfb18c23c176af4de953fdae2772a8941/cython-3.2.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f136f379a4a54246facd0eb6f1ee15c3837cb314ce87b677582ec014db4c6845", size = 3370134, upload-time = "2026-01-04T14:14:53.627Z" },
-    { url = "https://files.pythonhosted.org/packages/56/9e/539fb0d09e4f5251b5b14f8daf77e71fee021527f1013791038234618b6b/cython-3.2.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:35ab0632186057406ec729374c737c37051d2eacad9d515d94e5a3b3e58a9b02", size = 3537552, upload-time = "2026-01-04T14:14:56.852Z" },
-    { url = "https://files.pythonhosted.org/packages/10/c6/82d19a451c050d1be0f05b1a3302267463d391db548f013ee88b5348a8e9/cython-3.2.4-cp310-cp310-win_amd64.whl", hash = "sha256:ca2399dc75796b785f74fb85c938254fa10c80272004d573c455f9123eceed86", size = 2766191, upload-time = "2026-01-04T14:14:58.709Z" },
-    { url = "https://files.pythonhosted.org/packages/85/cc/8f06145ec3efa121c8b1b67f06a640386ddacd77ee3e574da582a21b14ee/cython-3.2.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ff9af2134c05e3734064808db95b4dd7341a39af06e8945d05ea358e1741aaed", size = 2953769, upload-time = "2026-01-04T14:15:00.361Z" },
-    { url = "https://files.pythonhosted.org/packages/55/b0/706cf830eddd831666208af1b3058c2e0758ae157590909c1f634b53bed9/cython-3.2.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:67922c9de058a0bfb72d2e75222c52d09395614108c68a76d9800f150296ddb3", size = 3243841, upload-time = "2026-01-04T14:15:02.066Z" },
-    { url = "https://files.pythonhosted.org/packages/ac/25/58893afd4ef45f79e3d4db82742fa4ff874b936d67a83c92939053920ccd/cython-3.2.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b362819d155fff1482575e804e43e3a8825332d32baa15245f4642022664a3f4", size = 3378083, upload-time = "2026-01-04T14:15:04.248Z" },
-    { url = "https://files.pythonhosted.org/packages/32/e4/424a004d7c0d8a4050c81846ebbd22272ececfa9a498cb340aa44fccbec2/cython-3.2.4-cp311-cp311-win_amd64.whl", hash = "sha256:1a64a112a34ec719b47c01395647e54fb4cf088a511613f9a3a5196694e8e382", size = 2769990, upload-time = "2026-01-04T14:15:06.53Z" },
-    { url = "https://files.pythonhosted.org/packages/91/4d/1eb0c7c196a136b1926f4d7f0492a96c6fabd604d77e6cd43b56a3a16d83/cython-3.2.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:64d7f71be3dd6d6d4a4c575bb3a4674ea06d1e1e5e4cd1b9882a2bc40ed3c4c9", size = 2970064, upload-time = "2026-01-04T14:15:08.567Z" },
-    { url = "https://files.pythonhosted.org/packages/03/1c/46e34b08bea19a1cdd1e938a4c123e6299241074642db9d81983cef95e9f/cython-3.2.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:869487ea41d004f8b92171f42271fbfadb1ec03bede3158705d16cd570d6b891", size = 3226757, upload-time = "2026-01-04T14:15:10.812Z" },
-    { url = "https://files.pythonhosted.org/packages/12/33/3298a44d201c45bcf0d769659725ae70e9c6c42adf8032f6d89c8241098d/cython-3.2.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:55b6c44cd30821f0b25220ceba6fe636ede48981d2a41b9bbfe3c7902ce44ea7", size = 3388969, upload-time = "2026-01-04T14:15:12.45Z" },
-    { url = "https://files.pythonhosted.org/packages/bb/f3/4275cd3ea0a4cf4606f9b92e7f8766478192010b95a7f516d1b7cf22cb10/cython-3.2.4-cp312-cp312-win_amd64.whl", hash = "sha256:767b143704bdd08a563153448955935844e53b852e54afdc552b43902ed1e235", size = 2756457, upload-time = "2026-01-04T14:15:14.67Z" },
-    { url = "https://files.pythonhosted.org/packages/18/b5/1cfca43b7d20a0fdb1eac67313d6bb6b18d18897f82dd0f17436bdd2ba7f/cython-3.2.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:28e8075087a59756f2d059273184b8b639fe0f16cf17470bd91c39921bc154e0", size = 2960506, upload-time = "2026-01-04T14:15:16.733Z" },
-    { url = "https://files.pythonhosted.org/packages/71/bb/8f28c39c342621047fea349a82fac712a5e2b37546d2f737bbde48d5143d/cython-3.2.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:03893c88299a2c868bb741ba6513357acd104e7c42265809fd58dce1456a36fc", size = 3213148, upload-time = "2026-01-04T14:15:18.804Z" },
-    { url = "https://files.pythonhosted.org/packages/7a/d2/16fa02f129ed2b627e88d9d9ebd5ade3eeb66392ae5ba85b259d2d52b047/cython-3.2.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f81eda419b5ada7b197bbc3c5f4494090e3884521ffd75a3876c93fbf66c9ca8", size = 3375764, upload-time = "2026-01-04T14:15:20.817Z" },
-    { url = "https://files.pythonhosted.org/packages/91/3f/deb8f023a5c10c0649eb81332a58c180fad27c7533bb4aae138b5bc34d92/cython-3.2.4-cp313-cp313-win_amd64.whl", hash = "sha256:83266c356c13c68ffe658b4905279c993d8a5337bb0160fa90c8a3e297ea9a2e", size = 2754238, upload-time = "2026-01-04T14:15:23.001Z" },
-    { url = "https://files.pythonhosted.org/packages/ee/d7/3bda3efce0c5c6ce79cc21285dbe6f60369c20364e112f5a506ee8a1b067/cython-3.2.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d4b4fd5332ab093131fa6172e8362f16adef3eac3179fd24bbdc392531cb82fa", size = 2971496, upload-time = "2026-01-04T14:15:25.038Z" },
-    { url = "https://files.pythonhosted.org/packages/89/ed/1021ffc80b9c4720b7ba869aea8422c82c84245ef117ebe47a556bdc00c3/cython-3.2.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e3b5ac54e95f034bc7fb07313996d27cbf71abc17b229b186c1540942d2dc28e", size = 3256146, upload-time = "2026-01-04T14:15:26.741Z" },
-    { url = "https://files.pythonhosted.org/packages/0c/51/ca221ec7e94b3c5dc4138dcdcbd41178df1729c1e88c5dfb25f9d30ba3da/cython-3.2.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f43be4eaa6afd58ce20d970bb1657a3627c44e1760630b82aa256ba74b4acb", size = 3383458, upload-time = "2026-01-04T14:15:28.425Z" },
-    { url = "https://files.pythonhosted.org/packages/79/2e/1388fc0243240cd54994bb74f26aaaf3b2e22f89d3a2cf8da06d75d46ca2/cython-3.2.4-cp314-cp314-win_amd64.whl", hash = "sha256:983f9d2bb8a896e16fa68f2b37866ded35fa980195eefe62f764ddc5f9f5ef8e", size = 2791241, upload-time = "2026-01-04T14:15:30.448Z" },
-    { url = "https://files.pythonhosted.org/packages/0a/8b/fd393f0923c82be4ec0db712fffb2ff0a7a131707b842c99bf24b549274d/cython-3.2.4-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:36bf3f5eb56d5281aafabecbaa6ed288bc11db87547bba4e1e52943ae6961ccf", size = 2875622, upload-time = "2026-01-04T14:15:39.749Z" },
-    { url = "https://files.pythonhosted.org/packages/73/48/48530d9b9d64ec11dbe0dd3178a5fe1e0b27977c1054ecffb82be81e9b6a/cython-3.2.4-cp39-abi3-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6d5267f22b6451eb1e2e1b88f6f78a2c9c8733a6ddefd4520d3968d26b824581", size = 3210669, upload-time = "2026-01-04T14:15:41.911Z" },
-    { url = "https://files.pythonhosted.org/packages/5e/91/4865fbfef1f6bb4f21d79c46104a53d1a3fa4348286237e15eafb26e0828/cython-3.2.4-cp39-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3b6e58f73a69230218d5381817850ce6d0da5bb7e87eb7d528c7027cbba40b06", size = 2856835, upload-time = "2026-01-04T14:15:43.815Z" },
-    { url = "https://files.pythonhosted.org/packages/fa/39/60317957dbef179572398253f29d28f75f94ab82d6d39ea3237fb6c89268/cython-3.2.4-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e71efb20048358a6b8ec604a0532961c50c067b5e63e345e2e359fff72feaee8", size = 2994408, upload-time = "2026-01-04T14:15:45.422Z" },
-    { url = "https://files.pythonhosted.org/packages/8d/30/7c24d9292650db4abebce98abc9b49c820d40fa7c87921c0a84c32f4efe7/cython-3.2.4-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:28b1e363b024c4b8dcf52ff68125e635cb9cb4b0ba997d628f25e32543a71103", size = 2891478, upload-time = "2026-01-04T14:15:47.394Z" },
-    { url = "https://files.pythonhosted.org/packages/86/70/03dc3c962cde9da37a93cca8360e576f904d5f9beecfc9d70b1f820d2e5f/cython-3.2.4-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:31a90b4a2c47bb6d56baeb926948348ec968e932c1ae2c53239164e3e8880ccf", size = 3225663, upload-time = "2026-01-04T14:15:49.446Z" },
-    { url = "https://files.pythonhosted.org/packages/b1/97/10b50c38313c37b1300325e2e53f48ea9a2c078a85c0c9572057135e31d5/cython-3.2.4-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e65e4773021f8dc8532010b4fbebe782c77f9a0817e93886e518c93bd6a44e9d", size = 3115628, upload-time = "2026-01-04T14:15:51.323Z" },
-    { url = "https://files.pythonhosted.org/packages/8f/b1/d6a353c9b147848122a0db370863601fdf56de2d983b5c4a6a11e6ee3cd7/cython-3.2.4-cp39-abi3-win32.whl", hash = "sha256:2b1f12c0e4798293d2754e73cd6f35fa5bbdf072bdc14bc6fc442c059ef2d290", size = 2437463, upload-time = "2026-01-04T14:15:53.787Z" },
-    { url = "https://files.pythonhosted.org/packages/2d/d8/319a1263b9c33b71343adfd407e5daffd453daef47ebc7b642820a8b68ed/cython-3.2.4-cp39-abi3-win_arm64.whl", hash = "sha256:3b8e62049afef9da931d55de82d8f46c9a147313b69d5ff6af6e9121d545ce7a", size = 2442754, upload-time = "2026-01-04T14:15:55.382Z" },
-    { url = "https://files.pythonhosted.org/packages/ff/fa/d3c15189f7c52aaefbaea76fb012119b04b9013f4bf446cb4eb4c26c4e6b/cython-3.2.4-py3-none-any.whl", hash = "sha256:732fc93bc33ae4b14f6afaca663b916c2fdd5dcbfad7114e17fb2434eeaea45c", size = 1257078, upload-time = "2026-01-04T14:14:12.373Z" },
-]
-
 [[package]]
 name = "dacite"
 version = "1.9.2"
@@ -1647,15 +1587,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/78/01/8da8dd078b354a89602a875d310a0d725dad92b5b4d61069576e0a0e02e4/einops_exts-0.0.4-py3-none-any.whl", hash = "sha256:6d310a4c858e459ebff8288580f90255d354cfa3bde22a53b59baae64b48cb95", size = 3925, upload-time = "2023-01-05T20:02:42.796Z" },
 ]
 
-[[package]]
-name = "emoji"
-version = "2.15.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/a2/78/0d2db9382c92a163d7095fc08efff7800880f830a152cfced40161e7638d/emoji-2.15.0.tar.gz", hash = "sha256:eae4ab7d86456a70a00a985125a03263a5eac54cd55e51d7e184b1ed3b6757e4", size = 615483, upload-time = "2025-09-21T12:13:02.755Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/e1/5e/4b5aaaabddfacfe36ba7768817bd1f71a7a810a43705e531f3ae4c690767/emoji-2.15.0-py3-none-any.whl", hash = "sha256:205296793d66a89d88af4688fa57fd6496732eb48917a87175a023c8138995eb", size = 608433, upload-time = "2025-09-21T12:13:01.197Z" },
-]
-
 [[package]]
 name = "en-core-web-sm"
 version = "3.8.0"
@@ -2208,15 +2139,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/88/dc/2bc81a01caa887ed58db3c725bebf1e98f37807a4d06c51ecaa85a7cabe0/gepa-0.0.17-py3-none-any.whl", hash = "sha256:0ea98f4179dbc8dd83bdf53494f302e663ee1da8300d086c4cc8ce4aefa4042c", size = 110464, upload-time = "2025-09-25T22:13:44.14Z" },
 ]
 
-[[package]]
-name = "gin-config"
-version = "0.5.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/b9/c4/5aaf77a77e7a2a1446f20580f8493adea9dcd3befcf61faf7ba5de65d7fc/gin-config-0.5.0.tar.gz", hash = "sha256:0c6ea5026ded927c8c93c990b01c695257c1df446e45e549a158cfbc79e19ed6", size = 56185, upload-time = "2021-11-03T08:11:34.812Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/11/8f/9f1894efa1bb15e98613244b24dfbacfe2309e0ac3cfc27d4c608c2270d2/gin_config-0.5.0-py3-none-any.whl", hash = "sha256:bddb7ca221ea2b46cdb59321e79fecf02d6e3b728906047fcd4076c297609fd6", size = 61327, upload-time = "2021-11-03T08:11:33.435Z" },
-]
-
 [[package]]
 name = "gitdb"
 version = "4.0.12"
@@ -4152,8 +4074,8 @@ dependencies = [
     { name = "nltk" },
     { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-7-medhelm-human-evaluation' and extra == 'extra-7-medhelm-ibm')" },
     { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-7-medhelm-human-evaluation' and extra == 'extra-7-medhelm-ibm')" },
-    { name = "pandas" },
     { name = "openpyxl" },
+    { name = "pandas" },
     { name = "pyarrow" },
     { name = "pyarrow-hotfix" },
     { name = "pyhocon" },
@@ -4234,6 +4156,7 @@ all = [
     { name = "opencc" },
     { name = "opencv-python" },
     { name = "opencv-python-headless" },
+    { name = "openpyxl" },
     { name = "pdf2image" },
     { name = "pillow" },
     { name = "pycocoevalcap" },
@@ -4506,7 +4429,6 @@ summarization = [
     { name = "protobuf" },
     { name = "rouge-score" },
     { name = "sentencepiece" },
-    { name = "summ-eval" },
 ]
 together = [
     { name = "together" },
@@ -4576,7 +4498,7 @@ requires-dist = [
     { name = "av", marker = "extra == 'audiolm'", specifier = "~=14.3" },
     { name = "awscli", marker = "extra == 'amazon'", specifier = "~=1.33" },
     { name = "bert-score", marker = "extra == 'medhelm'", specifier = "~=0.3.13" },
-    { name = "bert-score", marker = "extra == 'summarization'", specifier = "~=0.3" },
+    { name = "bert-score", marker = "extra == 'summarization'", specifier = "~=0.3.13" },
     { name = "boto3", marker = "extra == 'amazon'", specifier = "~=1.34" },
     { name = "botocore", marker = "extra == 'amazon'", specifier = "~=1.34" },
     { name = "bottle", specifier = "~=0.12.23" },
@@ -4698,11 +4620,11 @@ requires-dist = [
     { name = "opencv-python", marker = "python_full_version >= '3.10' and extra == 'heim'", specifier = ">=4.7.0.68,<4.8.2.0" },
     { name = "opencv-python-headless", marker = "python_full_version < '3.10' and extra == 'heim'", specifier = ">=4.7.0.68,<=4.11.0.86" },
     { name = "opencv-python-headless", marker = "extra == 'image2struct'", specifier = ">=4.7.0.68,<=4.11.0.86" },
+    { name = "openpyxl", specifier = "~=3.1" },
     { name = "openpyxl", marker = "extra == 'ibm-enterprise-scenarios'", specifier = "~=3.1" },
     { name = "openpyxl", marker = "extra == 'medhelm'", specifier = "~=3.1" },
     { name = "openpyxl", marker = "extra == 'scenarios'", specifier = "~=3.1" },
     { name = "pandas", specifier = "~=2.0" },
-    { name = "openpyxl", specifier = "~=3.1" },
     { name = "pdf2image", marker = "extra == 'image2struct'", specifier = "~=1.16" },
     { name = "pillow", marker = "extra == 'images'", specifier = ">=10.2" },
     { name = "protobuf", marker = "extra == 'summarization'" },
@@ -4713,12 +4635,12 @@ requires-dist = [
     { name = "pydantic", marker = "extra == 'openai'", specifier = "~=2.0" },
     { name = "pydub", marker = "extra == 'audiolm'", specifier = "~=0.25.1" },
     { name = "pyhocon", specifier = "~=0.3.59" },
-    { name = "python-docx", specifier = "~=1.1" },
     { name = "pymongo", marker = "extra == 'mongo'", specifier = "~=4.2" },
     { name = "pyonmttok", marker = "extra == 'seahelm'", specifier = "==1.37.0" },
     { name = "pypinyin", marker = "extra == 'cleva'", specifier = "~=0.49.0" },
     { name = "pythainlp", marker = "extra == 'seahelm'", specifier = "==5.0.0" },
     { name = "python-crfsuite", marker = "extra == 'seahelm'", specifier = "~=0.9.11" },
+    { name = "python-docx", specifier = "~=1.1" },
     { name = "python-docx", marker = "extra == 'medhelm'", specifier = "~=1.1" },
     { name = "pytorch-fid", marker = "extra == 'heim'", specifier = "~=0.3.0" },
     { name = "pytorch-lightning", marker = "extra == 'heim'", specifier = "~=2.0" },
@@ -4746,7 +4668,6 @@ requires-dist = [
     { name = "soundfile", marker = "extra == 'audiolm'", specifier = "~=0.12" },
     { name = "spacy", specifier = "~=3.5" },
     { name = "sqlitedict", specifier = ">=2.1.0,<3.0" },
-    { name = "summ-eval", marker = "extra == 'summarization'", specifier = "~=0.892" },
     { name = "surge-api", marker = "extra == 'human-evaluation'", specifier = "~=1.1" },
     { name = "tensorflow", marker = "extra == 'heim'", specifier = "~=2.11" },
     { name = "tiktoken", marker = "extra == 'openai'", specifier = "~=0.7" },
@@ -4873,16 +4794,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/a4/8e/469e5a4a2f5855992e425f3cb33804cc07bf18d48f2db061aec61ce50270/more_itertools-10.8.0-py3-none-any.whl", hash = "sha256:52d4362373dcf7c52546bc4af9a86ee7c4579df9a8dc268be0a2f949d376cc9b", size = 69667, upload-time = "2025-09-02T15:23:09.635Z" },
 ]
 
-[[package]]
-name = "moverscore"
-version = "1.0.3"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "portalocker" },
-    { name = "typing" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/d7/66/5ac942903a4d2d316f6c1df716e047e34fb0f9bfc7e70d44dab995708e9d/moverscore-1.0.3.tar.gz", hash = "sha256:99be483d512d4548a2191543f8c7dd6727d83163f92ded2d1958657b177479b8", size = 7665, upload-time = "2020-04-09T00:13:03.15Z" }
-
 [[package]]
 name = "mpmath"
 version = "1.3.0"
@@ -7163,16 +7074,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/a6/53/d78dc063216e62fc55f6b2eebb447f6a4b0a59f55c8406376f76bf959b08/pydub-0.25.1-py2.py3-none-any.whl", hash = "sha256:65617e33033874b59d87db603aa1ed450633288aefead953b30bded59cb599a6", size = 32327, upload-time = "2021-03-10T02:09:53.503Z" },
 ]
 
-[[package]]
-name = "pyemd"
-version = "0.5.1"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-7-medhelm-human-evaluation' and extra == 'extra-7-medhelm-ibm')" },
-    { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-7-medhelm-human-evaluation' and extra == 'extra-7-medhelm-ibm')" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/c0/c5/7fea8e7a71cd026b30ed3c40e4c5ea13a173e28f8855da17e25271e8f545/pyemd-0.5.1.tar.gz", hash = "sha256:fc81c2116f8573e559dfbb8d73e03d9f73c22d0770559f406516984302e07e70", size = 91498, upload-time = "2018-01-29T04:13:45.996Z" }
-
 [[package]]
 name = "pyflakes"
 version = "3.4.0"
@@ -7526,24 +7427,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/77/eb/cc6dbfe70d15318dbce82674b1e8057cef2634ca9f9121a16b8a06c630db/pytorch_lightning-2.6.0-py3-none-any.whl", hash = "sha256:ee72cff4b8c983ecfaae8599382544bd5236d9eb300adc7dd305f359195f4e79", size = 849476, upload-time = "2025-11-28T09:34:11.271Z" },
 ]
 
-[[package]]
-name = "pytorch-pretrained-bert"
-version = "0.6.2"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "boto3" },
-    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-7-medhelm-human-evaluation' and extra == 'extra-7-medhelm-ibm')" },
-    { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-7-medhelm-human-evaluation' and extra == 'extra-7-medhelm-ibm')" },
-    { name = "regex" },
-    { name = "requests" },
-    { name = "torch" },
-    { name = "tqdm" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/62/99/32f994e64c1f3c23f73ccff01ae7324ecd82ac2e5f16657940cb72832a06/pytorch_pretrained_bert-0.6.2.tar.gz", hash = "sha256:9cf7c6221e854071b9844f2a9a581e05a24777351618c010493d9c76601c6747", size = 127408, upload-time = "2019-04-25T19:42:04.571Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/d7/e0/c08d5553b89973d9a240605b9c12404bcf8227590de62bae27acbcfe076b/pytorch_pretrained_bert-0.6.2-py3-none-any.whl", hash = "sha256:ddd86f2f4ca595a2ad05d48c35d4f5c1c5f150bef79d56e3159701840a574d06", size = 123793, upload-time = "2019-04-25T19:42:02.054Z" },
-]
-
 [[package]]
 name = "pytrec-eval"
 version = "0.5"
@@ -8250,21 +8133,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/06/f2/6c90ccf3ad1d09a7d662a405b274f3c93b92df59c8d6a025d26aaf34d302/sacrebleu-2.6.0-py3-none-any.whl", hash = "sha256:3edc1531575cfe4ad04ce53491a9307e234af1c3f805a1f491cbec844229a8a8", size = 100785, upload-time = "2026-01-12T17:17:18.868Z" },
 ]
 
-[[package]]
-name = "sacremoses"
-version = "0.1.1"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "click" },
-    { name = "joblib" },
-    { name = "regex" },
-    { name = "tqdm" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/1d/51/fbdc4af4f6e85d26169e28be3763fe50ddfd0d4bf8b871422b0788dcc4d2/sacremoses-0.1.1.tar.gz", hash = "sha256:b6fd5d3a766b02154ed80b962ddca91e1fd25629c0978c7efba21ebccf663934", size = 883188, upload-time = "2023-10-30T15:56:20.187Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/0b/f0/89ee2bc9da434bd78464f288fdb346bc2932f2ee80a90b2a4bbbac262c74/sacremoses-0.1.1-py3-none-any.whl", hash = "sha256:31e04c98b169bfd902144824d191825cd69220cdb4ae4bcf1ec58a7db5587b1a", size = 897476, upload-time = "2023-10-30T15:56:18.121Z" },
-]
-
 [[package]]
 name = "safetensors"
 version = "0.7.0"
@@ -9261,60 +9129,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/34/ae/e3707f6c1bc6f7aa0df600ba8075bfb8a19252140cd595335be60e25f9ee/standard_sunau-3.13.0-py3-none-any.whl", hash = "sha256:53af624a9529c41062f4c2fd33837f297f3baa196b0cfceffea6555654602622", size = 7364, upload-time = "2024-10-30T16:01:28.003Z" },
 ]
 
-[[package]]
-name = "stanza"
-version = "1.11.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "emoji" },
-    { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-7-medhelm-human-evaluation' and extra == 'extra-7-medhelm-ibm')" },
-    { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-7-medhelm-human-evaluation' and extra == 'extra-7-medhelm-ibm')" },
-    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-7-medhelm-human-evaluation' and extra == 'extra-7-medhelm-ibm')" },
-    { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-7-medhelm-human-evaluation' and extra == 'extra-7-medhelm-ibm')" },
-    { name = "protobuf" },
-    { name = "requests" },
-    { name = "tomli", marker = "python_full_version < '3.11' or (extra == 'extra-7-medhelm-human-evaluation' and extra == 'extra-7-medhelm-ibm')" },
-    { name = "torch" },
-    { name = "tqdm" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/07/e5/acd22862a75f424d98bb690fec9ab292da6e797cab367fa8fa451c547637/stanza-1.11.0.tar.gz", hash = "sha256:42ba9d4752e74c4e1e6fc2ca96e98bb8fa194049782cc35fde2a5118fd5f75ab", size = 1484551, upload-time = "2025-10-05T06:44:03.665Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/e0/47/c6f8dd24ca100f6c260209b27be4d2e0ae68f13d4b2b4b1b343876c9e765/stanza-1.11.0-py3-none-any.whl", hash = "sha256:3a0bcf24830e32e88f6d0cff1e757661e53ed1b60149fa7f72211d61c6dab063", size = 1706081, upload-time = "2025-10-05T06:43:59.247Z" },
-]
-
-[[package]]
-name = "summ-eval"
-version = "0.892"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "bert-score" },
-    { name = "blanc" },
-    { name = "click" },
-    { name = "cython" },
-    { name = "gin-config" },
-    { name = "moverscore" },
-    { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-7-medhelm-human-evaluation' and extra == 'extra-7-medhelm-ibm')" },
-    { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-7-medhelm-human-evaluation' and extra == 'extra-7-medhelm-ibm')" },
-    { name = "nltk" },
-    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-7-medhelm-human-evaluation' and extra == 'extra-7-medhelm-ibm')" },
-    { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-7-medhelm-human-evaluation' and extra == 'extra-7-medhelm-ibm')" },
-    { name = "psutil" },
-    { name = "pyemd" },
-    { name = "pytorch-pretrained-bert" },
-    { name = "sacrebleu" },
-    { name = "sacremoses" },
-    { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-7-medhelm-human-evaluation' and extra == 'extra-7-medhelm-ibm')" },
-    { name = "scipy", version = "1.17.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-7-medhelm-human-evaluation' and extra == 'extra-7-medhelm-ibm')" },
-    { name = "six" },
-    { name = "spacy" },
-    { name = "stanza" },
-    { name = "transformers" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/ce/bd/cda7a3e42e0f9bd2cea98a3ccf39af467569f12134cdfaee1907a958491c/summ_eval-0.892.tar.gz", hash = "sha256:bb03541fb50a0670731501d88baebbaed0e6b35f629a57a545fcb339dfb9a014", size = 80469, upload-time = "2022-04-05T13:48:54.31Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/c0/aa/409b5b443ea0514c6174aefb26fdfba2de1309115077dc7b23762f50a8bf/summ_eval-0.892-py3-none-any.whl", hash = "sha256:3855e7d05956586c24ebe27a33513b103aca5766ebd5edf8690645c61b98fd61", size = 111895, upload-time = "2022-04-05T13:48:52.554Z" },
-]
-
 [[package]]
 name = "surge-api"
 version = "1.5.10"
@@ -10207,15 +10021,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/5e/86/a9ebfd509cbe74471106dffed320e208c72537f9aeb0a55eaa6b1b5e4d17/types_tabulate-0.9.0.20241207-py3-none-any.whl", hash = "sha256:b8dad1343c2a8ba5861c5441370c3e35908edd234ff036d4298708a1d4cf8a85", size = 8307, upload-time = "2024-12-07T02:54:41.031Z" },
 ]
 
-[[package]]
-name = "typing"
-version = "3.10.0.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/b0/1b/835d4431805939d2996f8772aca1d2313a57e8860fec0e48e8e7dfe3a477/typing-3.10.0.0.tar.gz", hash = "sha256:13b4ad211f54ddbf93e5901a9967b1e07720c1d1b78d596ac6a439641aa1b130", size = 78962, upload-time = "2021-05-01T18:03:58.186Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/f2/5d/865e17349564eb1772688d8afc5e3081a5964c640d64d1d2880ebaed002d/typing-3.10.0.0-py3-none-any.whl", hash = "sha256:12fbdfbe7d6cca1a42e485229afcb0b0c8259258cfb919b8a5e2a5c953742f89", size = 26320, upload-time = "2021-05-01T18:03:56.398Z" },
-]
-
 [[package]]
 name = "typing-extensions"
 version = "4.15.0"

From 9bb8af566a5de8ebe27f18784c9986f9f2a022a3 Mon Sep 17 00:00:00 2001
From: Iulian Igas <iulian@pacific.ai>
Date: Wed, 27 May 2026 10:47:39 +0300
Subject: [PATCH 3/4] fix(metrics): satisfy mypy in vendored DataStatsMetric

Use separate tokenized variables instead of reassigning str parameters.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 .../metrics/data_stats/data_stats_metric.py     | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/helm/benchmark/metrics/data_stats/data_stats_metric.py b/src/helm/benchmark/metrics/data_stats/data_stats_metric.py
index 4a2d98ff5cd..0ec9f3c7018 100644
--- a/src/helm/benchmark/metrics/data_stats/data_stats_metric.py
+++ b/src/helm/benchmark/metrics/data_stats/data_stats_metric.py
@@ -2,7 +2,7 @@
 # Vendored so medhelm[summarization] does not depend on summ-eval (pyemd==0.5.1, torch<2.0).
 
 from collections import Counter
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
 
 import spacy
 
@@ -31,11 +31,20 @@ def __init__(self, n_gram: int = 3, case: bool = False, tokenize: bool = True):
         self.tokenize = tokenize
 
     def evaluate_example(self, summary: str, input_text: str) -> Dict[str, float]:
+        text_for_fragments: Union[str, List[str]]
+        summary_for_fragments: Union[str, List[str]]
         if self.tokenize:
             nlp = _get_spacy_en()
-            input_text = [tok.text for tok in nlp(input_text, disable=["tagger", "parser", "ner", "textcat"])]
-            summary = [tok.text for tok in nlp(summary, disable=["tagger", "parser", "ner", "textcat"])]
-        fragments = Fragments(summary, input_text, case=self.case)
+            text_for_fragments = [
+                tok.text for tok in nlp(input_text, disable=["tagger", "parser", "ner", "textcat"])
+            ]
+            summary_for_fragments = [
+                tok.text for tok in nlp(summary, disable=["tagger", "parser", "ner", "textcat"])
+            ]
+        else:
+            text_for_fragments = input_text
+            summary_for_fragments = summary
+        fragments = Fragments(summary_for_fragments, text_for_fragments, case=self.case)
         score_dict: Dict[str, float] = {
             "coverage": fragments.coverage(),
             "density": fragments.density(),

From ea3b6d3ccb987437e089d785fbc777a1d530a708 Mon Sep 17 00:00:00 2001
From: Iulian Igas <iulian@pacific.ai>
Date: Wed, 27 May 2026 10:50:27 +0300
Subject: [PATCH 4/4] style(metrics): apply black formatting to DataStatsMetric

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 .../benchmark/metrics/data_stats/data_stats_metric.py     | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/helm/benchmark/metrics/data_stats/data_stats_metric.py b/src/helm/benchmark/metrics/data_stats/data_stats_metric.py
index 0ec9f3c7018..38e467151f0 100644
--- a/src/helm/benchmark/metrics/data_stats/data_stats_metric.py
+++ b/src/helm/benchmark/metrics/data_stats/data_stats_metric.py
@@ -35,12 +35,8 @@ def evaluate_example(self, summary: str, input_text: str) -> Dict[str, float]:
         summary_for_fragments: Union[str, List[str]]
         if self.tokenize:
             nlp = _get_spacy_en()
-            text_for_fragments = [
-                tok.text for tok in nlp(input_text, disable=["tagger", "parser", "ner", "textcat"])
-            ]
-            summary_for_fragments = [
-                tok.text for tok in nlp(summary, disable=["tagger", "parser", "ner", "textcat"])
-            ]
+            text_for_fragments = [tok.text for tok in nlp(input_text, disable=["tagger", "parser", "ner", "textcat"])]
+            summary_for_fragments = [tok.text for tok in nlp(summary, disable=["tagger", "parser", "ner", "textcat"])]
         else:
             text_for_fragments = input_text
             summary_for_fragments = summary